<commit_before>package httpclient_test

import (
	"net/http"
	"net/url"
	"path"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"gnd.la/net/httpclient"
	"gnd.la/util/urlutil"

	"github.com/elazarl/goproxy"
)

const (
	httpBin = "http://httpbin.org"
)

func testUserAgent(t *testing.T, c *httpclient.Client, exp string) {
	ep := urlutil.MustJoin(httpBin, "/user-agent")
	resp, err := c.Get(ep)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Close()
	var m map[string]interface{}
	if err := resp.JSONDecode(&m); err != nil {
		t.Fatal(err)
	}
	ua := m["user-agent"].(string)
	if idx := strings.Index(ua, " AppEngine-Google"); idx >= 0 {
		ua = ua[:idx]
	}
	if ua != exp {
		t.Errorf("expecting User-Agent %q, got %q instead", exp, ua)
	}
}

func TestUserAgent(t *testing.T) {
	const ua = "Gondolier"
	c := httpclient.New(nil)
	testUserAgent(t, c, httpclient.DefaultUserAgent)
	c.SetUserAgent(ua)
	testUserAgent(t, c, ua)
}

func decodeArgs(resp *httpclient.Response) (map[string]string, error) {
	var m map[string]interface{}
	if err := resp.JSONDecode(&m); err != nil {
		return nil, err
	}
	var args map[string]interface{}
	if strings.HasSuffix(resp.Request.URL.Path, "post") {
		args = m["form"].(map[string]interface{})
	} else {
		args = m["args"].(map[string]interface{})
	}
	values := make(map[string]string, len(args))
	for k, v := range args {
		values[k] = v.(string)
	}
	return values, nil
}

func testForm(t *testing.T, f func(string, url.Values) (*httpclient.Response, error), u string, data map[string]string, exp map[string]string) {
	form := make(url.Values)
	for k, v := range data {
		form.Add(k, v)
	}
	resp, err := f(u, form)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Close()
	args, err := decodeArgs(resp)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(args, exp) {
		t.Errorf("expecting values %v, got %v instead", exp, args)
	}
}

func TestGetForm(t *testing.T) {
	data := map[string]string{"a": "b", "c": "d"}
	f := httpclient.New(nil).GetForm
	testForm(t, f, urlutil.MustJoin(httpBin, "/get"), data, data)
	expect := map[string]string{"e": "f"}
	for k, v := range data {
		expect[k] = v
	}
	testForm(t, f, urlutil.MustJoin(httpBin, "/get?e=f"), data, expect)
}

func TestPostForm(t *testing.T) {
	data := map[string]string{"a": "b", "c": "d"}
	testForm(t, httpclient.New(nil).PostForm, urlutil.MustJoin(httpBin, "/post"), data, data)
}

func redirNumber(t *testing.T, url string) int {
	b := path.Base(url)
	val, err := strconv.Atoi(b)
	if err != nil {
		t.Fatal(err)
	}
	return val
}

func TestRedirect(t *testing.T) {
	start := urlutil.MustJoin(httpBin, "/redirect/6")
	end := urlutil.MustJoin(httpBin, "/get")
	c := httpclient.New(nil)
	resp, err := c.Get(start)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Close()
	if u := resp.Request.URL.String(); u != end {
		t.Errorf("expecting final url %q, got %q instead", end, u)
	}
	cur := redirNumber(t, start)
	next := start
	for {
		req, err := http.NewRequest("GET", next, nil)
		if err != nil {
			t.Fatal(err)
		}
		resp, err := c.Trip(req)
		if err != nil {
			t.Fatal(err)
		}
		if cur > 0 {
			redir, err := resp.Redirect()
			if err != nil {
				t.Fatal(err)
			}
			cur--
			if cur > 0 {
				rn := redirNumber(t, redir)
				if rn != cur {
					t.Fatalf("expecting redirect %d, got %d instead", cur, rn)
				}
			}
			next = redir
		} else {
			if resp.IsRedirect() {
				t.Error("unexpected redirect")
			}
			if u := resp.Request.URL.String(); u != end {
				t.Errorf("expecting final url %q, got %q instead", end, u)
			}
			break
		}
	}
}

func TestProxy(t *testing.T) {
	c := httpclient.New(nil)
	if !c.SupportsProxy() {
		t.Skipf("current run environment does not support support proxies")
	}
	const addr = "127.0.0.1:12345"
	proxy := goproxy.NewProxyHttpServer()
	count := 0
	proxy.OnRequest().DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			count++
			return r, nil
		})
	go func() {
		http.ListenAndServe(addr, proxy)
	}()
	time.Sleep(time.Millisecond)
	c.SetProxy(func(_ *http.Request) (*url.URL, error) {
		return url.Parse("http://" + addr)
	})
	testUserAgent(t, c, httpclient.DefaultUserAgent)
	if count != 1 {
		t.Errorf("expecting 1 proxy request, got %d instead", count)
	}
}
<commit_msg>Change the test to make sure Clone() keeps the client's settings<commit_after>package httpclient_test

import (
	"net/http"
	"net/url"
	"path"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"

	"gnd.la/net/httpclient"
	"gnd.la/util/urlutil"

	"github.com/elazarl/goproxy"
)

const (
	httpBin = "http://httpbin.org"
)

func testUserAgent(t *testing.T, c *httpclient.Client, exp string) {
	ep := urlutil.MustJoin(httpBin, "/user-agent")
	resp, err := c.Get(ep)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Close()
	var m map[string]interface{}
	if err := resp.JSONDecode(&m); err != nil {
		t.Fatal(err)
	}
	ua := m["user-agent"].(string)
	if idx := strings.Index(ua, " AppEngine-Google"); idx >= 0 {
		ua = ua[:idx]
	}
	if ua != exp {
		t.Errorf("expecting User-Agent %q, got %q instead", exp, ua)
	}
}

func TestUserAgent(t *testing.T) {
	const ua = "Gondolier"
	c := httpclient.New(nil)
	testUserAgent(t, c, httpclient.DefaultUserAgent)
	c.SetUserAgent(ua)
	testUserAgent(t, c, ua)
	testUserAgent(t, c.Clone(nil), ua)
}

func decodeArgs(resp *httpclient.Response) (map[string]string, error) {
	var m map[string]interface{}
	if err := resp.JSONDecode(&m); err != nil {
		return nil, err
	}
	var args map[string]interface{}
	if strings.HasSuffix(resp.Request.URL.Path, "post") {
		args = m["form"].(map[string]interface{})
	} else {
		args = m["args"].(map[string]interface{})
	}
	values := make(map[string]string, len(args))
	for k, v := range args {
		values[k] = v.(string)
	}
	return values, nil
}

func testForm(t *testing.T, f func(string, url.Values) (*httpclient.Response, error), u string, data map[string]string, exp map[string]string) {
	form := make(url.Values)
	for k, v := range data {
		form.Add(k, v)
	}
	resp, err := f(u, form)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Close()
	args, err := decodeArgs(resp)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(args, exp) {
		t.Errorf("expecting values %v, got %v instead", exp, args)
	}
}

func TestGetForm(t *testing.T) {
	data := map[string]string{"a": "b", "c": "d"}
	f := httpclient.New(nil).GetForm
	testForm(t, f, urlutil.MustJoin(httpBin, "/get"), data, data)
	expect := map[string]string{"e": "f"}
	for k, v := range data {
		expect[k] = v
	}
	testForm(t, f, urlutil.MustJoin(httpBin, "/get?e=f"), data, expect)
}

func TestPostForm(t *testing.T) {
	data := map[string]string{"a": "b", "c": "d"}
	testForm(t, httpclient.New(nil).PostForm, urlutil.MustJoin(httpBin, "/post"), data, data)
}

func redirNumber(t *testing.T, url string) int {
	b := path.Base(url)
	val, err := strconv.Atoi(b)
	if err != nil {
		t.Fatal(err)
	}
	return val
}

func TestRedirect(t *testing.T) {
	start := urlutil.MustJoin(httpBin, "/redirect/6")
	end := urlutil.MustJoin(httpBin, "/get")
	c := httpclient.New(nil)
	resp, err := c.Get(start)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Close()
	if u := resp.Request.URL.String(); u != end {
		t.Errorf("expecting final url %q, got %q instead", end, u)
	}
	cur := redirNumber(t, start)
	next := start
	for {
		req, err := http.NewRequest("GET", next, nil)
		if err != nil {
			t.Fatal(err)
		}
		resp, err := c.Trip(req)
		if err != nil {
			t.Fatal(err)
		}
		if cur > 0 {
			redir, err := resp.Redirect()
			if err != nil {
				t.Fatal(err)
			}
			cur--
			if cur > 0 {
				rn := redirNumber(t, redir)
				if rn != cur {
					t.Fatalf("expecting redirect %d, got %d instead", cur, rn)
				}
			}
			next = redir
		} else {
			if resp.IsRedirect() {
				t.Error("unexpected redirect")
			}
			if u := resp.Request.URL.String(); u != end {
				t.Errorf("expecting final url %q, got %q instead", end, u)
			}
			break
		}
	}
}

func TestProxy(t *testing.T) {
	c := httpclient.New(nil)
	if !c.SupportsProxy() {
		t.Skipf("current run environment does not support support proxies")
	}
	const addr = "127.0.0.1:12345"
	proxy := goproxy.NewProxyHttpServer()
	count := 0
	proxy.OnRequest().DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			count++
			return r, nil
		})
	go func() {
		http.ListenAndServe(addr, proxy)
	}()
	time.Sleep(time.Millisecond)
	c.SetProxy(func(_ *http.Request) (*url.URL, error) {
		return url.Parse("http://" + addr)
	})
	testUserAgent(t, c, httpclient.DefaultUserAgent)
	if count != 1 {
		t.Errorf("expecting 1 proxy request, got %d instead", count)
	}
	testUserAgent(t, c.Clone(nil), httpclient.DefaultUserAgent)
	if count != 2 {
		t.Errorf("expecting 2 proxy request, got %d instead", count)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/sselph/scraper/ovgdb"
	"github.com/syndtr/goleveldb/leveldb"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

const (
	query    = "SELECT releases.releaseID, roms.romID, releases.releaseTitleName, releases.releaseCoverFront, releases.releaseDescription, releases.releaseDeveloper, releases.releasePublisher, releases.releaseGenre, releases.releaseDate, roms.romDumpSource, roms.romHashSHA1, roms.romExtensionlessFileName, releases.releaseReferenceURL FROM releases INNER JOIN roms ON roms.romID = releases.romID"
	col      = 13
	apiURL   = "https://api.github.com/repos/OpenVGDB/OpenVGDB/releases?page=1&per_page=1"
	fileName = "openvgdb.sqlite"
	zipName  = "openvgdb.zip"
	metaName = "openvgdb-s.meta"
	dbName   = "ldb"
)

func parseDate(d string) string {
	dateFormats := []string{"Jan 2, 2006", "2006", "January 2006"}
	for _, f := range dateFormats {
		t, err := time.Parse(f, d)
		if err == nil {
			return t.Format("20060102T000000")
		}
	}
	return ""
}

func getGenre(g string) string {
	return strings.SplitN(g, ",", 2)[0]
}

// scanRow scans a row. NULL values will be empty strings.
func scanRow(rows *sql.Rows) ([]string, error) {
	var s []interface{}
	for i := 0; i < col; i++ {
		s = append(s, interface{}(&sql.NullString{}))
	}
	err := rows.Scan(s...)
	if err != nil {
		return nil, err
	}
	var r []string
	for _, v := range s {
		r = append(r, v.(*sql.NullString).String)
	}
	return r, nil
}

func queryDB(db *sql.DB, q string) ([]ovgdb.Game, error) {
	rows, err := db.Query(q)
	ret := []ovgdb.Game{}
	if err != nil {
		return ret, err
	}
	defer rows.Close()
	for rows.Next() {
		v, err := scanRow(rows)
		if err != nil {
			return ret, err
		}
		source := []string{"OpenVGDB"}
		if v[9] != "" {
			source = append(source, v[9])
		}
		if v[12] != "" {
			u, err := url.Parse(v[12])
			if err == nil {
				source = append(source, u.Host)
			}
		}
		g := ovgdb.Game{
			ReleaseID: v[0],
			RomID:     v[1],
			Name:      v[2],
			Art:       v[3],
			Desc:      v[4],
			Developer: v[5],
			Publisher: v[6],
			Genre:     getGenre(v[7]),
			Date:      parseDate(v[8]),
			Source:    strings.Join(source, ","),
			Hash:      strings.ToLower(v[10]),
			FileName:  v[11],
		}
		ret = append(ret, g)
	}
	return ret, nil
}

type Release struct {
	TagName string  `json:"tag_name"`
	Assets  []Asset `json:"assets"`
}

type Asset struct {
	Name        string `json:"name"`
	DownloadURL string `json:"browser_download_url"`
}

func getRelease() (*Release, error) {
	var releases []Release
	resp, err := http.Get(apiURL)
	if err != nil {
		return nil, fmt.Errorf("could not get releases, %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("got %v response", resp.Status)
	}

	if err = json.NewDecoder(resp.Body).Decode(&releases); err != nil {
		return nil, fmt.Errorf("could not unmarshall JSON, %v", err)
	}
	if len(releases) == 0 {
		return nil, fmt.Errorf("no releases found")
	}
	return &releases[0], nil
}

func updateDB(version, p string) error {
	log.Print("INFO: Checking for new OpenVGDB.")
	r, err := getRelease()
	if err != nil {
		return err
	}
	if r.TagName == version {
		log.Printf("INFO: OpenVGDB %s up to date.", version)
		return nil
	}
	log.Printf("INFO: Upgrading OpenGDB: %s -> %s.", version, r.TagName)
	if len(r.Assets) == 0 {
		return fmt.Errorf("no openvgdb found")
	}
	for _, v := range r.Assets {
		if v.Name != zipName {
			continue
		}
		resp, err := http.Get(v.DownloadURL)
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		zf := path.Join(p, zipName)
		err = ioutil.WriteFile(zf, b, 0664)
		if err != nil {
			return err
		}
		rc, err := zip.OpenReader(zf)
		if err != nil {
			return err
		}
		defer rc.Close()
		for _, v := range rc.Reader.File {
			if v.FileHeader.Name != fileName {
				continue
			}
			frc, err := v.Open()
			if err != nil {
				return err
			}
			defer frc.Close()
			b, err = ioutil.ReadAll(frc)
			if err != nil {
				return err
			}
			ioutil.WriteFile(path.Join(p, metaName), []byte(r.TagName), 0664)
			os.Remove(zf)
			log.Print("INFO: Upgrade Complete.")
			return ioutil.WriteFile(path.Join(p, fileName), b, 0664)
		}
	}
	return fmt.Errorf("no openvgdb found")
}

func mkDir(d string) error {
	fi, err := os.Stat(d)
	switch {
	case os.IsNotExist(err):
		return os.MkdirAll(d, 0775)
	case err != nil:
		return err
	case fi.IsDir():
		return nil
	}
	return fmt.Errorf("%s is a file not a directory.", d)
}

func exists(f string) bool {
	fi, err := os.Stat(f)
	return !os.IsNotExist(err) && fi.Size() > 0
}

func GetDB() (*sql.DB, error) {
	p, err := ovgdb.GetDBPath()
	if err != nil {
		return nil, err
	}
	err = mkDir(p)
	var version string
	if err != nil {
		return nil, err
	}
	fp := path.Join(p, fileName)
	mp := path.Join(p, metaName)
	if exists(fp) && exists(mp) {
		b, err := ioutil.ReadFile(mp)
		if err != nil {
			return nil, err
		}
		version = strings.Trim(string(b[:]), "\n\r")
	}
	err = updateDB(version, p)
	if err != nil {
		return nil, err
	}
	return sql.Open("sqlite3", fp)
}

func main() {
	db, err := GetDB()
	if err != nil {
		log.Fatal(err)
	}
	games, err := queryDB(db, query)
	if err != nil {
		log.Fatal(err)
	}
	p, err := ovgdb.GetDBPath()
	if err != nil {
		log.Fatal(err)
	}
	os.RemoveAll(path.Join(p, dbName))
	if err != nil {
		log.Fatal(err)
	}
	ldb, err := leveldb.OpenFile(path.Join(p, dbName), nil)
	if err != nil {
		log.Fatal(err)
	}
	defer ldb.Close()
	for _, g := range games {
		b, err := json.Marshal(g.ToSlice())
		if err != nil {
			log.Fatal(err)
		}
		batch := new(leveldb.Batch)
		i := []byte(g.ReleaseID)
		h := []byte(g.Hash)
		fn := []byte(strings.ToLower(g.FileName))
		batch.Put(i, b)
		batch.Put(h, i)
		batch.Put(fn, i)
		err = ldb.Write(batch, nil)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Filter out rows from ovgdb that don't have data.<commit_after>package main

import (
	"archive/zip"
	"database/sql"
	"encoding/json"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/sselph/scraper/ovgdb"
	"github.com/syndtr/goleveldb/leveldb"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"
	"time"
)

const (
	query    = "SELECT releases.releaseID, roms.romID, releases.releaseTitleName, releases.releaseCoverFront, releases.releaseDescription, releases.releaseDeveloper, releases.releasePublisher, releases.releaseGenre, releases.releaseDate, roms.romDumpSource, roms.romHashSHA1, roms.romExtensionlessFileName, releases.releaseReferenceURL FROM releases INNER JOIN roms ON roms.romID = releases.romID"
	col      = 13
	apiURL   = "https://api.github.com/repos/OpenVGDB/OpenVGDB/releases?page=1&per_page=1"
	fileName = "openvgdb.sqlite"
	zipName  = "openvgdb.zip"
	metaName = "openvgdb-s.meta"
	dbName   = "ldb"
)

func parseDate(d string) string {
	dateFormats := []string{"Jan 2, 2006", "2006", "January 2006"}
	for _, f := range dateFormats {
		t, err := time.Parse(f, d)
		if err == nil {
			return t.Format("20060102T000000")
		}
	}
	return ""
}

func getGenre(g string) string {
	return strings.SplitN(g, ",", 2)[0]
}

// scanRow scans a row. NULL values will be empty strings.
func scanRow(rows *sql.Rows) ([]string, error) {
	var s []interface{}
	for i := 0; i < col; i++ {
		s = append(s, interface{}(&sql.NullString{}))
	}
	err := rows.Scan(s...)
	if err != nil {
		return nil, err
	}
	var r []string
	for _, v := range s {
		r = append(r, v.(*sql.NullString).String)
	}
	return r, nil
}

func queryDB(db *sql.DB, q string) ([]ovgdb.Game, error) {
	rows, err := db.Query(q)
	ret := []ovgdb.Game{}
	if err != nil {
		return ret, err
	}
	defer rows.Close()
	for rows.Next() {
		v, err := scanRow(rows)
		if err != nil {
			return ret, err
		}
		source := []string{"OpenVGDB"}
		if v[9] != "" {
			source = append(source, v[9])
		}
		if v[12] != "" {
			u, err := url.Parse(v[12])
			if err == nil {
				source = append(source, u.Host)
			}
		}
		g := ovgdb.Game{
			ReleaseID: v[0],
			RomID:     v[1],
			Name:      v[2],
			Art:       v[3],
			Desc:      v[4],
			Developer: v[5],
			Publisher: v[6],
			Genre:     getGenre(v[7]),
			Date:      parseDate(v[8]),
			Source:    strings.Join(source, ","),
			Hash:      strings.ToLower(v[10]),
			FileName:  v[11],
		}
		// filter out rows that don't add much value.
		if g.Desc == "" && g.Art == "" && g.Developer == "" && g.Date == "" {
			continue
		}
		ret = append(ret, g)
	}
	return ret, nil
}

type Release struct {
	TagName string  `json:"tag_name"`
	Assets  []Asset `json:"assets"`
}

type Asset struct {
	Name        string `json:"name"`
	DownloadURL string `json:"browser_download_url"`
}

func getRelease() (*Release, error) {
	var releases []Release
	resp, err := http.Get(apiURL)
	if err != nil {
		return nil, fmt.Errorf("could not get releases, %v", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("got %v response", resp.Status)
	}

	if err = json.NewDecoder(resp.Body).Decode(&releases); err != nil {
		return nil, fmt.Errorf("could not unmarshall JSON, %v", err)
	}
	if len(releases) == 0 {
		return nil, fmt.Errorf("no releases found")
	}
	return &releases[0], nil
}

func updateDB(version, p string) error {
	log.Print("INFO: Checking for new OpenVGDB.")
	r, err := getRelease()
	if err != nil {
		return err
	}
	if r.TagName == version {
		log.Printf("INFO: OpenVGDB %s up to date.", version)
		return nil
	}
	log.Printf("INFO: Upgrading OpenGDB: %s -> %s.", version, r.TagName)
	if len(r.Assets) == 0 {
		return fmt.Errorf("no openvgdb found")
	}
	for _, v := range r.Assets {
		if v.Name != zipName {
			continue
		}
		resp, err := http.Get(v.DownloadURL)
		if err != nil {
			return err
		}
		defer resp.Body.Close()
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		zf := path.Join(p, zipName)
		err = ioutil.WriteFile(zf, b, 0664)
		if err != nil {
			return err
		}
		rc, err := zip.OpenReader(zf)
		if err != nil {
			return err
		}
		defer rc.Close()
		for _, v := range rc.Reader.File {
			if v.FileHeader.Name != fileName {
				continue
			}
			frc, err := v.Open()
			if err != nil {
				return err
			}
			defer frc.Close()
			b, err = ioutil.ReadAll(frc)
			if err != nil {
				return err
			}
			ioutil.WriteFile(path.Join(p, metaName), []byte(r.TagName), 0664)
			os.Remove(zf)
			log.Print("INFO: Upgrade Complete.")
			return ioutil.WriteFile(path.Join(p, fileName), b, 0664)
		}
	}
	return fmt.Errorf("no openvgdb found")
}

func mkDir(d string) error {
	fi, err := os.Stat(d)
	switch {
	case os.IsNotExist(err):
		return os.MkdirAll(d, 0775)
	case err != nil:
		return err
	case fi.IsDir():
		return nil
	}
	return fmt.Errorf("%s is a file not a directory.", d)
}

func exists(f string) bool {
	fi, err := os.Stat(f)
	return !os.IsNotExist(err) && fi.Size() > 0
}

func GetDB() (*sql.DB, error) {
	p, err := ovgdb.GetDBPath()
	if err != nil {
		return nil, err
	}
	err = mkDir(p)
	var version string
	if err != nil {
		return nil, err
	}
	fp := path.Join(p, fileName)
	mp := path.Join(p, metaName)
	if exists(fp) && exists(mp) {
		b, err := ioutil.ReadFile(mp)
		if err != nil {
			return nil, err
		}
		version = strings.Trim(string(b[:]), "\n\r")
	}
	err = updateDB(version, p)
	if err != nil {
		return nil, err
	}
	return sql.Open("sqlite3", fp)
}

func main() {
	db, err := GetDB()
	if err != nil {
		log.Fatal(err)
	}
	games, err := queryDB(db, query)
	if err != nil {
		log.Fatal(err)
	}
	p, err := ovgdb.GetDBPath()
	if err != nil {
		log.Fatal(err)
	}
	os.RemoveAll(path.Join(p, dbName))
	if err != nil {
		log.Fatal(err)
	}
	ldb, err := leveldb.OpenFile(path.Join(p, dbName), nil)
	if err != nil {
		log.Fatal(err)
	}
	defer ldb.Close()
	for _, g := range games {
		b, err := json.Marshal(g.ToSlice())
		if err != nil {
			log.Fatal(err)
		}
		batch := new(leveldb.Batch)
		i := []byte(g.ReleaseID)
		h := []byte(g.Hash)
		fn := []byte(strings.ToLower(g.FileName))
		batch.Put(i, b)
		batch.Put(h, i)
		batch.Put(fn, i)
		err = ldb.Write(batch, nil)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"testing"

	gc "launchpad.net/gocheck"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/environs/config"
	_ "launchpad.net/juju-core/environs/dummy"
)

func TestPackage(t *testing.T) {
	gc.TestingT(t)
}

type MetadataSuite struct{}

var _ = gc.Suite(&MetadataSuite{})

var (
	flagRunMain = flag.Bool("run-main", false, "Run the application's main function for recursive testing")
)

// Reentrancy point for testing (something as close as possible to) the juju
// tool itself.
func TestRunMain(t *testing.T) {
	if *flagRunMain {
		Main(flag.Args())
	}
}

func badrun(c *gc.C, exit int, args ...string) string {
	localArgs := append([]string{"-test.run", "TestRunMain", "-run-main", "--", "juju"}, args...)
	ps := exec.Command(os.Args[0], localArgs...)
	ps.Env = append(os.Environ(), "JUJU_HOME="+config.JujuHome())
	output, err := ps.CombinedOutput()
	if exit != 0 {
		c.Assert(err, gc.ErrorMatches, fmt.Sprintf("exit status %d", exit))
	}
	return string(output)
}

func (s *MetadataSuite) TestTearDown(c *gc.C) {
	loggo.ResetLoggers()
}

var commandNames = []string{
	"generate-image",
	"help",
	"validate-images",
}

func (s *MetadataSuite) TestHelpCommands(c *gc.C) {
	// Check that we have correctly registered all the commands
	// by checking the help output.
	defer config.SetJujuHome(config.SetJujuHome(c.MkDir()))
	out := badrun(c, 0, "help", "commands")
	lines := strings.Split(out, "\n")
	var names []string
	for _, line := range lines {
		f := strings.Fields(line)
		if len(f) == 0 {
			continue
		}
		names = append(names, f[0])
	}
	// The names should be output in alphabetical order, so don't sort.
	c.Assert(names, gc.DeepEquals, commandNames)
}

var globalFlags = []string{
	"--debug .*",
	"-h, --help .*",
	"--log-config .*",
	"--log-file .*",
	"-v, --verbose .*",
}

func (s *MetadataSuite) TestHelpGlobalOptions(c *gc.C) {
	// Check that we have correctly registered all the topics
	// by checking the help output.
	defer config.SetJujuHome(config.SetJujuHome(c.MkDir()))
	out := badrun(c, 0, "help", "global-options")
	c.Assert(out, gc.Matches, `Global Options

These options may be used with any command, and may appear in front of any
command\.(.|\n)*`)
	lines := strings.Split(out, "\n")
	var flags []string
	for _, line := range lines {
		f := strings.Fields(line)
		if len(f) == 0 || line[0] != '-' {
			continue
		}
		flags = append(flags, line)
	}
	c.Assert(len(flags), gc.Equals, len(globalFlags))
	for i, line := range flags {
		c.Assert(line, gc.Matches, globalFlags[i])
	}
}
<commit_msg>Fix test<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strings"
	"testing"

	gc "launchpad.net/gocheck"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/environs/config"
	_ "launchpad.net/juju-core/environs/dummy"
)

func TestPackage(t *testing.T) {
	gc.TestingT(t)
}

type MetadataSuite struct{}

var _ = gc.Suite(&MetadataSuite{})

var (
	flagRunMain = flag.Bool("run-main", false, "Run the application's main function for recursive testing")
)

// Reentrancy point for testing (something as close as possible to) the juju
// tool itself.
func TestRunMain(t *testing.T) {
	if *flagRunMain {
		Main(flag.Args())
	}
}

func badrun(c *gc.C, exit int, args ...string) string {
	localArgs := append([]string{"-test.run", "TestRunMain", "-run-main", "--", "juju"}, args...)
	ps := exec.Command(os.Args[0], localArgs...)
	ps.Env = append(os.Environ(), "JUJU_HOME="+config.JujuHome())
	output, err := ps.CombinedOutput()
	if exit != 0 {
		c.Assert(err, gc.ErrorMatches, fmt.Sprintf("exit status %d", exit))
	}
	return string(output)
}

func (s *MetadataSuite) TestTearDown(c *gc.C) {
	loggo.ResetLoggers()
}

var commandNames = []string{
	"generate-image",
	"help",
	"validate-images",
}

func (s *MetadataSuite) TestHelpCommands(c *gc.C) {
	// Check that we have correctly registered all the commands
	// by checking the help output.
	defer config.SetJujuHome(config.SetJujuHome(c.MkDir()))
	out := badrun(c, 0, "help", "commands")
	lines := strings.Split(out, "\n")
	var names []string
	for _, line := range lines {
		f := strings.Fields(line)
		if len(f) == 0 {
			continue
		}
		names = append(names, f[0])
	}
	// The names should be output in alphabetical order, so don't sort.
	c.Assert(names, gc.DeepEquals, commandNames)
}

var globalFlags = []string{
	"--debug .*",
	"--description .*",
	"-h, --help .*",
	"--log-config .*",
	"--log-file .*",
	"-v, --verbose .*",
}

func (s *MetadataSuite) TestHelpGlobalOptions(c *gc.C) {
	// Check that we have correctly registered all the topics
	// by checking the help output.
	defer config.SetJujuHome(config.SetJujuHome(c.MkDir()))
	out := badrun(c, 0, "help", "global-options")
	c.Assert(out, gc.Matches, `Global Options

These options may be used with any command, and may appear in front of any
command\.(.|\n)*`)
	lines := strings.Split(out, "\n")
	var flags []string
	for _, line := range lines {
		f := strings.Fields(line)
		if len(f) == 0 || line[0] != '-' {
			continue
		}
		flags = append(flags, line)
	}
	c.Assert(len(flags), gc.Equals, len(globalFlags))
	for i, line := range flags {
		c.Assert(line, gc.Matches, globalFlags[i])
	}
}
<|endoftext|>


<commit_before>package generate

import (
	"encoding/json"
	"io/ioutil"
	"testing"

	"github.com/go-swagger/go-swagger/scan"
	"github.com/stretchr/testify/assert"
	"gopkg.in/yaml.v2"
)

const (
	basePath       = "../../../../fixtures/goparsing/spec"
	jsonResultFile = "../../../../fixtures/goparsing/spec/api_spec.json"
	yamlResultFile = "../../../../fixtures/goparsing/spec/api_spec.yml"
)

func TestGenerateJSONSpec(t *testing.T) {
	opts := scan.Opts{
		BasePath: basePath,
	}

	swspec, err := scan.Application(opts)
	assert.NoError(t, err)

	data, err := marshalToJSONFormat(swspec, true)
	assert.NoError(t, err)

	expected, err := ioutil.ReadFile(jsonResultFile)
	assert.NoError(t, err)

	varifyJSONData(t, data, expected)
}

func TestGenerateYAMLSpec(t *testing.T) {
	opts := scan.Opts{
		BasePath: basePath,
	}

	swspec, err := scan.Application(opts)
	assert.NoError(t, err)

	data, err := marshalToYAMLFormat(swspec)
	assert.NoError(t, err)

	expected, err := ioutil.ReadFile(yamlResultFile)
	assert.NoError(t, err)

	varifyYAMLData(t, data, expected)
}

func varifyJSONData(t *testing.T, data, expectedJSON []byte) {
	var got interface{}
	var expected interface{}

	err := json.Unmarshal(data, &got)
	assert.NoError(t, err)

	err = json.Unmarshal(expectedJSON, &expected)
	assert.NoError(t, err)

	if !assert.ObjectsAreEqual(got, expected) {
		assert.Fail(t, "marshaled JSON data doesn't equal expected JSON data")
	}
}

func varifyYAMLData(t *testing.T, data, expectedYAML []byte) {
	var got interface{}
	var expected interface{}

	err := yaml.Unmarshal(data, &got)
	assert.NoError(t, err)

	err = yaml.Unmarshal(expectedYAML, &expected)
	assert.NoError(t, err)

	if !assert.ObjectsAreEqual(got, expected) {
		assert.Fail(t, "marshaled YAML data doesn't equal expected YAML data")
	}
}
<commit_msg>add more unit test<commit_after>package generate

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"testing"

	"github.com/go-swagger/go-swagger/scan"
	"github.com/jessevdk/go-flags"
	"github.com/stretchr/testify/assert"
	"gopkg.in/yaml.v2"
)

const (
	basePath       = "../../../../fixtures/goparsing/spec"
	jsonResultFile = basePath + "/api_spec.json"
	yamlResultFile = basePath + "/api_spec.yml"
)

func TestSpecFileExecute(t *testing.T) {
	files := []string{"", "spec.json", "spec.yml", "spec.yaml"}
	for _, outputFile := range files {
		spec := &SpecFile{
			BasePath: basePath,
			Output:   flags.Filename(outputFile),
		}

		err := spec.Execute(nil)
		assert.NoError(t, err)
		if outputFile != "" {
			os.Remove(outputFile)
		}
	}
}

func TestGenerateJSONSpec(t *testing.T) {
	opts := scan.Opts{
		BasePath: basePath,
	}

	swspec, err := scan.Application(opts)
	assert.NoError(t, err)

	data, err := marshalToJSONFormat(swspec, true)
	assert.NoError(t, err)

	expected, err := ioutil.ReadFile(jsonResultFile)
	assert.NoError(t, err)

	varifyJSONData(t, data, expected)
}

func TestGenerateYAMLSpec(t *testing.T) {
	opts := scan.Opts{
		BasePath: basePath,
	}

	swspec, err := scan.Application(opts)
	assert.NoError(t, err)

	data, err := marshalToYAMLFormat(swspec)
	assert.NoError(t, err)

	expected, err := ioutil.ReadFile(yamlResultFile)
	assert.NoError(t, err)

	varifyYAMLData(t, data, expected)
}

func varifyJSONData(t *testing.T, data, expectedJSON []byte) {
	var got interface{}
	var expected interface{}

	err := json.Unmarshal(data, &got)
	assert.NoError(t, err)

	err = json.Unmarshal(expectedJSON, &expected)
	assert.NoError(t, err)

	if !assert.ObjectsAreEqual(got, expected) {
		assert.Fail(t, "marshaled JSON data doesn't equal expected JSON data")
	}
}

func varifyYAMLData(t *testing.T, data, expectedYAML []byte) {
	var got interface{}
	var expected interface{}

	err := yaml.Unmarshal(data, &got)
	assert.NoError(t, err)

	err = yaml.Unmarshal(expectedYAML, &expected)
	assert.NoError(t, err)

	if !assert.ObjectsAreEqual(got, expected) {
		assert.Fail(t, "marshaled YAML data doesn't equal expected YAML data")
	}
}
<|endoftext|>


<commit_before>package connections

import (
	"errors"
	"sync"

	"github.com/sirupsen/logrus"
)

type ConnectionGroupManager struct {
	groups      map[int]*ConnectionGroup
	groupsMutex *sync.RWMutex
	groupLimit  int
	connsLimit  int
	connsCount  int
	logger      logrus.FieldLogger
}

func NewConnectionGroupManager(logger logrus.FieldLogger, groupLimit, connsLimit int) (*ConnectionGroupManager, error) {
	if groupLimit > 0 {
		return &ConnectionGroupManager{
			groups:      map[int]*ConnectionGroup{},
			groupsMutex: &sync.RWMutex{},
			groupLimit:  groupLimit,
			connsLimit:  connsLimit,
			logger:      logger,
		}, nil
	}

	return nil, errors.New("cannot create connection group manager: invalid group limit")
}

func (m *ConnectionGroupManager) unsafeIsFull() bool {
	return len(m.groups) == m.groupLimit
}

func (m *ConnectionGroupManager) IsFull() bool {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()
	return m.unsafeIsFull()
}

type ErrAddGroup string

func (e ErrAddGroup) Error() string {
	return "cannot add group: " + string(e)
}

var (
	ErrGroupLimitReached = ErrAddGroup("limit group count reached")
	ErrCannotGetID       = ErrAddGroup("cannot get id for group")
	ErrConnsLimitReached = ErrAddGroup("cannot reserve connections for group: connections count reached")
)

func (m *ConnectionGroupManager) Add(group *ConnectionGroup) (int, error) {
	m.groupsMutex.Lock()
	defer m.groupsMutex.Unlock()

	if m.unsafeIsFull() {
		return 0, ErrGroupLimitReached
	}

	if group.GetLimit() > m.connsLimit-m.connsCount {
		if m.connsLimit-m.connsCount == 0 {
			return 0, ErrConnsLimitReached
		}
		group.SetLimit(m.connsLimit - m.connsCount)
	}

	m.connsCount += group.GetLimit()

	for id := 0; id <= len(m.groups); id++ {
		if _, occupied := m.groups[id]; !occupied {
			m.groups[id] = group
			return id, nil
		}
	}

	return 0, ErrCannotGetID
}

type ErrDeleteGroup string

func (e ErrDeleteGroup) Error() string {
	return "cannot delete group: " + string(e)
}

var (
	ErrDeleteNotEmptyGroup = ErrDeleteGroup("group is not empty")
	ErrDeleteNotFoundGroup = ErrDeleteGroup("group not found")
)

func (m *ConnectionGroupManager) Delete(group *ConnectionGroup) error {
	if !group.IsEmpty() {
		return ErrDeleteNotEmptyGroup
	}

	m.groupsMutex.Lock()
	defer m.groupsMutex.Unlock()

	m.connsCount -= group.GetLimit()

	for id := range m.groups {
		if m.groups[id] == group {
			delete(m.groups, id)
			return nil
		}
	}

	return ErrDeleteNotFoundGroup
}

var ErrNotFoundGroup = errors.New("not found group")

func (m *ConnectionGroupManager) Get(id int) (*ConnectionGroup, error) {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()

	if group, ok := m.groups[id]; ok {
		return group, nil
	}

	return nil, ErrNotFoundGroup
}

func (m *ConnectionGroupManager) Groups() map[int]*ConnectionGroup {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()
	groups := map[int]*ConnectionGroup{}
	for id, group := range m.groups {
		groups[id] = group
	}
	return groups
}

func (m *ConnectionGroupManager) GroupLimit() int {
	return m.groupLimit
}

func (m *ConnectionGroupManager) GroupCount() int {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()
	return len(m.groups)
}

func (m *ConnectionGroupManager) Capacity() float32 {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()

	var count = 0
	for _, group := range m.groups {
		count += group.GetCount()
	}

	return float32(count) / float32(m.connsLimit)
}
<commit_msg>Fix deleting groups in ConnectionGroupManager<commit_after>package connections

import (
	"errors"
	"sync"

	"github.com/sirupsen/logrus"
)

type ConnectionGroupManager struct {
	groups      map[int]*ConnectionGroup
	groupsMutex *sync.RWMutex
	groupLimit  int
	connsLimit  int
	connsCount  int
	logger      logrus.FieldLogger
}

func NewConnectionGroupManager(logger logrus.FieldLogger, groupLimit, connsLimit int) (*ConnectionGroupManager, error) {
	if groupLimit > 0 {
		return &ConnectionGroupManager{
			groups:      map[int]*ConnectionGroup{},
			groupsMutex: &sync.RWMutex{},
			groupLimit:  groupLimit,
			connsLimit:  connsLimit,
			logger:      logger,
		}, nil
	}

	return nil, errors.New("cannot create connection group manager: invalid group limit")
}

func (m *ConnectionGroupManager) unsafeIsFull() bool {
	return len(m.groups) == m.groupLimit
}

func (m *ConnectionGroupManager) IsFull() bool {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()
	return m.unsafeIsFull()
}

type ErrAddGroup string

func (e ErrAddGroup) Error() string {
	return "cannot add group: " + string(e)
}

var (
	ErrGroupLimitReached = ErrAddGroup("limit group count reached")
	ErrCannotGetID       = ErrAddGroup("cannot get id for group")
	ErrConnsLimitReached = ErrAddGroup("cannot reserve connections for group: connections count reached")
)

func (m *ConnectionGroupManager) Add(group *ConnectionGroup) (int, error) {
	m.groupsMutex.Lock()
	defer m.groupsMutex.Unlock()

	if m.unsafeIsFull() {
		return 0, ErrGroupLimitReached
	}

	if group.GetLimit() > m.connsLimit-m.connsCount {
		if m.connsLimit-m.connsCount == 0 {
			return 0, ErrConnsLimitReached
		}
		group.SetLimit(m.connsLimit - m.connsCount)
	}

	m.connsCount += group.GetLimit()

	for id := 0; id <= len(m.groups); id++ {
		if _, occupied := m.groups[id]; !occupied {
			m.groups[id] = group
			return id, nil
		}
	}

	return 0, ErrCannotGetID
}

type ErrDeleteGroup string

func (e ErrDeleteGroup) Error() string {
	return "cannot delete group: " + string(e)
}

var (
	ErrDeleteNotEmptyGroup = ErrDeleteGroup("group is not empty")
	ErrDeleteNotFoundGroup = ErrDeleteGroup("group not found")
)

func (m *ConnectionGroupManager) Delete(group *ConnectionGroup) error {
	m.groupsMutex.Lock()
	defer m.groupsMutex.Unlock()

	if !group.IsEmpty() {
		return ErrDeleteNotEmptyGroup
	}

	for id := range m.groups {
		if m.groups[id] == group {
			delete(m.groups, id)
			m.connsCount -= group.GetLimit()

			return nil
		}
	}

	return ErrDeleteNotFoundGroup
}

var ErrNotFoundGroup = errors.New("not found group")

func (m *ConnectionGroupManager) Get(id int) (*ConnectionGroup, error) {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()

	if group, ok := m.groups[id]; ok {
		return group, nil
	}

	return nil, ErrNotFoundGroup
}

func (m *ConnectionGroupManager) Groups() map[int]*ConnectionGroup {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()
	groups := map[int]*ConnectionGroup{}
	for id, group := range m.groups {
		groups[id] = group
	}
	return groups
}

func (m *ConnectionGroupManager) GroupLimit() int {
	return m.groupLimit
}

func (m *ConnectionGroupManager) GroupCount() int {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()
	return len(m.groups)
}

func (m *ConnectionGroupManager) Capacity() float32 {
	m.groupsMutex.RLock()
	defer m.groupsMutex.RUnlock()

	var count = 0
	for _, group := range m.groups {
		count += group.GetCount()
	}

	return float32(count) / float32(m.connsLimit)
}
<|endoftext|>


<commit_before>package containermetrics_test

import (
	"errors"
	"time"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/containermetrics"
	efakes "github.com/cloudfoundry-incubator/executor/fakes"
	msfake "github.com/cloudfoundry/dropsonde/metric_sender/fake"
	dmetrics "github.com/cloudfoundry/dropsonde/metrics"
	"github.com/pivotal-golang/clock/fakeclock"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("StatsReporter", func() {
	var (
		logger *lagertest.TestLogger

		interval           time.Duration
		fakeClock          *fakeclock.FakeClock
		fakeExecutorClient *efakes.FakeClient
		fakeMetricSender   *msfake.FakeMetricSender

		workingListContainersStub func(executor.Tags) ([]executor.Container, error)

		process ifrit.Process
	)

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")

		interval = 10 * time.Second
		fakeClock = fakeclock.NewFakeClock(time.Unix(123, 456))
		fakeExecutorClient = new(efakes.FakeClient)

		fakeMetricSender = msfake.NewFakeMetricSender()
		dmetrics.Initialize(fakeMetricSender)

		containerResults := make(chan []executor.Container, 10)

		one := 1
		containerResults <- []executor.Container{
			{
				Guid: "guid-without-index",

				MemoryUsageInBytes: 123,
				DiskUsageInBytes:   456,
				TimeSpentInCPU:     100 * time.Second,

				MetricsConfig: executor.MetricsConfig{
					Guid: "metrics-guid-without-index",
				},
			},
			{
				Guid: "guid-with-no-metrics-guid",

				MemoryUsageInBytes: 1023,
				DiskUsageInBytes:   4056,
				TimeSpentInCPU:     1000 * time.Second,
			},
			{
				Guid: "guid-with-index",

				MemoryUsageInBytes: 321,
				DiskUsageInBytes:   654,
				TimeSpentInCPU:     100 * time.Second,

				MetricsConfig: executor.MetricsConfig{
					Guid:  "metrics-guid-with-index",
					Index: &one,
				},
			},
		}

		containerResults <- []executor.Container{
			{
				Guid: "guid-without-index",

				MemoryUsageInBytes: 1230,
				DiskUsageInBytes:   4560,
				TimeSpentInCPU:     105 * time.Second,

				MetricsConfig: executor.MetricsConfig{
					Guid: "metrics-guid-without-index",
				},
			},
			{
				Guid: "guid-with-index",

				MemoryUsageInBytes: 3210,
				DiskUsageInBytes:   6540,
				TimeSpentInCPU:     110 * time.Second,

				MetricsConfig: executor.MetricsConfig{
					Guid:  "metrics-guid-with-index",
					Index: &one,
				},
			},
		}

		containerResults <- []executor.Container{
			{
				Guid: "guid-without-index",

				MemoryUsageInBytes: 12300,
				DiskUsageInBytes:   45600,
				TimeSpentInCPU:     107 * time.Second,

				MetricsConfig: executor.MetricsConfig{
					Guid: "metrics-guid-without-index",
				},
			},
			{
				Guid: "guid-with-index",

				MemoryUsageInBytes: 32100,
				DiskUsageInBytes:   65400,
				TimeSpentInCPU:     112 * time.Second,

				MetricsConfig: executor.MetricsConfig{
					Guid:  "metrics-guid-with-index",
					Index: &one,
				},
			},
		}

		workingListContainersStub = func(executor.Tags) ([]executor.Container, error) {
			return <-containerResults, nil
		}

		fakeExecutorClient.ListContainersStub = workingListContainersStub

		process = ifrit.Invoke(containermetrics.NewStatsReporter(logger, interval, fakeClock, fakeExecutorClient))
	})

	AfterEach(func() {
		ginkgomon.Interrupt(process)
	})

	Context("when the interval elapses", func() {
		BeforeEach(func() {
			fakeClock.Increment(interval)
			Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(1))
		})

		It("emits memory and disk usage for each container, but no CPU", func() {
			Eventually(func() msfake.ContainerMetric {
				return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
			}).Should(Equal(msfake.ContainerMetric{
				ApplicationId: "metrics-guid-without-index",
				InstanceIndex: -1,
				CpuPercentage: 0.0,
				MemoryBytes:   123,
				DiskBytes:     456,
			}))

			Eventually(func() msfake.ContainerMetric {
				return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
			}).Should(Equal(msfake.ContainerMetric{
				ApplicationId: "metrics-guid-with-index",
				InstanceIndex: 1,
				CpuPercentage: 0.0,
				MemoryBytes:   321,
				DiskBytes:     654,
			}))
		})

		It("does not emit anything for containers with no metrics guid", func() {
			Consistently(func() msfake.ContainerMetric {
				return fakeMetricSender.GetContainerMetric("")
			}).Should(BeZero())
		})

		Context("and the interval elapses again", func() {
			BeforeEach(func() {
				fakeClock.Increment(interval)
				Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(2))
			})

			It("emits the new memory and disk usage, and the computed CPU percent", func() {
				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-without-index",
					InstanceIndex: -1,
					CpuPercentage: 50.0,
					MemoryBytes:   1230,
					DiskBytes:     4560,
				}))

				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-with-index",
					InstanceIndex: 1,
					CpuPercentage: 100.0,
					MemoryBytes:   3210,
					DiskBytes:     6540,
				}))
			})

			Context("and the interval elapses again", func() {
				BeforeEach(func() {
					fakeClock.Increment(interval)
					Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(3))
				})

				It("emits the new memory and disk usage, and the computed CPU percent", func() {
					Eventually(func() msfake.ContainerMetric {
						return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
					}).Should(Equal(msfake.ContainerMetric{
						ApplicationId: "metrics-guid-without-index",
						InstanceIndex: -1,
						CpuPercentage: 20.0,
						MemoryBytes:   12300,
						DiskBytes:     45600,
					}))

					Eventually(func() msfake.ContainerMetric {
						return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
					}).Should(Equal(msfake.ContainerMetric{
						ApplicationId: "metrics-guid-with-index",
						InstanceIndex: 1,
						CpuPercentage: 20.0,
						MemoryBytes:   32100,
						DiskBytes:     65400,
					}))
				})
			})
		})
	})

	Context("when listing containers fails", func() {
		BeforeEach(func() {
			fakeExecutorClient.ListContainersReturns(nil, errors.New("nope"))
			fakeClock.Increment(interval)
			Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(1))
		})

		It("does not blow up", func() {
			Consistently(process.Wait()).ShouldNot(Receive())
		})

		Context("and the interval elapses again, and it works that time", func() {
			BeforeEach(func() {
				fakeExecutorClient.ListContainersStub = workingListContainersStub
				fakeClock.Increment(interval)
				Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(2))
			})

			It("processes the containers happily", func() {
				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-without-index",
					InstanceIndex: -1,
					CpuPercentage: 0.0,
					MemoryBytes:   123,
					DiskBytes:     456,
				}))

				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-with-index",
					InstanceIndex: 1,
					CpuPercentage: 0.0,
					MemoryBytes:   321,
					DiskBytes:     654,
				}))
			})
		})
	})
})
<commit_msg>Use one way to send results<commit_after>package containermetrics_test

import (
	"errors"
	"time"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/containermetrics"
	efakes "github.com/cloudfoundry-incubator/executor/fakes"
	msfake "github.com/cloudfoundry/dropsonde/metric_sender/fake"
	dmetrics "github.com/cloudfoundry/dropsonde/metrics"
	"github.com/pivotal-golang/clock/fakeclock"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type listContainerResults struct {
	containers []executor.Container
	err        error
}

var _ = Describe("StatsReporter", func() {
	var (
		logger *lagertest.TestLogger

		interval           time.Duration
		fakeClock          *fakeclock.FakeClock
		fakeExecutorClient *efakes.FakeClient
		fakeMetricSender   *msfake.FakeMetricSender

		containerResults chan listContainerResults
		process          ifrit.Process
	)

	sendContainerResults := func() {
		one := 1
		containerResults <- listContainerResults{
			containers: []executor.Container{
				{
					Guid: "guid-without-index",

					MemoryUsageInBytes: 123,
					DiskUsageInBytes:   456,
					TimeSpentInCPU:     100 * time.Second,

					MetricsConfig: executor.MetricsConfig{
						Guid: "metrics-guid-without-index",
					},
				},
				{
					Guid: "guid-with-no-metrics-guid",

					MemoryUsageInBytes: 1023,
					DiskUsageInBytes:   4056,
					TimeSpentInCPU:     1000 * time.Second,
				},
				{
					Guid: "guid-with-index",

					MemoryUsageInBytes: 321,
					DiskUsageInBytes:   654,
					TimeSpentInCPU:     100 * time.Second,

					MetricsConfig: executor.MetricsConfig{
						Guid:  "metrics-guid-with-index",
						Index: &one,
					},
				},
			},
			err: nil,
		}

		containerResults <- listContainerResults{
			containers: []executor.Container{
				{
					Guid: "guid-without-index",

					MemoryUsageInBytes: 1230,
					DiskUsageInBytes:   4560,
					TimeSpentInCPU:     105 * time.Second,

					MetricsConfig: executor.MetricsConfig{
						Guid: "metrics-guid-without-index",
					},
				},
				{
					Guid: "guid-with-index",

					MemoryUsageInBytes: 3210,
					DiskUsageInBytes:   6540,
					TimeSpentInCPU:     110 * time.Second,

					MetricsConfig: executor.MetricsConfig{
						Guid:  "metrics-guid-with-index",
						Index: &one,
					},
				},
			},
			err: nil,
		}

		containerResults <- listContainerResults{
			containers: []executor.Container{
				{
					Guid: "guid-without-index",

					MemoryUsageInBytes: 12300,
					DiskUsageInBytes:   45600,
					TimeSpentInCPU:     107 * time.Second,

					MetricsConfig: executor.MetricsConfig{
						Guid: "metrics-guid-without-index",
					},
				},
				{
					Guid: "guid-with-index",

					MemoryUsageInBytes: 32100,
					DiskUsageInBytes:   65400,
					TimeSpentInCPU:     112 * time.Second,

					MetricsConfig: executor.MetricsConfig{
						Guid:  "metrics-guid-with-index",
						Index: &one,
					},
				},
			},
			err: nil,
		}
	}

	BeforeEach(func() {
		logger = lagertest.NewTestLogger("test")

		interval = 10 * time.Second
		fakeClock = fakeclock.NewFakeClock(time.Unix(123, 456))
		fakeExecutorClient = new(efakes.FakeClient)

		fakeMetricSender = msfake.NewFakeMetricSender()
		dmetrics.Initialize(fakeMetricSender)

		containerResults = make(chan listContainerResults, 10)

		fakeExecutorClient.ListContainersStub = func(executor.Tags) ([]executor.Container, error) {
			result := <-containerResults
			return result.containers, result.err
		}

		process = ifrit.Invoke(containermetrics.NewStatsReporter(logger, interval, fakeClock, fakeExecutorClient))
	})

	AfterEach(func() {
		ginkgomon.Interrupt(process)
	})

	Context("when the interval elapses", func() {
		BeforeEach(func() {
			sendContainerResults()

			fakeClock.Increment(interval)
			Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(1))
		})

		It("emits memory and disk usage for each container, but no CPU", func() {
			Eventually(func() msfake.ContainerMetric {
				return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
			}).Should(Equal(msfake.ContainerMetric{
				ApplicationId: "metrics-guid-without-index",
				InstanceIndex: -1,
				CpuPercentage: 0.0,
				MemoryBytes:   123,
				DiskBytes:     456,
			}))

			Eventually(func() msfake.ContainerMetric {
				return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
			}).Should(Equal(msfake.ContainerMetric{
				ApplicationId: "metrics-guid-with-index",
				InstanceIndex: 1,
				CpuPercentage: 0.0,
				MemoryBytes:   321,
				DiskBytes:     654,
			}))
		})

		It("does not emit anything for containers with no metrics guid", func() {
			Consistently(func() msfake.ContainerMetric {
				return fakeMetricSender.GetContainerMetric("")
			}).Should(BeZero())
		})

		Context("and the interval elapses again", func() {
			BeforeEach(func() {
				fakeClock.Increment(interval)
				Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(2))
			})

			It("emits the new memory and disk usage, and the computed CPU percent", func() {
				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-without-index",
					InstanceIndex: -1,
					CpuPercentage: 50.0,
					MemoryBytes:   1230,
					DiskBytes:     4560,
				}))

				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-with-index",
					InstanceIndex: 1,
					CpuPercentage: 100.0,
					MemoryBytes:   3210,
					DiskBytes:     6540,
				}))
			})

			Context("and the interval elapses again", func() {
				BeforeEach(func() {
					fakeClock.Increment(interval)
					Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(3))
				})

				It("emits the new memory and disk usage, and the computed CPU percent", func() {
					Eventually(func() msfake.ContainerMetric {
						return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
					}).Should(Equal(msfake.ContainerMetric{
						ApplicationId: "metrics-guid-without-index",
						InstanceIndex: -1,
						CpuPercentage: 20.0,
						MemoryBytes:   12300,
						DiskBytes:     45600,
					}))

					Eventually(func() msfake.ContainerMetric {
						return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
					}).Should(Equal(msfake.ContainerMetric{
						ApplicationId: "metrics-guid-with-index",
						InstanceIndex: 1,
						CpuPercentage: 20.0,
						MemoryBytes:   32100,
						DiskBytes:     65400,
					}))
				})
			})
		})
	})

	Context("when listing containers fails", func() {
		BeforeEach(func() {
			containerResults <- listContainerResults{containers: nil, err: errors.New("nope")}
			fakeClock.Increment(interval)
			Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(1))
		})

		It("does not blow up", func() {
			Consistently(process.Wait()).ShouldNot(Receive())
		})

		Context("and the interval elapses again, and it works that time", func() {
			BeforeEach(func() {
				sendContainerResults()
				fakeClock.Increment(interval)
				Eventually(fakeExecutorClient.ListContainersCallCount).Should(Equal(2))
			})

			It("processes the containers happily", func() {
				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-without-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-without-index",
					InstanceIndex: -1,
					CpuPercentage: 0.0,
					MemoryBytes:   123,
					DiskBytes:     456,
				}))

				Eventually(func() msfake.ContainerMetric {
					return fakeMetricSender.GetContainerMetric("metrics-guid-with-index")
				}).Should(Equal(msfake.ContainerMetric{
					ApplicationId: "metrics-guid-with-index",
					InstanceIndex: 1,
					CpuPercentage: 0.0,
					MemoryBytes:   321,
					DiskBytes:     654,
				}))
			})
		})
	})
})
<|endoftext|>


<commit_before>package nomad

import (
	"testing"

	memdb "github.com/hashicorp/go-memdb"
	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
	"github.com/hashicorp/nomad/client"
	"github.com/hashicorp/nomad/client/config"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
)

func TestClientCSIController_AttachVolume_Local(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s, cleanupS := TestServer(t, nil)
	defer cleanupS()
	codec := rpcClient(t, s)
	testutil.WaitForLeader(t, s.RPC)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s.config.RPCAddr.String()}
	})
	defer cleanupC()

	testutil.WaitForResult(func() (bool, error) {
		nodes := s.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		require.Fail("should have a client")
	})

	req := &cstructs.ClientCSIControllerAttachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSIController_AttachVolume_Forwarded(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS1()
	s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS2()
	TestJoin(t, s1, s2)
	testutil.WaitForLeader(t, s1.RPC)
	testutil.WaitForLeader(t, s2.RPC)
	codec := rpcClient(t, s2)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s2.config.RPCAddr.String()}
		c.GCDiskUsageThreshold = 100.0
	})
	defer cleanupC()

	testutil.WaitForResult(func() (bool, error) {
		nodes := s2.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		require.Fail("should have a client")
	})

	// Force remove the connection locally in case it exists
	s1.nodeConnsLock.Lock()
	delete(s1.nodeConns, c.NodeID())
	s1.nodeConnsLock.Unlock()

	req := &cstructs.ClientCSIControllerAttachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSIController_DetachVolume_Local(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s, cleanupS := TestServer(t, nil)
	defer cleanupS()
	codec := rpcClient(t, s)
	testutil.WaitForLeader(t, s.RPC)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s.config.RPCAddr.String()}
	})
	defer cleanupC()

	testutil.WaitForResult(func() (bool, error) {
		nodes := s.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		require.Fail("should have a client")
	})

	req := &cstructs.ClientCSIControllerDetachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSIController_DetachVolume_Forwarded(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS1()
	s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS2()
	TestJoin(t, s1, s2)
	testutil.WaitForLeader(t, s1.RPC)
	testutil.WaitForLeader(t, s2.RPC)
	codec := rpcClient(t, s2)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s2.config.RPCAddr.String()}
		c.GCDiskUsageThreshold = 100.0
	})
	defer cleanupC()

	testutil.WaitForResult(func() (bool, error) {
		nodes := s2.connectedNodes()
		return len(nodes) == 1, nil
	}, func(err error) {
		require.Fail("should have a client")
	})

	// Force remove the connection locally in case it exists
	s1.nodeConnsLock.Lock()
	delete(s1.nodeConns, c.NodeID())
	s1.nodeConnsLock.Unlock()

	req := &cstructs.ClientCSIControllerDetachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSI_NodeForControllerPlugin(t *testing.T) {
	t.Parallel()
	srv, shutdown := TestServer(t, func(c *Config) {})
	testutil.WaitForLeader(t, srv.RPC)
	defer shutdown()

	plugins := map[string]*structs.CSIInfo{
		"minnie": {PluginID: "minnie",
			Healthy:                  true,
			ControllerInfo:           &structs.CSIControllerInfo{},
			NodeInfo:                 &structs.CSINodeInfo{},
			RequiresControllerPlugin: true,
		},
	}
	state := srv.fsm.State()

	node1 := mock.Node()
	node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions
	node1.CSIControllerPlugins = plugins
	node2 := mock.Node()
	node2.CSIControllerPlugins = plugins
	node2.ID = uuid.Generate()
	node3 := mock.Node()
	node3.ID = uuid.Generate()

	err := state.UpsertNode(1002, node1)
	require.NoError(t, err)
	err = state.UpsertNode(1003, node2)
	require.NoError(t, err)
	err = state.UpsertNode(1004, node3)
	require.NoError(t, err)

	ws := memdb.NewWatchSet()

	plugin, err := state.CSIPluginByID(ws, "minnie")
	require.NoError(t, err)
	nodeID, err := srv.staticEndpoints.ClientCSI.nodeForController(plugin.ID, "")

	// only node1 has both the controller and a recent Nomad version
	require.Equal(t, nodeID, node1.ID)
}
<commit_msg>tests: wait until clients are in the state store<commit_after>package nomad

import (
	"fmt"
	"testing"

	memdb "github.com/hashicorp/go-memdb"
	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
	"github.com/hashicorp/nomad/client"
	"github.com/hashicorp/nomad/client/config"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/helper/uuid"
	"github.com/hashicorp/nomad/nomad/mock"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/testutil"
	"github.com/stretchr/testify/require"
)

func TestClientCSIController_AttachVolume_Local(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s, cleanupS := TestServer(t, nil)
	defer cleanupS()
	codec := rpcClient(t, s)
	testutil.WaitForLeader(t, s.RPC)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s.config.RPCAddr.String()}
	})
	defer cleanupC()

	waitForNodes(t, s, 1)

	req := &cstructs.ClientCSIControllerAttachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSIController_AttachVolume_Forwarded(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS1()
	s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS2()
	TestJoin(t, s1, s2)
	testutil.WaitForLeader(t, s1.RPC)
	testutil.WaitForLeader(t, s2.RPC)
	codec := rpcClient(t, s2)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s2.config.RPCAddr.String()}
		c.GCDiskUsageThreshold = 100.0
	})
	defer cleanupC()

	waitForNodes(t, s2, 1)

	// Force remove the connection locally in case it exists
	s1.nodeConnsLock.Lock()
	delete(s1.nodeConns, c.NodeID())
	s1.nodeConnsLock.Unlock()

	req := &cstructs.ClientCSIControllerAttachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSIController_DetachVolume_Local(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s, cleanupS := TestServer(t, nil)
	defer cleanupS()
	codec := rpcClient(t, s)
	testutil.WaitForLeader(t, s.RPC)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s.config.RPCAddr.String()}
	})
	defer cleanupC()

	waitForNodes(t, s, 1)

	req := &cstructs.ClientCSIControllerDetachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSIController_DetachVolume_Forwarded(t *testing.T) {
	t.Parallel()
	require := require.New(t)

	// Start a server and client
	s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS1()
	s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 })
	defer cleanupS2()
	TestJoin(t, s1, s2)
	testutil.WaitForLeader(t, s1.RPC)
	testutil.WaitForLeader(t, s2.RPC)
	codec := rpcClient(t, s2)

	c, cleanupC := client.TestClient(t, func(c *config.Config) {
		c.Servers = []string{s2.config.RPCAddr.String()}
		c.GCDiskUsageThreshold = 100.0
	})
	defer cleanupC()

	waitForNodes(t, s2, 1)

	// Force remove the connection locally in case it exists
	s1.nodeConnsLock.Lock()
	delete(s1.nodeConns, c.NodeID())
	s1.nodeConnsLock.Unlock()

	req := &cstructs.ClientCSIControllerDetachVolumeRequest{
		CSIControllerQuery: cstructs.CSIControllerQuery{ControllerNodeID: c.NodeID()},
	}

	// Fetch the response
	var resp structs.GenericResponse
	err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp)
	require.NotNil(err)
	// Should recieve an error from the client endpoint
	require.Contains(err.Error(), "must specify plugin name to dispense")
}

func TestClientCSI_NodeForControllerPlugin(t *testing.T) {
	t.Parallel()
	srv, shutdown := TestServer(t, func(c *Config) {})
	testutil.WaitForLeader(t, srv.RPC)
	defer shutdown()

	plugins := map[string]*structs.CSIInfo{
		"minnie": {PluginID: "minnie",
			Healthy:                  true,
			ControllerInfo:           &structs.CSIControllerInfo{},
			NodeInfo:                 &structs.CSINodeInfo{},
			RequiresControllerPlugin: true,
		},
	}
	state := srv.fsm.State()

	node1 := mock.Node()
	node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions
	node1.CSIControllerPlugins = plugins
	node2 := mock.Node()
	node2.CSIControllerPlugins = plugins
	node2.ID = uuid.Generate()
	node3 := mock.Node()
	node3.ID = uuid.Generate()

	err := state.UpsertNode(1002, node1)
	require.NoError(t, err)
	err = state.UpsertNode(1003, node2)
	require.NoError(t, err)
	err = state.UpsertNode(1004, node3)
	require.NoError(t, err)

	ws := memdb.NewWatchSet()

	plugin, err := state.CSIPluginByID(ws, "minnie")
	require.NoError(t, err)
	nodeID, err := srv.staticEndpoints.ClientCSI.nodeForController(plugin.ID, "")

	// only node1 has both the controller and a recent Nomad version
	require.Equal(t, nodeID, node1.ID)
}

// waitForNodes waits until the server is connected to expectedNodes
// clients and they are in the state store
func waitForNodes(t *testing.T, s *Server, expectedNodes int) {
	codec := rpcClient(t, s)

	testutil.WaitForResult(func() (bool, error) {
		connNodes := s.connectedNodes()
		if len(connNodes) != expectedNodes {
			return false, fmt.Errorf("expected %d nodes but found %d", expectedNodes, len(connNodes))

		}

		get := &structs.NodeListRequest{
			QueryOptions: structs.QueryOptions{Region: "global"},
		}
		var resp structs.NodeListResponse
		err := msgpackrpc.CallWithCodec(codec, "Node.List", get, &resp)
		if err != nil {
			return false, err
		}

		if err != nil {
			return false, fmt.Errorf("failed to list nodes: %v", err)
		}
		if len(resp.Nodes) != 1 {
			return false, fmt.Errorf("expected %d nodes but found %d", 1, len(resp.Nodes))
		}

		return true, nil
	}, func(err error) {
		require.NoError(t, err)
	})
}
<|endoftext|>


<commit_before>// Copyright 2019 The Cloud Robotics Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/googlecloudrobotics/core/src/go/pkg/gcr"
	"github.com/googlecloudrobotics/core/src/go/pkg/kubeutils"
	"github.com/googlecloudrobotics/core/src/go/pkg/robotauth"
	"github.com/googlecloudrobotics/core/src/go/pkg/setup"

	"golang.org/x/oauth2"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

var (
	domain              = flag.String("domain", "", "Domain for the Cloud Robotics project (default: www.endpoints.<project>.cloud.goog)")
	project             = flag.String("project", "", "Project ID for the Google Cloud Platform")
	projectNumber       = flag.Uint64("project-number", 0, "Project Number for the Google Cloud Platform")
	robotName           = flag.String("robot-name", "", "Robot name")
	robotRole           = flag.String("robot-role", "", "Robot role. Optional if the robot is already registered.")
	robotType           = flag.String("robot-type", "", "Robot type. Optional if the robot is already registered.")
	robotAuthentication = flag.Bool("robot-authentication", true, "Set up robot authentication.")
	appManagement       = flag.Bool("app-management", true, "Set up app management.")
)

const (
	filesDir      = "/setup-robot-files"
	helmPath      = filesDir + "/helm"
	numDNSRetries = 6
)

func parseFlags() {
	flag.Parse()

	if *project == "" {
		log.Fatal("--project is required.")
	}
	if *robotName == "" {
		log.Fatal("--robot-name is required.")
	}
	if *domain == "" {
		*domain = fmt.Sprintf("www.endpoints.%s.cloud.goog", *project)
	}
}

// Since this might be the first interaction with the cluster, manually resolve the
// domain name with retries to give a better error in the case of failure.
func waitForDNS() error {
	log.Printf("DNS lookup for %q", *domain)
	delay := time.Second
	for i := 0; i < numDNSRetries; i++ {
		ips, err := net.LookupIP(*domain)
		if err == nil {
			// Check that the results contain an ipv4 addr. Initially, coredns may only
			// return ipv6 addresses in which case helm will fail.
			for _, ip := range ips {
				if ip.To4() != nil {
					return nil
				}
			}
		}
		log.Printf("... Retry dns for %q", *domain)
		time.Sleep(delay)
		delay += delay
	}
	return fmt.Errorf("DNS lookup for %q failed", *domain)
}

func main() {
	parseFlags()
	envToken := os.Getenv("ACCESS_TOKEN")
	if envToken == "" {
		log.Fatal("ACCESS_TOKEN environment variable is required.")
	}

	// Set up the OAuth2 token source.
	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: envToken})

	if *robotRole != "" || *robotType != "" {
		if err := createOrUpdateRobot(tokenSource); err != nil {
			log.Fatalf("Failed to update robot CR %v: %v", *robotName, err)
		}
	}

	// Connect to the surrounding k8s cluster.
	localConfig, err := rest.InClusterConfig()
	if err != nil {
		log.Fatal("Failed to load in-cluster config: ", err)
	}
	k8sLocalClientSet, err := kubernetes.NewForConfig(localConfig)
	if err != nil {
		log.Fatal("Failed to create kubernetes client set: ", err)
	}

	if err := waitForDNS(); err != nil {
		log.Fatalf("Failed to resolve cloud cluster: %s. Please retry in 5 minutes.", err)
	}

	if *robotAuthentication {
		// Set up robot authentication.
		auth := &robotauth.RobotAuth{
			RobotName:           *robotName,
			ProjectId:           *project,
			Domain:              *domain,
			PublicKeyRegistryId: fmt.Sprintf("robot-%s", *robotName),
		}
		client := oauth2.NewClient(context.Background(), tokenSource)
		if err := setup.CreateAndPublishCredentialsToCloud(client, auth); err != nil {
			log.Fatal(err)
		}
		if err := auth.StoreInK8sSecret(k8sLocalClientSet); err != nil {
			log.Fatal(fmt.Errorf("Failed to write auth secret: %v", err))
		}
		if err := gcr.UpdateGcrCredentials(k8sLocalClientSet, auth); err != nil {
			log.Fatal(err)
		}
	}

	// Create service account and role binding for Tiller.
	// (this isn't strictly necessary until we're using auth properly, but it's
	//  one less thing to fix when RBAC is used properly)
	if _, err := k8sLocalClientSet.CoreV1().ServiceAccounts("kube-system").Create(&corev1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "tiller",
			Namespace: "kube-system",
		},
	}); err != nil {
		log.Println("Failed to create tiller service account: ", err)
	}
	if _, err := k8sLocalClientSet.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "tiller",
			Namespace: "kube-system",
		},
		RoleRef: rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "ClusterRole",
			Name:     "cluster-admin",
		},
		Subjects: []rbacv1.Subject{{
			Kind:      "ServiceAccount",
			Name:      "tiller",
			Namespace: "kube-system",
		}},
	}); err != nil {
		log.Println("Failed to create tiller role binding: ", err)
	}

	log.Println("Initializing Helm")
	output, err := exec.Command(
		helmPath,
		"init",
		"--history-max=10",
		"--upgrade",
		"--force-upgrade",
		"--wait",
		"--service-account=tiller").CombinedOutput()
	if err != nil {
		log.Fatalf("Helm init failed: %v. Helm output:\n%s\n", err, output)
	}

	// Clean up deprecated releases.
	deleteReleaseIfPresent("robot-cluster")

	installChartOrDie("robot-base", "base-robot-0.0.1.tgz")
}

func helmValuesStringFromMap(varMap map[string]string) string {
	varList := []string{}
	for k, v := range varMap {
		varList = append(varList, fmt.Sprintf("%s=%s", k, v))
	}
	return strings.Join(varList, ",")
}

func deleteReleaseIfPresent(name string) {
	output, err := exec.Command(
		helmPath,
		"delete",
		"--purge",
		name).CombinedOutput()
	if err != nil {
		log.Printf("Helm delete of %s failed: %v. Helm output:\n%s\n", name, err, output)
	} else {
		log.Printf("%s\nSuccessfully removed %s Helm release", output, name)
	}
}

func installChartOrDie(name string, chartPath string) {
	log.Printf("Installing %s Helm chart from %s", name, chartPath)
	vars := helmValuesStringFromMap(map[string]string{
		"domain":               *domain,
		"project":              *project,
		"project_number":       strconv.FormatUint(*projectNumber, 10),
		"app_management":       strconv.FormatBool(*appManagement),
		"robot_authentication": strconv.FormatBool(*robotAuthentication),
		"robot.name":           *robotName,
	})
	output, err := exec.Command(
		helmPath,
		"upgrade",
		"--install",
		"--force",
		"--set-string",
		vars,
		name,
		filepath.Join(filesDir, chartPath)).CombinedOutput()
	if err != nil {
		log.Fatalf("Helm install of %s failed: %v. Helm output:\n%s\n", name, err, output)
	} else {
		log.Printf("%s\nSuccessfully installed %s Helm chart", output, name)
	}
}

func createOrUpdateRobot(tokenSource oauth2.TokenSource) error {
	// Set up client for cloud k8s cluster (needed only to obtain list of robots).
	k8sCloudCfg := kubeutils.BuildCloudKubernetesConfig(tokenSource, *domain)
	k8sDynamicClient, err := dynamic.NewForConfig(k8sCloudCfg)
	if err != nil {
		return err
	}
	robotGVR := schema.GroupVersionResource{
		Group:    "registry.cloudrobotics.com",
		Version:  "v1alpha1",
		Resource: "robots"}
	robotClient := k8sDynamicClient.Resource(robotGVR).Namespace("default")
	robot, err := robotClient.Get(*robotName, metav1.GetOptions{})
	if err != nil {
		if s, ok := err.(*apierrors.StatusError); ok && s.ErrStatus.Reason == metav1.StatusReasonNotFound {
			robot := &unstructured.Unstructured{}
			robot.SetKind("Robot")
			robot.SetAPIVersion("registry.cloudrobotics.com/v1alpha1")
			robot.SetName(*robotName)
			robot.SetLabels(map[string]string{"cloudrobotics.com/robot-name": *robotName})
			robot.Object["spec"] = map[string]string{
				"role":    *robotRole,
				"type":    *robotType,
				"project": *project,
			}
			robot.Object["status"] = make(map[string]interface{})
			_, err := robotClient.Create(robot, metav1.CreateOptions{})
			return err
		} else {
			return fmt.Errorf("Failed to get robot %v: %v", *robotName, err)
		}
	}
	spec, ok := robot.Object["spec"].(map[string]interface{})
	if !ok {
		return fmt.Errorf("unmarshaling robot failed: spec is not a map")
	}
	spec["role"] = *robotRole
	spec["type"] = *robotType
	spec["project"] = *project
	_, err = robotClient.Update(robot, metav1.UpdateOptions{})
	return err
}
<commit_msg>Wait for DNS before updating robot CR<commit_after>// Copyright 2019 The Cloud Robotics Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/googlecloudrobotics/core/src/go/pkg/gcr"
	"github.com/googlecloudrobotics/core/src/go/pkg/kubeutils"
	"github.com/googlecloudrobotics/core/src/go/pkg/robotauth"
	"github.com/googlecloudrobotics/core/src/go/pkg/setup"

	"golang.org/x/oauth2"
	corev1 "k8s.io/api/core/v1"
	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

var (
	domain              = flag.String("domain", "", "Domain for the Cloud Robotics project (default: www.endpoints.<project>.cloud.goog)")
	project             = flag.String("project", "", "Project ID for the Google Cloud Platform")
	projectNumber       = flag.Uint64("project-number", 0, "Project Number for the Google Cloud Platform")
	robotName           = flag.String("robot-name", "", "Robot name")
	robotRole           = flag.String("robot-role", "", "Robot role. Optional if the robot is already registered.")
	robotType           = flag.String("robot-type", "", "Robot type. Optional if the robot is already registered.")
	robotAuthentication = flag.Bool("robot-authentication", true, "Set up robot authentication.")
	appManagement       = flag.Bool("app-management", true, "Set up app management.")
)

const (
	filesDir      = "/setup-robot-files"
	helmPath      = filesDir + "/helm"
	numDNSRetries = 6
)

func parseFlags() {
	flag.Parse()

	if *project == "" {
		log.Fatal("--project is required.")
	}
	if *robotName == "" {
		log.Fatal("--robot-name is required.")
	}
	if *domain == "" {
		*domain = fmt.Sprintf("www.endpoints.%s.cloud.goog", *project)
	}
}

// Since this might be the first interaction with the cluster, manually resolve the
// domain name with retries to give a better error in the case of failure.
func waitForDNS() error {
	log.Printf("DNS lookup for %q", *domain)
	delay := time.Second
	for i := 0; i < numDNSRetries; i++ {
		ips, err := net.LookupIP(*domain)
		if err == nil {
			// Check that the results contain an ipv4 addr. Initially, coredns may only
			// return ipv6 addresses in which case helm will fail.
			for _, ip := range ips {
				if ip.To4() != nil {
					return nil
				}
			}
		}
		log.Printf("... Retry dns for %q", *domain)
		time.Sleep(delay)
		delay += delay
	}
	return fmt.Errorf("DNS lookup for %q failed", *domain)
}

func main() {
	parseFlags()
	envToken := os.Getenv("ACCESS_TOKEN")
	if envToken == "" {
		log.Fatal("ACCESS_TOKEN environment variable is required.")
	}

	if err := waitForDNS(); err != nil {
		log.Fatalf("Failed to resolve cloud cluster: %s. Please retry in 5 minutes.", err)
	}

	// Set up the OAuth2 token source.
	tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: envToken})

	if *robotRole != "" || *robotType != "" {
		if err := createOrUpdateRobot(tokenSource); err != nil {
			log.Fatalf("Failed to update robot CR %v: %v", *robotName, err)
		}
	}

	// Connect to the surrounding k8s cluster.
	localConfig, err := rest.InClusterConfig()
	if err != nil {
		log.Fatal("Failed to load in-cluster config: ", err)
	}
	k8sLocalClientSet, err := kubernetes.NewForConfig(localConfig)
	if err != nil {
		log.Fatal("Failed to create kubernetes client set: ", err)
	}

	if *robotAuthentication {
		// Set up robot authentication.
		auth := &robotauth.RobotAuth{
			RobotName:           *robotName,
			ProjectId:           *project,
			Domain:              *domain,
			PublicKeyRegistryId: fmt.Sprintf("robot-%s", *robotName),
		}
		client := oauth2.NewClient(context.Background(), tokenSource)
		if err := setup.CreateAndPublishCredentialsToCloud(client, auth); err != nil {
			log.Fatal(err)
		}
		if err := auth.StoreInK8sSecret(k8sLocalClientSet); err != nil {
			log.Fatal(fmt.Errorf("Failed to write auth secret: %v", err))
		}
		if err := gcr.UpdateGcrCredentials(k8sLocalClientSet, auth); err != nil {
			log.Fatal(err)
		}
	}

	// Create service account and role binding for Tiller.
	// (this isn't strictly necessary until we're using auth properly, but it's
	//  one less thing to fix when RBAC is used properly)
	if _, err := k8sLocalClientSet.CoreV1().ServiceAccounts("kube-system").Create(&corev1.ServiceAccount{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "tiller",
			Namespace: "kube-system",
		},
	}); err != nil {
		log.Println("Failed to create tiller service account: ", err)
	}
	if _, err := k8sLocalClientSet.RbacV1().ClusterRoleBindings().Create(&rbacv1.ClusterRoleBinding{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "tiller",
			Namespace: "kube-system",
		},
		RoleRef: rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "ClusterRole",
			Name:     "cluster-admin",
		},
		Subjects: []rbacv1.Subject{{
			Kind:      "ServiceAccount",
			Name:      "tiller",
			Namespace: "kube-system",
		}},
	}); err != nil {
		log.Println("Failed to create tiller role binding: ", err)
	}

	log.Println("Initializing Helm")
	output, err := exec.Command(
		helmPath,
		"init",
		"--history-max=10",
		"--upgrade",
		"--force-upgrade",
		"--wait",
		"--service-account=tiller").CombinedOutput()
	if err != nil {
		log.Fatalf("Helm init failed: %v. Helm output:\n%s\n", err, output)
	}

	// Clean up deprecated releases.
	deleteReleaseIfPresent("robot-cluster")

	installChartOrDie("robot-base", "base-robot-0.0.1.tgz")
}

func helmValuesStringFromMap(varMap map[string]string) string {
	varList := []string{}
	for k, v := range varMap {
		varList = append(varList, fmt.Sprintf("%s=%s", k, v))
	}
	return strings.Join(varList, ",")
}

func deleteReleaseIfPresent(name string) {
	output, err := exec.Command(
		helmPath,
		"delete",
		"--purge",
		name).CombinedOutput()
	if err != nil {
		log.Printf("Helm delete of %s failed: %v. Helm output:\n%s\n", name, err, output)
	} else {
		log.Printf("%s\nSuccessfully removed %s Helm release", output, name)
	}
}

func installChartOrDie(name string, chartPath string) {
	log.Printf("Installing %s Helm chart from %s", name, chartPath)
	vars := helmValuesStringFromMap(map[string]string{
		"domain":               *domain,
		"project":              *project,
		"project_number":       strconv.FormatUint(*projectNumber, 10),
		"app_management":       strconv.FormatBool(*appManagement),
		"robot_authentication": strconv.FormatBool(*robotAuthentication),
		"robot.name":           *robotName,
	})
	output, err := exec.Command(
		helmPath,
		"upgrade",
		"--install",
		"--force",
		"--set-string",
		vars,
		name,
		filepath.Join(filesDir, chartPath)).CombinedOutput()
	if err != nil {
		log.Fatalf("Helm install of %s failed: %v. Helm output:\n%s\n", name, err, output)
	} else {
		log.Printf("%s\nSuccessfully installed %s Helm chart", output, name)
	}
}

func createOrUpdateRobot(tokenSource oauth2.TokenSource) error {
	// Set up client for cloud k8s cluster (needed only to obtain list of robots).
	k8sCloudCfg := kubeutils.BuildCloudKubernetesConfig(tokenSource, *domain)
	k8sDynamicClient, err := dynamic.NewForConfig(k8sCloudCfg)
	if err != nil {
		return err
	}
	robotGVR := schema.GroupVersionResource{
		Group:    "registry.cloudrobotics.com",
		Version:  "v1alpha1",
		Resource: "robots"}
	robotClient := k8sDynamicClient.Resource(robotGVR).Namespace("default")
	robot, err := robotClient.Get(*robotName, metav1.GetOptions{})
	if err != nil {
		if s, ok := err.(*apierrors.StatusError); ok && s.ErrStatus.Reason == metav1.StatusReasonNotFound {
			robot := &unstructured.Unstructured{}
			robot.SetKind("Robot")
			robot.SetAPIVersion("registry.cloudrobotics.com/v1alpha1")
			robot.SetName(*robotName)
			robot.SetLabels(map[string]string{"cloudrobotics.com/robot-name": *robotName})
			robot.Object["spec"] = map[string]string{
				"role":    *robotRole,
				"type":    *robotType,
				"project": *project,
			}
			robot.Object["status"] = make(map[string]interface{})
			_, err := robotClient.Create(robot, metav1.CreateOptions{})
			return err
		} else {
			return fmt.Errorf("Failed to get robot %v: %v", *robotName, err)
		}
	}
	spec, ok := robot.Object["spec"].(map[string]interface{})
	if !ok {
		return fmt.Errorf("unmarshaling robot failed: spec is not a map")
	}
	spec["role"] = *robotRole
	spec["type"] = *robotType
	spec["project"] = *project
	_, err = robotClient.Update(robot, metav1.UpdateOptions{})
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"container/list"
	"fmt"
	"os"
	"sort"
	"strconv"

	"github.com/jyggen/advent-of-go/internal/solver"
	"github.com/jyggen/advent-of-go/internal/utils"
)

func main() {
	p1, p2, err := solver.SolveFromFile(os.Stdin, SolvePart1, SolvePart2)
	if err != nil {
		panic(err)
	}

	fmt.Println(p1)
	fmt.Println(p2)
}

func filterCorrupted(input string) ([][]rune, int) {
	lines := utils.ToRuneSlice(input, "\n")
	score := 0
	incomplete := make([][]rune, 0, len(lines))

	for _, runes := range lines {
		state := list.New()
		corrupted := false

		for _, r := range runes {
			switch r {
			case '(':
				state.PushBack(0)
			case ')':
				e := state.Back()

				if e.Value.(int) != 0 {
					score += 3
					corrupted = true
					break
				}

				state.Remove(e)
			case '[':
				state.PushBack(1)
			case ']':
				e := state.Back()

				if e.Value.(int) != 1 {
					score += 57
					corrupted = true
					break
				}

				state.Remove(e)
			case '{':
				state.PushBack(2)
			case '}':
				e := state.Back()

				if e.Value.(int) != 2 {
					score += 1197
					corrupted = true
					break
				}

				state.Remove(e)
			case '<':
				state.PushBack(3)
			case '>':
				e := state.Back()

				if e.Value.(int) != 3 {
					score += 25137
					corrupted = true
					break
				}

				state.Remove(e)
			}
		}

		if !corrupted {
			incomplete = append(incomplete, runes)
		}
	}

	return incomplete, score
}

func SolvePart1(input string) (string, error) {
	_, score := filterCorrupted(input)

	return strconv.Itoa(score), nil
}

func SolvePart2(input string) (string, error) {
	incomplete, _ := filterCorrupted(input)
	scores := make([]int, 0, len(incomplete))

	for _, runes := range incomplete {
		state := list.New()
		score := 0

		for _, r := range runes {
			switch r {
			case '(':
				state.PushBack(0)
			case ')':
				e := state.Back()
				state.Remove(e)
			case '[':
				state.PushBack(1)
			case ']':
				e := state.Back()
				state.Remove(e)
			case '{':
				state.PushBack(2)
			case '}':
				e := state.Back()
				state.Remove(e)
			case '<':
				state.PushBack(3)
			case '>':
				e := state.Back()
				state.Remove(e)
			}
		}

		for state.Len() > 0 {
			e := state.Back()
			score *= 5

			switch e.Value.(int) {
			case 0:
				score += 1
			case 1:
				score += 2
			case 2:
				score += 3
			case 3:
				score += 4
			}

			state.Remove(e)
		}

		scores = append(scores, score)
	}

	sort.Ints(scores)

	return strconv.Itoa(scores[len(scores)/2]), nil
}
<commit_msg>Clean up 2021.10<commit_after>package main

import (
	"container/list"
	"fmt"
	"os"
	"sort"
	"strconv"

	"github.com/jyggen/advent-of-go/internal/solver"
	"github.com/jyggen/advent-of-go/internal/utils"
)

func main() {
	p1, p2, err := solver.SolveFromFile(os.Stdin, SolvePart1, SolvePart2)
	if err != nil {
		panic(err)
	}

	fmt.Println(p1)
	fmt.Println(p2)
}

func filterCorrupted(input string) ([][]rune, int) {
	lines := utils.ToRuneSlice(input, "\n")
	score := 0
	incomplete := make([][]rune, 0, len(lines))
	pairs := map[rune]rune{
		'(': ')',
		'[': ']',
		'{': '}',
		'<': '>',
	}
	points := map[rune]int{
		')': 3,
		']': 57,
		'}': 1197,
		'>': 25137,
	}

	for _, runes := range lines {
		state := list.New()
		corrupted := false

		for _, r := range runes {
			switch r {
			case '(', '[', '{', '<':
				state.PushBack(r)
			case ')', ']', '}', '>':
				e := state.Back()

				if pairs[e.Value.(rune)] != r {
					score += points[r]
					corrupted = true
					break
				}

				state.Remove(e)
			}
		}

		if !corrupted {
			incomplete = append(incomplete, runes)
		}
	}

	return incomplete, score
}

func SolvePart1(input string) (string, error) {
	_, score := filterCorrupted(input)

	return strconv.Itoa(score), nil
}

func SolvePart2(input string) (string, error) {
	incomplete, _ := filterCorrupted(input)
	scores := make([]int, 0, len(incomplete))
	points := map[rune]int{
		'(': 1,
		'[': 2,
		'{': 3,
		'<': 4,
	}

	for _, runes := range incomplete {
		state := list.New()
		score := 0

		for _, r := range runes {
			switch r {
			case '(', '[', '{', '<':
				state.PushBack(r)
			case ')', ']', '}', '>':
				state.Remove(state.Back())
			}
		}

		for state.Len() > 0 {
			e := state.Back()
			score *= 5
			score += points[e.Value.(rune)]

			state.Remove(e)
		}

		scores = append(scores, score)
	}

	sort.Ints(scores)

	return strconv.Itoa(scores[len(scores)/2]), nil
}
<|endoftext|>


<commit_before>package parser

import (
	"testing"

	"github.com/fatih/color"
)

var items = map[itemType]string{
	itemVersion:  "itemVersion",
	itemOperator: "itemOperator",
	itemSet:      "itemSet",
	itemRange:    "itemRange",
	itemAdvanced: "itemAdvanced",
	itemError:    "itemError",
	itemEOF:      "itemEOF",
}

type results []itemType

type lexerTestables struct {
	expected bool
	value    string
	result   results
}

var constraints = []*lexerTestables{
	{true, "1.0.0 || >=2.5.0 || 5.0.0 - 7.2.3",
		results{
			itemVersion,
			itemRange,
			itemOperator,
			itemVersion,
			itemRange,
			itemVersion,
			itemAdvanced,
			itemVersion,
		},
	},
	// Operators
	{true, "=2.3.2",
		results{itemOperator, itemVersion},
	},
	{true, "<=1.2.3",
		results{itemOperator, itemVersion},
	},
	{true, ">=1.2.3",
		results{itemOperator, itemVersion},
	},
	// Sets
	{true, "5.3.5 4.3.5",
		results{itemVersion, itemSet, itemVersion},
	},
	//Ranges
	{true, "5.3.5||4.3.5",
		results{itemVersion, itemRange, itemVersion},
	},
	{true, "5.3.5 ||4.3.5",
		results{itemVersion, itemRange, itemVersion},
	},
	{true, "5.3.5|| 4.3.5",
		results{itemVersion, itemRange, itemVersion},
	},
	// Tilde and Caret Ranges
	{false, "~ 1.2.3",
		results{itemAdvanced},
	},
	{true, "~1.2.3",
		results{itemAdvanced, itemVersion},
	},
	{true, "^4.5.2-alpha.1",
		results{itemAdvanced, itemVersion},
	},
	{false, ">= 1.2.3",
		results{},
	},
	// X-Ranges
	{true, "*",
		results{itemAdvanced},
	},
	{true, "1.0",
		results{itemAdvanced},
	},
	{true, "1.x",
		results{itemAdvanced},
	},
	{false, "1.x+98uihuhyg",
		results{},
	},
	{true, "1.*.2",
		results{itemAdvanced},
	},
	{true, "1.*.2 || 1.x.4",
		results{itemAdvanced, itemRange, itemAdvanced},
	},
	{true, "1.*.2-beta",
		results{itemAdvanced},
	},
	{true, "*.1.2",
		results{itemAdvanced},
	},
	{false, "1x.2.*",
		results{},
	},
	{false, "1.x2.*",
		results{},
	},
	{false, "1...1",
		results{},
	},
	{false, "1.x.",
		results{},
	},

	// Assorted syntax errors
	{false, "1.2.3 >=",
		results{itemVersion, itemSet},
	},
	{false, "5.3.5 |1| 4.3.5",
		results{itemVersion},
	},
	{false, "5. 4.4",
		results{},
	},
	{false, "<1<1",
		results{itemOperator},
	},
	{false, "<1||",
		results{itemOperator, itemVersion},
	},
	{false, "M",
		results{},
	},
}

func init() {
	// Appends appropriate end token based on expected result.
	for _, c := range constraints {
		if c.expected {
			c.result = append(c.result, itemEOF)
		} else {
			c.result = append(c.result, itemError)
		}
	}
}

var cyan = color.New(color.FgCyan).SprintFunc()
var yellow = color.New(color.FgYellow).SprintFunc()

func TestLexer(t *testing.T) {
	for _, c := range constraints {
		_, ch := lex(c.value)
		result := true
		x := 0
		for i := range ch {

			result = (i.typ != itemError)

			if i.typ != c.result[x] {
				t.Logf("lex(%v) => %v(%v), want %v \n", cyan(c.value), items[i.typ], yellow(i), items[c.result[x]])
			}
			x++
		}
		if result != c.expected {
			t.Logf("lex(%v) => %t, want %t \n", cyan(c.value), result, c.expected)
		}
	}
}

// Poor implementation, just for initial testing.
func BenchmarkLexerComplex(b *testing.B) {
	const VERSION = "1.0.0 || >=2.5.0 || 5.0.0 - 7.2.3 || ~4.3.1 ^2.1.1"

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		_, ch := lex(VERSION)
		for {
			_, ok := <-ch
			if ok == false {
				//fmt.Printf("%v: '%v' \n", items[s.typ], s)
				//} else {
				break
			}
		}
	}
}

// Poor implementation, just for initial testing.
func BenchmarkLexerSimple(b *testing.B) {
	const VERSION = "1.0.0"

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		_, ch := lex(VERSION)
		for {
			_, ok := <-ch
			if ok == false {
				//fmt.Printf("%v: '%v' \n", items[s.typ], s)
				//} else {
				break
			}
		}
	}
}
<commit_msg>Added more tests.<commit_after>package parser

import (
	"testing"

	"github.com/fatih/color"
)

var items = map[itemType]string{
	itemVersion:  "itemVersion",
	itemOperator: "itemOperator",
	itemSet:      "itemSet",
	itemRange:    "itemRange",
	itemAdvanced: "itemAdvanced",
	itemError:    "itemError",
	itemEOF:      "itemEOF",
}

type results []itemType

type lexerTestables struct {
	expected bool
	value    string
	result   results
}

var constraints = []*lexerTestables{
	{true, "1.0.0 || >=2.5.0 || 5.0.0 - 7.2.3",
		results{
			itemVersion,
			itemRange,
			itemOperator,
			itemVersion,
			itemRange,
			itemVersion,
			itemAdvanced,
			itemVersion,
		},
	},
	// Operators
	{true, "=2.3.2",
		results{itemOperator, itemVersion},
	},
	{true, "<=1.2.3",
		results{itemOperator, itemVersion},
	},
	{true, ">=1.2.3",
		results{itemOperator, itemVersion},
	},
	// Sets
	{true, "5.3.5 4.3.5",
		results{itemVersion, itemSet, itemVersion},
	},
	//Ranges
	{true, "5.3.5||4.3.5",
		results{itemVersion, itemRange, itemVersion},
	},
	{true, "5.3.5 ||4.3.5",
		results{itemVersion, itemRange, itemVersion},
	},
	{true, "5.3.5|| 4.3.5",
		results{itemVersion, itemRange, itemVersion},
	},
	// Tilde and Caret Ranges
	{false, "~ 1.2.3",
		results{itemAdvanced},
	},
	{true, "~1.2.3",
		results{itemAdvanced, itemVersion},
	},
	{true, "^4.5.2-alpha.1",
		results{itemAdvanced, itemVersion},
	},
	{false, ">= 1.2.3",
		results{},
	},
	// X-Ranges
	{true, "*",
		results{itemAdvanced},
	},
	{false, "**",
		results{},
	},
	{true, "1.0",
		results{itemAdvanced},
	},
	{true, "1.x",
		results{itemAdvanced},
	},
	{true, "*.x",
		results{itemAdvanced},
	},
	{false, "1.x+98uihuhyg",
		results{},
	},
	{true, "1.*.2",
		results{itemAdvanced},
	},
	{true, "1.*.2 || 1.x.4",
		results{itemAdvanced, itemRange, itemAdvanced},
	},
	{true, "1.*.2-beta",
		results{itemAdvanced},
	},
	{true, "*.1.2",
		results{itemAdvanced},
	},
	{false, "1x.2.*",
		results{},
	},
	{false, "1.x2.*",
		results{},
	},
	{false, "1...1",
		results{},
	},
	{false, "1.x.",
		results{},
	},

	// Assorted syntax errors
	{false, "1.2.3 >=",
		results{itemVersion, itemSet},
	},
	{false, "5.3.5 |1| 4.3.5",
		results{itemVersion},
	},
	{false, "5. 4.4",
		results{},
	},
	{false, "<1<1",
		results{itemOperator},
	},
	{false, "<1||",
		results{itemOperator, itemVersion},
	},
	{false, "M",
		results{},
	},
}

func init() {
	// Appends appropriate end token based on expected result.
	for _, c := range constraints {
		if c.expected {
			c.result = append(c.result, itemEOF)
		} else {
			c.result = append(c.result, itemError)
		}
	}
}

var cyan = color.New(color.FgCyan).SprintFunc()
var yellow = color.New(color.FgYellow).SprintFunc()

func TestLexer(t *testing.T) {
	for _, c := range constraints {
		_, ch := lex(c.value)
		result := true
		x := 0
		for i := range ch {

			result = (i.typ != itemError)

			if i.typ != c.result[x] {
				t.Logf("lex(%v) => %v(%v), want %v \n", cyan(c.value), items[i.typ], yellow(i), items[c.result[x]])
			}
			x++
		}
		if result != c.expected {
			t.Logf("lex(%v) => %t, want %t \n", cyan(c.value), result, c.expected)
		}
	}
}

// Poor implementation, just for initial testing.
func BenchmarkLexerComplex(b *testing.B) {
	const VERSION = "1.0.0 || >=2.5.0 || 5.0.0 - 7.2.3 || ~4.3.1 ^2.1.1"

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		_, ch := lex(VERSION)
		for {
			_, ok := <-ch
			if ok == false {
				//fmt.Printf("%v: '%v' \n", items[s.typ], s)
				//} else {
				break
			}
		}
	}
}

// Poor implementation, just for initial testing.
func BenchmarkLexerSimple(b *testing.B) {
	const VERSION = "1.0.0"

	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		_, ch := lex(VERSION)
		for {
			_, ok := <-ch
			if ok == false {
				//fmt.Printf("%v: '%v' \n", items[s.typ], s)
				//} else {
				break
			}
		}
	}
}
<|endoftext|>