<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"github.com/vharitonsky/iniflags"
	mwclient "cgt.name/pkg/go-mwclient"
	"github.com/garyhouston/takenwith/mwlib"
	"log"
	"os"
)

// Flags are just to get the user details, usually from the configuration file.
func getUser() string {
	iniflags.SetConfigFile(mwlib.GetWorkingDir() + "/wikilogin.conf")
	var user string
	flag.StringVar(&user, "user", "nobody@example.com", "Operator's email address or Wiki user name.")
	iniflags.Parse()
	return user
}

// This login program must be run before using the main bot. It saves
// cookies into a file in the bot's directory.
func main() {
	userinfo := getUser()
	if len(os.Args) != 1 {
		log.Fatal("Usage: ", os.Args[0], " -help: display options")
	}
	client, err := mwclient.New("https://commons.wikimedia.org/w/api.php", "wikilogin " + userinfo)
	if err != nil {
		panic(err)
	}

	scanner := bufio.NewScanner(os.Stdin)
	fmt.Print("Username: ")
	scanner.Scan()
	user := scanner.Text()
	fmt.Print("Password: ")
	scanner.Scan()
	password := scanner.Text()

	client.Maxlag.On = true

	err = client.Login(user, password)
	if err != nil {
		log.Fatal(err)
	}
	cookies := client.DumpCookies()
	cookieFile := mwlib.GetWorkingDir() + "/cookies"
	writer, err := os.Create(cookieFile)
	if err != nil {
		panic(err)
	}
	for i := range cookies {
		writer.WriteString(cookies[i].Name)
		writer.WriteString(" ")
		writer.WriteString(cookies[i].Value)
		writer.WriteString("\n")
	}
}
<commit_msg>go fmt<commit_after>package main

import (
	"bufio"
	mwclient "cgt.name/pkg/go-mwclient"
	"flag"
	"fmt"
	"github.com/garyhouston/takenwith/mwlib"
	"github.com/vharitonsky/iniflags"
	"log"
	"os"
)

// Flags are just to get the user details, usually from the configuration file.
func getUser() string {
	iniflags.SetConfigFile(mwlib.GetWorkingDir() + "/wikilogin.conf")
	var user string
	flag.StringVar(&user, "user", "nobody@example.com", "Operator's email address or Wiki user name.")
	iniflags.Parse()
	return user
}

// This login program must be run before using the main bot. It saves
// cookies into a file in the bot's directory.
func main() {
	userinfo := getUser()
	if len(os.Args) != 1 {
		log.Fatal("Usage: ", os.Args[0], " -help: display options")
	}
	client, err := mwclient.New("https://commons.wikimedia.org/w/api.php", "wikilogin "+userinfo)
	if err != nil {
		panic(err)
	}

	scanner := bufio.NewScanner(os.Stdin)
	fmt.Print("Username: ")
	scanner.Scan()
	user := scanner.Text()
	fmt.Print("Password: ")
	scanner.Scan()
	password := scanner.Text()

	client.Maxlag.On = true

	err = client.Login(user, password)
	if err != nil {
		log.Fatal(err)
	}
	cookies := client.DumpCookies()
	cookieFile := mwlib.GetWorkingDir() + "/cookies"
	writer, err := os.Create(cookieFile)
	if err != nil {
		panic(err)
	}
	for i := range cookies {
		writer.WriteString(cookies[i].Name)
		writer.WriteString(" ")
		writer.WriteString(cookies[i].Value)
		writer.WriteString("\n")
	}
}
<|endoftext|>


<commit_before>package torrent_test

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenkalti/log"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/resume/torrentresume"
	"github.com/cenkalti/rain/torrent"
	"github.com/crosbymichael/tracker/registry/inmem"
	"github.com/crosbymichael/tracker/server"
)

var (
	trackerAddr    = ":5000"
	torrentFile    = filepath.Join("testdata", "10mb.torrent")
	torrentDataDir = "testdata"
	torrentName    = "10mb"
	timeout        = 10 * time.Second
)

func init() {
	logger.SetLogLevel(log.DEBUG)
}

func newResumeFile(t *testing.T) *torrentresume.TorrentResume {
	resumeFile, err := ioutil.TempFile("", "rain-test-")
	if err != nil {
		t.Fatal(err)
	}
	resumePath := resumeFile.Name()
	println("XXX", resumePath)
	res, err := torrentresume.New(resumePath)
	if err != nil {
		t.Fatal(err)
	}
	return res
	// TODO remove tempfile
}

func TestDownloadTorrent(t *testing.T) {
	where, err := ioutil.TempDir("", "rain-")
	if err != nil {
		t.Fatal(err)
	}

	res1 := newResumeFile(t)
	defer res1.Close()

	defer startTracker(t)()

	f, err := os.Open(torrentFile)
	if err != nil {
		t.Fatal(err)
	}

	t1, err := torrent.New(f, torrentDataDir, 6881, res1)
	if err != nil {
		t.Fatal(err)
	}
	defer t1.Close()
	t1.Start()

	// Wait for seeder to announce to tracker.
	time.Sleep(100 * time.Millisecond)

	res2 := newResumeFile(t)
	defer res2.Close()

	f.Seek(0, io.SeekStart)
	t2, err := torrent.New(f, where, 6882, res2)
	if err != nil {
		t.Fatal(err)
	}
	defer t2.Close()
	t2.Start()

	select {
	case <-t2.NotifyComplete():
	case err = <-t2.NotifyError():
		t.Fatal(err)
	case <-time.After(timeout):
		panic("download did not finish")
	}

	cmd := exec.Command("diff", "-rq",
		filepath.Join(torrentDataDir, torrentName),
		filepath.Join(where, torrentName))
	err = cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

	err = os.RemoveAll(where)
	if err != nil {
		t.Fatal(err)
	}
}

func startTracker(t *testing.T) (stop func()) {
	logger := logrus.New()
	logger.Level = logrus.DebugLevel
	registry := inmem.New()
	s := server.New(120, 30, registry, logger)
	srv := http.Server{
		Addr:    trackerAddr,
		Handler: s,
	}
	go func() {
		err := srv.ListenAndServe()
		if err == http.ErrServerClosed {
			return
		}
		t.Fatal(err)
	}()
	return func() {
		ctx, cancel := context.WithCancel(context.Background())
		cancel()
		err := srv.Shutdown(ctx)
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestDownloadMagnet(t *testing.T) {
	where, err := ioutil.TempDir("", "rain-")
	if err != nil {
		t.Fatal(err)
	}

	res1 := newResumeFile(t)
	defer res1.Close()

	defer startTracker(t)()

	f, err := os.Open(torrentFile)
	if err != nil {
		t.Fatal(err)
	}

	t1, err := torrent.New(f, torrentDataDir, 6881, res1)
	if err != nil {
		t.Fatal(err)
	}
	defer t1.Close()
	t1.Start()

	// Wait for seeder to announce to tracker.
	time.Sleep(100 * time.Millisecond)

	res2 := newResumeFile(t)
	defer res2.Close()

	f.Seek(0, io.SeekStart)
	magnetLink := "magnet:?xt=urn:btih:0a8e2e8c9371a91e9047ed189ceffbc460803262&dn=10mb&tr=http%3A%2F%2F127.0.0.1%3A5000%2Fannounce"
	t2, err := torrent.NewMagnet(magnetLink, where, 6882, res2)
	if err != nil {
		t.Fatal(err)
	}
	defer t2.Close()
	t2.Start()

	select {
	case <-t2.NotifyComplete():
	case err = <-t2.NotifyError():
		t.Fatal(err)
	case <-time.After(timeout):
		panic("download did not finish")
	}

	cmd := exec.Command("diff", "-rq",
		filepath.Join(torrentDataDir, torrentName),
		filepath.Join(where, torrentName))
	err = cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

	err = os.RemoveAll(where)
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>fix tests<commit_after>package torrent_test

import (
	"context"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/cenkalti/log"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/resume/torrentresume"
	"github.com/cenkalti/rain/torrent"
	"github.com/crosbymichael/tracker/registry/inmem"
	"github.com/crosbymichael/tracker/server"
)

var (
	trackerAddr    = ":5000"
	torrentFile    = filepath.Join("testdata", "10mb.torrent")
	torrentDataDir = "testdata"
	torrentName    = "10mb"
	timeout        = 10 * time.Second
)

func init() {
	logger.SetLogLevel(log.DEBUG)
}

func newResumeFile(t *testing.T) *torrentresume.TorrentResume {
	resumeFile, err := ioutil.TempFile("", "rain-test-")
	if err != nil {
		t.Fatal(err)
	}
	resumePath := resumeFile.Name()
	println("XXX", resumePath)
	res, err := torrentresume.New(resumePath)
	if err != nil {
		t.Fatal(err)
	}
	return res
	// TODO remove tempfile
}

func TestDownloadTorrent(t *testing.T) {
	where, err := ioutil.TempDir("", "rain-")
	if err != nil {
		t.Fatal(err)
	}

	res1 := newResumeFile(t)
	defer res1.Close()

	defer startTracker(t)()

	f, err := os.Open(torrentFile)
	if err != nil {
		t.Fatal(err)
	}

	t1, err := torrent.New(f, torrentDataDir, 6881, res1)
	if err != nil {
		t.Fatal(err)
	}
	defer t1.Close()
	t1.Start()

	// Wait for seeder to announce to tracker.
	time.Sleep(100 * time.Millisecond)

	res2 := newResumeFile(t)
	defer res2.Close()

	f.Seek(0, io.SeekStart)
	t2, err := torrent.New(f, where, 6882, res2)
	if err != nil {
		t.Fatal(err)
	}
	defer t2.Close()
	t2.Start()

	select {
	case <-t2.NotifyComplete():
	case err = <-t2.NotifyError():
		t.Fatal(err)
	case <-time.After(timeout):
		panic("download did not finish")
	}

	cmd := exec.Command("diff", "-rq",
		filepath.Join(torrentDataDir, torrentName),
		filepath.Join(where, torrentName))
	err = cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

	err = os.RemoveAll(where)
	if err != nil {
		t.Fatal(err)
	}
}

func startTracker(t *testing.T) (stop func()) {
	logger := logrus.New()
	logger.Level = logrus.DebugLevel
	registry := inmem.New()
	s := server.New(120, 30, registry, logger)
	srv := http.Server{
		Addr:    trackerAddr,
		Handler: s,
	}
	go func() {
		err := srv.ListenAndServe()
		if err == http.ErrServerClosed {
			return
		}
		t.Fatal(err)
	}()
	return func() {
		err := srv.Shutdown(context.Background())
		if err != nil {
			t.Fatal(err)
		}
	}
}

func TestDownloadMagnet(t *testing.T) {
	where, err := ioutil.TempDir("", "rain-")
	if err != nil {
		t.Fatal(err)
	}

	res1 := newResumeFile(t)
	defer res1.Close()

	defer startTracker(t)()

	f, err := os.Open(torrentFile)
	if err != nil {
		t.Fatal(err)
	}

	t1, err := torrent.New(f, torrentDataDir, 6881, res1)
	if err != nil {
		t.Fatal(err)
	}
	defer t1.Close()
	t1.Start()

	// Wait for seeder to announce to tracker.
	time.Sleep(100 * time.Millisecond)

	res2 := newResumeFile(t)
	defer res2.Close()

	f.Seek(0, io.SeekStart)
	magnetLink := "magnet:?xt=urn:btih:0a8e2e8c9371a91e9047ed189ceffbc460803262&dn=10mb&tr=http%3A%2F%2F127.0.0.1%3A5000%2Fannounce"
	t2, err := torrent.NewMagnet(magnetLink, where, 6882, res2)
	if err != nil {
		t.Fatal(err)
	}
	defer t2.Close()
	t2.Start()

	select {
	case <-t2.NotifyComplete():
	case err = <-t2.NotifyError():
		t.Fatal(err)
	case <-time.After(timeout):
		panic("download did not finish")
	}

	cmd := exec.Command("diff", "-rq",
		filepath.Join(torrentDataDir, torrentName),
		filepath.Join(where, torrentName))
	err = cmd.Run()
	if err != nil {
		t.Fatal(err)
	}

	err = os.RemoveAll(where)
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package protocol

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
)

// This is sent into the function
// All HTTP request headers should be set in env
type jsonio struct {
	Headers    http.Header `json:"headers,omitempty"`
	Body       string      `json:"body"`
	StatusCode int         `json:"status_code,omitempty"`
}

// JSONProtocol converts stdin/stdout streams from HTTP into JSON format.
type JSONProtocol struct {
	in  io.Writer
	out io.Reader
}

func (p *JSONProtocol) IsStreamable() bool {
	return true
}

func writeString(err error, dst io.Writer, str string) error {
	if err != nil {
		return err
	}
	_, err = io.WriteString(dst, str)
	return err
}

func (h *JSONProtocol) DumpJSON(req *http.Request) error {
	stdin := json.NewEncoder(h.in)
	bb := new(bytes.Buffer)
	_, err := bb.ReadFrom(req.Body)
	err = writeString(err, h.in, "{")
	err = writeString(err, h.in, `"body":`)
	err = stdin.Encode(bb.String())
	if err != nil {
		return err
	}
	err = writeString(err, h.in, ",")
	defer bb.Reset()
	err = writeString(err, h.in, `"headers":`)
	err = stdin.Encode(req.Header)
	if err != nil {
		return err
	}
	err = writeString(err, h.in, "}")
	return err
}

func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error {
	err := h.DumpJSON(req)
	if err != nil {
		return err
	}
	jout := new(jsonio)
	dec := json.NewDecoder(h.out)
	if err := dec.Decode(jout); err != nil {
		return err
	}
	if rw, ok := w.(http.ResponseWriter); ok {
		// this has to be done for pulling out:
		// - status code
		// - body
		// - headers
		for k, vs := range jout.Headers {
			for _, v := range vs {
				rw.Header().Add(k, v) // on top of any specified on the route
			}
		}
		rw.WriteHeader(jout.StatusCode)
		_, err = io.WriteString(rw, jout.Body) // TODO timeout
		if err != nil {
			return err
		}
	} else {
		// logs can just copy the full thing in there, headers and all.
		err = json.NewEncoder(w).Encode(jout)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Error before sending json object if something bad happend with reading a request body<commit_after>package protocol

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
)

// This is sent into the function
// All HTTP request headers should be set in env
type jsonio struct {
	Headers    http.Header `json:"headers,omitempty"`
	Body       string      `json:"body"`
	StatusCode int         `json:"status_code,omitempty"`
}

// JSONProtocol converts stdin/stdout streams from HTTP into JSON format.
type JSONProtocol struct {
	in  io.Writer
	out io.Reader
}

func (p *JSONProtocol) IsStreamable() bool {
	return true
}

func writeString(err error, dst io.Writer, str string) error {
	if err != nil {
		return err
	}
	_, err = io.WriteString(dst, str)
	return err
}

func (h *JSONProtocol) DumpJSON(req *http.Request) error {
	stdin := json.NewEncoder(h.in)
	bb := new(bytes.Buffer)
	_, err := bb.ReadFrom(req.Body)
	if err != nil {
		return err
	}
	err = writeString(err, h.in, "{")
	err = writeString(err, h.in, `"body":`)
	err = stdin.Encode(bb.String())
	if err != nil {
		return err
	}
	err = writeString(err, h.in, ",")
	defer bb.Reset()
	err = writeString(err, h.in, `"headers":`)
	err = stdin.Encode(req.Header)
	if err != nil {
		return err
	}
	err = writeString(err, h.in, "}")
	return err
}

func (h *JSONProtocol) Dispatch(w io.Writer, req *http.Request) error {
	err := h.DumpJSON(req)
	if err != nil {
		return err
	}
	jout := new(jsonio)
	dec := json.NewDecoder(h.out)
	if err := dec.Decode(jout); err != nil {
		return err
	}
	if rw, ok := w.(http.ResponseWriter); ok {
		// this has to be done for pulling out:
		// - status code
		// - body
		// - headers
		for k, vs := range jout.Headers {
			for _, v := range vs {
				rw.Header().Add(k, v) // on top of any specified on the route
			}
		}
		rw.WriteHeader(jout.StatusCode)
		_, err = io.WriteString(rw, jout.Body) // TODO timeout
		if err != nil {
			return err
		}
	} else {
		// logs can just copy the full thing in there, headers and all.
		err = json.NewEncoder(w).Encode(jout)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package git

import (
	"os/exec"

	"github.com/pkg/errors"
	"sigs.k8s.io/kustomize/api/filesys"
)

// Cloner is a function that can clone a git repo.
type Cloner func(repoSpec *RepoSpec) error

// ClonerUsingGitExec uses a local git install, as opposed
// to say, some remote API, to obtain a local clone of
// a remote repo.
func ClonerUsingGitExec(repoSpec *RepoSpec) error {
	gitProgram, err := exec.LookPath("git")
	if err != nil {
		return errors.Wrap(err, "no 'git' program on path")
	}
	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
	if err != nil {
		return err
	}

	if repoSpec.Ref == "" {
		repoSpec.Ref = "master"
	}
	cmd := exec.Command(
		gitProgram,
		"clone",
		"--depth=1",
		repoSpec.CloneSpec(),
		repoSpec.Dir.String())
	_, err = cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(
			err,
			"trouble cloning git repo %v in %s",
			repoSpec.CloneSpec(), repoSpec.Dir.String())
	}

	cmd = exec.Command(
		gitProgram,
		"fetch",
		"--depth=1",
		"origin",
		repoSpec.Ref)
	cmd.Dir = repoSpec.Dir.String()
	_, err = cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"checkout",
		repoSpec.Ref)
	cmd.Dir = repoSpec.Dir.String()
	_, err = cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "trouble checking out %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"submodule",
		"update",
		"--init",
		"--recursive")
	cmd.Dir = repoSpec.Dir.String()
	_, err = cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
	}

	return nil
}

// DoNothingCloner returns a cloner that only sets
// cloneDir field in the repoSpec.  It's assumed that
// the cloneDir is associated with some fake filesystem
// used in a test.
func DoNothingCloner(dir filesys.ConfirmedDir) Cloner {
	return func(rs *RepoSpec) error {
		rs.Dir = dir
		return nil
	}
}
<commit_msg>fix: re-add log outputs<commit_after>// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0

package git

import (
	"log"
	"os/exec"

	"github.com/pkg/errors"
	"sigs.k8s.io/kustomize/api/filesys"
)

// Cloner is a function that can clone a git repo.
type Cloner func(repoSpec *RepoSpec) error

// ClonerUsingGitExec uses a local git install, as opposed
// to say, some remote API, to obtain a local clone of
// a remote repo.
func ClonerUsingGitExec(repoSpec *RepoSpec) error {
	gitProgram, err := exec.LookPath("git")
	if err != nil {
		return errors.Wrap(err, "no 'git' program on path")
	}
	repoSpec.Dir, err = filesys.NewTmpConfirmedDir()
	if err != nil {
		return err
	}

	if repoSpec.Ref == "" {
		repoSpec.Ref = "master"
	}
	cmd := exec.Command(
		gitProgram,
		"clone",
		"--depth=1",
		repoSpec.CloneSpec(),
		repoSpec.Dir.String())
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error cloning git repo: %s", out)
		return errors.Wrapf(
			err,
			"trouble cloning git repo %v in %s",
			repoSpec.CloneSpec(), repoSpec.Dir.String())
	}

	cmd = exec.Command(
		gitProgram,
		"fetch",
		"--depth=1",
		"origin",
		repoSpec.Ref)
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error fetching ref: %s", out)
		return errors.Wrapf(err, "trouble fetching %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"checkout",
		repoSpec.Ref)
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error checking out ref: %s", out)
		return errors.Wrapf(err, "trouble checking out %s", repoSpec.Ref)
	}

	cmd = exec.Command(
		gitProgram,
		"submodule",
		"update",
		"--init",
		"--recursive")
	cmd.Dir = repoSpec.Dir.String()
	out, err = cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error fetching submodules: %s", out)
		return errors.Wrapf(err, "trouble fetching submodules for %s", repoSpec.CloneSpec())
	}

	return nil
}

// DoNothingCloner returns a cloner that only sets
// cloneDir field in the repoSpec.  It's assumed that
// the cloneDir is associated with some fake filesystem
// used in a test.
func DoNothingCloner(dir filesys.ConfirmedDir) Cloner {
	return func(rs *RepoSpec) error {
		rs.Dir = dir
		return nil
	}
}
<|endoftext|>


<commit_before>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// APTFixityChecker performs ongoing fixity checks on files stored in S3.
type APTFixityChecker struct {
	Context            *context.Context
	FixityChannel      chan *models.FixityResult
	RecordChannel      chan *models.FixityResult
	PostProcessChannel chan *models.FixityResult
}

func NewAPTFixityChecker(_context *context.Context) *APTFixityChecker {
	checker := &APTFixityChecker{
		Context: _context,
	}
	workerBufferSize := _context.Config.FixityWorker.Workers * 10
	checker.FixityChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.RecordChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.PostProcessChannel = make(chan *models.FixityResult, workerBufferSize)
	for i := 0; i < _context.Config.StoreWorker.Workers; i++ {
		go checker.checkFixity()
		go checker.record()
		go checker.postProcess()
	}
	return checker
}

func (checker *APTFixityChecker) HandleMessage(message *nsq.Message) error {
	fixityResult := checker.buildFixityResult(message)
	if fixityResult.Error != nil {
		checker.Context.MessageLog.Error(fixityResult.Error.Error())
		return fixityResult.Error
	}
	// Check syncmap to see if this item is already in process.

	// We'll ping NSQ manually when we need to.
	message.DisableAutoResponse()
	checker.Context.MessageLog.Info("Putting %s into fixity channel",
		fixityResult.GenericFile.Identifier)

	checker.FixityChannel <- fixityResult
	return nil
}

func (checker *APTFixityChecker) checkFixity() {
	for fixityResult := range checker.FixityChannel {
		// Here's where we do the actual digest calculation.
		checker.getFixityValueOfS3File(fixityResult)
		if fixityResult.Error != nil {
			checker.PostProcessChannel <- fixityResult
		} else {
			checker.RecordChannel <- fixityResult
		}
	}
}

func (checker *APTFixityChecker) record() {
	for fixityResult := range checker.RecordChannel {
		// Create PREMIS event saying whether fixity event
		// succeeded or failed.
		event, err := models.NewEventGenericFileFixityCheck(
			time.Now().UTC(),
			constants.AlgSha256,
			fixityResult.Sha256,
			fixityResult.Sha256 == fixityResult.PharosSha256())
		if err != nil {
			fixityResult.Error = fmt.Errorf("Could not create Premis Event for %s: %v",
				fixityResult.GenericFile.Identifier, err)
		} else {
			resp := checker.Context.PharosClient.PremisEventSave(event)
			if resp.Error != nil {
				fixityResult.Error = fmt.Errorf("After completing fixity check for %s, "+
					"could not save PremisEvent to Pharos: %v. Event data: %v",
					fixityResult.GenericFile.Identifier, resp.Error, event)
			} else {
				checker.Context.MessageLog.Info("Completing fixity check for %s, "+
					"and saved PremisEvent %s to Pharos",
					fixityResult.GenericFile.Identifier, event.Identifier)
			}
		}
		checker.PostProcessChannel <- fixityResult
	}
}

func (checker *APTFixityChecker) postProcess() {
	for fixityResult := range checker.PostProcessChannel {
		// Finish or requeue NSQ
		if fixityResult.Error != nil {
			if fixityResult.ErrorIsFatal {
				checker.Context.MessageLog.Error("%s (FATAL)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Finish()
			} else {
				checker.Context.MessageLog.Error("%s (transient)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Requeue(1 * time.Minute)
			}
		} else {
			if fixityResult.PharosSha256() == fixityResult.Sha256 {
				checker.Context.MessageLog.Info("Fixity check complete for %s. Fixity %s matches.",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256)
			} else {
				checker.Context.MessageLog.Warning("Fixity check complete for %s. S3 fixity %s "+
					"DOES NOT MATCH PHAROS FIXITY %s",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256,
					fixityResult.PharosSha256())
			}
			fixityResult.NSQMessage.Finish()
		}
	}
}

// getFixityValueOfS3File calculates the sha256 digest of an S3 file.
// The downloader streams the file from S3 to /dev/null, because
// we don't need to have the file on disk. We can calculate the
// digest from the stream. We get the file from S3/Virginia, not
// Glacier/Oregon! When this is done, the fixity value will be in
// fixityResult.Sha256.
func (checker *APTFixityChecker) getFixityValueOfS3File(fixityResult *models.FixityResult) {
	bucket, key, err := fixityResult.BucketAndKey()
	if err != nil {
		fixityResult.Error = fmt.Errorf("Can't get S3 bucket and key names for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		fixityResult.ErrorIsFatal = true
		return
	}
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		bucket,      // should be S3 preservation bucket
		key,         // s3 key to fetch
		"/dev/null", // local path at which to save the s3 file
		false,       // don't calculate md5 digest
		true)        // do calculate sha256 digest
	downloader.Fetch()
	if downloader.ErrorMessage != "" {
		fixityResult.Error = fmt.Errorf("Error fetching file %s (%s/%s) from S3: %s",
			fixityResult.GenericFile.Identifier, bucket, key,
			downloader.ErrorMessage)
		if strings.Contains(downloader.ErrorMessage, "NoSuchKey") {
			fixityResult.ErrorIsFatal = true
		}
		return
	}
	fixityResult.S3FileExists = true
	fixityResult.Sha256 = downloader.Sha256Digest
	return
}

func (checker *APTFixityChecker) buildFixityResult(message *nsq.Message) *models.FixityResult {
	fixityResult := models.NewFixityResult(message)
	gfIdentifier := strings.TrimSpace(string(message.Body))
	// Get GenericFile with checksums (param includeRelations = true)
	resp := checker.Context.PharosClient.GenericFileGet(gfIdentifier, true)
	if resp.Error != nil {
		fixityResult.Error = fmt.Errorf("Can't get generic file %s from Pharos: %v", resp.Error.Error())
		if resp.Response.StatusCode == 404 {
			fixityResult.ErrorIsFatal = true
		}
		return fixityResult
	}
	fixityResult.GenericFile = resp.GenericFile()
	if fixityResult.GenericFile.URI == "" {
		fixityResult.Error = fmt.Errorf("GenericFile %s has no S3 URI.", fixityResult.GenericFile.Identifier)
		fixityResult.ErrorIsFatal = true
	}
	return fixityResult
}
<commit_msg>More work on fixity checker<commit_after>package workers

import (
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/network"
	"github.com/nsqio/go-nsq"
	"strings"
	"time"
)

// APTFixityChecker performs ongoing fixity checks on files stored in S3.
type APTFixityChecker struct {
	Context            *context.Context
	FixityChannel      chan *models.FixityResult
	RecordChannel      chan *models.FixityResult
	PostProcessChannel chan *models.FixityResult
	ItemsInProcess     *models.SynchronizedMap
}

func NewAPTFixityChecker(_context *context.Context) *APTFixityChecker {
	checker := &APTFixityChecker{
		Context:        _context,
		ItemsInProcess: models.NewSynchronizedMap(),
	}
	workerBufferSize := _context.Config.FixityWorker.Workers * 10
	checker.FixityChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.RecordChannel = make(chan *models.FixityResult, workerBufferSize)
	checker.PostProcessChannel = make(chan *models.FixityResult, workerBufferSize)
	for i := 0; i < _context.Config.StoreWorker.Workers; i++ {
		go checker.checkFixity()
		go checker.record()
		go checker.postProcess()
	}
	return checker
}

func (checker *APTFixityChecker) HandleMessage(message *nsq.Message) error {
	fixityResult := checker.buildFixityResult(message)
	if fixityResult.Error != nil {
		checker.Context.MessageLog.Error("Cannot process %s: %v",
			string(message.Body), fixityResult.Error.Error())
		message.Finish()
		return nil // Should we return an error to NSQ?
	}
	// Check syncmap to see if this item is already in process.
	startedAt := checker.ItemsInProcess.Get(fixityResult.GenericFile.Identifier)
	if startedAt != "" {
		checker.Context.MessageLog.Info("Skipping %s: already in process as of %s.",
			fixityResult.GenericFile.Identifier, startedAt)
		message.Finish()
		return nil
	}

	// Note that we're working on this.
	checker.ItemsInProcess.Add(fixityResult.GenericFile.Identifier, time.Now().UTC().Format(time.RFC3339))
	checker.Context.MessageLog.Info("Added %s to items in process", fixityResult.GenericFile.Identifier)

	// We'll ping NSQ manually when we need to.
	message.DisableAutoResponse()
	checker.Context.MessageLog.Info("Putting %s into fixity channel",
		fixityResult.GenericFile.Identifier)

	checker.FixityChannel <- fixityResult
	return nil
}

func (checker *APTFixityChecker) checkFixity() {
	for fixityResult := range checker.FixityChannel {
		// Here's where we do the actual digest calculation.
		checker.getFixityValueOfS3File(fixityResult)
		if fixityResult.Error != nil {
			checker.PostProcessChannel <- fixityResult
		} else {
			checker.RecordChannel <- fixityResult
		}
	}
}

func (checker *APTFixityChecker) record() {
	for fixityResult := range checker.RecordChannel {
		// Create PREMIS event saying whether fixity event
		// succeeded or failed.
		event, err := models.NewEventGenericFileFixityCheck(
			time.Now().UTC(),
			constants.AlgSha256,
			fixityResult.Sha256,
			fixityResult.Sha256 == fixityResult.PharosSha256())
		if err != nil {
			fixityResult.Error = fmt.Errorf("Could not create Premis Event for %s: %v",
				fixityResult.GenericFile.Identifier, err)
		} else {
			resp := checker.Context.PharosClient.PremisEventSave(event)
			if resp.Error != nil {
				fixityResult.Error = fmt.Errorf("After completing fixity check for %s, "+
					"could not save PremisEvent to Pharos: %v. Event data: %v",
					fixityResult.GenericFile.Identifier, resp.Error, event)
			} else {
				checker.Context.MessageLog.Info("Completing fixity check for %s, "+
					"and saved PremisEvent %s to Pharos",
					fixityResult.GenericFile.Identifier, event.Identifier)
			}
		}
		checker.PostProcessChannel <- fixityResult
	}
}

func (checker *APTFixityChecker) postProcess() {
	for fixityResult := range checker.PostProcessChannel {
		// Finish or requeue NSQ
		if fixityResult.Error != nil {
			if fixityResult.ErrorIsFatal {
				checker.Context.MessageLog.Error("%s (FATAL)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Finish()
			} else {
				checker.Context.MessageLog.Error("%s (transient)", fixityResult.Error.Error())
				fixityResult.NSQMessage.Requeue(1 * time.Minute)
			}
		} else {
			if fixityResult.PharosSha256() == fixityResult.Sha256 {
				checker.Context.MessageLog.Info("Fixity check complete for %s. Fixity %s matches.",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256)
			} else {
				checker.Context.MessageLog.Warning("Fixity check complete for %s. S3 fixity %s "+
					"DOES NOT MATCH PHAROS FIXITY %s",
					fixityResult.GenericFile.Identifier, fixityResult.Sha256,
					fixityResult.PharosSha256())
			}
			fixityResult.NSQMessage.Finish()
		}
		checker.ItemsInProcess.Delete(fixityResult.GenericFile.Identifier)
		checker.Context.MessageLog.Info("Removed %s from items in process", fixityResult.GenericFile.Identifier)
	}
}

// getFixityValueOfS3File calculates the sha256 digest of an S3 file.
// The downloader streams the file from S3 to /dev/null, because
// we don't need to have the file on disk. We can calculate the
// digest from the stream. We get the file from S3/Virginia, not
// Glacier/Oregon! When this is done, the fixity value will be in
// fixityResult.Sha256.
func (checker *APTFixityChecker) getFixityValueOfS3File(fixityResult *models.FixityResult) {
	bucket, key, err := fixityResult.BucketAndKey()
	if err != nil {
		fixityResult.Error = fmt.Errorf("Can't get S3 bucket and key names for %s: %v",
			fixityResult.GenericFile.Identifier, err)
		fixityResult.ErrorIsFatal = true
		return
	}
	downloader := network.NewS3Download(
		constants.AWSVirginia,
		bucket,      // should be S3 preservation bucket
		key,         // s3 key to fetch
		"/dev/null", // local path at which to save the s3 file
		false,       // don't calculate md5 digest
		true)        // do calculate sha256 digest
	downloader.Fetch()
	if downloader.ErrorMessage != "" {
		fixityResult.Error = fmt.Errorf("Error fetching file %s (%s/%s) from S3: %s",
			fixityResult.GenericFile.Identifier, bucket, key,
			downloader.ErrorMessage)
		if strings.Contains(downloader.ErrorMessage, "NoSuchKey") {
			fixityResult.ErrorIsFatal = true
		}
		return
	}
	fixityResult.S3FileExists = true
	fixityResult.Sha256 = downloader.Sha256Digest
	return
}

func (checker *APTFixityChecker) buildFixityResult(message *nsq.Message) *models.FixityResult {
	fixityResult := models.NewFixityResult(message)
	gfIdentifier := strings.TrimSpace(string(message.Body))
	// Get GenericFile with checksums (param includeRelations = true)
	resp := checker.Context.PharosClient.GenericFileGet(gfIdentifier, true)
	if resp.Error != nil {
		fixityResult.Error = fmt.Errorf("Can't get generic file %s from Pharos: %v", resp.Error.Error())
		if resp.Response.StatusCode == 404 {
			fixityResult.ErrorIsFatal = true
		}
		return fixityResult
	}
	fixityResult.GenericFile = resp.GenericFile()
	if fixityResult.GenericFile.URI == "" {
		fixityResult.Error = fmt.Errorf("GenericFile %s has no S3 URI.", fixityResult.GenericFile.Identifier)
		fixityResult.ErrorIsFatal = true
	}
	return fixityResult
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"fmt"

	"github.com/google/gapid/gapis/api"
)

func (c ErrorCode) Err() error {
	switch c {
	case ErrSuccess:
		return nil
	case ErrAborted:
		return api.ErrCmdAborted{}
	default:
		return fmt.Errorf("Unknown error code %v", c)
	}
}
<commit_msg>gapil/compiler: Declare errAborted global error.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package compiler

import (
	"fmt"

	"github.com/google/gapid/gapis/api"
)

var errAborted error = api.ErrCmdAborted{}

func (c ErrorCode) Err() error {
	switch c {
	case ErrSuccess:
		return nil
	case ErrAborted:
		return errAborted
	default:
		return fmt.Errorf("Unknown error code %v", c)
	}
}
<|endoftext|>


<commit_before>package isolated

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	. "code.cloudfoundry.org/cli/cf/util/testhelpers/matchers"
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("apply-manifest command", func() {
	var (
		orgName      string
		spaceName    string
		appName      string
		manifestPath string
		appDir       string
	)

	BeforeEach(func() {
		orgName = helpers.NewOrgName()
		spaceName = helpers.NewSpaceName()
		appName = helpers.PrefixedRandomName("app")

		var err error
		appDir, err = ioutil.TempDir("", "simple-app")
		Expect(err).NotTo(HaveOccurred())

		manifestPath = filepath.Join(appDir, "manifest.yml")
		// Ensure the file exists at the minimum
		helpers.WriteManifest(manifestPath, map[string]interface{}{})
	})

	AfterEach(func() {
		Expect(os.RemoveAll(appDir)).ToNot(HaveOccurred())
	})

	Describe("help", func() {
		It("appears in cf help -a", func() {
			session := helpers.CF("help", "-a")
			Eventually(session).Should(Exit(0))
			Expect(session).To(HaveCommandInCategoryWithDescription("apply-manifest", "APPS", "Apply manifest properties to a space"))
		})

		When("--help flag is set", func() {
			It("displays command usage to output", func() {
				session := helpers.CF("apply-manifest", "--help")

				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("apply-manifest - Apply manifest properties to a space"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf apply-manifest -f APP_MANIFEST_PATH"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("create-app, create-app-manifest, push"))

				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(true, true, ReadOnlyOrg, "apply-manifest", "-f", manifestPath)
		})
	})

	When("the environment is set up correctly", func() {
		BeforeEach(func() {
			helpers.SetupCF(orgName, spaceName)
		})

		AfterEach(func() {
			helpers.QuickDeleteOrg(orgName)
		})

		When("the app name in the manifest is missing", func() {
			BeforeEach(func() {
				helpers.WriteManifest(manifestPath, map[string]interface{}{
					"applications": []map[string]interface{}{
						{
							"instances": 3,
						},
					},
				})
			})

			It("reports an error", func() {
				session := helpers.CF("apply-manifest", "-f", manifestPath)
				Eventually(session.Err).Should(Say("Found an application with no name specified"))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Exit(1))
			})
		})

		When("there is a CC error", func() {
			BeforeEach(func() {
				helpers.WriteManifest(manifestPath, map[string]interface{}{
					"applications": []map[string]interface{}{
						{
							"name":      appName,
							"instances": -1,
						},
					},
				})
			})

			It("displays the error", func() {
				session := helpers.CF("apply-manifest", "-f", manifestPath)
				Eventually(session.Err).Should(Say("Instances must be greater than or equal to 0"))
				Eventually(session).Should(Say("FAILED"))

				Eventually(session).Should(Exit(1))
			})
		})

		When("-f is provided", func() {
			When("the -f flag is not given an arg", func() {
				It("tells the user that the flag requires an arg, prints help text, and exits 1", func() {
					session := helpers.CF("apply-manifest", "-f")

					Eventually(session.Err).Should(Say("Incorrect Usage: expected argument for flag `-f'"))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the -f flag points to a directory that does not have a manifest.yml file", func() {
				var (
					emptyDir string
					err      error
				)

				BeforeEach(func() {
					emptyDir, err = ioutil.TempDir("", "empty")
					Expect(err).NotTo(HaveOccurred())
				})

				AfterEach(func() {
					Expect(os.RemoveAll(emptyDir)).ToNot(HaveOccurred())
				})

				It("tells the user that the provided path doesn't exist, prints help text, and exits 1", func() {
					session := helpers.CF("apply-manifest", "-f", emptyDir)

					Eventually(session.Err).Should(Say("Incorrect Usage: The specified directory '%s' does not contain a file named 'manifest.yml'.", emptyDir))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the -f flag points to a file that does not exist", func() {
				It("tells the user that the provided path doesn't exist, prints help text, and exits 1", func() {
					session := helpers.CF("apply-manifest", "-f", "path/that/does/not/exist")

					Eventually(session.Err).Should(Say("Incorrect Usage: The specified path 'path/that/does/not/exist' does not exist."))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the manifest exists where -f points", func() {
				It("applies the manifest successfully", func() {
					userName, _ := helpers.GetCredentials()
					helpers.WriteManifest(filepath.Join(appDir, "manifest.yml"), map[string]interface{}{
						"applications": []map[string]interface{}{
							{
								"name":      appName,
								"instances": 3,
							},
						},
					})

					session := helpers.CF("apply-manifest", "-f", appDir)
					Eventually(session).Should(Say("Applying manifest %s in org %s / space %s as %s...", regexp.QuoteMeta(manifestPath), orgName, spaceName, userName))
					Eventually(session).Should(Exit())

					session = helpers.CF("app", appName)
					Eventually(session).Should(Say(`instances:\s+%s`, `\d/3`))
					Eventually(session).Should(Exit())
				})
			})
		})

		When("-f is not provided", func() {
			When("a properly formatted manifest is present in the pwd", func() {
				It("autodetects and applies the manifest", func() {
					userName, _ := helpers.GetCredentials()
					helpers.WriteManifest(filepath.Join(appDir, "manifest.yml"), map[string]interface{}{
						"applications": []map[string]interface{}{
							{
								"name":      appName,
								"instances": 3,
							},
						},
					})

					session := helpers.CustomCF(helpers.CFEnv{WorkingDirectory: appDir}, "apply-manifest")
					Eventually(session).Should(Say("Applying manifest %s in org %s / space %s as %s...", regexp.QuoteMeta(manifestPath), orgName, spaceName, userName))
					Eventually(session).Should(Exit())

					session = helpers.CF("app", appName)
					Eventually(session).Should(Say(`instances:\s+%s`, `\d/3`))
					Eventually(session).Should(Exit())
				})
			})

			When("the current directory does not have a manifest", func() {
				It("fails nicely", func() {
					currentDir, err := os.Getwd()
					Expect(err).NotTo(HaveOccurred())
					session := helpers.CF("apply-manifest")

					Eventually(session.Err).Should(Say(`Could not find 'manifest.yml' file in %s`, currentDir))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session).Should(Exit(1))
				})
			})
		})
	})
})
<commit_msg>Use SayPath instead of Say to account for Windows paths<commit_after>package isolated

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	. "code.cloudfoundry.org/cli/cf/util/testhelpers/matchers"
	"code.cloudfoundry.org/cli/integration/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("apply-manifest command", func() {
	var (
		orgName      string
		spaceName    string
		appName      string
		manifestPath string
		appDir       string
	)

	BeforeEach(func() {
		orgName = helpers.NewOrgName()
		spaceName = helpers.NewSpaceName()
		appName = helpers.PrefixedRandomName("app")

		var err error
		appDir, err = ioutil.TempDir("", "simple-app")
		Expect(err).NotTo(HaveOccurred())

		manifestPath = filepath.Join(appDir, "manifest.yml")
		// Ensure the file exists at the minimum
		helpers.WriteManifest(manifestPath, map[string]interface{}{})
	})

	AfterEach(func() {
		Expect(os.RemoveAll(appDir)).ToNot(HaveOccurred())
	})

	Describe("help", func() {
		It("appears in cf help -a", func() {
			session := helpers.CF("help", "-a")
			Eventually(session).Should(Exit(0))
			Expect(session).To(HaveCommandInCategoryWithDescription("apply-manifest", "APPS", "Apply manifest properties to a space"))
		})

		When("--help flag is set", func() {
			It("displays command usage to output", func() {
				session := helpers.CF("apply-manifest", "--help")

				Eventually(session).Should(Say("NAME:"))
				Eventually(session).Should(Say("apply-manifest - Apply manifest properties to a space"))
				Eventually(session).Should(Say("USAGE:"))
				Eventually(session).Should(Say("cf apply-manifest -f APP_MANIFEST_PATH"))
				Eventually(session).Should(Say("SEE ALSO:"))
				Eventually(session).Should(Say("create-app, create-app-manifest, push"))

				Eventually(session).Should(Exit(0))
			})
		})
	})

	When("the environment is not setup correctly", func() {
		It("fails with the appropriate errors", func() {
			helpers.CheckEnvironmentTargetedCorrectly(true, true, ReadOnlyOrg, "apply-manifest", "-f", manifestPath)
		})
	})

	When("the environment is set up correctly", func() {
		BeforeEach(func() {
			helpers.SetupCF(orgName, spaceName)
		})

		AfterEach(func() {
			helpers.QuickDeleteOrg(orgName)
		})

		When("the app name in the manifest is missing", func() {
			BeforeEach(func() {
				helpers.WriteManifest(manifestPath, map[string]interface{}{
					"applications": []map[string]interface{}{
						{
							"instances": 3,
						},
					},
				})
			})

			It("reports an error", func() {
				session := helpers.CF("apply-manifest", "-f", manifestPath)
				Eventually(session.Err).Should(Say("Found an application with no name specified"))
				Eventually(session).Should(Say("FAILED"))
				Eventually(session).Should(Exit(1))
			})
		})

		When("there is a CC error", func() {
			BeforeEach(func() {
				helpers.WriteManifest(manifestPath, map[string]interface{}{
					"applications": []map[string]interface{}{
						{
							"name":      appName,
							"instances": -1,
						},
					},
				})
			})

			It("displays the error", func() {
				session := helpers.CF("apply-manifest", "-f", manifestPath)
				Eventually(session.Err).Should(Say("Instances must be greater than or equal to 0"))
				Eventually(session).Should(Say("FAILED"))

				Eventually(session).Should(Exit(1))
			})
		})

		When("-f is provided", func() {
			When("the -f flag is not given an arg", func() {
				It("tells the user that the flag requires an arg, prints help text, and exits 1", func() {
					session := helpers.CF("apply-manifest", "-f")

					Eventually(session.Err).Should(Say("Incorrect Usage: expected argument for flag `-f'"))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the -f flag points to a directory that does not have a manifest.yml file", func() {
				var (
					emptyDir string
					err      error
				)

				BeforeEach(func() {
					emptyDir, err = ioutil.TempDir("", "empty")
					Expect(err).NotTo(HaveOccurred())
				})

				AfterEach(func() {
					Expect(os.RemoveAll(emptyDir)).ToNot(HaveOccurred())
				})

				It("tells the user that the provided path doesn't exist, prints help text, and exits 1", func() {
					session := helpers.CF("apply-manifest", "-f", emptyDir)

					Eventually(session.Err).Should(Say("Incorrect Usage: The specified directory '%s' does not contain a file named 'manifest.yml'.", emptyDir))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the -f flag points to a file that does not exist", func() {
				It("tells the user that the provided path doesn't exist, prints help text, and exits 1", func() {
					session := helpers.CF("apply-manifest", "-f", "path/that/does/not/exist")

					Eventually(session.Err).Should(Say("Incorrect Usage: The specified path 'path/that/does/not/exist' does not exist."))
					Eventually(session).Should(Say("NAME:"))
					Eventually(session).Should(Exit(1))
				})
			})

			When("the manifest exists where -f points", func() {
				It("applies the manifest successfully", func() {
					userName, _ := helpers.GetCredentials()
					helpers.WriteManifest(filepath.Join(appDir, "manifest.yml"), map[string]interface{}{
						"applications": []map[string]interface{}{
							{
								"name":      appName,
								"instances": 3,
							},
						},
					})

					session := helpers.CF("apply-manifest", "-f", appDir)
					Eventually(session).Should(Say("Applying manifest %s in org %s / space %s as %s...", regexp.QuoteMeta(manifestPath), orgName, spaceName, userName))
					Eventually(session).Should(Exit())

					session = helpers.CF("app", appName)
					Eventually(session).Should(Say(`instances:\s+%s`, `\d/3`))
					Eventually(session).Should(Exit())
				})
			})
		})

		When("-f is not provided", func() {
			When("a properly formatted manifest is present in the pwd", func() {
				It("autodetects and applies the manifest", func() {
					userName, _ := helpers.GetCredentials()
					helpers.WriteManifest(filepath.Join(appDir, "manifest.yml"), map[string]interface{}{
						"applications": []map[string]interface{}{
							{
								"name":      appName,
								"instances": 3,
							},
						},
					})

					session := helpers.CustomCF(helpers.CFEnv{WorkingDirectory: appDir}, "apply-manifest")
					Eventually(session).Should(Say("Applying manifest %s in org %s / space %s as %s...", regexp.QuoteMeta(manifestPath), orgName, spaceName, userName))
					Eventually(session).Should(Exit())

					session = helpers.CF("app", appName)
					Eventually(session).Should(Say(`instances:\s+%s`, `\d/3`))
					Eventually(session).Should(Exit())
				})
			})

			When("the current directory does not have a manifest", func() {
				It("fails nicely", func() {
					currentDir, err := os.Getwd()
					Expect(err).NotTo(HaveOccurred())
					session := helpers.CF("apply-manifest")

					Eventually(session.Err).Should(helpers.SayPath(`Could not find 'manifest.yml' file in %s`, currentDir))
					Eventually(session).Should(Say("FAILED"))
					Eventually(session).Should(Exit(1))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avm

import (
	"math"
	"testing"

	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/utils/crypto"
	"github.com/ava-labs/gecko/utils/units"
	"github.com/ava-labs/gecko/vms/components/ava"
	"github.com/ava-labs/gecko/vms/secp256k1fx"
)

func TestPrefixedSetsAndGets(t *testing.T) {
	_, _, vm := GenesisVM(t)
	ctx := vm.ctx
	defer func() {
		vm.Shutdown()
		ctx.Lock.Unlock()
	}()

	state := vm.state

	vm.codec.RegisterType(&ava.TestVerifiable{})

	utxo := &ava.UTXO{
		UTXOID: ava.UTXOID{
			TxID:        ids.Empty,
			OutputIndex: 1,
		},
		Asset: ava.Asset{ID: ids.Empty},
		Out:   &ava.TestVerifiable{},
	}

	tx := &Tx{UnsignedTx: &BaseTx{
		NetID: networkID,
		BCID:  chainID,
		Ins: []*ava.TransferableInput{{
			UTXOID: ava.UTXOID{
				TxID:        ids.Empty,
				OutputIndex: 0,
			},
			Asset: ava.Asset{ID: asset},
			In: &secp256k1fx.TransferInput{
				Amt: 20 * units.KiloAva,
				Input: secp256k1fx.Input{
					SigIndices: []uint32{
						0,
					},
				},
			},
		}},
	}}

	unsignedBytes, err := vm.codec.Marshal(tx.UnsignedTx)
	if err != nil {
		t.Fatal(err)
	}

	key := keys[0]
	sig, err := key.Sign(unsignedBytes)
	if err != nil {
		t.Fatal(err)
	}
	fixedSig := [crypto.SECP256K1RSigLen]byte{}
	copy(fixedSig[:], sig)

	tx.Creds = append(tx.Creds, &secp256k1fx.Credential{
		Sigs: [][crypto.SECP256K1RSigLen]byte{
			fixedSig,
		},
	})

	b, err := vm.codec.Marshal(tx)
	if err != nil {
		t.Fatal(err)
	}
	tx.Initialize(b)

	if err := state.SetUTXO(ids.Empty, utxo); err != nil {
		t.Fatal(err)
	}
	if err := state.SetTx(ids.Empty, tx); err != nil {
		t.Fatal(err)
	}
	if err := state.SetStatus(ids.Empty, choices.Accepted); err != nil {
		t.Fatal(err)
	}

	resultUTXO, err := state.UTXO(ids.Empty)
	if err != nil {
		t.Fatal(err)
	}
	resultTx, err := state.Tx(ids.Empty)
	if err != nil {
		t.Fatal(err)
	}
	resultStatus, err := state.Status(ids.Empty)
	if err != nil {
		t.Fatal(err)
	}

	if resultUTXO.OutputIndex != 1 {
		t.Fatalf("Wrong UTXO returned")
	}
	if !resultTx.ID().Equals(tx.ID()) {
		t.Fatalf("Wrong Tx returned")
	}
	if resultStatus != choices.Accepted {
		t.Fatalf("Wrong Status returned")
	}
}

func TestPrefixedFundingNoAddresses(t *testing.T) {
	_, _, vm := GenesisVM(t)
	ctx := vm.ctx
	defer func() {
		vm.Shutdown()
		ctx.Lock.Unlock()
	}()

	state := vm.state

	vm.codec.RegisterType(&ava.TestVerifiable{})

	utxo := &ava.UTXO{
		UTXOID: ava.UTXOID{
			TxID:        ids.Empty,
			OutputIndex: 1,
		},
		Asset: ava.Asset{ID: ids.Empty},
		Out:   &ava.TestVerifiable{},
	}

	if err := state.FundUTXO(utxo); err != nil {
		t.Fatal(err)
	}
	if err := state.SpendUTXO(utxo.InputID()); err != nil {
		t.Fatal(err)
	}
}

func TestPrefixedFundingAddresses(t *testing.T) {
	_, _, vm := GenesisVM(t)
	ctx := vm.ctx
	defer func() {
		vm.Shutdown()
		ctx.Lock.Unlock()
	}()

	state := vm.state

	vm.codec.RegisterType(&ava.TestAddressable{})

	utxo := &ava.UTXO{
		UTXOID: ava.UTXOID{
			TxID:        ids.Empty,
			OutputIndex: 1,
		},
		Asset: ava.Asset{ID: ids.Empty},
		Out: &ava.TestAddressable{
			Addrs: [][]byte{{0}},
		},
	}

	if err := state.FundUTXO(utxo); err != nil {
		t.Fatal(err)
	}
	funds, err := state.Funds([]byte{0}, ids.Empty, math.MaxInt32)
	if err != nil {
		t.Fatal(err)
	}
	if len(funds) != 1 {
		t.Fatalf("Should have returned 1 utxoIDs")
	}
	if utxoID := funds[0]; !utxoID.Equals(utxo.InputID()) {
		t.Fatalf("Returned wrong utxoID")
	}
	if err := state.SpendUTXO(utxo.InputID()); err != nil {
		t.Fatal(err)
	}
	funds, err = state.Funds([]byte{0}, ids.Empty, -1)
	if err != nil {
		t.Fatal(err)
	}
	if len(funds) != 0 {
		t.Fatalf("Should have returned 0 utxoIDs")
	}
}
<commit_msg>fix typo in test<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package avm

import (
	"math"
	"testing"

	"github.com/ava-labs/gecko/ids"
	"github.com/ava-labs/gecko/snow/choices"
	"github.com/ava-labs/gecko/utils/crypto"
	"github.com/ava-labs/gecko/utils/units"
	"github.com/ava-labs/gecko/vms/components/ava"
	"github.com/ava-labs/gecko/vms/secp256k1fx"
)

func TestPrefixedSetsAndGets(t *testing.T) {
	_, _, vm := GenesisVM(t)
	ctx := vm.ctx
	defer func() {
		vm.Shutdown()
		ctx.Lock.Unlock()
	}()

	state := vm.state

	vm.codec.RegisterType(&ava.TestVerifiable{})

	utxo := &ava.UTXO{
		UTXOID: ava.UTXOID{
			TxID:        ids.Empty,
			OutputIndex: 1,
		},
		Asset: ava.Asset{ID: ids.Empty},
		Out:   &ava.TestVerifiable{},
	}

	tx := &Tx{UnsignedTx: &BaseTx{
		NetID: networkID,
		BCID:  chainID,
		Ins: []*ava.TransferableInput{{
			UTXOID: ava.UTXOID{
				TxID:        ids.Empty,
				OutputIndex: 0,
			},
			Asset: ava.Asset{ID: asset},
			In: &secp256k1fx.TransferInput{
				Amt: 20 * units.KiloAva,
				Input: secp256k1fx.Input{
					SigIndices: []uint32{
						0,
					},
				},
			},
		}},
	}}

	unsignedBytes, err := vm.codec.Marshal(tx.UnsignedTx)
	if err != nil {
		t.Fatal(err)
	}

	key := keys[0]
	sig, err := key.Sign(unsignedBytes)
	if err != nil {
		t.Fatal(err)
	}
	fixedSig := [crypto.SECP256K1RSigLen]byte{}
	copy(fixedSig[:], sig)

	tx.Creds = append(tx.Creds, &secp256k1fx.Credential{
		Sigs: [][crypto.SECP256K1RSigLen]byte{
			fixedSig,
		},
	})

	b, err := vm.codec.Marshal(tx)
	if err != nil {
		t.Fatal(err)
	}
	tx.Initialize(b)

	if err := state.SetUTXO(ids.Empty, utxo); err != nil {
		t.Fatal(err)
	}
	if err := state.SetTx(ids.Empty, tx); err != nil {
		t.Fatal(err)
	}
	if err := state.SetStatus(ids.Empty, choices.Accepted); err != nil {
		t.Fatal(err)
	}

	resultUTXO, err := state.UTXO(ids.Empty)
	if err != nil {
		t.Fatal(err)
	}
	resultTx, err := state.Tx(ids.Empty)
	if err != nil {
		t.Fatal(err)
	}
	resultStatus, err := state.Status(ids.Empty)
	if err != nil {
		t.Fatal(err)
	}

	if resultUTXO.OutputIndex != 1 {
		t.Fatalf("Wrong UTXO returned")
	}
	if !resultTx.ID().Equals(tx.ID()) {
		t.Fatalf("Wrong Tx returned")
	}
	if resultStatus != choices.Accepted {
		t.Fatalf("Wrong Status returned")
	}
}

func TestPrefixedFundingNoAddresses(t *testing.T) {
	_, _, vm := GenesisVM(t)
	ctx := vm.ctx
	defer func() {
		vm.Shutdown()
		ctx.Lock.Unlock()
	}()

	state := vm.state

	vm.codec.RegisterType(&ava.TestVerifiable{})

	utxo := &ava.UTXO{
		UTXOID: ava.UTXOID{
			TxID:        ids.Empty,
			OutputIndex: 1,
		},
		Asset: ava.Asset{ID: ids.Empty},
		Out:   &ava.TestVerifiable{},
	}

	if err := state.FundUTXO(utxo); err != nil {
		t.Fatal(err)
	}
	if err := state.SpendUTXO(utxo.InputID()); err != nil {
		t.Fatal(err)
	}
}

func TestPrefixedFundingAddresses(t *testing.T) {
	_, _, vm := GenesisVM(t)
	ctx := vm.ctx
	defer func() {
		vm.Shutdown()
		ctx.Lock.Unlock()
	}()

	state := vm.state

	vm.codec.RegisterType(&ava.TestAddressable{})

	utxo := &ava.UTXO{
		UTXOID: ava.UTXOID{
			TxID:        ids.Empty,
			OutputIndex: 1,
		},
		Asset: ava.Asset{ID: ids.Empty},
		Out: &ava.TestAddressable{
			Addrs: [][]byte{{0}},
		},
	}

	if err := state.FundUTXO(utxo); err != nil {
		t.Fatal(err)
	}
	funds, err := state.Funds([]byte{0}, ids.Empty, math.MaxInt32)
	if err != nil {
		t.Fatal(err)
	}
	if len(funds) != 1 {
		t.Fatalf("Should have returned 1 utxoIDs")
	}
	if utxoID := funds[0]; !utxoID.Equals(utxo.InputID()) {
		t.Fatalf("Returned wrong utxoID")
	}
	if err := state.SpendUTXO(utxo.InputID()); err != nil {
		t.Fatal(err)
	}
	funds, err = state.Funds([]byte{0}, ids.Empty, math.MaxInt32)
	if err != nil {
		t.Fatal(err)
	}
	if len(funds) != 0 {
		t.Fatalf("Should have returned 0 utxoIDs")
	}
}
<|endoftext|>


<commit_before>package workerpool

import (
	"sync"
	"testing"
	"time"
)

const max = 20

func TestExample(t *testing.T) {
	wp := New(2)
	requests := []string{"alpha", "beta", "gamma", "delta", "epsilon"}

	rspChan := make(chan string, len(requests))
	for _, r := range requests {
		r := r
		wp.Submit(func() {
			rspChan <- r
		})
	}

	wp.StopWait()

	close(rspChan)
	rspSet := map[string]struct{}{}
	for rsp := range rspChan {
		rspSet[rsp] = struct{}{}
	}
	if len(rspSet) < len(requests) {
		t.Fatal("Did not handle all requests")
	}
	for _, req := range requests {
		if _, ok := rspSet[req]; !ok {
			t.Fatal("Missing expected valus:", req)
		}
	}
}

func TestMaxWorkers(t *testing.T) {
	t.Parallel()

	wp := New(0)
	if wp.maxWorkers != 1 {
		t.Fatal("should have created one worker")
	}

	wp = New(max)
	defer wp.Stop()

	started := make(chan struct{}, max)
	sync := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			started <- struct{}{}
			<-sync
		})
	}

	// Wait for all queued tasks to be dispatched to workers.
	timeout := time.After(5 * time.Second)
	for startCount := 0; startCount < max; {
		select {
		case <-started:
			startCount++
		case <-timeout:
			t.Fatal("timed out waiting for workers to start")
		}
	}

	// Release workers.
	close(sync)
}

func TestReuseWorkers(t *testing.T) {
	t.Parallel()

	wp := New(5)
	defer wp.Stop()

	sync := make(chan struct{})

	// Cause worker to be created, and available for reuse before next task.
	for i := 0; i < 10; i++ {
		wp.Submit(func() { <-sync })
		sync <- struct{}{}
		time.Sleep(100 * time.Millisecond)
	}

	// If the same worker was always reused, then only one worker would have
	// been created and there should only be one ready.
	if countReady(wp) > 1 {
		t.Fatal("Worker not reused")
	}
}

func TestWorkerTimeout(t *testing.T) {
	t.Parallel()

	wp := New(max)
	defer wp.Stop()

	sync := make(chan struct{})
	started := make(chan struct{}, max)
	// Cause workers to be created.  Workers wait on channel, keeping them busy
	// and causing the worker pool to create more.
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			started <- struct{}{}
			<-sync
		})
	}

	// Wait for tasks to start.
	for i := 0; i < max; i++ {
		<-started
	}

	if anyReady(wp) {
		t.Fatal("number of ready workers should be zero")
	}
	// Release workers.
	close(sync)

	if countReady(wp) != max {
		t.Fatal("Expected", max, "ready workers")
	}

	// Check that a worker timed out.
	time.Sleep((idleTimeoutSec + 1) * time.Second)
	if countReady(wp) != max-1 {
		t.Fatal("First worker did not timeout")
	}

	// Check that another worker timed out.
	time.Sleep((idleTimeoutSec + 1) * time.Second)
	if countReady(wp) != max-2 {
		t.Fatal("Second worker did not timeout")
	}
}

func TestStop(t *testing.T) {
	t.Parallel()

	wp := New(max)
	defer wp.Stop()

	started := make(chan struct{}, max)
	sync := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			started <- struct{}{}
			<-sync
		})
	}

	// Wait for all queued tasks to be dispatched to workers.
	timeout := time.After(5 * time.Second)
	for startCount := 0; startCount < max; {
		select {
		case <-started:
			startCount++
		case <-timeout:
			t.Fatal("timed out waiting for workers to start")
		}
	}

	// Release workers.
	close(sync)

	if wp.Stopped() {
		t.Error("pool should not be stopped")
	}

	wp.Stop()
	if anyReady(wp) {
		t.Error("should have zero workers after stop")
	}

	if !wp.Stopped() {
		t.Error("pool should be stopped")
	}

	// Start workers, and have them all wait on a channel before completing.
	wp = New(5)
	sync = make(chan struct{})
	finished := make(chan struct{}, max)
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			<-sync
			finished <- struct{}{}
		})
	}

	// Call Stop() and see that only the already running tasks were completed.
	go func() {
		time.Sleep(10000 * time.Millisecond)
		close(sync)
	}()
	wp.Stop()
	var count int
Count:
	for count < max {
		select {
		case <-finished:
			count++
		default:
			break Count
		}
	}
	if count > 5 {
		t.Error("Should not have completed any queued tasks, did", count)
	}

	// Check that calling Stop() againg is OK.
	wp.Stop()
}

func TestStopWait(t *testing.T) {
	t.Parallel()

	// Start workers, and have them all wait on a channel before completing.
	wp := New(5)
	sync := make(chan struct{})
	finished := make(chan struct{}, max)
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			<-sync
			finished <- struct{}{}
		})
	}

	// Call StopWait() and see that all tasks were completed.
	go func() {
		time.Sleep(10 * time.Millisecond)
		close(sync)
	}()
	wp.StopWait()
	for count := 0; count < max; count++ {
		select {
		case <-finished:
		default:
			t.Error("Should have completed all queued tasks")
		}
	}

	if anyReady(wp) {
		t.Error("should have zero workers after stopwait")
	}

	if !wp.Stopped() {
		t.Error("pool should be stopped")
	}

	// Make sure that calling StopWait() with no queued tasks is OK.
	wp = New(5)
	wp.StopWait()

	if anyReady(wp) {
		t.Error("should have zero workers after stopwait")
	}

	// Check that calling StopWait() againg is OK.
	wp.StopWait()
}

func TestSubmitWait(t *testing.T) {
	wp := New(1)
	defer wp.Stop()

	// Check that these are noop.
	wp.Submit(nil)
	wp.SubmitWait(nil)

	done1 := make(chan struct{})
	wp.Submit(func() {
		time.Sleep(100 * time.Millisecond)
		close(done1)
	})
	select {
	case <-done1:
		t.Fatal("Submit did not return immediately")
	default:
	}

	done2 := make(chan struct{})
	wp.SubmitWait(func() {
		time.Sleep(100 * time.Millisecond)
		close(done2)
	})
	select {
	case <-done2:
	default:
		t.Fatal("SubmitWait did not wait for function to execute")
	}
}

func TestOverflow(t *testing.T) {
	wp := New(2)
	releaseChan := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < 64; i++ {
		wp.Submit(func() { <-releaseChan })
	}

	// Start a goroutine to free the workers after calling stop.  This way
	// the dispatcher can exit, then when this goroutine runs, the workerpool
	// can exit.
	go func() {
		<-time.After(time.Millisecond)
		close(releaseChan)
	}()
	wp.Stop()

	// Now that the worker pool has exited, it is safe to inspect its waiting
	// queue without causing a race.
	qlen := wp.waitingQueue.Len()
	if qlen != 62 {
		t.Fatal("Expected 62 tasks in waiting queue, have", qlen)
	}
}

func TestStopRace(t *testing.T) {
	wp := New(20)
	releaseChan := make(chan struct{})
	workRelChan := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < 20; i++ {
		wp.Submit(func() { <-workRelChan })
	}

	time.Sleep(5 * time.Second)
	for i := 0; i < 64; i++ {
		go func() {
			<-releaseChan
			wp.Stop()
		}()
	}

	close(workRelChan)
	close(releaseChan)
}

func anyReady(w *WorkerPool) bool {
	select {
	case wkCh := <-w.readyWorkers:
		w.readyWorkers <- wkCh
		return true
	default:
	}
	return false
}

func countReady(w *WorkerPool) int {
	// Try to pull max workers off of ready queue.
	timeout := time.After(5 * time.Second)
	readyTmp := make(chan chan func(), max)
	var readyCount int
	for i := 0; i < max; i++ {
		select {
		case wkCh := <-w.readyWorkers:
			readyTmp <- wkCh
			readyCount++
		case <-timeout:
			readyCount = i
			i = max
		}
	}

	// Restore ready workers.
	close(readyTmp)
	go func() {
		for r := range readyTmp {
			w.readyWorkers <- r
		}
	}()
	return readyCount
}

/*

Run benchmarking with: go test -bench '.'

*/

func BenchmarkEnqueue(b *testing.B) {
	wp := New(1)
	defer wp.Stop()
	releaseChan := make(chan struct{})

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() { <-releaseChan })
	}
	close(releaseChan)
}

func BenchmarkEnqueue2(b *testing.B) {
	wp := New(2)
	defer wp.Stop()
	releaseChan := make(chan struct{})

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		for i := 0; i < 64; i++ {
			wp.Submit(func() { <-releaseChan })
		}
		for i := 0; i < 64; i++ {
			releaseChan <- struct{}{}
		}
	}
	close(releaseChan)
}

func BenchmarkExecute1Worker(b *testing.B) {
	wp := New(1)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute2Worker(b *testing.B) {
	wp := New(2)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute4Workers(b *testing.B) {
	wp := New(4)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute16Workers(b *testing.B) {
	wp := New(16)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute64Workers(b *testing.B) {
	wp := New(64)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}
<commit_msg>Tests for queue size methods<commit_after>package workerpool

import (
	"sync"
	"testing"
	"time"
)

const max = 20

func TestExample(t *testing.T) {
	wp := New(2)
	requests := []string{"alpha", "beta", "gamma", "delta", "epsilon"}

	rspChan := make(chan string, len(requests))
	for _, r := range requests {
		r := r
		wp.Submit(func() {
			rspChan <- r
		})
	}

	wp.StopWait()

	close(rspChan)
	rspSet := map[string]struct{}{}
	for rsp := range rspChan {
		rspSet[rsp] = struct{}{}
	}
	if len(rspSet) < len(requests) {
		t.Fatal("Did not handle all requests")
	}
	for _, req := range requests {
		if _, ok := rspSet[req]; !ok {
			t.Fatal("Missing expected valus:", req)
		}
	}
}

func TestMaxWorkers(t *testing.T) {
	t.Parallel()

	wp := New(0)
	if wp.maxWorkers != 1 {
		t.Fatal("should have created one worker")
	}

	wp = New(max)
	defer wp.Stop()

	started := make(chan struct{}, max)
	sync := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			started <- struct{}{}
			<-sync
		})
	}

	// Wait for all queued tasks to be dispatched to workers.
	timeout := time.After(5 * time.Second)
	taskq, wq := wp.QueueSize()
	if taskq != wp.TaksQueueSize() {
		t.Fatal("Internal task queue and produced task queue has different length")
	}
	if wq != wp.WaitingQueueSize() {
		t.Fatal("Working Queue size returned should not be 0")
		panic("WRONG")
	}
	for startCount := 0; startCount < max; {
		select {
		case <-started:
			startCount++
		case <-timeout:
			t.Fatal("timed out waiting for workers to start")
		}
	}

	// Release workers.
	close(sync)
}

func TestReuseWorkers(t *testing.T) {
	t.Parallel()

	wp := New(5)
	defer wp.Stop()

	sync := make(chan struct{})

	// Cause worker to be created, and available for reuse before next task.
	for i := 0; i < 10; i++ {
		wp.Submit(func() { <-sync })
		sync <- struct{}{}
		time.Sleep(100 * time.Millisecond)
	}

	// If the same worker was always reused, then only one worker would have
	// been created and there should only be one ready.
	if countReady(wp) > 1 {
		t.Fatal("Worker not reused")
	}
}

func TestWorkerTimeout(t *testing.T) {
	t.Parallel()

	wp := New(max)
	defer wp.Stop()

	sync := make(chan struct{})
	started := make(chan struct{}, max)
	// Cause workers to be created.  Workers wait on channel, keeping them busy
	// and causing the worker pool to create more.
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			started <- struct{}{}
			<-sync
		})
	}

	// Wait for tasks to start.
	for i := 0; i < max; i++ {
		<-started
	}

	if anyReady(wp) {
		t.Fatal("number of ready workers should be zero")
	}
	// Release workers.
	close(sync)

	if countReady(wp) != max {
		t.Fatal("Expected", max, "ready workers")
	}

	// Check that a worker timed out.
	time.Sleep((idleTimeoutSec + 1) * time.Second)
	if countReady(wp) != max-1 {
		t.Fatal("First worker did not timeout")
	}

	// Check that another worker timed out.
	time.Sleep((idleTimeoutSec + 1) * time.Second)
	if countReady(wp) != max-2 {
		t.Fatal("Second worker did not timeout")
	}
}

func TestStop(t *testing.T) {
	t.Parallel()

	wp := New(max)
	defer wp.Stop()

	started := make(chan struct{}, max)
	sync := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			started <- struct{}{}
			<-sync
		})
	}

	// Wait for all queued tasks to be dispatched to workers.
	timeout := time.After(5 * time.Second)
	for startCount := 0; startCount < max; {
		select {
		case <-started:
			startCount++
		case <-timeout:
			t.Fatal("timed out waiting for workers to start")
		}
	}

	// Release workers.
	close(sync)

	if wp.Stopped() {
		t.Error("pool should not be stopped")
	}

	wp.Stop()
	if anyReady(wp) {
		t.Error("should have zero workers after stop")
	}

	if !wp.Stopped() {
		t.Error("pool should be stopped")
	}

	// Start workers, and have them all wait on a channel before completing.
	wp = New(5)
	sync = make(chan struct{})
	finished := make(chan struct{}, max)
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			<-sync
			finished <- struct{}{}
		})
	}

	// Call Stop() and see that only the already running tasks were completed.
	go func() {
		time.Sleep(10000 * time.Millisecond)
		close(sync)
	}()
	wp.Stop()
	var count int
Count:
	for count < max {
		select {
		case <-finished:
			count++
		default:
			break Count
		}
	}
	if count > 5 {
		t.Error("Should not have completed any queued tasks, did", count)
	}

	// Check that calling Stop() againg is OK.
	wp.Stop()
}

func TestStopWait(t *testing.T) {
	t.Parallel()

	// Start workers, and have them all wait on a channel before completing.
	wp := New(5)
	sync := make(chan struct{})
	finished := make(chan struct{}, max)
	for i := 0; i < max; i++ {
		wp.Submit(func() {
			<-sync
			finished <- struct{}{}
		})
	}

	// Call StopWait() and see that all tasks were completed.
	go func() {
		time.Sleep(10 * time.Millisecond)
		close(sync)
	}()
	wp.StopWait()
	for count := 0; count < max; count++ {
		select {
		case <-finished:
		default:
			t.Error("Should have completed all queued tasks")
		}
	}

	if anyReady(wp) {
		t.Error("should have zero workers after stopwait")
	}

	if !wp.Stopped() {
		t.Error("pool should be stopped")
	}

	// Make sure that calling StopWait() with no queued tasks is OK.
	wp = New(5)
	wp.StopWait()

	if anyReady(wp) {
		t.Error("should have zero workers after stopwait")
	}

	// Check that calling StopWait() againg is OK.
	wp.StopWait()
}

func TestSubmitWait(t *testing.T) {
	wp := New(1)
	defer wp.Stop()

	// Check that these are noop.
	wp.Submit(nil)
	wp.SubmitWait(nil)

	done1 := make(chan struct{})
	wp.Submit(func() {
		time.Sleep(100 * time.Millisecond)
		close(done1)
	})
	select {
	case <-done1:
		t.Fatal("Submit did not return immediately")
	default:
	}

	done2 := make(chan struct{})
	wp.SubmitWait(func() {
		time.Sleep(100 * time.Millisecond)
		close(done2)
	})
	select {
	case <-done2:
	default:
		t.Fatal("SubmitWait did not wait for function to execute")
	}
}

func TestOverflow(t *testing.T) {
	wp := New(2)
	releaseChan := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < 64; i++ {
		wp.Submit(func() { <-releaseChan })
	}

	// Start a goroutine to free the workers after calling stop.  This way
	// the dispatcher can exit, then when this goroutine runs, the workerpool
	// can exit.
	go func() {
		<-time.After(time.Millisecond)
		close(releaseChan)
	}()
	wp.Stop()

	// Now that the worker pool has exited, it is safe to inspect its waiting
	// queue without causing a race.
	qlen := wp.waitingQueue.Len()
	if qlen != 62 {
		t.Fatal("Expected 62 tasks in waiting queue, have", qlen)
	}
}

func TestStopRace(t *testing.T) {
	wp := New(20)
	releaseChan := make(chan struct{})
	workRelChan := make(chan struct{})

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < 20; i++ {
		wp.Submit(func() { <-workRelChan })
	}

	time.Sleep(5 * time.Second)
	for i := 0; i < 64; i++ {
		go func() {
			<-releaseChan
			wp.Stop()
		}()
	}

	close(workRelChan)
	close(releaseChan)
}

func anyReady(w *WorkerPool) bool {
	select {
	case wkCh := <-w.readyWorkers:
		w.readyWorkers <- wkCh
		return true
	default:
	}
	return false
}

func countReady(w *WorkerPool) int {
	// Try to pull max workers off of ready queue.
	timeout := time.After(5 * time.Second)
	readyTmp := make(chan chan func(), max)
	var readyCount int
	for i := 0; i < max; i++ {
		select {
		case wkCh := <-w.readyWorkers:
			readyTmp <- wkCh
			readyCount++
		case <-timeout:
			readyCount = i
			i = max
		}
	}

	// Restore ready workers.
	close(readyTmp)
	go func() {
		for r := range readyTmp {
			w.readyWorkers <- r
		}
	}()
	return readyCount
}

/*

Run benchmarking with: go test -bench '.'

*/

func BenchmarkEnqueue(b *testing.B) {
	wp := New(1)
	defer wp.Stop()
	releaseChan := make(chan struct{})

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() { <-releaseChan })
	}
	close(releaseChan)
}

func BenchmarkEnqueue2(b *testing.B) {
	wp := New(2)
	defer wp.Stop()
	releaseChan := make(chan struct{})

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		for i := 0; i < 64; i++ {
			wp.Submit(func() { <-releaseChan })
		}
		for i := 0; i < 64; i++ {
			releaseChan <- struct{}{}
		}
	}
	close(releaseChan)
}

func BenchmarkExecute1Worker(b *testing.B) {
	wp := New(1)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute2Worker(b *testing.B) {
	wp := New(2)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute4Workers(b *testing.B) {
	wp := New(4)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute16Workers(b *testing.B) {
	wp := New(16)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}

func BenchmarkExecute64Workers(b *testing.B) {
	wp := New(64)
	defer wp.Stop()
	var allDone sync.WaitGroup
	allDone.Add(b.N)

	b.ResetTimer()

	// Start workers, and have them all wait on a channel before completing.
	for i := 0; i < b.N; i++ {
		wp.Submit(func() {
			time.Sleep(time.Millisecond)
			allDone.Done()
		})
	}
	allDone.Wait()
}
<|endoftext|>


<commit_before>package btree

import (
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/bradberger/gocache/cache"
	"github.com/google/btree"

	"stathat.com/c/consistent"
)

var (
	_ cache.Cache = (*Tree)(nil)
	_ btree.Item  = (*item)(nil)
)

type item struct {
	key   string
	value interface{}
}

func (i *item) Less(than btree.Item) bool {
	if than == nil {
		return true
	}
	return strings.Compare(i.key, than.(*item).key) == -1
}

type Container struct {
	shards map[string]*Tree
	ch     *consistent.Consistent
}

func NewContainer(numShards int) *Container {
	c := &Container{shards: make(map[string]*Tree, numShards), ch: consistent.New()}
	c.ch.NumberOfReplicas = 1
	for i := 0; i < numShards; i++ {
		k := fmt.Sprintf("shard-%d", i)
		c.shards[k] = New()
		c.ch.Add(k)
	}
	return c
}

func (c *Container) tree(key string) (*Tree, error) {
	name, err := c.ch.Get(key)
	if err != nil {
		return nil, err
	}
	return c.shards[name], nil
}

func (c *Container) Get(key string, dstVal interface{}) (err error) {
	t, err := c.tree(key)
	if err != nil {
		return err
	}
	return t.Get(key, dstVal)
}

func (c *Container) Set(key string, val interface{}, exp time.Duration) error {
	t, err := c.tree(key)
	if err != nil {
		return err
	}
	return t.Set(key, val, exp)
}

func (c *Container) Del(key string) error {
	t, err := c.tree(key)
	if err != nil {
		return err
	}
	return t.Del(key)
}

func (c *Container) Exists(key string) bool {
	t, err := c.tree(key)
	if err != nil {
		return false
	}
	return t.Exists(key)
}

// Tree is a wrapper for github.com/google/btree which adds expiration times and parallel thread write safetey
type Tree struct {
	tree *btree.BTree

	exp map[string]time.Time

	sync.RWMutex
}

// New returns a newly initialized btree based cache with a default degree of 2
func New() *Tree {
	return &Tree{tree: btree.New(2), exp: make(map[string]time.Time)}
}

// Get implements the "cache.Cache".Get interface
func (t *Tree) Get(key string, dstVal interface{}) error {
	i, ok := t.tree.Get(&item{key: key}).(*item)
	if !ok || i == nil || i.value == nil {
		return cache.ErrNotFound
	}
	return cache.Copy(i.value, dstVal)
}

// Set implements the "cache.Cache".Set interface
func (t *Tree) Set(key string, val interface{}, exp time.Duration) error {
	if reflect.ValueOf(val).Kind() == reflect.Ptr {
		val = reflect.ValueOf(val).Elem().Interface()
	}
	t.Lock()
	t.tree.ReplaceOrInsert(&item{key: key, value: val})
	if exp > 0 {
		t.exp[key] = time.Now().Add(exp)
		go t.expire(key, exp)
	}
	t.Unlock()
	return nil
}

func (t *Tree) expire(key string, delay time.Duration) {
	select {
	case <-time.After(delay):
		t.RLock()
		exp, _ := t.exp[key]
		t.RUnlock()
		if exp.Before(time.Now()) {
			t.Del(key)
		}
	}
}

// Del implements the "cache.Cache".Del interface
func (t *Tree) Del(key string) error {
	t.Lock()
	t.tree.Delete(&item{key: key})
	delete(t.exp, key)
	t.Unlock()
	return nil
}

// Exists implements the "cache.Cache".Exists interface
func (t *Tree) Exists(key string) bool {
	return t.tree.Has(&item{key: key})
}

// Tree returns the underlying btree
func (t *Tree) Tree() *btree.BTree {
	return t.tree
}
<commit_msg>Fixes one race<commit_after>package btree

import (
	"fmt"
	"reflect"
	"strings"
	"sync"
	"time"

	"github.com/bradberger/gocache/cache"
	"github.com/google/btree"

	"stathat.com/c/consistent"
)

var (
	_ cache.Cache = (*Tree)(nil)
	_ btree.Item  = (*item)(nil)
)

type item struct {
	key   string
	value interface{}
}

func (i *item) Less(than btree.Item) bool {
	if than == nil {
		return true
	}
	return strings.Compare(i.key, than.(*item).key) == -1
}

type Container struct {
	shards map[string]*Tree
	ch     *consistent.Consistent
}

func NewContainer(numShards int) *Container {
	c := &Container{shards: make(map[string]*Tree, numShards), ch: consistent.New()}
	c.ch.NumberOfReplicas = 1
	for i := 0; i < numShards; i++ {
		k := fmt.Sprintf("shard-%d", i)
		c.shards[k] = New()
		c.ch.Add(k)
	}
	return c
}

func (c *Container) tree(key string) (*Tree, error) {
	name, err := c.ch.Get(key)
	if err != nil {
		return nil, err
	}
	return c.shards[name], nil
}

func (c *Container) Get(key string, dstVal interface{}) (err error) {
	t, err := c.tree(key)
	if err != nil {
		return err
	}
	return t.Get(key, dstVal)
}

func (c *Container) Set(key string, val interface{}, exp time.Duration) error {
	t, err := c.tree(key)
	if err != nil {
		return err
	}
	return t.Set(key, val, exp)
}

func (c *Container) Del(key string) error {
	t, err := c.tree(key)
	if err != nil {
		return err
	}
	return t.Del(key)
}

func (c *Container) Exists(key string) bool {
	t, err := c.tree(key)
	if err != nil {
		return false
	}
	return t.Exists(key)
}

// Tree is a wrapper for github.com/google/btree which adds expiration times and parallel thread write safetey
type Tree struct {
	tree *btree.BTree

	exp map[string]time.Time

	sync.RWMutex
}

// New returns a newly initialized btree based cache with a default degree of 2
func New() *Tree {
	return &Tree{tree: btree.New(2), exp: make(map[string]time.Time)}
}

// Get implements the "cache.Cache".Get interface
func (t *Tree) Get(key string, dstVal interface{}) error {
	i, ok := t.tree.Get(&item{key: key}).(*item)
	if !ok || i == nil || i.value == nil {
		return cache.ErrNotFound
	}
	return cache.Copy(i.value, dstVal)
}

// Set implements the "cache.Cache".Set interface
func (t *Tree) Set(key string, val interface{}, exp time.Duration) error {
	if reflect.ValueOf(val).Kind() == reflect.Ptr {
		val = reflect.ValueOf(val).Elem().Interface()
	}
	t.Lock()
	t.tree.ReplaceOrInsert(&item{key: key, value: val})
	if exp > 0 {
		t.exp[key] = time.Now().Add(exp)
		go t.expire(key, exp)
	}
	t.Unlock()
	return nil
}

func (t *Tree) expire(key string, delay time.Duration) {
	select {
	case <-time.After(delay):
		t.RLock()
		exp, _ := t.exp[key]
		t.RUnlock()
		if exp.Before(time.Now()) {
			t.Del(key)
		}
	}
}

// Del implements the "cache.Cache".Del interface
func (t *Tree) Del(key string) error {
	t.Lock()
	t.tree.Delete(&item{key: key})
	delete(t.exp, key)
	t.Unlock()
	return nil
}

// Exists implements the "cache.Cache".Exists interface
func (t *Tree) Exists(key string) (exists bool) {
	t.RLock()
	exists = t.tree.Has(&item{key: key})
	t.RUnlock()
	return
}

// Tree returns the underlying btree
func (t *Tree) Tree() *btree.BTree {
	return t.tree
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package spaces

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/params"
)

var logger = loggo.GetLogger("juju.apiserver.spaces")

func init() {
	// TODO(dimitern): Uncomment once *state.State implements Backing.
	// common.RegisterStandardFacade("Spaces", 1, NewAPI)
}

// API defines the methods the Spaces API facade implements.
type API interface {
	CreateSpaces(params.CreateSpacesParams) (params.ErrorResults, error)
}

// spacesAPI implements the API interface.
type spacesAPI struct {
	backing    common.NetworkBacking
	resources  *common.Resources
	authorizer common.Authorizer
}

var _ API = (*spacesAPI)(nil)

// NewAPI creates a new server-side Spaces API facade.
func NewAPI(backing common.NetworkBacking, resources *common.Resources, authorizer common.Authorizer) (API, error) {
	// Only clients can access the Spaces facade.
	if !authorizer.AuthClient() {
		return nil, common.ErrPerm
	}
	return &spacesAPI{
		backing:    backing,
		resources:  resources,
		authorizer: authorizer,
	}, nil
}

// CreateSpaces creates a new Juju network space, associating the
// specified subnets with it (optional; can be empty).
func (api *spacesAPI) CreateSpaces(args params.CreateSpacesParams) (params.ErrorResults, error) {
	results := params.ErrorResults{}

	for _, space := range args.Spaces {
		err := api.createOneSpace(space)
		errorResult := params.ErrorResult{}
		if err != nil {
			errors.Trace(err)
			errorResult.Error = common.ServerError(err)
		}

		results.Results = append(results.Results, errorResult)
	}

	return results, nil
}

func (api *spacesAPI) createOneSpace(args params.CreateSpaceParams) error {
	if len(args.SubnetTags) == 0 {
		return errors.NotValidf("calling CreateSpaces with zero subnets is") // ... not valid.
	}

	// Validate the args, assemble information for api.backing.AddSpaces
	var subnets []string

	spaceTag, err := names.ParseSpaceTag(args.SpaceTag)
	if err != nil {
		return errors.Annotate(err, "given SpaceTag is invalid")
	}

	for _, tag := range args.SubnetTags {
		if subnetTag, err := names.ParseSubnetTag(tag); err != nil {
			return errors.Annotate(err, "given SubnetTag is invalid")
		} else {
			subnets = append(subnets, subnetTag.Id())
		}
	}

	// Add the validated space
	if err := api.backing.AddSpace(spaceTag.Id(), subnets, args.Public); err != nil {
		return errors.Annotate(err, "cannot create space")
	}
	return nil
}

func (api *spacesAPI) ListSpaces() (params.SpaceListResults, error) {
	results := params.SpaceListResults{}
	spaces, err := api.baking.AllSpaces()
	if err != nil {
		results.Error = common.ServerError(err)
		return results, errors.Annotate(err, "cannot list spaces")
	}
	return results, nil
}
<commit_msg>Typo correction<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package spaces

import (
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"

	"github.com/juju/juju/apiserver/common"
	"github.com/juju/juju/apiserver/params"
)

var logger = loggo.GetLogger("juju.apiserver.spaces")

func init() {
	// TODO(dimitern): Uncomment once *state.State implements Backing.
	// common.RegisterStandardFacade("Spaces", 1, NewAPI)
}

// API defines the methods the Spaces API facade implements.
type API interface {
	CreateSpaces(params.CreateSpacesParams) (params.ErrorResults, error)
}

// spacesAPI implements the API interface.
type spacesAPI struct {
	backing    common.NetworkBacking
	resources  *common.Resources
	authorizer common.Authorizer
}

var _ API = (*spacesAPI)(nil)

// NewAPI creates a new server-side Spaces API facade.
func NewAPI(backing common.NetworkBacking, resources *common.Resources, authorizer common.Authorizer) (API, error) {
	// Only clients can access the Spaces facade.
	if !authorizer.AuthClient() {
		return nil, common.ErrPerm
	}
	return &spacesAPI{
		backing:    backing,
		resources:  resources,
		authorizer: authorizer,
	}, nil
}

// CreateSpaces creates a new Juju network space, associating the
// specified subnets with it (optional; can be empty).
func (api *spacesAPI) CreateSpaces(args params.CreateSpacesParams) (params.ErrorResults, error) {
	results := params.ErrorResults{}

	for _, space := range args.Spaces {
		err := api.createOneSpace(space)
		errorResult := params.ErrorResult{}
		if err != nil {
			errors.Trace(err)
			errorResult.Error = common.ServerError(err)
		}

		results.Results = append(results.Results, errorResult)
	}

	return results, nil
}

func (api *spacesAPI) createOneSpace(args params.CreateSpaceParams) error {
	if len(args.SubnetTags) == 0 {
		return errors.NotValidf("calling CreateSpaces with zero subnets is") // ... not valid.
	}

	// Validate the args, assemble information for api.backing.AddSpaces
	var subnets []string

	spaceTag, err := names.ParseSpaceTag(args.SpaceTag)
	if err != nil {
		return errors.Annotate(err, "given SpaceTag is invalid")
	}

	for _, tag := range args.SubnetTags {
		if subnetTag, err := names.ParseSubnetTag(tag); err != nil {
			return errors.Annotate(err, "given SubnetTag is invalid")
		} else {
			subnets = append(subnets, subnetTag.Id())
		}
	}

	// Add the validated space
	if err := api.backing.AddSpace(spaceTag.Id(), subnets, args.Public); err != nil {
		return errors.Annotate(err, "cannot create space")
	}
	return nil
}

func (api *spacesAPI) ListSpaces() (params.SpaceListResults, error) {
	results := params.SpaceListResults{}
	spaces, err := api.backing.AllSpaces()
	if err != nil {
		results.Error = common.ServerError(err)
		return results, errors.Annotate(err, "cannot list spaces")
	}
	return results, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/format"
	"os"
	"strconv"
	"text/scanner"

	"github.com/crgimenes/goConfig"
)

type config struct {
	InputFile  string `json:"i" cfg:"i"`
	OutputFile string `json:"o" cfg:"o"`
}

type fileInfo struct {
	Version     string
	PackageName string
	ObjectName  string
	Height      int
	Width       int
}

const alert = "/* Automatically generated, do not change manually. */"

func main() {

	cfg := config{}

	goConfig.PrefixEnv = "BMV"
	err := goConfig.Parse(&cfg)
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	if cfg.InputFile == "" {
		goConfig.Usage()
		os.Exit(1)
	}

	err = parse(cfg.InputFile)
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

}

func parse(fileName string) (err error) {

	var out string
	var iFile *os.File
	var hCount int
	out = alert + "\n\n"
	//var oFile *os.File

	iFile, err = os.Open(fileName)
	if err != nil {
		return
	}
	defer func() {
		if e := iFile.Close(); e != nil {
			err = e
			return
		}
	}()

	fi := fileInfo{}

	var s scanner.Scanner
	s.Init(iFile)
	s.Filename = fileName
	s.Mode = scanner.ScanIdents |
		scanner.ScanFloats |
		scanner.ScanChars |
		scanner.ScanStrings |
		scanner.ScanRawStrings |
		scanner.ScanComments
	var tok rune

	ntok := 0
	for tok != scanner.EOF {
		tok = s.Scan()
		if tok == scanner.Comment {
			out += s.TokenText() + "\n"
		} else {

			switch ntok {
			case 0:
				fi.Version = s.TokenText()
			case 1:
				var h int
				h, err = strconv.Atoi(s.TokenText())
				if err != nil {
					return
				}
				fi.Height = h
			case 2:
				var w int
				w, err = strconv.Atoi(s.TokenText())
				if err != nil {
					return
				}
				fi.Width = w
			case 3:
				fi.PackageName = s.TokenText()
				out += "package " + fi.PackageName + "\n"
			case 4:
				fi.ObjectName = s.TokenText()
				out += "var " + fi.ObjectName + "Height = " + strconv.Itoa(fi.Height) + "\n"
				out += "var " + fi.ObjectName + "Width = " + strconv.Itoa(fi.Height) + "\n"
				out += "var " + fi.ObjectName + " [][]byte" + "\n\n"
				out += "func Load" + fi.ObjectName + "() {\n\n"
				out += fi.ObjectName + " = [][]byte{\n"
			default:
				//fmt.Println("At position", s.Pos(), ":", s.TokenText(), ntok)
				bs := s.TokenText()
				if bs == "" {
					continue
				}
				if hCount == 0 {
					out += "{\n"
				}

				if len(bs) > fi.Width {
					err = fmt.Errorf("Error at %v", s.Pos())
					println(err.Error())
					return
				}

				var r int64
				r, err = strconv.ParseInt(bs, 2, 64)
				if err != nil {
					println(err.Error())
					return
				}
				out += fmt.Sprintf("0x%02X, // %v\n", r, bs)

				hCount++
				if hCount >= fi.Height {
					hCount = 0
					out += "},\n"
				}
			}

			ntok++
		}

	}
	out += "}}\n"
	//println(out)

	arr := []byte(out)
	arr, err = format.Source(arr)
	if err != nil {
		return
	}
	println(string(arr))

	return
}
<commit_msg>update to struct<commit_after>package main

import (
	"fmt"
	"go/format"
	"os"
	"strconv"
	"text/scanner"

	"github.com/crgimenes/goConfig"
)

type config struct {
	InputFile  string `json:"i" cfg:"i"`
	OutputFile string `json:"o" cfg:"o"`
}

type fileInfo struct {
	Version     string
	PackageName string
	ObjectName  string
	Height      int
	Width       int
}

const alert = "/* Automatically generated, do not change manually. */"

func main() {

	cfg := config{}

	goConfig.PrefixEnv = "BMV"
	err := goConfig.Parse(&cfg)
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

	if cfg.InputFile == "" {
		goConfig.Usage()
		os.Exit(1)
	}

	err = parse(cfg.InputFile)
	if err != nil {
		println(err.Error())
		os.Exit(1)
	}

}

func parse(fileName string) (err error) {

	var out string
	var iFile *os.File
	var hCount int
	out = alert + "\n\n"
	//var oFile *os.File

	iFile, err = os.Open(fileName)
	if err != nil {
		return
	}
	defer func() {
		if e := iFile.Close(); e != nil {
			err = e
			return
		}
	}()

	fi := fileInfo{}

	var s scanner.Scanner
	s.Init(iFile)
	s.Filename = fileName
	s.Mode = scanner.ScanIdents |
		scanner.ScanFloats |
		scanner.ScanChars |
		scanner.ScanStrings |
		scanner.ScanRawStrings |
		scanner.ScanComments
	var tok rune

	ntok := 0
	for tok != scanner.EOF {
		tok = s.Scan()
		if tok == scanner.Comment {
			out += s.TokenText() + "\n"
		} else {

			switch ntok {
			case 0:
				fi.Version = s.TokenText()
			case 1:
				var h int
				h, err = strconv.Atoi(s.TokenText())
				if err != nil {
					return
				}
				fi.Height = h
			case 2:
				var w int
				w, err = strconv.Atoi(s.TokenText())
				if err != nil {
					return
				}
				fi.Width = w
			case 3:
				fi.PackageName = s.TokenText()
				out += "package " + fi.PackageName + "\n"
			case 4:
				fi.ObjectName = s.TokenText()
				out += "type " + fi.ObjectName + " struct {\n"
				out += "Height int\n"
				out += "Width int\n"
				out += "Bitmap [][]byte\n}\n\n"
				out += "func (f *" + fi.ObjectName + ")Load(){\n\n"
				out += "Height = " + strconv.Itoa(fi.Height) + "\n"
				out += "Width = " + strconv.Itoa(fi.Width) + "\n"
				out += "f.Bitmap = [][]byte{\n"
			default:
				//fmt.Println("At position", s.Pos(), ":", s.TokenText(), ntok)
				bs := s.TokenText()
				if bs == "" {
					continue
				}
				if hCount == 0 {
					out += "{\n"
				}

				if len(bs) > fi.Width {
					err = fmt.Errorf("Error at %v", s.Pos())
					println(err.Error())
					return
				}

				var r int64
				r, err = strconv.ParseInt(bs, 2, 64)
				if err != nil {
					println(err.Error())
					return
				}
				out += fmt.Sprintf("0x%02X, // %v\n", r, bs)

				hCount++
				if hCount >= fi.Height {
					hCount = 0
					out += "},\n"
				}
			}

			ntok++
		}

	}
	out += "}}\n"
	//println(out)

	arr := []byte(out)
	arr, err = format.Source(arr)
	if err != nil {
		return
	}
	println(string(arr))

	return
}
<|endoftext|>