<commit_before>/*Package configurationFile represents Encodable/Decodable Golang structures from/to TOML structures.
 *
 *The global structure is ConfigurationFile, which is the simple way to store accurtely your local informations.
 *
 */
package configurationFile

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/user"

	"fmt"

	"path/filepath"

	"sync"

	"github.com/BurntSushi/toml"
	"github.com/k0pernicus/goyave/consts"
	"github.com/k0pernicus/goyave/gitManip"
	"github.com/k0pernicus/goyave/traces"
	"github.com/k0pernicus/goyave/utils"
)

/*GetConfigurationFileContent get the content of the local configuration file.
 *If no configuration file has been found, create a default one and set the bytes array.
 */
func GetConfigurationFileContent(filePointer *os.File, bytesArray *[]byte) {
	fileState, err := filePointer.Stat()
	// If the file is empty, get the default structure and save it
	if err != nil || fileState.Size() == 0 {
		traces.WarningTracer.Println("No (or empty) configuration file - creating default one...")
		var fileBuffer bytes.Buffer
		cLocalhost := utils.GetHostname()
		cUser, err := user.Current()
		var cUserName string
		if err != nil {
			cUserName = consts.DefaultUserName
		} else {
			cUserName = cUser.Username
		}
		defaultStructure := Default(cUserName, cLocalhost)
		defaultStructure.Encode(&fileBuffer)
		*bytesArray = fileBuffer.Bytes()
	} else {
		b, _ := ioutil.ReadAll(filePointer)
		*bytesArray = b
	}
}

/*ConfigurationFile represents the TOML structure of the Goyave configuration file
 *
 *Properties:
 *	Author:
 *		The name of the user
 *  Local:
 *		Local informations
 *  Repositories:
 *		Local git repositories
 *	VisibleRepositories:
 *		A list of local ** visible ** git repositories (** used localy **)
 *	Groups:
 *		A list of groups
 *	locker:
 *		Mutex to perform concurrent RW on map data structures
 */
type ConfigurationFile struct {
	Author              string
	Local               LocalInformations        `toml:"local"`
	Repositories        map[string]GitRepository `toml:"repositories"`
	VisibleRepositories VisibleRepositories      `toml:"-"`
	Groups              map[string]Group         `toml:"group"`
	locker              sync.RWMutex             `toml:"-"`
}

/*Default is a constructor for ConfigurationFile
 *
 *Parameters:
 *	author:
 *		The name of the user
 *  hostname:
 *		The machine hostname
 */
func Default(author string, hostname string) *ConfigurationFile {
	return &ConfigurationFile{
		Author: author,
		Local: LocalInformations{
			DefaultTarget: consts.VisibleFlag,
			Group:         hostname,
		},
		Groups: map[string]Group{
			hostname: []string{},
		},
	}
}

/*AddRepository append the given repository to the list of local repositories, if it does not exists
 */
func (c *ConfigurationFile) AddRepository(path, target string) error {
	name := filepath.Base(path)
	hostname := utils.GetHostname()
	c.locker.Lock()
	defer c.locker.Unlock()
	robj, ok := c.Repositories[name]
	// If the repository exists and the path is ok, stop
	if ok && robj.Paths[hostname].Path == path {
		return nil
	}
	// Initialize the new GroupPath structure
	cgroup := GroupPath{
		Name: name,
		Path: path,
	}
	// If the repository exists but the path is not ok, update it
	if ok {
		robj.Paths[hostname] = cgroup
		return nil
	}
	// Otherwise, create a new GitRepository structure, and append it in the Repositories field
	c.Repositories[name] = GitRepository{
		Name: name,
		Paths: map[string]GroupPath{
			hostname: cgroup,
		},
		URL: gitManip.GetRemoteURL(path),
	}
	// If the user wants to add automatically new repositories as repositories to "follow", change
	// his flag as a "visible" repository
	if target == consts.VisibleFlag {
		c.Groups[hostname] = append(c.Groups[hostname], name)
	}
	return nil
}

/*GetPath returns the local path file, for a given repository
 */
func (c *ConfigurationFile) GetPath(repository string) (string, bool) {
	gobj, ok := c.VisibleRepositories[repository]
	return gobj, ok
}

/*Process initializes useful fields in the data structure
 */
func (c *ConfigurationFile) Process() error {
	// If the configuration file is new, initialize the map and finish here
	if c.Repositories == nil {
		c.Repositories = make(map[string]GitRepository)
		return nil
	}
	// Otherwise, initialize useful fields
	hostname := utils.GetHostname()
	vrepositories, ok := c.Groups[hostname]
	if !ok {
		return fmt.Errorf("the hostname %s has not been found - please to launch 'crawl' before", hostname)
	}
	c.VisibleRepositories = make(VisibleRepositories)
	for _, repository := range vrepositories {
		c.VisibleRepositories[repository] = c.Repositories[repository].Paths[hostname].Path
	}
	return nil
}

/*VisibleRepositories is a map structure to store, for each repository name (and the hostname), the associated path
 */
type VisibleRepositories map[string]string

/*Method that returns if a repository, identified by his name and his path (optional), exists in the given structure
 * If path is empty (empty string), the function will only check the name
 */
func (v VisibleRepositories) exists(name, path string) bool {
	_, ok := v[name]
	if !ok || path == "" {
		return ok
	}
	return v[name] == path
}

/*GitRepository represents the structure of a local git repository
 *
 *Properties:
 *	Name:
 * 		The custom name of the repository
 *  Paths:
 *		Path per group name
 *	URL:
 *		The remote URL of the repository (from origin)
 */
type GitRepository struct {
	Name  string               `toml:"name"`
	Paths map[string]GroupPath `toml:"paths"`
	URL   string               `toml:"url"`
}

/*GroupPath represents the structure of a local path, using a given group
 *
 *Properties:
 *  Name:
 *		The name of the local git repository
 *	Path:
 *		A string that points to the local git repository
 */
type GroupPath struct {
	Name string
	Path string
}

/*Group represents a group of git repositories names
 */
type Group []string

/*LocalInformations represents your local configuration of Goyave
 *
 *Properties:
 *  DefaultEntry:
 *		The default entry to store a git repository (hidden or visible)
 *	Group:
 *		The current group name.
 */
type LocalInformations struct {
	DefaultTarget string
	Group         string
}

/*DecodeString is a function to decode an entire string (which is the content of a given TOML file) to a ConfigurationFile structure
 */
func DecodeString(c *ConfigurationFile, data string) error {
	_, err := toml.Decode(data, *c)
	return err
}

/*DecodeBytesArray is a function to decode an entire string (which is the content of a given TOML file) to a ConfigurationFile structure
 */
func DecodeBytesArray(c *ConfigurationFile, data []byte) error {
	_, err := toml.Decode(string(data[:]), *c)
	return err
}

/*Encode is a function to encode a ConfigurationFile structure to a byffer of bytes
 */
func (c *ConfigurationFile) Encode(buffer *bytes.Buffer) error {
	return toml.NewEncoder(buffer).Encode(c)
}
<commit_msg>Debugging when crawling on a new computer<commit_after>/*Package configurationFile represents Encodable/Decodable Golang structures from/to TOML structures.
 *
 *The global structure is ConfigurationFile, which is the simple way to store accurtely your local informations.
 *
 */
package configurationFile

import (
	"bytes"
	"io/ioutil"
	"os"
	"os/user"

	"path/filepath"

	"sync"

	"github.com/BurntSushi/toml"
	"github.com/k0pernicus/goyave/consts"
	"github.com/k0pernicus/goyave/gitManip"
	"github.com/k0pernicus/goyave/traces"
	"github.com/k0pernicus/goyave/utils"
)

/*GetConfigurationFileContent get the content of the local configuration file.
 *If no configuration file has been found, create a default one and set the bytes array.
 */
func GetConfigurationFileContent(filePointer *os.File, bytesArray *[]byte) {
	fileState, err := filePointer.Stat()
	// If the file is empty, get the default structure and save it
	if err != nil || fileState.Size() == 0 {
		traces.WarningTracer.Println("No (or empty) configuration file - creating default one...")
		var fileBuffer bytes.Buffer
		cLocalhost := utils.GetHostname()
		cUser, err := user.Current()
		var cUserName string
		if err != nil {
			cUserName = consts.DefaultUserName
		} else {
			cUserName = cUser.Username
		}
		defaultStructure := Default(cUserName, cLocalhost)
		defaultStructure.Encode(&fileBuffer)
		*bytesArray = fileBuffer.Bytes()
	} else {
		b, _ := ioutil.ReadAll(filePointer)
		*bytesArray = b
	}
}

/*ConfigurationFile represents the TOML structure of the Goyave configuration file
 *
 *Properties:
 *	Author:
 *		The name of the user
 *  Local:
 *		Local informations
 *  Repositories:
 *		Local git repositories
 *	VisibleRepositories:
 *		A list of local ** visible ** git repositories (** used localy **)
 *	Groups:
 *		A list of groups
 *	locker:
 *		Mutex to perform concurrent RW on map data structures
 */
type ConfigurationFile struct {
	Author              string
	Local               LocalInformations        `toml:"local"`
	Repositories        map[string]GitRepository `toml:"repositories"`
	VisibleRepositories VisibleRepositories      `toml:"-"`
	Groups              map[string]Group         `toml:"group"`
	locker              sync.RWMutex             `toml:"-"`
}

/*Default is a constructor for ConfigurationFile
 *
 *Parameters:
 *	author:
 *		The name of the user
 *  hostname:
 *		The machine hostname
 */
func Default(author string, hostname string) *ConfigurationFile {
	return &ConfigurationFile{
		Author: author,
		Local: LocalInformations{
			DefaultTarget: consts.VisibleFlag,
			Group:         hostname,
		},
		Groups: map[string]Group{
			hostname: []string{},
		},
	}
}

/*AddRepository append the given repository to the list of local repositories, if it does not exists
 */
func (c *ConfigurationFile) AddRepository(path, target string) error {
	name := filepath.Base(path)
	hostname := utils.GetHostname()
	c.locker.Lock()
	defer c.locker.Unlock()
	robj, ok := c.Repositories[name]
	// If the repository exists and the path is ok, stop
	if ok && robj.Paths[hostname].Path == path {
		return nil
	}
	// Initialize the new GroupPath structure
	cgroup := GroupPath{
		Name: name,
		Path: path,
	}
	// If the repository exists but the path is not ok, update it
	if ok {
		robj.Paths[hostname] = cgroup
		return nil
	}
	// Otherwise, create a new GitRepository structure, and append it in the Repositories field
	c.Repositories[name] = GitRepository{
		Name: name,
		Paths: map[string]GroupPath{
			hostname: cgroup,
		},
		URL: gitManip.GetRemoteURL(path),
	}
	// If the user wants to add automatically new repositories as repositories to "follow", change
	// his flag as a "visible" repository
	if target == consts.VisibleFlag {
		c.Groups[hostname] = append(c.Groups[hostname], name)
	}
	return nil
}

/*GetPath returns the local path file, for a given repository
 */
func (c *ConfigurationFile) GetPath(repository string) (string, bool) {
	gobj, ok := c.VisibleRepositories[repository]
	return gobj, ok
}

/*Process initializes useful fields in the data structure
 */
func (c *ConfigurationFile) Process() {
	// If the configuration file is new, initialize the map and finish here
	if c.Repositories == nil {
		c.Repositories = make(map[string]GitRepository)
	}
	// Otherwise, initialize useful fields
	hostname := utils.GetHostname()
	vrepositories, ok := c.Groups[hostname]
	if !ok {
		traces.InfoTracer.Printf("creating new group '%s'\n", hostname)
		c.Groups[hostname] = []string{}
	}
	c.VisibleRepositories = make(VisibleRepositories)
	for _, repository := range vrepositories {
		c.VisibleRepositories[repository] = c.Repositories[repository].Paths[hostname].Path
	}
}

/*VisibleRepositories is a map structure to store, for each repository name (and the hostname), the associated path
 */
type VisibleRepositories map[string]string

/*Method that returns if a repository, identified by his name and his path (optional), exists in the given structure
 * If path is empty (empty string), the function will only check the name
 */
func (v VisibleRepositories) exists(name, path string) bool {
	_, ok := v[name]
	if !ok || path == "" {
		return ok
	}
	return v[name] == path
}

/*GitRepository represents the structure of a local git repository
 *
 *Properties:
 *	Name:
 * 		The custom name of the repository
 *  Paths:
 *		Path per group name
 *	URL:
 *		The remote URL of the repository (from origin)
 */
type GitRepository struct {
	Name  string               `toml:"name"`
	Paths map[string]GroupPath `toml:"paths"`
	URL   string               `toml:"url"`
}

/*GroupPath represents the structure of a local path, using a given group
 *
 *Properties:
 *  Name:
 *		The name of the local git repository
 *	Path:
 *		A string that points to the local git repository
 */
type GroupPath struct {
	Name string
	Path string
}

/*Group represents a group of git repositories names
 */
type Group []string

/*LocalInformations represents your local configuration of Goyave
 *
 *Properties:
 *  DefaultEntry:
 *		The default entry to store a git repository (hidden or visible)
 *	Group:
 *		The current group name.
 */
type LocalInformations struct {
	DefaultTarget string
	Group         string
}

/*DecodeString is a function to decode an entire string (which is the content of a given TOML file) to a ConfigurationFile structure
 */
func DecodeString(c *ConfigurationFile, data string) error {
	_, err := toml.Decode(data, *c)
	return err
}

/*DecodeBytesArray is a function to decode an entire string (which is the content of a given TOML file) to a ConfigurationFile structure
 */
func DecodeBytesArray(c *ConfigurationFile, data []byte) error {
	_, err := toml.Decode(string(data[:]), *c)
	return err
}

/*Encode is a function to encode a ConfigurationFile structure to a byffer of bytes
 */
func (c *ConfigurationFile) Encode(buffer *bytes.Buffer) error {
	return toml.NewEncoder(buffer).Encode(c)
}
<|endoftext|>


<commit_before>package filesystem

import (
	"io"
	"os"
	"path/filepath"
)

type localFSClient struct {
}

func NewLocalFSClient() Client {
	return &localFSClient{}
}

func (c *localFSClient) OpenReadCloser(name string) (io.ReadCloser, error) {
	return os.Open(name)
}

func (c *localFSClient) OpenWriteCloser(name string) (io.WriteCloser, error) {
	exist, err := c.Exists(name)
	if err != nil {
		return nil, err
	}
	if !exist {
		f, err := os.Create(name)
		if err != nil {
			return nil, err
		}
		return f, nil
	}
	return os.Open(name)
}

func (c *localFSClient) Exists(name string) (bool, error) {
	_, err := os.Stat(name)
	return existCommon(err)
}

func (c *localFSClient) Rename(oldpath, newpath string) error {
	return os.Rename(oldpath, newpath)
}

func (c *localFSClient) Glob(pattern string) (matches []string, err error) {
	return filepath.Glob(pattern)
}

func (c *localFSClient) Remove(name string) error {
	return os.Remove(name)
}

func existCommon(err error) (bool, error) {
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
<commit_msg>filesystem: Open file with O_WRONLY in OpenWriteCloser()<commit_after>package filesystem

import (
	"io"
	"os"
	"path/filepath"
)

type localFSClient struct {
}

func NewLocalFSClient() Client {
	return &localFSClient{}
}

func (c *localFSClient) OpenReadCloser(name string) (io.ReadCloser, error) {
	return os.Open(name)
}

func (c *localFSClient) OpenWriteCloser(name string) (io.WriteCloser, error) {
	exist, err := c.Exists(name)
	if err != nil {
		return nil, err
	}
	if !exist {
		f, err := os.Create(name)
		if err != nil {
			return nil, err
		}
		return f, nil
	}
	return os.OpenFile(name, os.O_WRONLY, 0)
}

func (c *localFSClient) Exists(name string) (bool, error) {
	_, err := os.Stat(name)
	return existCommon(err)
}

func (c *localFSClient) Rename(oldpath, newpath string) error {
	return os.Rename(oldpath, newpath)
}

func (c *localFSClient) Glob(pattern string) (matches []string, err error) {
	return filepath.Glob(pattern)
}

func (c *localFSClient) Remove(name string) error {
	return os.Remove(name)
}

func existCommon(err error) (bool, error) {
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"errors"
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Throttle
	var throttleCalled bool
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		AssertFalse(throttleCalled)
		throttleCalled = true

		AssertEq(t.ctx, ctx)
		AssertEq(readSize, tokens)

		return
	}

	// Call
	t.reader.Read(make([]byte, readSize))

	ExpectTrue(throttleCalled)
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	// Throttle
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 1))

	ExpectEq(0, n)
	ExpectThat(err, Error(HasSubstr("throttle")))
	ExpectThat(err, Error(HasSubstr("cancel")))
}

func (t *ThrottledReaderTest) CallsWrapped() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(buf, p)

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	// Wrapped
	expectedErr := errors.New("taco")
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = expectedErr
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = io.EOF
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = len(p)
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, readSize))

	ExpectEq(readSize, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondFails() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	AssertTrue(false, "TODO")
}
<commit_msg>ThrottledReaderTest.WrappedReturnsShortRead_CallsAgain<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ratelimit_test

import (
	"errors"
	"io"
	"testing"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/ratelimit"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestThrottledReader(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// An io.Reader that defers to a function.
type funcReader struct {
	f func([]byte) (int, error)
}

func (fr *funcReader) Read(p []byte) (n int, err error) {
	n, err = fr.f(p)
	return
}

// A throttler that defers to a function.
type funcThrottle struct {
	f func(context.Context, uint64) bool
}

func (ft *funcThrottle) Capacity() (c uint64) {
	return 1024
}

func (ft *funcThrottle) Wait(
	ctx context.Context,
	tokens uint64) (ok bool) {
	ok = ft.f(ctx, tokens)
	return
}

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type ThrottledReaderTest struct {
	ctx context.Context

	wrapped  funcReader
	throttle funcThrottle

	reader io.Reader
}

var _ SetUpInterface = &ThrottledReaderTest{}

func init() { RegisterTestSuite(&ThrottledReaderTest{}) }

func (t *ThrottledReaderTest) SetUp(ti *TestInfo) {
	t.ctx = ti.Ctx

	// Set up the default throttle function.
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		ok = true
		return
	}

	// Set up the reader.
	t.reader = ratelimit.ThrottledReader(t.ctx, &t.wrapped, &t.throttle)
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *ThrottledReaderTest) CallsThrottle() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Throttle
	var throttleCalled bool
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		AssertFalse(throttleCalled)
		throttleCalled = true

		AssertEq(t.ctx, ctx)
		AssertEq(readSize, tokens)

		return
	}

	// Call
	t.reader.Read(make([]byte, readSize))

	ExpectTrue(throttleCalled)
}

func (t *ThrottledReaderTest) ThrottleSaysCancelled() {
	// Throttle
	t.throttle.f = func(ctx context.Context, tokens uint64) (ok bool) {
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 1))

	ExpectEq(0, n)
	ExpectThat(err, Error(HasSubstr("throttle")))
	ExpectThat(err, Error(HasSubstr("cancel")))
}

func (t *ThrottledReaderTest) CallsWrapped() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var readCalled bool
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertFalse(readCalled)
		readCalled = true

		AssertEq(buf, p)

		err = errors.New("")
		return
	}

	// Call
	t.reader.Read(buf)

	ExpectTrue(readCalled)
}

func (t *ThrottledReaderTest) WrappedReturnsError() {
	// Wrapped
	expectedErr := errors.New("taco")
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = expectedErr
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(expectedErr, err)
}

func (t *ThrottledReaderTest) WrappedReturnsEOF() {
	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = 11
		err = io.EOF
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, 16))

	ExpectEq(11, n)
	ExpectEq(io.EOF, err)
}

func (t *ThrottledReaderTest) WrappedReturnsFullRead() {
	const readSize = 17
	AssertLe(readSize, t.throttle.Capacity())

	// Wrapped
	t.wrapped.f = func(p []byte) (n int, err error) {
		n = len(p)
		return
	}

	// Call
	n, err := t.reader.Read(make([]byte, readSize))

	ExpectEq(readSize, n)
	ExpectEq(nil, err)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_CallsAgain() {
	buf := make([]byte, 16)
	AssertLe(len(buf), t.throttle.Capacity())

	// Wrapped
	var callCount int
	t.wrapped.f = func(p []byte) (n int, err error) {
		AssertLt(callCount, 2)
		switch callCount {
		case 0:
			callCount++
			n = 2

		case 1:
			callCount++
			AssertEq(&buf[2], &p[0])
			err = errors.New("")
		}

		return
	}

	// Call
	t.reader.Read(buf)

	ExpectEq(2, callCount)
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondFails() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) WrappedReturnsShortRead_SecondSuceeds() {
	AssertTrue(false, "TODO")
}

func (t *ThrottledReaderTest) ReadSizeIsAboveThrottleCapacity() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>package ast

// TokenType is the type of tokem that will be returned by the Scanner.
type TokenType int

// The following are the token types that are recognized by the scanner
const (
	EOF TokenType = iota
	Comment
	Section
	WhiteSpace
	NLine
	Ident
	Assign   // =
	LBrace   // [
	RBrace   // ]
	LBracket // )
	RBracket // (
	Exclam   // !
)

// Token is the identifier for a chunk of text.
type Token struct {
	Type   TokenType
	Text   string
	Line   int
	Column int
}
<commit_msg>Add Begin and End fields to Token<commit_after>package ast

// TokenType is the type of tokem that will be returned by the Scanner.
type TokenType int

// The following are the token types that are recognized by the scanner
const (
	EOF TokenType = iota
	Comment
	Section
	WhiteSpace
	NLine
	Ident
	Assign   // =
	LBrace   // [
	RBrace   // ]
	LBracket // )
	RBracket // (
	Exclam   // !
)

// Token is the identifier for a chunk of text.
type Token struct {
	Type   TokenType
	Text   string
	Line   int
	Column int
	Begin  int
	End    int
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"fmt"
	"net/url"
	"log"
	"time"
	"bufio"
	"strings"

	"github.com/ChimeraCoder/anaconda"
)

func init() {
	consumer_key := os.Getenv("TWITTER_CONSUMER_KEY")
	consumer_secret := os.Getenv("TWITTER_CONSUMER_SECRET")
	access_token := os.Getenv("TWITTER_ACCESS_TOKEN")
	access_token_secret := os.Getenv("TWITTER_ACCESS_TOKEN_SECRET")

	if consumer_key == "" || consumer_secret == "" {
		log.Fatalln("consumer tokens left blank")
	}

	if access_token == "" || access_token_secret == "" {
		log.Fatalln("access tokens left blank")
	}
	fmt.Println("Consumer tokens and keys available in environment \n Initiating connection with twitter api")
}
//streamListener will listen to tweets
func streamListener(stream chan anaconda.Tweet) {
	consumer_key := os.Getenv("TWITTER_CONSUMER_KEY")
	consumer_secret := os.Getenv("TWITTER_CONSUMER_SECRET")
	access_token := os.Getenv("TWITTER_ACCESS_TOKEN")
	access_token_secret := os.Getenv("TWITTER_ACCESS_TOKEN_SECRET")

	anaconda.SetConsumerKey(consumer_key)
	anaconda.SetConsumerSecret(consumer_secret)
	api := anaconda.NewTwitterApi(access_token, access_token_secret)
	v := url.Values{}

	twitterStream := api.PublicStreamSample(v)
	fmt.Println("Starting streaming tweets ...")
	for t := range twitterStream.C {
		switch v := t.(type) {
			case anaconda.Tweet:
				if v.Lang == "en"{ //Capture only Tweets in English
					stream <- v
				}
		}
	}
}
//streamWriter will listen to channel and process/extract
//necessary data from tweets
//func streamWriter(stream chan anaconda.Tweet) {

//}
func main() {

	//Get stream
	filename := "tweetStream-" + time.Now().Format(time.RFC3339) + ".csv"
	fmt.Printf("Writting tweets to %s\n",filename)
	fout, err := os.Create(filename)
	if err != nil {
		log.Panic(err)
	}
	defer func() {
		if err := fout.Close(); err != nil {
			log.Panic(err)
		}
		log.Printf("Closed file out")
	}()
	writer := bufio.NewWriter(fout)
	defer func() {
		if err := writer.Flush(); err != nil {
			log.Panic(err)
		}
		log.Printf("flushed contents of writer")
	}()
	fmt.Fprintln(writer, "Text, User.TimeZone, CreatedAt, longitude, latitude")
	stream := make(chan anaconda.Tweet, 10) //buffered channel size 10
	go streamListener(stream)
	//go streamWriter(stream)
	for v := range(stream){
		tweetText := strings.Replace(v.Text, ",", " ", -1) //Remove all commas iside tweet text
		if v.HasCoordinates() { //Assumption that coordinates are type point
			longitude, err1 := v.Longitude()
			latitude, err2 := v.Latitude()
			if err1 != nil || err2 != nil { // ideally this should never happen
				fmt.Fprintf(writer, "%s, %s, %s, , \n",tweetText, v.User.TimeZone, v.CreatedAt)
			} else {
				fmt.Fprintf(writer, "%s, %s, %s, %v, %v \n",tweetText, v.User.TimeZone, v.CreatedAt, longitude, latitude)
			}
		} else {
			fmt.Fprintf(writer, "%s, %s, %s, , \n",tweetText, v.User.TimeZone, v.CreatedAt)
		}

	}
}
<commit_msg>MODIFY strings.Replace with NewReplacer<commit_after>package main

import (
	"os"
	"fmt"
	"net/url"
	"log"
	"time"
	"bufio"
	"strings"

	"github.com/ChimeraCoder/anaconda"
)

func init() {
	consumer_key := os.Getenv("TWITTER_CONSUMER_KEY")
	consumer_secret := os.Getenv("TWITTER_CONSUMER_SECRET")
	access_token := os.Getenv("TWITTER_ACCESS_TOKEN")
	access_token_secret := os.Getenv("TWITTER_ACCESS_TOKEN_SECRET")

	if consumer_key == "" || consumer_secret == "" {
		log.Fatalln("consumer tokens left blank")
	}

	if access_token == "" || access_token_secret == "" {
		log.Fatalln("access tokens left blank")
	}
	fmt.Println("Consumer tokens and keys available in environment \n Initiating connection with twitter api")
}
//streamListener will listen to tweets
func streamListener(stream chan anaconda.Tweet) {
	consumer_key := os.Getenv("TWITTER_CONSUMER_KEY")
	consumer_secret := os.Getenv("TWITTER_CONSUMER_SECRET")
	access_token := os.Getenv("TWITTER_ACCESS_TOKEN")
	access_token_secret := os.Getenv("TWITTER_ACCESS_TOKEN_SECRET")

	anaconda.SetConsumerKey(consumer_key)
	anaconda.SetConsumerSecret(consumer_secret)
	api := anaconda.NewTwitterApi(access_token, access_token_secret)
	v := url.Values{}

	twitterStream := api.PublicStreamSample(v)
	fmt.Println("Starting streaming tweets ...")
	for t := range twitterStream.C {
		switch v := t.(type) {
			case anaconda.Tweet:
				if v.Lang == "en"{ //Capture only Tweets in English
					stream <- v
				}
		}
	}
}
//streamWriter will listen to channel and process/extract
//necessary data from tweets
//func streamWriter(stream chan anaconda.Tweet) {

//}
func main() {

	//Get stream
	filename := "tweetStream-" + time.Now().Format(time.RFC3339) + ".csv"
	fmt.Printf("Writting tweets to %s\n",filename)
	fout, err := os.Create(filename)
	if err != nil {
		log.Panic(err)
	}
	defer func() {
		if err := fout.Close(); err != nil {
			log.Panic(err)
		}
		log.Printf("Closed file out")
	}()
	writer := bufio.NewWriter(fout)
	defer func() {
		if err := writer.Flush(); err != nil {
			log.Panic(err)
		}
		log.Printf("flushed contents of writer")
	}()
	fmt.Fprintln(writer, "Text, User.TimeZone, CreatedAt, longitude, latitude")

	r := strings.NewReplacer(",", " ", "\n", " ") //NewReplacer will remove occurences of comma and newlines

	stream := make(chan anaconda.Tweet, 10) //buffered channel size 10
	go streamListener(stream)
	//go streamWriter(stream)
	for v := range(stream){
		tweetText := r.Replace(v.Text)
		if v.HasCoordinates() { //Assumption that coordinates are type point
			longitude, err1 := v.Longitude()
			latitude, err2 := v.Latitude()
			if err1 != nil || err2 != nil { // ideally this should never happen
				fmt.Fprintf(writer, "%s, %s, %s, , \n",tweetText, v.User.TimeZone, v.CreatedAt)
			} else {
				fmt.Fprintf(writer, "%s, %s, %s, %v, %v \n",tweetText, v.User.TimeZone, v.CreatedAt, longitude, latitude)
			}
		} else {
			fmt.Fprintf(writer, "%s, %s, %s, , \n",tweetText, v.User.TimeZone, v.CreatedAt)
		}

	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

type StringArray []string

func (a *StringArray) Set(s string) error {
	*a = append(*a, s)
	return nil
}

func (a *StringArray) String() string {
	return fmt.Sprint(*a)
}
<commit_msg>split -k command line arguments on comma<commit_after>package main

import (
	"fmt"
	"strings"
)

type StringArray []string

func (a *StringArray) Set(s string) error {
	for _, ss := range strings.Split(s, ",") {
		*a = append(*a, ss)
	}
	return nil
}

func (a *StringArray) String() string {
	return fmt.Sprint(*a)
}
<|endoftext|>


<commit_before>package out_test

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"time"

	"terraform-resource/models"
	"terraform-resource/out"
	"terraform-resource/test/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Out Plan", func() {

	var (
		backendType   string
		backendConfig map[string]interface{}
		envName       string
		stateFilePath string
		planFilePath  string
		s3ObjectPath  string
		workingDir    string
		workspacePath string
	)

	BeforeEach(func() {
		region := os.Getenv("AWS_REGION") // optional
		if region == "" {
			region = "us-east-1"
		}

		workspacePath = helpers.RandomString("out-backend-test")

		envName = helpers.RandomString("out-test")
		stateFilePath = path.Join(workspacePath, envName, "terraform.tfstate")
		planFilePath = path.Join(workspacePath, fmt.Sprintf("%s-plan", envName), "terraform.tfstate")
		s3ObjectPath = path.Join(bucketPath, helpers.RandomString("out-test"))

		backendType = "s3"
		backendConfig = map[string]interface{}{
			"bucket":               bucket,
			"key":                  "terraform.tfstate",
			"access_key":           accessKey,
			"secret_key":           secretKey,
			"region":               region,
			"workspace_key_prefix": workspacePath,
		}

		var err error
		workingDir, err = ioutil.TempDir(os.TempDir(), "terraform-resource-out-test")
		Expect(err).ToNot(HaveOccurred())

		// ensure relative paths resolve correctly
		err = os.Chdir(workingDir)
		Expect(err).ToNot(HaveOccurred())

		fixturesDir := path.Join(helpers.ProjectRoot(), "fixtures")
		err = exec.Command("cp", "-r", fixturesDir, workingDir).Run()
		Expect(err).ToNot(HaveOccurred())

		err = helpers.DownloadStatefulPlugin(workingDir)
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		_ = os.RemoveAll(workingDir)
		awsVerifier.DeleteObjectFromS3(bucket, s3ObjectPath)
		awsVerifier.DeleteObjectFromS3(bucket, planFilePath)
		awsVerifier.DeleteObjectFromS3(bucket, stateFilePath)
	})

	It("plan infrastructure and apply it", func() {
		planOutRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		applyRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:  "fixtures/aws/",
					PlanRun: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
				},
			},
		}

		By("running 'out' to create the plan file")

		planrunner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		planOutput, err := planrunner.Run(planOutRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file exists")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)
		defer awsVerifier.DeleteObjectFromS3(bucket, planFilePath)

		Expect(planOutput.Version.EnvName).To(Equal(planOutRequest.Params.EnvName))
		Expect(planOutput.Version.PlanOnly).To(Equal("true"), "Expected PlanOnly to be true, but was false")
		Expect(planOutput.Version.Serial).To(BeEmpty())
		Expect(planOutput.Version.PlanChecksum).To(MatchRegexp("[0-9|a-f]+"))

		Expect(path.Join(os.TempDir(), "tf-plan.log")).To(BeAnExistingFile())

		By("ensuring s3 file does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			s3ObjectPath,
		)

		By("applying the plan")

		applyrunner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		createOutput, err := applyrunner.Run(applyRequest)
		Expect(err).ToNot(HaveOccurred())

		Expect(createOutput.Version.PlanOnly).To(BeEmpty())
		Expect(createOutput.Version.Serial).ToNot(BeEmpty())
		Expect(createOutput.Version.PlanChecksum).To(BeEmpty())

		Expect(createOutput.Metadata).ToNot(BeEmpty())
		fields := map[string]interface{}{}
		for _, field := range createOutput.Metadata {
			fields[field.Name] = field.Value
		}
		Expect(fields["env_name"]).To(Equal(envName))
		expectedMD5 := fmt.Sprintf("%x", md5.Sum([]byte("terraform-is-neat")))
		Expect(fields["content_md5"]).To(Equal(expectedMD5))

		awsVerifier.ExpectS3FileToExist(
			bucket,
			s3ObjectPath,
		)

		By("ensuring that plan file no longer exists after the apply")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)
		Expect(err).ToNot(HaveOccurred())
	})

	It("takes the existing statefile into account when generating a plan", func() {
		initialApplyRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source: "fixtures/aws/",
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		planRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		applyPlanRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:  "fixtures/aws/",
					PlanRun: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
				},
			},
		}

		By("ensuring plan and state files does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			stateFilePath,
		)
		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to create the statefile")

		runner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		_, err := runner.Run(initialApplyRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that statefile exists and plan does not")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			stateFilePath,
		)
		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		initialLastModified := awsVerifier.GetLastModifiedFromS3(bucket, s3ObjectPath)

		time.Sleep(1 * time.Second) // ensure LastModified has time to change

		By("creating the plan")

		_, err = runner.Run(planRequest)
		Expect(err).ToNot(HaveOccurred())

		awsVerifier.ExpectS3FileToExist(
			bucket,
			s3ObjectPath,
		)

		By("ensuring that state and plan files exist")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			stateFilePath,
		)
		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		By("applying the plan")

		_, err = runner.Run(applyPlanRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that existing statefile was used and S3 Object was unchanged")
		finalLastModified := awsVerifier.GetLastModifiedFromS3(bucket, s3ObjectPath)
		Expect(finalLastModified).To(Equal(initialLastModified))
	})

	It("allows re-generating a plan", func() {
		initialPlanRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		planRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		By("ensuring plan files does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to create the plan file")

		runner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		_, err := runner.Run(initialPlanRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan exists")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		initialLastModified := awsVerifier.GetLastModifiedFromS3(bucket, planFilePath)

		time.Sleep(1 * time.Second) // ensure LastModified has time to change

		By("recreating the plan")

		_, err = runner.Run(planRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file still exists")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		finalLastModified := awsVerifier.GetLastModifiedFromS3(bucket, planFilePath)
		Expect(finalLastModified).ToNot(Equal(initialLastModified))
	})

	It("plan should be deleted on destroy", func() {
		planOutRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		By("ensuring state and plan file does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to create the plan file")

		planrunner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		_, err := planrunner.Run(planOutRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file exists with valid version (LastModified)")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to delete the plan file")

		planOutRequest.Params.Terraform.PlanOnly = false
		planOutRequest.Params.Action = models.DestroyAction
		_, err = planrunner.Run(planOutRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file no longer exists")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)
	})
})
<commit_msg>Ensure plan log is deleted at end of test<commit_after>package out_test

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"time"

	"terraform-resource/models"
	"terraform-resource/out"
	"terraform-resource/test/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Out Plan", func() {

	var (
		backendType   string
		backendConfig map[string]interface{}
		envName       string
		stateFilePath string
		planFilePath  string
		s3ObjectPath  string
		workingDir    string
		workspacePath string
	)

	BeforeEach(func() {
		region := os.Getenv("AWS_REGION") // optional
		if region == "" {
			region = "us-east-1"
		}

		workspacePath = helpers.RandomString("out-backend-test")

		envName = helpers.RandomString("out-test")
		stateFilePath = path.Join(workspacePath, envName, "terraform.tfstate")
		planFilePath = path.Join(workspacePath, fmt.Sprintf("%s-plan", envName), "terraform.tfstate")
		s3ObjectPath = path.Join(bucketPath, helpers.RandomString("out-test"))

		backendType = "s3"
		backendConfig = map[string]interface{}{
			"bucket":               bucket,
			"key":                  "terraform.tfstate",
			"access_key":           accessKey,
			"secret_key":           secretKey,
			"region":               region,
			"workspace_key_prefix": workspacePath,
		}

		var err error
		workingDir, err = ioutil.TempDir(os.TempDir(), "terraform-resource-out-test")
		Expect(err).ToNot(HaveOccurred())

		// ensure relative paths resolve correctly
		err = os.Chdir(workingDir)
		Expect(err).ToNot(HaveOccurred())

		fixturesDir := path.Join(helpers.ProjectRoot(), "fixtures")
		err = exec.Command("cp", "-r", fixturesDir, workingDir).Run()
		Expect(err).ToNot(HaveOccurred())

		err = helpers.DownloadStatefulPlugin(workingDir)
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		_ = os.RemoveAll(workingDir)
		awsVerifier.DeleteObjectFromS3(bucket, s3ObjectPath)
		awsVerifier.DeleteObjectFromS3(bucket, planFilePath)
		awsVerifier.DeleteObjectFromS3(bucket, stateFilePath)
	})

	It("plan infrastructure and apply it", func() {
		planOutRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		applyRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:  "fixtures/aws/",
					PlanRun: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
				},
			},
		}

		By("running 'out' to create the plan file")

		planrunner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		planOutput, err := planrunner.Run(planOutRequest)
		Expect(err).ToNot(HaveOccurred())

		defer os.RemoveAll(path.Join(os.TempDir(), "tf-plan.log"))

		By("ensuring that plan file exists")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)
		defer awsVerifier.DeleteObjectFromS3(bucket, planFilePath)

		Expect(planOutput.Version.EnvName).To(Equal(planOutRequest.Params.EnvName))
		Expect(planOutput.Version.PlanOnly).To(Equal("true"), "Expected PlanOnly to be true, but was false")
		Expect(planOutput.Version.Serial).To(BeEmpty())
		Expect(planOutput.Version.PlanChecksum).To(MatchRegexp("[0-9|a-f]+"))

		Expect(path.Join(os.TempDir(), "tf-plan.log")).To(BeAnExistingFile())

		By("ensuring s3 file does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			s3ObjectPath,
		)

		By("applying the plan")

		applyrunner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		createOutput, err := applyrunner.Run(applyRequest)
		Expect(err).ToNot(HaveOccurred())

		Expect(createOutput.Version.PlanOnly).To(BeEmpty())
		Expect(createOutput.Version.Serial).ToNot(BeEmpty())
		Expect(createOutput.Version.PlanChecksum).To(BeEmpty())

		Expect(createOutput.Metadata).ToNot(BeEmpty())
		fields := map[string]interface{}{}
		for _, field := range createOutput.Metadata {
			fields[field.Name] = field.Value
		}
		Expect(fields["env_name"]).To(Equal(envName))
		expectedMD5 := fmt.Sprintf("%x", md5.Sum([]byte("terraform-is-neat")))
		Expect(fields["content_md5"]).To(Equal(expectedMD5))

		awsVerifier.ExpectS3FileToExist(
			bucket,
			s3ObjectPath,
		)

		By("ensuring that plan file no longer exists after the apply")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)
		Expect(err).ToNot(HaveOccurred())
	})

	It("takes the existing statefile into account when generating a plan", func() {
		initialApplyRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source: "fixtures/aws/",
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		planRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		applyPlanRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:  "fixtures/aws/",
					PlanRun: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
				},
			},
		}

		By("ensuring plan and state files does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			stateFilePath,
		)
		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to create the statefile")

		runner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		_, err := runner.Run(initialApplyRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that statefile exists and plan does not")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			stateFilePath,
		)
		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		initialLastModified := awsVerifier.GetLastModifiedFromS3(bucket, s3ObjectPath)

		time.Sleep(1 * time.Second) // ensure LastModified has time to change

		By("creating the plan")

		_, err = runner.Run(planRequest)
		Expect(err).ToNot(HaveOccurred())

		awsVerifier.ExpectS3FileToExist(
			bucket,
			s3ObjectPath,
		)

		By("ensuring that state and plan files exist")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			stateFilePath,
		)
		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		By("applying the plan")

		_, err = runner.Run(applyPlanRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that existing statefile was used and S3 Object was unchanged")
		finalLastModified := awsVerifier.GetLastModifiedFromS3(bucket, s3ObjectPath)
		Expect(finalLastModified).To(Equal(initialLastModified))
	})

	It("allows re-generating a plan", func() {
		initialPlanRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		planRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		By("ensuring plan files does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to create the plan file")

		runner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		_, err := runner.Run(initialPlanRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan exists")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		initialLastModified := awsVerifier.GetLastModifiedFromS3(bucket, planFilePath)

		time.Sleep(1 * time.Second) // ensure LastModified has time to change

		By("recreating the plan")

		_, err = runner.Run(planRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file still exists")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		finalLastModified := awsVerifier.GetLastModifiedFromS3(bucket, planFilePath)
		Expect(finalLastModified).ToNot(Equal(initialLastModified))
	})

	It("plan should be deleted on destroy", func() {
		planOutRequest := models.OutRequest{
			Source: models.Source{
				Terraform: models.Terraform{
					BackendType:   backendType,
					BackendConfig: backendConfig,
				},
			},
			Params: models.OutParams{
				EnvName: envName,
				Terraform: models.Terraform{
					Source:   "fixtures/aws/",
					PlanOnly: true,
					Env: map[string]string{
						"HOME": workingDir, // in prod plugin is installed system-wide
					},
					Vars: map[string]interface{}{
						"access_key":     accessKey,
						"secret_key":     secretKey,
						"bucket":         bucket,
						"object_key":     s3ObjectPath,
						"object_content": "terraform-is-neat",
						"region":         region,
					},
				},
			},
		}

		By("ensuring state and plan file does not already exist")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to create the plan file")

		planrunner := out.Runner{
			SourceDir: workingDir,
			LogWriter: GinkgoWriter,
		}
		_, err := planrunner.Run(planOutRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file exists with valid version (LastModified)")

		awsVerifier.ExpectS3FileToExist(
			bucket,
			planFilePath,
		)

		By("running 'out' to delete the plan file")

		planOutRequest.Params.Terraform.PlanOnly = false
		planOutRequest.Params.Action = models.DestroyAction
		_, err = planrunner.Run(planOutRequest)
		Expect(err).ToNot(HaveOccurred())

		By("ensuring that plan file no longer exists")

		awsVerifier.ExpectS3FileToNotExist(
			bucket,
			planFilePath,
		)
	})
})
<|endoftext|>


<commit_before>package importgraph_test

import (
	"go/build"
	"runtime"
	"sort"
	"testing"

	"code.google.com/p/go.tools/refactor/importgraph"

	_ "crypto/hmac" // just for test, below
)

const this = "code.google.com/p/go.tools/refactor/importgraph"

func TestBuild(t *testing.T) {
	saved := runtime.GOMAXPROCS(8) // Build is highly parallel
	defer runtime.GOMAXPROCS(saved)

	forward, reverse, errors := importgraph.Build(&build.Default)

	// Test direct edges.
	// We throw in crypto/hmac to prove that external test files
	// (such as this one) are inspected.
	for _, p := range []string{"go/build", "runtime", "testing", "crypto/hmac"} {
		if !forward[this][p] {
			t.Errorf("forward[importgraph][%s] not found", p)
		}
		if !reverse[p][this] {
			t.Errorf("reverse[%s][importgraph] not found", p)
		}
	}

	// Test non-existent direct edges
	for _, p := range []string{"fmt", "errors", "reflect"} {
		if forward[this][p] {
			t.Errorf("unexpected: forward[importgraph][%s] found", p)
		}
		if reverse[p][this] {
			t.Errorf("unexpected: reverse[%s][importgraph] found", p)
		}
	}

	// Test Search is reflexive.
	if !forward.Search(this)[this] {
		t.Errorf("irreflexive: forward.Search(importgraph)[importgraph] not found")
	}
	if !reverse.Search(this)[this] {
		t.Errorf("irrefexive: reverse.Search(importgraph)[importgraph] not found")
	}

	// Test Search is transitive.  (There is no direct edge to these packages.)
	for _, p := range []string{"errors", "reflect", "unsafe"} {
		if !forward.Search(this)[p] {
			t.Errorf("intransitive: forward.Search(importgraph)[%s] not found", p)
		}
		if !reverse.Search(p)[this] {
			t.Errorf("intransitive: reverse.Search(%s)[importgraph] not found", p)
		}
	}

	// debugging
	if false {
		for path, err := range errors {
			t.Logf("%s: %s", path, err)
		}
		printSorted := func(direction string, g importgraph.Graph, start string) {
			t.Log(direction)
			var pkgs []string
			for pkg := range g.Search(start) {
				pkgs = append(pkgs, pkg)
			}
			sort.Strings(pkgs)
			for _, pkg := range pkgs {
				t.Logf("\t%s", pkg)
			}
		}
		printSorted("forward", forward, this)
		printSorted("forward", reverse, this)
	}
}
<commit_msg>refactor/importgraph: add test of cycles<commit_after>package importgraph_test

import (
	"go/build"
	"runtime"
	"sort"
	"testing"

	"code.google.com/p/go.tools/refactor/importgraph"

	_ "crypto/hmac" // just for test, below
)

const this = "code.google.com/p/go.tools/refactor/importgraph"

func TestBuild(t *testing.T) {
	saved := runtime.GOMAXPROCS(8) // Build is highly parallel
	defer runtime.GOMAXPROCS(saved)

	forward, reverse, errors := importgraph.Build(&build.Default)

	// Test direct edges.
	// We throw in crypto/hmac to prove that external test files
	// (such as this one) are inspected.
	for _, p := range []string{"go/build", "runtime", "testing", "crypto/hmac"} {
		if !forward[this][p] {
			t.Errorf("forward[importgraph][%s] not found", p)
		}
		if !reverse[p][this] {
			t.Errorf("reverse[%s][importgraph] not found", p)
		}
	}

	// Test non-existent direct edges
	for _, p := range []string{"fmt", "errors", "reflect"} {
		if forward[this][p] {
			t.Errorf("unexpected: forward[importgraph][%s] found", p)
		}
		if reverse[p][this] {
			t.Errorf("unexpected: reverse[%s][importgraph] found", p)
		}
	}

	// Test Search is reflexive.
	if !forward.Search(this)[this] {
		t.Errorf("irreflexive: forward.Search(importgraph)[importgraph] not found")
	}
	if !reverse.Search(this)[this] {
		t.Errorf("irrefexive: reverse.Search(importgraph)[importgraph] not found")
	}

	// Test Search is transitive.  (There is no direct edge to these packages.)
	for _, p := range []string{"errors", "reflect", "unsafe"} {
		if !forward.Search(this)[p] {
			t.Errorf("intransitive: forward.Search(importgraph)[%s] not found", p)
		}
		if !reverse.Search(p)[this] {
			t.Errorf("intransitive: reverse.Search(%s)[importgraph] not found", p)
		}
	}

	// Test strongly-connected components.  Because A's external
	// test package can depend on B, and vice versa, most of the
	// standard libraries are mutually dependent when their external
	// tests are considered.
	//
	// For any nodes x, y in the same SCC, y appears in the results
	// of both forward and reverse searches starting from x
	if !forward.Search("fmt")["io"] ||
		!forward.Search("io")["fmt"] ||
		!reverse.Search("fmt")["io"] ||
		!reverse.Search("io")["fmt"] {
		t.Errorf("fmt and io are not mutually reachable despite being in the same SCC")
	}

	// debugging
	if false {
		for path, err := range errors {
			t.Logf("%s: %s", path, err)
		}
		printSorted := func(direction string, g importgraph.Graph, start string) {
			t.Log(direction)
			var pkgs []string
			for pkg := range g.Search(start) {
				pkgs = append(pkgs, pkg)
			}
			sort.Strings(pkgs)
			for _, pkg := range pkgs {
				t.Logf("\t%s", pkg)
			}
		}
		printSorted("forward", forward, this)
		printSorted("reverse", reverse, this)
	}
}
<|endoftext|>


<commit_before>package sub

// BUG(natefinch) This is a bug description.  Bugs are an exception to the rule
// about only attached comments getting displayed in godoc.  Simply make a
// comment that starts with BUG(foo) where foo is the name of the person
// responsible for the bug, and it'll show up in the list of bugs in godoc.  You
// can click on the link at the beginning of the bug description to go right to
// the place where the BUG comment is located in the code.

// Constants are sorted to the top of the code documentation.
//
// Formatting
//
// Because it is declared on its own and not in a const() block, Applename gets
// full formatting support.
const AppleName = "apple"

// This text is a doc comment on the const block as a whole.  Unlike the
// comments inside the block, this text has full formatting.
//
// This can be a nice way to write a single comment about a group of variables.
const (
	// PearName is an exported constant used to create pear foods. Docs on
	// constants and variables enclosed in a block like this one get printed out
	// as-is.
	PearName = "pear"

	// PineappleName is the name of a pineapple.
	PineappleName = "pineapple"
)

// Apple is a global variable in the package.
//
// Because apple is declared on its own, rather in a block of var (), its
// documentation gets full formatting support.
var Apple = NewFood(AppleName)

// This is a block of variables with a formatted comment.
//
// Pretty nice.
var (
	// Pear is another global variable in the package.
	Pear = NewFood(PearName)
)

// Food is a type to show how type documentation works.
//
// Formatting
//
// You always get full formatting on type, function, and method docs.
//
// Below is the auto-embedded example from the function ExampleFood in
// more_examples_test.go.
type Food struct {
	// Name holds the common name of the food.
	//
	// Alas, no formatting here.
	Name string

	// weight is an unexported field that won't show up in documentation.
	weight int
}

// NewFood creates a new Food with the given name and a calculated weight.
//
// Godoc sorts "constructor" style functions like this, that return a type
// defined in the package, under the type itself.
//
// Note that you do not need to (and should not) make separate lines describing
// each parameter to the function.  Simply refer to the parameters by their name
// when you need to reference them in the rest of the documentation.
func NewFood(name string) Food {
	return Food{name, len(name)}
}

// Weight reports the weight of the food in letters.
//
// When decribing a function without parameters that simply returns data, it is
// traditional to write the doc saying the function reports foo, rather than
// *returns* foo.
//
// The example below intentionally fails during go test, to show you what that
// looks like.  The test will report:
//
//   --- FAIL: ExampleFood_Weight (4.933us)
//   got:
//   9
//   want:
//   10
//   FAIL
//   exit status 1
func (f Food) Weight() int {
	return f.weight
}

// Zap consumes the food and prints out a nasty message.
//
// This is just another top level function to show how the alphabetical sorting
// works.
func Zap(f Food) {
	fmt.Printf("I hate %ss!", f.Name)

}

// Eat consumes the food and prints out a nice message.
//
// This is an example of a function that isn't a "constructor" style function,
// so it will be sorted at the top level of the documentation.
//
// Functions get sorted before types.
func Eat(f Food) {
	fmt.Printf("I love %ss!", f.Name)
}
<commit_msg>a little more description about bugs, and note that they don't show up on the command in godoc.org<commit_after>package sub

// BUG(natefinch) This is a bug description. Bugs are an exception to the rule
// about only attached comments getting displayed in godoc.  Simply make a
// comment that starts with BUG(foo) where foo is the name of the person
// responsible for the bug, and it'll show up in the list of bugs in godoc.
// Note that bug descriptions are plaintext only.  You can click on the link at
// the beginning of the bug description to go right to the place where the BUG
// comment is located in the code.

// BUG(godoc.org) Note that bugs in the command (application) package do not
// show up on godoc.org.  This appears to be a bug in godoc.org, as they do show
// up in local godoc.

// Constants are sorted to the top of the code documentation.
//
// Formatting
//
// Because it is declared on its own and not in a const() block, Applename gets
// full formatting support.
const AppleName = "apple"

// This text is a doc comment on the const block as a whole.  Unlike the
// comments inside the block, this text has full formatting.
//
// This can be a nice way to write a single comment about a group of variables.
const (
	// PearName is an exported constant used to create pear foods. Docs on
	// constants and variables enclosed in a block like this one get printed out
	// as-is.
	PearName = "pear"

	// PineappleName is the name of a pineapple.
	PineappleName = "pineapple"
)

// Apple is a global variable in the package.
//
// Because apple is declared on its own, rather in a block of var (), its
// documentation gets full formatting support.
var Apple = NewFood(AppleName)

// This is a block of variables with a formatted comment.
//
// Pretty nice.
var (
	// Pear is another global variable in the package.
	Pear = NewFood(PearName)
)

// Food is a type to show how type documentation works.
//
// Formatting
//
// You always get full formatting on type, function, and method docs.
//
// Below is the auto-embedded example from the function ExampleFood in
// more_examples_test.go.
type Food struct {
	// Name holds the common name of the food.
	//
	// Alas, no formatting here.
	Name string

	// weight is an unexported field that won't show up in documentation.
	weight int
}

// NewFood creates a new Food with the given name and a calculated weight.
//
// Godoc sorts "constructor" style functions like this, that return a type
// defined in the package, under the type itself.
//
// Note that you do not need to (and should not) make separate lines describing
// each parameter to the function.  Simply refer to the parameters by their name
// when you need to reference them in the rest of the documentation.
func NewFood(name string) Food {
	return Food{name, len(name)}
}

// Weight reports the weight of the food in letters.
//
// When decribing a function without parameters that simply returns data, it is
// traditional to write the doc saying the function reports foo, rather than
// *returns* foo.
//
// The example below intentionally fails during go test, to show you what that
// looks like.  The test will report:
//
//   --- FAIL: ExampleFood_Weight (4.933us)
//   got:
//   9
//   want:
//   10
//   FAIL
//   exit status 1
func (f Food) Weight() int {
	return f.weight
}

// Zap consumes the food and prints out a nasty message.
//
// This is just another top level function to show how the alphabetical sorting
// works.
func Zap(f Food) {
	fmt.Printf("I hate %ss!", f.Name)

}

// Eat consumes the food and prints out a nice message.
//
// This is an example of a function that isn't a "constructor" style function,
// so it will be sorted at the top level of the documentation.
//
// Functions get sorted before types.
func Eat(f Food) {
	fmt.Printf("I love %ss!", f.Name)
}
<|endoftext|>


<commit_before>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/goharbor/harbor/src/lib/errors"
	"github.com/goharbor/harbor/src/pkg/blob/models"

	"github.com/docker/distribution/manifest/schema2"
	"github.com/goharbor/harbor/src/pkg/blob"
	"github.com/goharbor/harbor/src/pkg/distribution"
	htesting "github.com/goharbor/harbor/src/testing"
	"github.com/goharbor/harbor/src/testing/mock"
	blobtesting "github.com/goharbor/harbor/src/testing/pkg/blob"
	"github.com/google/uuid"
	"github.com/stretchr/testify/suite"

	pkg_blob "github.com/goharbor/harbor/src/pkg/blob"
)

type ControllerTestSuite struct {
	htesting.Suite
}

func (suite *ControllerTestSuite) SetupSuite() {
	suite.Suite.SetupSuite()
	suite.Suite.ClearTables = []string{"blob", "artifact_blob", "project_blob"}
}

func (suite *ControllerTestSuite) prepareBlob() string {

	ctx := suite.Context()
	digest := suite.DigestString()

	_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)

	return digest
}

func (suite *ControllerTestSuite) TestAttachToArtifact() {
	ctx := suite.Context()

	artifactDigest := suite.DigestString()
	blobDigests := []string{
		suite.prepareBlob(),
		suite.prepareBlob(),
		suite.prepareBlob(),
	}

	suite.Nil(Ctl.AssociateWithArtifact(ctx, blobDigests, artifactDigest))

	for _, digest := range blobDigests {
		exist, err := Ctl.Exist(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
		suite.Nil(err)
		suite.True(exist)
	}

	suite.Nil(Ctl.AssociateWithArtifact(ctx, blobDigests, artifactDigest))
}

func (suite *ControllerTestSuite) TestAttachToProjectByDigest() {
	suite.WithProject(func(projectID int64, projectName string) {
		ctx := suite.Context()

		digest := suite.prepareBlob()
		suite.Nil(Ctl.AssociateWithProjectByDigest(ctx, digest, projectID))

		exist, err := Ctl.Exist(ctx, digest, IsAssociatedWithProject(projectID))
		suite.Nil(err)
		suite.True(exist)
	})
}

func (suite *ControllerTestSuite) TestCalculateTotalSizeByProject() {
	suite.WithProject(func(projectID int64, projectName string) {
		ctx := suite.Context()

		id1, _ := Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeForeignLayer, 100)
		Ctl.AssociateWithProjectByID(ctx, id1, projectID)
		id2, _ := Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeLayer, 100)
		Ctl.AssociateWithProjectByID(ctx, id2, projectID)

		{
			size, err := Ctl.CalculateTotalSizeByProject(ctx, projectID, true)
			suite.Nil(err)
			suite.Equal(int64(100), size)
		}

		{
			size, err := Ctl.CalculateTotalSizeByProject(ctx, projectID, false)
			suite.Nil(err)
			suite.Equal(int64(200), size)
		}
	})
}

func (suite *ControllerTestSuite) TestCalculateTotalSize() {
	ctx := suite.Context()
	size1, err := Ctl.CalculateTotalSize(ctx, true)
	Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeForeignLayer, 100)
	Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeLayer, 100)
	size2, err := Ctl.CalculateTotalSize(ctx, false)
	suite.Nil(err)
	suite.Equal(int64(200), size2-size1)
}

func (suite *ControllerTestSuite) TestEnsure() {
	ctx := suite.Context()

	digest := suite.DigestString()

	_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)

	exist, err := Ctl.Exist(ctx, digest)
	suite.Nil(err)
	suite.True(exist)

	_, err = Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)
}

func (suite *ControllerTestSuite) TestExist() {
	ctx := suite.Context()

	exist, err := Ctl.Exist(ctx, suite.DigestString())
	suite.Nil(err)
	suite.False(exist)
}

func (suite *ControllerTestSuite) TestFindMissingAssociationsForProjectByArtifact() {
	blobMgr := &blobtesting.Manager{}

	ctl := &controller{blobMgr: blobMgr}

	ctx := context.TODO()
	projectID := int64(1)

	{
		blobs, err := ctl.FindMissingAssociationsForProject(ctx, projectID, nil)
		suite.Nil(err)
		suite.Len(blobs, 0)
	}

	blobs := []*blob.Blob{{Digest: "1"}, {Digest: "2"}, {Digest: "3"}}

	{
		mock.OnAnything(blobMgr, "List").Return(nil, nil).Once()
		missing, err := ctl.FindMissingAssociationsForProject(ctx, projectID, blobs)
		suite.Nil(err)
		suite.Len(missing, len(blobs))
	}

	{
		mock.OnAnything(blobMgr, "List").Return(blobs, nil).Once()
		missing, err := ctl.FindMissingAssociationsForProject(ctx, projectID, blobs)
		suite.Nil(err)
		suite.Len(missing, 0)
	}

	{
		associated := []*blob.Blob{{Digest: "1"}}
		mock.OnAnything(blobMgr, "List").Return(associated, nil).Once()
		missing, err := ctl.FindMissingAssociationsForProject(ctx, projectID, blobs)
		suite.Nil(err)
		suite.Len(missing, len(blobs)-len(associated))
	}
}

func (suite *ControllerTestSuite) TestGet() {
	ctx := suite.Context()

	{
		digest := suite.prepareBlob()
		blob, err := Ctl.Get(ctx, digest)
		suite.Nil(err)
		suite.Equal(digest, blob.Digest)
		suite.Equal(int64(100), blob.Size)
		suite.Equal("application/octet-stream", blob.ContentType)
	}

	{
		digest := suite.prepareBlob()
		artifactDigest := suite.DigestString()

		_, err := Ctl.Get(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
		suite.NotNil(err)

		Ctl.AssociateWithArtifact(ctx, []string{digest}, artifactDigest)

		blob, err := Ctl.Get(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
		suite.Nil(err)
		suite.Equal(digest, blob.Digest)
		suite.Equal(int64(100), blob.Size)
		suite.Equal("application/octet-stream", blob.ContentType)
	}

	{
		digest := suite.prepareBlob()

		suite.WithProject(func(projectID int64, projectName string) {
			_, err := Ctl.Get(ctx, digest, IsAssociatedWithProject(projectID))
			suite.NotNil(err)

			Ctl.AssociateWithProjectByDigest(ctx, digest, projectID)

			blob, err := Ctl.Get(ctx, digest, IsAssociatedWithProject(projectID))
			suite.Nil(err)
			suite.Equal(digest, blob.Digest)
			suite.Equal(int64(100), blob.Size)
			suite.Equal("application/octet-stream", blob.ContentType)
		})
	}
}

func (suite *ControllerTestSuite) TestSync() {
	var references []distribution.Descriptor
	for i := 0; i < 5; i++ {
		references = append(references, distribution.Descriptor{
			MediaType: fmt.Sprintf("media type %d", i),
			Digest:    suite.Digest(),
			Size:      int64(100 + i),
		})
	}

	suite.WithProject(func(projectID int64, projectName string) {
		ctx := suite.Context()

		{
			suite.Nil(Ctl.Sync(ctx, references))
			for _, reference := range references {
				blob, err := Ctl.Get(ctx, reference.Digest.String())
				suite.Nil(err)
				suite.Equal(reference.MediaType, blob.ContentType)
				suite.Equal(reference.Digest.String(), blob.Digest)
				suite.Equal(reference.Size, blob.Size)
			}
		}

		{
			references[0].MediaType = "media type"

			references = append(references, distribution.Descriptor{
				MediaType: "media type",
				Digest:    suite.Digest(),
				Size:      int64(100),
			})

			suite.Nil(Ctl.Sync(ctx, references))
		}
	})
}

func (suite *ControllerTestSuite) TestGetSetAcceptedBlobSize() {
	ctx := context.TODO()
	{
		sessionID := uuid.New().String()

		size, err := Ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(0), size)

		suite.Nil(Ctl.SetAcceptedBlobSize(ctx, sessionID, 100))

		size, err = Ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(100), size)
	}

	{
		// test blob size expired
		ctl := &controller{blobSizeExpiration: time.Second * 5}

		sessionID := uuid.New().String()

		size, err := ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(0), size)

		suite.Nil(ctl.SetAcceptedBlobSize(ctx, sessionID, 100))

		size, err = ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(100), size)

		time.Sleep(time.Second * 10)

		size, err = ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(0), size)
	}
}

func (suite *ControllerTestSuite) TestTouch() {
	ctx := suite.Context()

	err := Ctl.Touch(ctx, &blob.Blob{
		Status: models.StatusNone,
	})
	suite.NotNil(err)
	suite.True(errors.IsNotFoundErr(err))

	digest := suite.prepareBlob()
	blob, err := Ctl.Get(ctx, digest)
	suite.Nil(err)
	blob.Status = models.StatusDelete
	_, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), blob)
	suite.Nil(err)

	err = Ctl.Touch(ctx, blob)
	suite.Nil(err)
	suite.Equal(blob.Status, models.StatusNone)
}

func (suite *ControllerTestSuite) TestFail() {
	ctx := suite.Context()

	err := Ctl.Fail(ctx, &blob.Blob{
		Status: models.StatusNone,
	})
	suite.NotNil(err)
	suite.True(errors.IsNotFoundErr(err))

	digest := suite.prepareBlob()
	blob, err := Ctl.Get(ctx, digest)
	suite.Nil(err)

	blob.Status = models.StatusDelete
	_, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), blob)
	suite.Nil(err)

	// StatusDelete cannot be marked as StatusDeleteFailed
	err = Ctl.Fail(ctx, blob)
	suite.NotNil(err)
	suite.True(errors.IsNotFoundErr(err))

	blob.Status = models.StatusDeleting
	_, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), blob)
	suite.Nil(err)

	err = Ctl.Fail(ctx, blob)
	suite.Nil(err)

	blobAfter, err := Ctl.Get(ctx, digest)
	suite.Nil(err)
	suite.Equal(models.StatusDeleteFailed, blobAfter.Status)
}

func (suite *ControllerTestSuite) TestDelete() {
	ctx := suite.Context()

	digest := suite.DigestString()
	_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)

	blob, err := Ctl.Get(ctx, digest)
	suite.Nil(err)
	suite.Equal(digest, blob.Digest)

	err = Ctl.Delete(ctx, blob.ID)
	suite.Nil(err)

	exist, err := Ctl.Exist(ctx, digest)
	suite.Nil(err)
	suite.False(exist)
}

func TestControllerTestSuite(t *testing.T) {
	suite.Run(t, &ControllerTestSuite{})
}
<commit_msg>controller/blob: fix dropped test error<commit_after>// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/goharbor/harbor/src/lib/errors"
	"github.com/goharbor/harbor/src/pkg/blob/models"

	"github.com/docker/distribution/manifest/schema2"
	"github.com/goharbor/harbor/src/pkg/blob"
	"github.com/goharbor/harbor/src/pkg/distribution"
	htesting "github.com/goharbor/harbor/src/testing"
	"github.com/goharbor/harbor/src/testing/mock"
	blobtesting "github.com/goharbor/harbor/src/testing/pkg/blob"
	"github.com/google/uuid"
	"github.com/stretchr/testify/suite"

	pkg_blob "github.com/goharbor/harbor/src/pkg/blob"
)

type ControllerTestSuite struct {
	htesting.Suite
}

func (suite *ControllerTestSuite) SetupSuite() {
	suite.Suite.SetupSuite()
	suite.Suite.ClearTables = []string{"blob", "artifact_blob", "project_blob"}
}

func (suite *ControllerTestSuite) prepareBlob() string {

	ctx := suite.Context()
	digest := suite.DigestString()

	_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)

	return digest
}

func (suite *ControllerTestSuite) TestAttachToArtifact() {
	ctx := suite.Context()

	artifactDigest := suite.DigestString()
	blobDigests := []string{
		suite.prepareBlob(),
		suite.prepareBlob(),
		suite.prepareBlob(),
	}

	suite.Nil(Ctl.AssociateWithArtifact(ctx, blobDigests, artifactDigest))

	for _, digest := range blobDigests {
		exist, err := Ctl.Exist(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
		suite.Nil(err)
		suite.True(exist)
	}

	suite.Nil(Ctl.AssociateWithArtifact(ctx, blobDigests, artifactDigest))
}

func (suite *ControllerTestSuite) TestAttachToProjectByDigest() {
	suite.WithProject(func(projectID int64, projectName string) {
		ctx := suite.Context()

		digest := suite.prepareBlob()
		suite.Nil(Ctl.AssociateWithProjectByDigest(ctx, digest, projectID))

		exist, err := Ctl.Exist(ctx, digest, IsAssociatedWithProject(projectID))
		suite.Nil(err)
		suite.True(exist)
	})
}

func (suite *ControllerTestSuite) TestCalculateTotalSizeByProject() {
	suite.WithProject(func(projectID int64, projectName string) {
		ctx := suite.Context()

		id1, _ := Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeForeignLayer, 100)
		Ctl.AssociateWithProjectByID(ctx, id1, projectID)
		id2, _ := Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeLayer, 100)
		Ctl.AssociateWithProjectByID(ctx, id2, projectID)

		{
			size, err := Ctl.CalculateTotalSizeByProject(ctx, projectID, true)
			suite.Nil(err)
			suite.Equal(int64(100), size)
		}

		{
			size, err := Ctl.CalculateTotalSizeByProject(ctx, projectID, false)
			suite.Nil(err)
			suite.Equal(int64(200), size)
		}
	})
}

func (suite *ControllerTestSuite) TestCalculateTotalSize() {
	ctx := suite.Context()
	size1, err := Ctl.CalculateTotalSize(ctx, true)
	suite.Nil(err)
	Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeForeignLayer, 100)
	Ctl.Ensure(ctx, suite.DigestString(), schema2.MediaTypeLayer, 100)
	size2, err := Ctl.CalculateTotalSize(ctx, false)
	suite.Nil(err)
	suite.Equal(int64(200), size2-size1)
}

func (suite *ControllerTestSuite) TestEnsure() {
	ctx := suite.Context()

	digest := suite.DigestString()

	_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)

	exist, err := Ctl.Exist(ctx, digest)
	suite.Nil(err)
	suite.True(exist)

	_, err = Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)
}

func (suite *ControllerTestSuite) TestExist() {
	ctx := suite.Context()

	exist, err := Ctl.Exist(ctx, suite.DigestString())
	suite.Nil(err)
	suite.False(exist)
}

func (suite *ControllerTestSuite) TestFindMissingAssociationsForProjectByArtifact() {
	blobMgr := &blobtesting.Manager{}

	ctl := &controller{blobMgr: blobMgr}

	ctx := context.TODO()
	projectID := int64(1)

	{
		blobs, err := ctl.FindMissingAssociationsForProject(ctx, projectID, nil)
		suite.Nil(err)
		suite.Len(blobs, 0)
	}

	blobs := []*blob.Blob{{Digest: "1"}, {Digest: "2"}, {Digest: "3"}}

	{
		mock.OnAnything(blobMgr, "List").Return(nil, nil).Once()
		missing, err := ctl.FindMissingAssociationsForProject(ctx, projectID, blobs)
		suite.Nil(err)
		suite.Len(missing, len(blobs))
	}

	{
		mock.OnAnything(blobMgr, "List").Return(blobs, nil).Once()
		missing, err := ctl.FindMissingAssociationsForProject(ctx, projectID, blobs)
		suite.Nil(err)
		suite.Len(missing, 0)
	}

	{
		associated := []*blob.Blob{{Digest: "1"}}
		mock.OnAnything(blobMgr, "List").Return(associated, nil).Once()
		missing, err := ctl.FindMissingAssociationsForProject(ctx, projectID, blobs)
		suite.Nil(err)
		suite.Len(missing, len(blobs)-len(associated))
	}
}

func (suite *ControllerTestSuite) TestGet() {
	ctx := suite.Context()

	{
		digest := suite.prepareBlob()
		blob, err := Ctl.Get(ctx, digest)
		suite.Nil(err)
		suite.Equal(digest, blob.Digest)
		suite.Equal(int64(100), blob.Size)
		suite.Equal("application/octet-stream", blob.ContentType)
	}

	{
		digest := suite.prepareBlob()
		artifactDigest := suite.DigestString()

		_, err := Ctl.Get(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
		suite.NotNil(err)

		Ctl.AssociateWithArtifact(ctx, []string{digest}, artifactDigest)

		blob, err := Ctl.Get(ctx, digest, IsAssociatedWithArtifact(artifactDigest))
		suite.Nil(err)
		suite.Equal(digest, blob.Digest)
		suite.Equal(int64(100), blob.Size)
		suite.Equal("application/octet-stream", blob.ContentType)
	}

	{
		digest := suite.prepareBlob()

		suite.WithProject(func(projectID int64, projectName string) {
			_, err := Ctl.Get(ctx, digest, IsAssociatedWithProject(projectID))
			suite.NotNil(err)

			Ctl.AssociateWithProjectByDigest(ctx, digest, projectID)

			blob, err := Ctl.Get(ctx, digest, IsAssociatedWithProject(projectID))
			suite.Nil(err)
			suite.Equal(digest, blob.Digest)
			suite.Equal(int64(100), blob.Size)
			suite.Equal("application/octet-stream", blob.ContentType)
		})
	}
}

func (suite *ControllerTestSuite) TestSync() {
	var references []distribution.Descriptor
	for i := 0; i < 5; i++ {
		references = append(references, distribution.Descriptor{
			MediaType: fmt.Sprintf("media type %d", i),
			Digest:    suite.Digest(),
			Size:      int64(100 + i),
		})
	}

	suite.WithProject(func(projectID int64, projectName string) {
		ctx := suite.Context()

		{
			suite.Nil(Ctl.Sync(ctx, references))
			for _, reference := range references {
				blob, err := Ctl.Get(ctx, reference.Digest.String())
				suite.Nil(err)
				suite.Equal(reference.MediaType, blob.ContentType)
				suite.Equal(reference.Digest.String(), blob.Digest)
				suite.Equal(reference.Size, blob.Size)
			}
		}

		{
			references[0].MediaType = "media type"

			references = append(references, distribution.Descriptor{
				MediaType: "media type",
				Digest:    suite.Digest(),
				Size:      int64(100),
			})

			suite.Nil(Ctl.Sync(ctx, references))
		}
	})
}

func (suite *ControllerTestSuite) TestGetSetAcceptedBlobSize() {
	ctx := context.TODO()
	{
		sessionID := uuid.New().String()

		size, err := Ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(0), size)

		suite.Nil(Ctl.SetAcceptedBlobSize(ctx, sessionID, 100))

		size, err = Ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(100), size)
	}

	{
		// test blob size expired
		ctl := &controller{blobSizeExpiration: time.Second * 5}

		sessionID := uuid.New().String()

		size, err := ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(0), size)

		suite.Nil(ctl.SetAcceptedBlobSize(ctx, sessionID, 100))

		size, err = ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(100), size)

		time.Sleep(time.Second * 10)

		size, err = ctl.GetAcceptedBlobSize(ctx, sessionID)
		suite.Nil(err)
		suite.Equal(int64(0), size)
	}
}

func (suite *ControllerTestSuite) TestTouch() {
	ctx := suite.Context()

	err := Ctl.Touch(ctx, &blob.Blob{
		Status: models.StatusNone,
	})
	suite.NotNil(err)
	suite.True(errors.IsNotFoundErr(err))

	digest := suite.prepareBlob()
	blob, err := Ctl.Get(ctx, digest)
	suite.Nil(err)
	blob.Status = models.StatusDelete
	_, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), blob)
	suite.Nil(err)

	err = Ctl.Touch(ctx, blob)
	suite.Nil(err)
	suite.Equal(blob.Status, models.StatusNone)
}

func (suite *ControllerTestSuite) TestFail() {
	ctx := suite.Context()

	err := Ctl.Fail(ctx, &blob.Blob{
		Status: models.StatusNone,
	})
	suite.NotNil(err)
	suite.True(errors.IsNotFoundErr(err))

	digest := suite.prepareBlob()
	blob, err := Ctl.Get(ctx, digest)
	suite.Nil(err)

	blob.Status = models.StatusDelete
	_, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), blob)
	suite.Nil(err)

	// StatusDelete cannot be marked as StatusDeleteFailed
	err = Ctl.Fail(ctx, blob)
	suite.NotNil(err)
	suite.True(errors.IsNotFoundErr(err))

	blob.Status = models.StatusDeleting
	_, err = pkg_blob.Mgr.UpdateBlobStatus(suite.Context(), blob)
	suite.Nil(err)

	err = Ctl.Fail(ctx, blob)
	suite.Nil(err)

	blobAfter, err := Ctl.Get(ctx, digest)
	suite.Nil(err)
	suite.Equal(models.StatusDeleteFailed, blobAfter.Status)
}

func (suite *ControllerTestSuite) TestDelete() {
	ctx := suite.Context()

	digest := suite.DigestString()
	_, err := Ctl.Ensure(ctx, digest, "application/octet-stream", 100)
	suite.Nil(err)

	blob, err := Ctl.Get(ctx, digest)
	suite.Nil(err)
	suite.Equal(digest, blob.Digest)

	err = Ctl.Delete(ctx, blob.ID)
	suite.Nil(err)

	exist, err := Ctl.Exist(ctx, digest)
	suite.Nil(err)
	suite.False(exist)
}

func TestControllerTestSuite(t *testing.T) {
	suite.Run(t, &ControllerTestSuite{})
}
<|endoftext|>


<commit_before>// Copyright 2015 PLUMgrid
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// vim: set ts=8:sts=8:sw=8:noet

package hover

import (
	"fmt"
	"strings"
	"syscall"

	"github.com/docker/libkv"
	"github.com/docker/libkv/store"
	"github.com/docker/libkv/store/boltdb"
	"github.com/vishvananda/netlink"
)

/*
#cgo LDFLAGS: -lbcc
#include <bcc/bpf_common.h>
#include <bcc/libbpf.h>
*/
import "C"

type PatchPanel struct {
	adapter *BpfAdapter
	modules AdapterTable
	kv      store.Store
}

func NewPatchPanel() (pp *PatchPanel, err error) {
	var id string
	id, err = NewUUID4()
	if err != nil {
		return
	}

	boltdb.Register()
	kv, err := libkv.NewStore(store.BOLTDB, []string{"/tmp/hover.db"}, &store.Config{Bucket: "patch"})
	if err != nil {
		Warn.Print(err)
		return
	}
	pp = &PatchPanel{
		kv: kv,
	}
	defer func() {
		if err != nil {
			pp.Close()
			pp = nil
		}
	}()
	pp.adapter = &BpfAdapter{
		uuid:   id,
		name:   "patch",
		config: make(map[string]interface{}),
	}
	code := strings.Join([]string{iomoduleH, patchC}, "\n")
	pp.adapter.bpf = NewBpfModule(code, []string{"-w"})
	if pp.adapter.bpf == nil {
		err = fmt.Errorf("PatchPanel: unable to load core module")
		return
	}
	pp.modules = pp.adapter.Table("modules")
	if pp.modules == nil {
		err = fmt.Errorf("PatchPanel: Unable to load modules table")
		return
	}
	Debug.Printf("Patch panel modules table loaded: %v\n", pp.modules.Config())
	return
}

func (p *PatchPanel) Close() {
	if p.adapter != nil {
		p.adapter.Close()
	}
}

func ensureQdisc(link netlink.Link) (netlink.Qdisc, error) {
	qHandle := netlink.MakeHandle(0xffff, 0)
	qds, err := netlink.QdiscList(link)
	if err != nil {
		return nil, err
	}
	for _, q := range qds {
		if q.Attrs().Handle == qHandle {
			//Debug.Printf("Found existing ingress qdisc %x\n", q.Attrs().Handle)
			return q, nil
		}
	}
	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: netlink.QdiscAttrs{
			LinkIndex: link.Attrs().Index,
			Handle:    qHandle,
			Parent:    netlink.HANDLE_CLSACT,
		},
		QdiscType: "clsact",
	}
	if err := netlink.QdiscAdd(qdisc); err != nil {
		return nil, fmt.Errorf("failed ensuring qdisc: %v", err)
	}
	return qdisc, nil
}

func ensureIngressFd(link netlink.Link, fd int) error {
	_, err := ensureQdisc(link)
	if err != nil {
		return err
	}
	fHandle := netlink.MakeHandle(0, 1)
	filter := &netlink.BpfFilter{
		FilterAttrs: netlink.FilterAttrs{
			LinkIndex: link.Attrs().Index,
			Parent:    netlink.HANDLE_MIN_INGRESS,
			Handle:    fHandle,
			Priority:  1,
			Protocol:  syscall.ETH_P_ALL,
		},
		Fd:           fd,
		DirectAction: true,
	}
	filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_INGRESS)
	if err != nil {
		return fmt.Errorf("failed fetching ingress filter list: %s", err)
	}
	for _, f := range filters {
		if f.Attrs().Handle == fHandle {
			return nil
		}
	}
	if err := netlink.FilterAdd(filter); err != nil {
		return fmt.Errorf("failed adding ingress filter: %s", err)
	}
	//Debug.Printf("ensureIngressFd(%s) success\n", link.Attrs().Name)
	return nil
}

func ensureEgressFd(link netlink.Link, fd int) error {
	_, err := ensureQdisc(link)
	if err != nil {
		return err
	}
	fHandle := netlink.MakeHandle(0, 2)
	filter := &netlink.BpfFilter{
		FilterAttrs: netlink.FilterAttrs{
			LinkIndex: link.Attrs().Index,
			Parent:    netlink.HANDLE_MIN_EGRESS,
			Handle:    fHandle,
			Priority:  1,
			Protocol:  syscall.ETH_P_ALL,
		},
		Fd:           fd,
		DirectAction: true,
	}
	filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS)
	if err != nil {
		return fmt.Errorf("failed fetching egress filter list: %s", err)
	}
	for _, f := range filters {
		if f.Attrs().Handle == fHandle {
			return nil
		}
	}
	if err := netlink.FilterAdd(filter); err != nil {
		return fmt.Errorf("failed adding egress filter: %v", err)
	}
	return nil
}
<commit_msg>Remove libkv dependency<commit_after>// Copyright 2015 PLUMgrid
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// vim: set ts=8:sts=8:sw=8:noet

package hover

import (
	"fmt"
	"strings"
	"syscall"

	"github.com/vishvananda/netlink"
)

/*
#cgo LDFLAGS: -lbcc
#include <bcc/bpf_common.h>
#include <bcc/libbpf.h>
*/
import "C"

type PatchPanel struct {
	adapter *BpfAdapter
	modules AdapterTable
}

func NewPatchPanel() (pp *PatchPanel, err error) {
	var id string
	id, err = NewUUID4()
	if err != nil {
		return
	}

	if err != nil {
		Warn.Print(err)
		return
	}
	pp = &PatchPanel{}
	defer func() {
		if err != nil {
			pp.Close()
			pp = nil
		}
	}()
	pp.adapter = &BpfAdapter{
		uuid:   id,
		name:   "patch",
		config: make(map[string]interface{}),
	}
	code := strings.Join([]string{iomoduleH, patchC}, "\n")
	pp.adapter.bpf = NewBpfModule(code, []string{"-w"})
	if pp.adapter.bpf == nil {
		err = fmt.Errorf("PatchPanel: unable to load core module")
		return
	}
	pp.modules = pp.adapter.Table("modules")
	if pp.modules == nil {
		err = fmt.Errorf("PatchPanel: Unable to load modules table")
		return
	}
	Debug.Printf("Patch panel modules table loaded: %v\n", pp.modules.Config())
	return
}

func (p *PatchPanel) Close() {
	if p.adapter != nil {
		p.adapter.Close()
	}
}

func ensureQdisc(link netlink.Link) (netlink.Qdisc, error) {
	qHandle := netlink.MakeHandle(0xffff, 0)
	qds, err := netlink.QdiscList(link)
	if err != nil {
		return nil, err
	}
	for _, q := range qds {
		if q.Attrs().Handle == qHandle {
			//Debug.Printf("Found existing ingress qdisc %x\n", q.Attrs().Handle)
			return q, nil
		}
	}
	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: netlink.QdiscAttrs{
			LinkIndex: link.Attrs().Index,
			Handle:    qHandle,
			Parent:    netlink.HANDLE_CLSACT,
		},
		QdiscType: "clsact",
	}
	if err := netlink.QdiscAdd(qdisc); err != nil {
		return nil, fmt.Errorf("failed ensuring qdisc: %v", err)
	}
	return qdisc, nil
}

func ensureIngressFd(link netlink.Link, fd int) error {
	_, err := ensureQdisc(link)
	if err != nil {
		return err
	}
	fHandle := netlink.MakeHandle(0, 1)
	filter := &netlink.BpfFilter{
		FilterAttrs: netlink.FilterAttrs{
			LinkIndex: link.Attrs().Index,
			Parent:    netlink.HANDLE_MIN_INGRESS,
			Handle:    fHandle,
			Priority:  1,
			Protocol:  syscall.ETH_P_ALL,
		},
		Fd:           fd,
		DirectAction: true,
	}
	filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_INGRESS)
	if err != nil {
		return fmt.Errorf("failed fetching ingress filter list: %s", err)
	}
	for _, f := range filters {
		if f.Attrs().Handle == fHandle {
			return nil
		}
	}
	if err := netlink.FilterAdd(filter); err != nil {
		return fmt.Errorf("failed adding ingress filter: %s", err)
	}
	//Debug.Printf("ensureIngressFd(%s) success\n", link.Attrs().Name)
	return nil
}

func ensureEgressFd(link netlink.Link, fd int) error {
	_, err := ensureQdisc(link)
	if err != nil {
		return err
	}
	fHandle := netlink.MakeHandle(0, 2)
	filter := &netlink.BpfFilter{
		FilterAttrs: netlink.FilterAttrs{
			LinkIndex: link.Attrs().Index,
			Parent:    netlink.HANDLE_MIN_EGRESS,
			Handle:    fHandle,
			Priority:  1,
			Protocol:  syscall.ETH_P_ALL,
		},
		Fd:           fd,
		DirectAction: true,
	}
	filters, err := netlink.FilterList(link, netlink.HANDLE_MIN_EGRESS)
	if err != nil {
		return fmt.Errorf("failed fetching egress filter list: %s", err)
	}
	for _, f := range filters {
		if f.Attrs().Handle == fHandle {
			return nil
		}
	}
	if err := netlink.FilterAdd(filter); err != nil {
		return fmt.Errorf("failed adding egress filter: %v", err)
	}
	return nil
}
<|endoftext|>