<commit_before>package logprefix

import (
	"fmt"
	"sort"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/shirou/gopsutil/host"
	schematypes "github.com/taskcluster/go-schematypes"
	"github.com/taskcluster/taskcluster-worker/plugins"
)

type provider struct {
	plugins.PluginProviderBase
}

type plugin struct {
	plugins.PluginBase
	keys      map[string]string
	bootTime  string
	taskCount int64 // Count number of tasks processed
}

func init() {
	plugins.Register("logprefix", provider{})
}

func (provider) ConfigSchema() schematypes.Schema {
	return configSchema
}

func (provider) NewPlugin(options plugins.PluginOptions) (plugins.Plugin, error) {
	keys := make(map[string]string)
	schematypes.MustValidateAndMap(configSchema, options.Config, &keys)

	// Print a neat message to make debugging config easier.
	// Presumably, config files inject stuff into this section using
	// transforms, so it's nice to have some log.
	for k, v := range keys {
		options.Monitor.Infof("Prefixing task logs: '%s' = '%s'", k, v)
	}

	// Obtain the system boottime
	boottime, err := host.BootTime()
	if err != nil {
		return nil, errors.Wrap(err, "unable to determine the system boot time")
	}

	return &plugin{
		keys:     keys,
		bootTime: time.Unix(int64(boottime), 0).Format(time.RFC3339),
	}, nil
}

func (p *plugin) NewTaskPlugin(options plugins.TaskPluginOptions) (plugins.TaskPlugin, error) {
	// Increment number of tasks processed, get the count and subtract one as we
	// want to report zero for the first task
	taskCount := atomic.AddInt64(&p.taskCount, 1) - 1

	keys := map[string]string{
		"TaskId":         options.TaskContext.TaskID,
		"RunId":          strconv.Itoa(options.TaskContext.RunID),
		"HostBootTime":   p.bootTime,
		"TasksSinceBoot": strconv.FormatInt(taskCount, 10),
	}

	// Construct list of all keys (so we can sort it)
	var allKeys []string
	for k := range keys {
		allKeys = append(allKeys, k)
	}
	for k := range p.keys {
		if !stringContains(allKeys, k) {
			allKeys = append(allKeys, k)
		} else {
			debug("overwriting: %s", k)
		}
	}
	// Sort list of allKeys (to ensure consistency)
	sort.Strings(allKeys)

	// Print keys to task log
	for _, k := range allKeys {
		v, ok := p.keys[k]
		if !ok {
			v = keys[k]
		}

		options.TaskContext.Log(fmt.Sprintf("%s: %s", k, v))
	}

	// Return a plugin that does nothing
	return plugins.TaskPluginBase{}, nil
}

func stringContains(list []string, element string) bool {
	for _, e := range list {
		if e == element {
			return true
		}
	}
	return false
}
<commit_msg>TasksSinceBoot -> TasksSinceStartup<commit_after>package logprefix

import (
	"fmt"
	"sort"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/shirou/gopsutil/host"
	schematypes "github.com/taskcluster/go-schematypes"
	"github.com/taskcluster/taskcluster-worker/plugins"
)

type provider struct {
	plugins.PluginProviderBase
}

type plugin struct {
	plugins.PluginBase
	keys      map[string]string
	bootTime  string
	taskCount int64 // Count number of tasks processed
}

func init() {
	plugins.Register("logprefix", provider{})
}

func (provider) ConfigSchema() schematypes.Schema {
	return configSchema
}

func (provider) NewPlugin(options plugins.PluginOptions) (plugins.Plugin, error) {
	keys := make(map[string]string)
	schematypes.MustValidateAndMap(configSchema, options.Config, &keys)

	// Print a neat message to make debugging config easier.
	// Presumably, config files inject stuff into this section using
	// transforms, so it's nice to have some log.
	for k, v := range keys {
		options.Monitor.Infof("Prefixing task logs: '%s' = '%s'", k, v)
	}

	// Obtain the system boottime
	boottime, err := host.BootTime()
	if err != nil {
		return nil, errors.Wrap(err, "unable to determine the system boot time")
	}

	return &plugin{
		keys:     keys,
		bootTime: time.Unix(int64(boottime), 0).Format(time.RFC3339),
	}, nil
}

func (p *plugin) NewTaskPlugin(options plugins.TaskPluginOptions) (plugins.TaskPlugin, error) {
	// Increment number of tasks processed, get the count and subtract one as we
	// want to report zero for the first task
	taskCount := atomic.AddInt64(&p.taskCount, 1) - 1

	keys := map[string]string{
		"TaskId":            options.TaskContext.TaskID,
		"RunId":             strconv.Itoa(options.TaskContext.RunID),
		"HostBootTime":      p.bootTime,
		"TasksSinceStartup": strconv.FormatInt(taskCount, 10),
	}

	// Construct list of all keys (so we can sort it)
	var allKeys []string
	for k := range keys {
		allKeys = append(allKeys, k)
	}
	for k := range p.keys {
		if !stringContains(allKeys, k) {
			allKeys = append(allKeys, k)
		} else {
			debug("overwriting: %s", k)
		}
	}
	// Sort list of allKeys (to ensure consistency)
	sort.Strings(allKeys)

	// Print keys to task log
	for _, k := range allKeys {
		v, ok := p.keys[k]
		if !ok {
			v = keys[k]
		}

		options.TaskContext.Log(fmt.Sprintf("%s: %s", k, v))
	}

	// Return a plugin that does nothing
	return plugins.TaskPluginBase{}, nil
}

func stringContains(list []string, element string) bool {
	for _, e := range list {
		if e == element {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package gogs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// Permission represents a API permission.
type Permission struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// Repository represents a API repository.
type Repository struct {
	Id          int64      `json:"id"`
	Owner       User       `json:"owner"`
	FullName    string     `json:"full_name"`
	Private     bool       `json:"private"`
	Fork        bool       `json:"fork"`
	HtmlUrl     string     `json:"html_url"`
	CloneUrl    string     `json:"clone_url"`
	SshUrl      string     `json:"ssh_url"`
	Permissions Permission `json:"permissions"`
}

// ListMyRepos lists all repositories for the authenticated user that has access to.
func (c *Client) ListMyRepos() ([]*Repository, error) {
	repos := make([]*Repository, 0, 10)
	err := c.getParsedResponse("GET", "/user/repos", nil, nil, &repos)
	return repos, err
}

type CreateRepoOption struct {
	Name        string `json:"name" binding:"Required"`
	Description string `json:"description" binding:"MaxSize(255)"`
	Private     bool   `json:"private"`
	AutoInit    bool   `json:"auto_init"`
	Gitignores  string `json:"gitignores"`
	License     string `json:"license"`
	Readme      string `json:"readme"`
}

// CreateRepo creates a repository for authenticated user.
func (c *Client) CreateRepo(opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", "/user/repos",
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}

// CreateOrgRepo creates an organization repository for authenticated user.
func (c *Client) CreateOrgRepo(org string, opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", fmt.Sprintf("/org/%s/repos", org),
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}

// GetRepo returns information of a repository of given owner.
func (c *Client) GetRepo(owner, reponame string) (*Repository, error) {
	repo := new(Repository)
	return repo, c.getParsedResponse("GET", fmt.Sprintf("/repos/%s/%s", owner, reponame),
		http.Header{"content-type": []string{"application/json"}}, nil, repo)
}

// DeleteRepo deletes a repository of user or organization.
func (c *Client) DeleteRepo(owner, repo string) error {
	_, err := c.getResponse("DELETE", fmt.Sprintf("/repos/%s/%s", owner, repo), nil, nil)
	return err
}

type MigrateRepoOption struct {
	CloneAddr    string `json:"clone_addr" binding:"Required"`
	AuthUsername string `json:"auth_username"`
	AuthPassword string `json:"auth_password"`
	Uid          int    `json:"uid" binding:"Required"`
	RepoName     string `json:"repo_name" binding:"Required"`
	Mirror       bool   `json:"mirror"`
	Private      bool   `json:"private"`
	Description  string `json:"description"`
}

// MigrateRepo migrates a repository from other Git hosting sources for the
// authenticated user.
//
// To migrate a repository for a organization, the authenticated user must be a
// owner of the specified organization.
func (c *Client) MigrateRepo(opt MigrateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", "/repos/migrate",
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}
<commit_msg>Rename Uid -> UID<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package gogs

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// Permission represents a API permission.
type Permission struct {
	Admin bool `json:"admin"`
	Push  bool `json:"push"`
	Pull  bool `json:"pull"`
}

// Repository represents a API repository.
type Repository struct {
	Id          int64      `json:"id"`
	Owner       User       `json:"owner"`
	FullName    string     `json:"full_name"`
	Private     bool       `json:"private"`
	Fork        bool       `json:"fork"`
	HtmlUrl     string     `json:"html_url"`
	CloneUrl    string     `json:"clone_url"`
	SshUrl      string     `json:"ssh_url"`
	Permissions Permission `json:"permissions"`
}

// ListMyRepos lists all repositories for the authenticated user that has access to.
func (c *Client) ListMyRepos() ([]*Repository, error) {
	repos := make([]*Repository, 0, 10)
	err := c.getParsedResponse("GET", "/user/repos", nil, nil, &repos)
	return repos, err
}

type CreateRepoOption struct {
	Name        string `json:"name" binding:"Required"`
	Description string `json:"description" binding:"MaxSize(255)"`
	Private     bool   `json:"private"`
	AutoInit    bool   `json:"auto_init"`
	Gitignores  string `json:"gitignores"`
	License     string `json:"license"`
	Readme      string `json:"readme"`
}

// CreateRepo creates a repository for authenticated user.
func (c *Client) CreateRepo(opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", "/user/repos",
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}

// CreateOrgRepo creates an organization repository for authenticated user.
func (c *Client) CreateOrgRepo(org string, opt CreateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", fmt.Sprintf("/org/%s/repos", org),
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}

// GetRepo returns information of a repository of given owner.
func (c *Client) GetRepo(owner, reponame string) (*Repository, error) {
	repo := new(Repository)
	return repo, c.getParsedResponse("GET", fmt.Sprintf("/repos/%s/%s", owner, reponame),
		http.Header{"content-type": []string{"application/json"}}, nil, repo)
}

// DeleteRepo deletes a repository of user or organization.
func (c *Client) DeleteRepo(owner, repo string) error {
	_, err := c.getResponse("DELETE", fmt.Sprintf("/repos/%s/%s", owner, repo), nil, nil)
	return err
}

type MigrateRepoOption struct {
	CloneAddr    string `json:"clone_addr" binding:"Required"`
	AuthUsername string `json:"auth_username"`
	AuthPassword string `json:"auth_password"`
	UID          int    `json:"uid" binding:"Required"`
	RepoName     string `json:"repo_name" binding:"Required"`
	Mirror       bool   `json:"mirror"`
	Private      bool   `json:"private"`
	Description  string `json:"description"`
}

// MigrateRepo migrates a repository from other Git hosting sources for the
// authenticated user.
//
// To migrate a repository for a organization, the authenticated user must be a
// owner of the specified organization.
func (c *Client) MigrateRepo(opt MigrateRepoOption) (*Repository, error) {
	body, err := json.Marshal(&opt)
	if err != nil {
		return nil, err
	}
	repo := new(Repository)
	return repo, c.getParsedResponse("POST", "/repos/migrate",
		http.Header{"content-type": []string{"application/json"}}, bytes.NewReader(body), repo)
}
<|endoftext|>


<commit_before>package storage

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type SuperBlock struct {
	Version     Version
	ReplicaType ReplicationType
}

func (s *SuperBlock) Bytes() []byte {
	header := make([]byte, SuperBlockSize)
	header[0] = byte(s.Version)
	header[1] = s.ReplicaType.Byte()
	return header
}

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	SuperBlock

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: replicationType}
	e = v.load(true)
	return
}
func LoadVolumeOnly(dirname string, id VolumeId) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: CopyNil}
	e = v.load(false)
	return
}
func (v *Volume) load(alsoLoadIndex bool) error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	if v.ReplicaType == CopyNil {
		e = v.readSuperBlock()
	} else {
		e = v.maybeWriteSuperBlock()
	}
	if e == nil && alsoLoadIndex {
		indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
		if ie != nil {
			return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
		}
		v.nm, e = LoadNeedleMap(indexFile)
	}
	return e
}
func (v *Volume) Version() Version {
	return v.SuperBlock.Version
}
func (v *Volume) Size() int64 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	fmt.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
	return -1
}
func (v *Volume) Close() {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() error {
	stat, e := v.dataFile.Stat()
	if e != nil {
		fmt.Printf("failed to stat datafile %s: %s", v.dataFile, e)
		return e
	}
	if stat.Size() == 0 {
		v.SuperBlock.Version = CurrentVersion
		_, e = v.dataFile.Write(v.SuperBlock.Bytes())
	}
	return e
}
func (v *Volume) readSuperBlock() (err error) {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.SuperBlock, err = ParseSuperBlock(header)
	return err
}
func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
	superBlock.Version = Version(header[0])
	if superBlock.ReplicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		err = fmt.Errorf("cannot read replica type: %s", err)
	}
	return
}
func (v *Volume) NeedToReplicate() bool {
	return v.ReplicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) (size uint32, err error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	var offset int64
	if offset, err = v.dataFile.Seek(0, 2); err != nil {
		return
	}
	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
		v.dataFile.Truncate(offset)
		return
	}
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
		_, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size)
	}
	return
}
func (v *Volume) delete(n *Needle) (uint32, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		if err := v.nm.Delete(n.Id); err != nil {
			return 0, err
		}
		offset, err := v.dataFile.Seek(int64(nv.Offset*NeedlePaddingSize), 0)
		if err != nil {
			return 0, fmt.Errorf("cannot get datafile (%s) position: %s", v.dataFile, err)
		}
		if _, err = n.Append(v.dataFile, v.Version()); err != nil {
			v.dataFile.Truncate(offset)
			return 0, err
		}
		return nv.Size, err
	}
	return 0, nil
}

func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0)
		return n.Read(v.dataFile, nv.Size, v.Version())
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) garbageLevel() float64 {
	return float64(v.nm.deletionByteCounter) / float64(v.ContentSize())
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return e
	}
	if e = v.load(true); e != nil {
		return e
	}
	return nil
}

func ScanVolumeFile(dirname string, id VolumeId,
	visitSuperBlock func(SuperBlock) error,
	visitNeedle func(n *Needle, offset uint32) error) (err error) {
	var v *Volume
	if v, err = LoadVolumeOnly(dirname, id); err != nil {
		return
	}
	if err = visitSuperBlock(v.SuperBlock); err != nil {
		return
	}

	version := v.Version()

	offset := uint32(SuperBlockSize)
	n, rest, e := ReadNeedleHeader(v.dataFile, version)
	if e != nil {
		err = fmt.Errorf("cannot read needle header: %s", e)
		return
	}
	for n != nil {
		if err = n.ReadNeedleBody(v.dataFile, version, rest); err != nil {
			err = fmt.Errorf("cannot read needle body: %s", err)
			return
		}
		if err = visitNeedle(n, offset); err != nil {
			return
		}
		offset += NeedleHeaderSize + rest
		if n, rest, err = ReadNeedleHeader(v.dataFile, version); err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("cannot read needle header: %s", err)
		}
	}

	return
}

func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
	var (
		dst, idx *os.File
	)
	if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer dst.Close()

	if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer idx.Close()

	nm := NewNeedleMap(idx)
	new_offset := uint32(SuperBlockSize)

	err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error {
		_, err = dst.Write(superBlock.Bytes())
		return err
	}, func(n *Needle, offset uint32) error {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if ok && nv.Offset*NeedlePaddingSize == offset {
			if nv.Size > 0 {
				if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil {
					return fmt.Errorf("cannot put needle: %s", err)
				}
				if _, err = n.Append(dst, v.Version()); err != nil {
					return fmt.Errorf("cannot append needle: %s", err)
				}
				new_offset += n.DiskSize()
				//log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
		}
		return nil
	})

	return
}
func (v *Volume) ContentSize() uint64 {
	return v.nm.fileByteCounter
}
<commit_msg>do not truncate file if any error happens. Truncating will lose all files after this file entry.<commit_after>package storage

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type SuperBlock struct {
	Version     Version
	ReplicaType ReplicationType
}

func (s *SuperBlock) Bytes() []byte {
	header := make([]byte, SuperBlockSize)
	header[0] = byte(s.Version)
	header[1] = s.ReplicaType.Byte()
	return header
}

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	SuperBlock

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: replicationType}
	e = v.load(true)
	return
}
func LoadVolumeOnly(dirname string, id VolumeId) (v *Volume, e error) {
	v = &Volume{dir: dirname, Id: id}
	v.SuperBlock = SuperBlock{ReplicaType: CopyNil}
	e = v.load(false)
	return
}
func (v *Volume) load(alsoLoadIndex bool) error {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
	}
	if v.ReplicaType == CopyNil {
		e = v.readSuperBlock()
	} else {
		e = v.maybeWriteSuperBlock()
	}
	if e == nil && alsoLoadIndex {
		indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
		if ie != nil {
			return fmt.Errorf("cannot create Volume Data %s.dat: %s", fileName, e)
		}
		v.nm, e = LoadNeedleMap(indexFile)
	}
	return e
}
func (v *Volume) Version() Version {
	return v.SuperBlock.Version
}
func (v *Volume) Size() int64 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	fmt.Printf("Failed to read file size %s %s\n", v.dataFile.Name(), e.Error())
	return -1
}
func (v *Volume) Close() {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() error {
	stat, e := v.dataFile.Stat()
	if e != nil {
		fmt.Printf("failed to stat datafile %s: %s", v.dataFile, e)
		return e
	}
	if stat.Size() == 0 {
		v.SuperBlock.Version = CurrentVersion
		_, e = v.dataFile.Write(v.SuperBlock.Bytes())
	}
	return e
}
func (v *Volume) readSuperBlock() (err error) {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, e := v.dataFile.Read(header); e != nil {
		return fmt.Errorf("cannot read superblock: %s", e)
	}
	v.SuperBlock, err = ParseSuperBlock(header)
	return err
}
func ParseSuperBlock(header []byte) (superBlock SuperBlock, err error) {
	superBlock.Version = Version(header[0])
	if superBlock.ReplicaType, err = NewReplicationTypeFromByte(header[1]); err != nil {
		err = fmt.Errorf("cannot read replica type: %s", err)
	}
	return
}
func (v *Volume) NeedToReplicate() bool {
	return v.ReplicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) (size uint32, err error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	var offset int64
	if offset, err = v.dataFile.Seek(0, 2); err != nil {
		return
	}
	if size, err = n.Append(v.dataFile, v.Version()); err != nil {
		v.dataFile.Truncate(offset)
		return
	}
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*NeedlePaddingSize < offset {
		_, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size)
	}
	return
}
func (v *Volume) delete(n *Needle) (uint32, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//fmt.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*NeedlePaddingSize), 0)
		_, err := n.Append(v.dataFile, v.Version())
		return nv.Size, err
	}
	return 0, nil
}

func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*NeedlePaddingSize, 0)
		return n.Read(v.dataFile, nv.Size, v.Version())
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) garbageLevel() float64 {
	return float64(v.nm.deletionByteCounter) / float64(v.ContentSize())
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	var e error
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat")); e != nil {
		return e
	}
	if e = os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx")); e != nil {
		return e
	}
	if e = v.load(true); e != nil {
		return e
	}
	return nil
}

func ScanVolumeFile(dirname string, id VolumeId,
	visitSuperBlock func(SuperBlock) error,
	visitNeedle func(n *Needle, offset uint32) error) (err error) {
	var v *Volume
	if v, err = LoadVolumeOnly(dirname, id); err != nil {
		return
	}
	if err = visitSuperBlock(v.SuperBlock); err != nil {
		return
	}

	version := v.Version()

	offset := uint32(SuperBlockSize)
	n, rest, e := ReadNeedleHeader(v.dataFile, version)
	if e != nil {
		err = fmt.Errorf("cannot read needle header: %s", e)
		return
	}
	for n != nil {
		if err = n.ReadNeedleBody(v.dataFile, version, rest); err != nil {
			err = fmt.Errorf("cannot read needle body: %s", err)
			return
		}
		if err = visitNeedle(n, offset); err != nil {
			return
		}
		offset += NeedleHeaderSize + rest
		if n, rest, err = ReadNeedleHeader(v.dataFile, version); err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("cannot read needle header: %s", err)
		}
	}

	return
}

func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err error) {
	var (
		dst, idx *os.File
	)
	if dst, err = os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer dst.Close()

	if idx, err = os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644); err != nil {
		return
	}
	defer idx.Close()

	nm := NewNeedleMap(idx)
	new_offset := uint32(SuperBlockSize)

	err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error {
		_, err = dst.Write(superBlock.Bytes())
		return err
	}, func(n *Needle, offset uint32) error {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if ok && nv.Offset*NeedlePaddingSize == offset {
			if nv.Size > 0 {
				if _, err = nm.Put(n.Id, new_offset/NeedlePaddingSize, n.Size); err != nil {
					return fmt.Errorf("cannot put needle: %s", err)
				}
				if _, err = n.Append(dst, v.Version()); err != nil {
					return fmt.Errorf("cannot append needle: %s", err)
				}
				new_offset += n.DiskSize()
				//log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
		}
		return nil
	})

	return
}
func (v *Volume) ContentSize() uint64 {
	return v.nm.fileByteCounter
}
<|endoftext|>


<commit_before>// A quick and easy way to setup a RESTful JSON API
//
// Go-JSON-REST is a thin layer on top of net/http that helps building RESTful JSON APIs easily.
// It provides fast URL routing using https://github.com/ant0ine/go-urlrouter, and helpers to deal
// with JSON requests and responses. It is not a high-level REST framework that transparently maps
// HTTP requests to language procedure calls, on the opposite, you constantly have access to the
// underlying net/http objects.
//
// Example:
//
//      package main
//
//      import (
//              "github.com/ant0ine/go-json-rest"
//              "net/http"
//      )
//
//      type User struct {
//              Id   string
//              Name string
//      }
//
//      func GetUser(w *rest.ResponseWriter, req *rest.Request) {
//              user := User{
//                      Id:   req.PathParam("id"),
//                      Name: "Antoine",
//              }
//              w.WriteJson(&user)
//      }
//
//      func main() {
//              handler := ResourceHandler{}
//              handler.SetRoutes(
//                      rest.Route{"GET", "/users/:id", GetUser},
//              )
//              http.ListenAndServe(":8080", &handler)
//      }
//
package rest

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/ant0ine/go-urlrouter"
	"io/ioutil"
	"net/http"
	"reflect"
	"runtime/debug"
	"strings"
)

// Implement the http.Handler interface and act as a router for the defined Routes.
// The defaults are intended to be developemnt friendly, for production you may want
// to turn on gzip and disable the JSON indentation.
type ResourceHandler struct {
	router urlrouter.Router

	// If true and if the client accepts the Gzip encoding, the response payloads
	// will be compressed using gzip, and the corresponding response header will set.
	EnableGzip bool

	// If true the JSON payload will be written in one line with no space.
	DisableJsonIndent bool

	// If true, when a "panic" happens, the error string and the stack trace will be
	// printed in the 500 response body.
	EnableResponseStackTrace bool
}

// Used with SetRoutes.
type Route struct {
	HttpMethod string
	PathExp    string
	Func       func(*ResponseWriter, *Request)
}

// Create a Route that points to an object method. It can be convenient to point to an object method instead
// of a function, this helper makes it easy by passing the object instance and the method name as parameters.
func RouteObjectMethod(http_method string, path_exp string, object_instance interface{}, object_method string) Route {

	value := reflect.ValueOf(object_instance)
	func_value := value.MethodByName(object_method)
	if func_value.IsValid() == false {
		panic(fmt.Sprintf(
			"Cannot find the object method %s on %s",
			object_method,
			value,
		))
	}
	route_func := func(w *ResponseWriter, r *Request) {
		func_value.Call([]reflect.Value{
			reflect.ValueOf(w),
			reflect.ValueOf(r),
		})
	}

	return Route{
		HttpMethod: http_method,
		PathExp:    path_exp,
		Func:       route_func,
	}
}

// Define the Routes. The order the Routes matters,
// if a request matches multiple Routes, the first one will be used.
// Note that the underlying router is https://github.com/ant0ine/go-urlrouter.
func (self *ResourceHandler) SetRoutes(routes ...Route) error {
	self.router = urlrouter.Router{
		Routes: []urlrouter.Route{},
	}
	for _, route := range routes {
		// make sure the method is uppercase
		http_method := strings.ToUpper(route.HttpMethod)

		self.router.Routes = append(
			self.router.Routes,
			urlrouter.Route{
				PathExp: http_method + route.PathExp,
				Dest:    route.Func,
			},
		)
	}
	return self.router.Start()
}

// This makes ResourceHandler implement the http.Handler interface
func (self *ResourceHandler) ServeHTTP(orig_writer http.ResponseWriter, orig_request *http.Request) {

	// catch user code's panic, and convert to http response
	defer func() {
		if r := recover(); r != nil {
			message := "Internal Server Error"
			if self.EnableResponseStackTrace {
				message = fmt.Sprintf("%s\n\n%s", r, debug.Stack())
			}
			http.Error(orig_writer, message, http.StatusInternalServerError)
		}
	}()

	// find the route
	route, params, err := self.router.FindRoute(
		strings.ToUpper(orig_request.Method) + orig_request.URL.Path,
	)
	if err != nil {
		// should never happen has the URL has already been parsed
		panic(err)
	}
	if route == nil {
		// no route found
		http.NotFound(orig_writer, orig_request)
		return
	}

	// determine if gzip is needed
	is_gzipped := self.EnableGzip == true &&
		strings.Contains(orig_request.Header.Get("Accept-Encoding"), "gzip")

	is_indented := !self.DisableJsonIndent

	request := Request{
		orig_request,
		params,
	}

	writer := ResponseWriter{
		orig_writer,
		is_gzipped,
		is_indented,
	}

	// run the user code
	handler := route.Dest.(func(*ResponseWriter, *Request))
	handler(&writer, &request)
}

// Inherit from http.Request, and provide additional methods.
type Request struct {
	*http.Request
	// map of parameters that have been matched in the URL Path.
	PathParams map[string]string
}

// Provide a convenient access to the PathParams map
func (self *Request) PathParam(name string) string {
	return self.PathParams[name]
}

// Read the request body and decode the JSON using json.Unmarshal
func (self *Request) DecodeJsonPayload(v interface{}) error {
	content, err := ioutil.ReadAll(self.Body)
	self.Body.Close()
	if err != nil {
		return err
	}
	err = json.Unmarshal(content, v)
	if err != nil {
		return err
	}
	return nil
}

// Inherit from an object implementing the http.ResponseWriter interface, and provide additional methods.
type ResponseWriter struct {
	http.ResponseWriter
	is_gzipped  bool
	is_indented bool
}

// Overloading of the http.ResponseWriter method.
// Provide additional capabilities, like transparent gzip encoding.
func (self *ResponseWriter) Write(b []byte) (int, error) {
	if self.is_gzipped {
		self.Header().Set("Content-Encoding", "gzip")
		gzip_writer := gzip.NewWriter(self.ResponseWriter)
		defer gzip_writer.Close()
		return gzip_writer.Write(b)
	}
	return self.ResponseWriter.Write(b)
}

// Encode the object in JSON, set the content-type header,
// and call Write
func (self *ResponseWriter) WriteJson(v interface{}) error {
	self.Header().Set("content-type", "application/json")
	var b []byte
	var err error
	if self.is_indented {
		b, err = json.MarshalIndent(v, "", "  ")
	} else {
		b, err = json.Marshal(v)
	}
	if err != nil {
		return err
	}
	self.Write(b)
	return nil
}
<commit_msg>add basic logging<commit_after>// A quick and easy way to setup a RESTful JSON API
//
// Go-JSON-REST is a thin layer on top of net/http that helps building RESTful JSON APIs easily.
// It provides fast URL routing using https://github.com/ant0ine/go-urlrouter, and helpers to deal
// with JSON requests and responses. It is not a high-level REST framework that transparently maps
// HTTP requests to language procedure calls, on the opposite, you constantly have access to the
// underlying net/http objects.
//
// Example:
//
//      package main
//
//      import (
//              "github.com/ant0ine/go-json-rest"
//              "net/http"
//      )
//
//      type User struct {
//              Id   string
//              Name string
//      }
//
//      func GetUser(w *rest.ResponseWriter, req *rest.Request) {
//              user := User{
//                      Id:   req.PathParam("id"),
//                      Name: "Antoine",
//              }
//              w.WriteJson(&user)
//      }
//
//      func main() {
//              handler := ResourceHandler{}
//              handler.SetRoutes(
//                      rest.Route{"GET", "/users/:id", GetUser},
//              )
//              http.ListenAndServe(":8080", &handler)
//      }
//
package rest

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/ant0ine/go-urlrouter"
	"io/ioutil"
	"log"
	"net/http"
	"reflect"
	"runtime/debug"
	"strings"
)

// Implement the http.Handler interface and act as a router for the defined Routes.
// The defaults are intended to be developemnt friendly, for production you may want
// to turn on gzip and disable the JSON indentation.
type ResourceHandler struct {
	router urlrouter.Router

	// If true and if the client accepts the Gzip encoding, the response payloads
	// will be compressed using gzip, and the corresponding response header will set.
	EnableGzip bool

	// If true the JSON payload will be written in one line with no space.
	DisableJsonIndent bool

	// If true, when a "panic" happens, the error string and the stack trace will be
	// printed in the 500 response body.
	EnableResponseStackTrace bool
}

// Used with SetRoutes.
type Route struct {
	HttpMethod string
	PathExp    string
	Func       func(*ResponseWriter, *Request)
}

// Create a Route that points to an object method. It can be convenient to point to an object method instead
// of a function, this helper makes it easy by passing the object instance and the method name as parameters.
func RouteObjectMethod(http_method string, path_exp string, object_instance interface{}, object_method string) Route {

	value := reflect.ValueOf(object_instance)
	func_value := value.MethodByName(object_method)
	if func_value.IsValid() == false {
		panic(fmt.Sprintf(
			"Cannot find the object method %s on %s",
			object_method,
			value,
		))
	}
	route_func := func(w *ResponseWriter, r *Request) {
		func_value.Call([]reflect.Value{
			reflect.ValueOf(w),
			reflect.ValueOf(r),
		})
	}

	return Route{
		HttpMethod: http_method,
		PathExp:    path_exp,
		Func:       route_func,
	}
}

// Define the Routes. The order the Routes matters,
// if a request matches multiple Routes, the first one will be used.
// Note that the underlying router is https://github.com/ant0ine/go-urlrouter.
func (self *ResourceHandler) SetRoutes(routes ...Route) error {
	self.router = urlrouter.Router{
		Routes: []urlrouter.Route{},
	}
	for _, route := range routes {
		// make sure the method is uppercase
		http_method := strings.ToUpper(route.HttpMethod)

		self.router.Routes = append(
			self.router.Routes,
			urlrouter.Route{
				PathExp: http_method + route.PathExp,
				Dest:    route.Func,
			},
		)
	}
	return self.router.Start()
}

// This makes ResourceHandler implement the http.Handler interface
func (self *ResourceHandler) ServeHTTP(orig_writer http.ResponseWriter, orig_request *http.Request) {

	// catch user code's panic, and convert to http response
	defer func() {
		if r := recover(); r != nil {
			trace := debug.Stack()
			log.Printf("%s\n%s", r, trace)

			// 500 response
			message := "Internal Server Error"
			if self.EnableResponseStackTrace {
				message = fmt.Sprintf("%s\n\n%s", r, trace)
			}
			http.Error(orig_writer, message, http.StatusInternalServerError)
		}
	}()

	// find the route
	route, params, err := self.router.FindRoute(
		strings.ToUpper(orig_request.Method) + orig_request.URL.Path,
	)
	if err != nil {
		// should never happen has the URL has already been parsed
		panic(err)
	}
	if route == nil {
		// no route found
		log.Printf("%s %s => No Route Found (404)", orig_request.Method, orig_request.URL)
		http.NotFound(orig_writer, orig_request)
		return
	}

	// determine if gzip is needed
	is_gzipped := self.EnableGzip == true &&
		strings.Contains(orig_request.Header.Get("Accept-Encoding"), "gzip")

	is_indented := !self.DisableJsonIndent

	request := Request{
		orig_request,
		params,
	}

	writer := ResponseWriter{
		orig_writer,
		is_gzipped,
		is_indented,
	}

	// run the user code
	handler := route.Dest.(func(*ResponseWriter, *Request))
	log.Printf("%s %s => Dispatching...", orig_request.Method, orig_request.URL)
	handler(&writer, &request)
}

// Inherit from http.Request, and provide additional methods.
type Request struct {
	*http.Request
	// map of parameters that have been matched in the URL Path.
	PathParams map[string]string
}

// Provide a convenient access to the PathParams map
func (self *Request) PathParam(name string) string {
	return self.PathParams[name]
}

// Read the request body and decode the JSON using json.Unmarshal
func (self *Request) DecodeJsonPayload(v interface{}) error {
	content, err := ioutil.ReadAll(self.Body)
	self.Body.Close()
	if err != nil {
		return err
	}
	err = json.Unmarshal(content, v)
	if err != nil {
		return err
	}
	return nil
}

// Inherit from an object implementing the http.ResponseWriter interface, and provide additional methods.
type ResponseWriter struct {
	http.ResponseWriter
	is_gzipped  bool
	is_indented bool
}

// Overloading of the http.ResponseWriter method.
// Provide additional capabilities, like transparent gzip encoding.
func (self *ResponseWriter) Write(b []byte) (int, error) {
	if self.is_gzipped {
		self.Header().Set("Content-Encoding", "gzip")
		gzip_writer := gzip.NewWriter(self.ResponseWriter)
		defer gzip_writer.Close()
		return gzip_writer.Write(b)
	}
	return self.ResponseWriter.Write(b)
}

// Encode the object in JSON, set the content-type header,
// and call Write
func (self *ResponseWriter) WriteJson(v interface{}) error {
	self.Header().Set("content-type", "application/json")
	var b []byte
	var err error
	if self.is_indented {
		b, err = json.MarshalIndent(v, "", "  ")
	} else {
		b, err = json.Marshal(v)
	}
	if err != nil {
		return err
	}
	self.Write(b)
	return nil
}
<|endoftext|>


<commit_before>// +build scale

/*
 * Copyright (C) 2017 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package tests

import (
	"errors"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/skydive-project/skydive/api"
	gclient "github.com/skydive-project/skydive/cmd/client"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/flow"
	shttp "github.com/skydive-project/skydive/http"
	"github.com/skydive-project/skydive/tests/helper"
	"github.com/skydive-project/skydive/topology/graph"
)

func TestScaleHA(t *testing.T) {
	gopath := os.Getenv("GOPATH")
	scale := gopath + "/src/github.com/skydive-project/skydive/scripts/scale.sh"

	setupCmds := []helper.Cmd{
		{fmt.Sprintf("%s start 2 2 2", scale), true},
		{"sleep 30", false},
	}
	helper.ExecCmds(t, setupCmds...)

	tearDownCmds := []helper.Cmd{
		{fmt.Sprintf("%s stop 2 4 2", scale), false},
	}
	defer helper.ExecCmds(t, tearDownCmds...)

	// Load Agent-1 as default config for our client
	config.InitConfig("file", []string{"/tmp/skydive-scale/agent-1.yml"})
	authOptions := &shttp.AuthenticationOpts{}

	client, err := api.NewCrudClientFromConfig(authOptions)
	if err != nil {
		t.Fatalf("Failed to create client: %s", err.Error())
	}

	// switch to the other analyzer
	os.Setenv("SKYDIVE_ANALYZERS", "localhost:8084")

	gh := gclient.NewGremlinQueryHelper(authOptions)

	var retry func() error
	var nodes []*graph.Node
	var flows []*flow.Flow

	checkHostNodes := func(nodeExpected int) {
		t.Logf("Check for host node: %d", nodeExpected)
		retry = func() error {
			if nodes, err = gh.GetNodes(`g.V().Has("Type", "host")`); err != nil {
				return err
			}

			if len(nodes) != nodeExpected {
				return fmt.Errorf("Should return %d host nodes got : %v", nodeExpected, nodes)
			}

			return nil
		}
		if err = common.Retry(retry, 10, 5*time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	checkICMPv4Flows := func(flowExpected int) {
		t.Logf("Check for flows: %d", flowExpected)
		retry = func() error {
			if flows, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/ICMPv4')"); err != nil {
				return err
			}

			// two capture 2 flows
			if len(flows) != flowExpected {
				return fmt.Errorf("Should get %d ICMPv4 flow got %d : %v", flowExpected, len(flows), flows)
			}

			return nil
		}
		if err = common.Retry(retry, 10, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}

		// check in the storage
		retry = func() error {
			if flows, err = gh.GetFlows("G.At('-1s', 300).Flows().Has('LayersPath', 'Ethernet/IPv4/ICMPv4')"); err != nil {
				return err
			}

			if len(flows) != flowExpected {
				return fmt.Errorf("Should get %d ICMPv4 flow from datastore got %d : %v", flowExpected, len(flows), flows)
			}

			return nil
		}
		if err = common.Retry(retry, 40, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	checkIPerfFlows := func(flowExpected int) {
		t.Logf("Check for flows: %d", flowExpected)
		retry = func() error {
			if flows, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/TCP').Has('Transport.B', '5001')"); err != nil {
				return err
			}

			// two capture 2 flows
			if len(flows) != flowExpected {
				var flowsTCP []*flow.Flow
				if flowsTCP, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/TCP')"); err != nil {
					return err
				}
				return fmt.Errorf("Should get %d iperf(tcp/5001) flow got %d : %v", flowExpected, len(flows), flowsTCP)
			}

			for _, f := range flows {
				if f.SocketA == nil || f.SocketA.Process != "/usr/bin/iperf" || f.SocketB == nil || f.SocketB.Process != "/usr/bin/iperf" {
					return fmt.Errorf("Should get iperf exe %v", f)
				}
				if f.SocketA.Name != "iperf" || f.SocketB.Name != "iperf" {
					return fmt.Errorf("Should get iperf thread name %v", f)
				}
			}
			return nil
		}
		if err = common.Retry(retry, 10, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}

		// check in the storage
		retry = func() error {
			if flows, err = gh.GetFlows("G.At('-1s', 300).Flows().Has('LayersPath', 'Ethernet/IPv4/TCP').Has('Transport.B', '5001')"); err != nil {
				return err
			}

			if len(flows) != flowExpected {
				var flowsTCP []*flow.Flow
				if flowsTCP, err = gh.GetFlows("G.At('-1s', 300).Flows().Has('LayersPath', 'Ethernet/IPv4/TCP')"); err != nil {
					return err
				}
				return fmt.Errorf("Should get %d iperf(tcp/5001) flow from datastore got %d : %#+v", flowExpected, len(flows), flowsTCP)
			}

			for _, f := range flows {
				if f.SocketA.Process != "/usr/bin/iperf" || f.SocketB.Process != "/usr/bin/iperf" {
					return fmt.Errorf("Should get iperf exe %v", f)
				}
				if f.SocketA.Name != "iperf" || f.SocketB.Name != "iperf" {
					return fmt.Errorf("Should get iperf thread name %v", f)
				}
			}
			return nil
		}
		if err = common.Retry(retry, 40, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	// test if we have our 2 hosts
	checkHostNodes(2)

	// before creating the capture check that ovs if ready and the connectivity is ok
	common.Retry(func() error {
		pings := []helper.Cmd{
			{fmt.Sprintf("%s ping agent-1-vm1 agent-2-vm1 -c 1", scale), false},
			{fmt.Sprintf("%s ping agent-3-vm1 agent-1-vm1 -c 1", scale), false},
		}
		return helper.ExecCmds(t, pings...)
	}, 30, time.Second)

	// start a capture
	capture := api.NewCapture("g.V().Has('Type', 'netns', 'Name', 'vm1').Out().Has('Name', 'eth0')", "")
	capture.SocketInfo = true
	if err = client.Create("capture", capture); err != nil {
		t.Fatal(err)
	}

	checkCaptures := func(captureExpected int) {
		t.Logf("Check for captures: %d", captureExpected)
		retry = func() error {
			if nodes, err = gh.GetNodes(`g.V().HasKey("Capture.ID")`); err != nil {
				return err
			}

			if len(nodes) != captureExpected {
				return fmt.Errorf("Should return %d capture got : %v", captureExpected, nodes)
			}

			return nil
		}

		if err = common.Retry(retry, 10, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	// check that we have 2 captures, one per vm1
	checkCaptures(2)

	// generate some packet and wait for seeing them, to be sure that the capture is started
	waitForFirstFlows := func() error {
		t.Logf("Wait for first flows...")

		retry = func() error {
			if flows, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/ICMPv4')"); err != nil {
				return err
			}

			if len(flows) != 2 {
				return errors.New("Should get at least one flow")
			}
			return nil
		}
		if err = common.Retry(retry, 10, time.Second); err != nil {
			return err
		}
		return nil
	}

	firstFlowsFnc := func() error {
		setupCmds = []helper.Cmd{
			{fmt.Sprintf("%s ping agent-1-vm1 agent-2-vm1 -c 1", scale), false},
		}
		helper.ExecCmds(t, setupCmds...)

		return waitForFirstFlows()
	}
	if err = common.Retry(firstFlowsFnc, 10, time.Second); err != nil {
		helper.ExecCmds(t, tearDownCmds...)
		t.Fatalf(err.Error())
	}

	// generate some packet, do not check because connectivity is not ensured
	for i := 0; i != 30; i++ {
		setupCmds = []helper.Cmd{
			{fmt.Sprintf("%s ping agent-1-vm1 agent-2-vm1 -c 1", scale), false},
		}
		helper.ExecCmds(t, setupCmds...)
	}

	// 60 flows expected as we have two captures
	checkICMPv4Flows(60 + 2)

	// increase the agent number
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s start 2 4 2", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if we have now 4 hosts
	checkHostNodes(4)

	// check that we have 4 captures, one per vm1
	checkCaptures(4)

	// kill the last agent
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s stop-agent 4", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if we have now 3 hosts
	checkHostNodes(3)

	// switch back to the first analyzer
	os.Setenv("SKYDIVE_ANALYZERS", "localhost:8082")

	// test if we have now 3 hosts
	checkHostNodes(3)

	// destroy the second analyzer
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s stop-analyzer 2", scale), false},
		{"sleep 5", false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if the remaining analyzer have a correct graph
	checkHostNodes(3)

	// generate more icmp traffic
	for i := 0; i != 30; i++ {
		setupCmds = []helper.Cmd{
			{fmt.Sprintf("%s ping agent-3-vm1 agent-1-vm1 -c 1", scale), false},
		}
		helper.ExecCmds(t, setupCmds...)
	}

	// 4*30 expected because the gremlin expression matches all the eth0
	checkICMPv4Flows(120 + 2)

	// iperf test  10 sec, 1Mbits/s
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s iperf agent-3-vm1 agent-1-vm1", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)
	checkIPerfFlows(2)

	// delete the capture to check that all captures will be delete at the agent side
	client.Delete("capture", capture.ID())
	checkCaptures(0)

	// delete an agent
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s stop-agent 1", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if we have now 2 hosts
	checkHostNodes(2)
}
<commit_msg>test: scale do not check SocketA/B in live<commit_after>// +build scale

/*
 * Copyright (C) 2017 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package tests

import (
	"errors"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/skydive-project/skydive/api"
	gclient "github.com/skydive-project/skydive/cmd/client"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/flow"
	shttp "github.com/skydive-project/skydive/http"
	"github.com/skydive-project/skydive/tests/helper"
	"github.com/skydive-project/skydive/topology/graph"
)

func TestScaleHA(t *testing.T) {
	gopath := os.Getenv("GOPATH")
	scale := gopath + "/src/github.com/skydive-project/skydive/scripts/scale.sh"

	setupCmds := []helper.Cmd{
		{fmt.Sprintf("%s start 2 2 2", scale), true},
		{"sleep 30", false},
	}
	helper.ExecCmds(t, setupCmds...)

	tearDownCmds := []helper.Cmd{
		{fmt.Sprintf("%s stop 2 4 2", scale), false},
	}
	defer helper.ExecCmds(t, tearDownCmds...)

	// Load Agent-1 as default config for our client
	config.InitConfig("file", []string{"/tmp/skydive-scale/agent-1.yml"})
	authOptions := &shttp.AuthenticationOpts{}

	client, err := api.NewCrudClientFromConfig(authOptions)
	if err != nil {
		t.Fatalf("Failed to create client: %s", err.Error())
	}

	// switch to the other analyzer
	os.Setenv("SKYDIVE_ANALYZERS", "localhost:8084")

	gh := gclient.NewGremlinQueryHelper(authOptions)

	var retry func() error
	var nodes []*graph.Node
	var flows []*flow.Flow

	checkHostNodes := func(nodeExpected int) {
		t.Logf("Check for host node: %d", nodeExpected)
		retry = func() error {
			if nodes, err = gh.GetNodes(`g.V().Has("Type", "host")`); err != nil {
				return err
			}

			if len(nodes) != nodeExpected {
				return fmt.Errorf("Should return %d host nodes got : %v", nodeExpected, nodes)
			}

			return nil
		}
		if err = common.Retry(retry, 10, 5*time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	checkICMPv4Flows := func(flowExpected int) {
		t.Logf("Check for flows: %d", flowExpected)
		retry = func() error {
			if flows, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/ICMPv4')"); err != nil {
				return err
			}

			// two capture 2 flows
			if len(flows) != flowExpected {
				return fmt.Errorf("Should get %d ICMPv4 flow got %d : %v", flowExpected, len(flows), flows)
			}

			return nil
		}
		if err = common.Retry(retry, 10, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}

		// check in the storage
		retry = func() error {
			if flows, err = gh.GetFlows("G.At('-1s', 300).Flows().Has('LayersPath', 'Ethernet/IPv4/ICMPv4')"); err != nil {
				return err
			}

			if len(flows) != flowExpected {
				return fmt.Errorf("Should get %d ICMPv4 flow from datastore got %d : %v", flowExpected, len(flows), flows)
			}

			return nil
		}
		if err = common.Retry(retry, 40, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	checkIPerfFlows := func(flowExpected int) {
		t.Logf("Check for flows: %d", flowExpected)
		retry = func() error {
			if flows, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/TCP').Has('Transport.B', '5001')"); err != nil {
				return err
			}

			// two capture 2 flows
			if len(flows) != flowExpected {
				var flowsTCP []*flow.Flow
				if flowsTCP, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/TCP')"); err != nil {
					return err
				}
				return fmt.Errorf("Should get %d iperf(tcp/5001) flow got %d : %v", flowExpected, len(flows), flowsTCP)
			}

			return nil
		}
		if err = common.Retry(retry, 10, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}

		// check in the storage
		retry = func() error {
			if flows, err = gh.GetFlows("G.At('-1s', 300).Flows().Has('LayersPath', 'Ethernet/IPv4/TCP').Has('Transport.B', '5001')"); err != nil {
				return err
			}

			if len(flows) != flowExpected {
				var flowsTCP []*flow.Flow
				if flowsTCP, err = gh.GetFlows("G.At('-1s', 300).Flows().Has('LayersPath', 'Ethernet/IPv4/TCP')"); err != nil {
					return err
				}
				return fmt.Errorf("Should get %d iperf(tcp/5001) flow from datastore got %d : %#+v", flowExpected, len(flows), flowsTCP)
			}

			for _, f := range flows {
				if f.SocketA == nil || f.SocketA.Process != "/usr/bin/iperf" || f.SocketB == nil || f.SocketB.Process != "/usr/bin/iperf" {
					return fmt.Errorf("Should get iperf exe as socket info %v", f)
				}
				if f.SocketA.Name != "iperf" || f.SocketB.Name != "iperf" {
					return fmt.Errorf("Should get iperf thread name %v", f)
				}
			}
			return nil
		}
		if err = common.Retry(retry, 40, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	// test if we have our 2 hosts
	checkHostNodes(2)

	// before creating the capture check that ovs if ready and the connectivity is ok
	common.Retry(func() error {
		pings := []helper.Cmd{
			{fmt.Sprintf("%s ping agent-1-vm1 agent-2-vm1 -c 1", scale), false},
			{fmt.Sprintf("%s ping agent-3-vm1 agent-1-vm1 -c 1", scale), false},
		}
		return helper.ExecCmds(t, pings...)
	}, 30, time.Second)

	// start a capture
	capture := api.NewCapture("g.V().Has('Type', 'netns', 'Name', 'vm1').Out().Has('Name', 'eth0')", "")
	capture.SocketInfo = true
	capture.Type = "pcap"
	if err = client.Create("capture", capture); err != nil {
		t.Fatal(err)
	}

	checkCaptures := func(captureExpected int) {
		t.Logf("Check for captures: %d", captureExpected)
		retry = func() error {
			if nodes, err = gh.GetNodes(`g.V().HasKey("Capture.ID")`); err != nil {
				return err
			}

			if len(nodes) != captureExpected {
				return fmt.Errorf("Should return %d capture got : %v", captureExpected, nodes)
			}

			return nil
		}

		if err = common.Retry(retry, 10, time.Second); err != nil {
			helper.ExecCmds(t, tearDownCmds...)
			t.Fatalf(err.Error())
		}
	}

	// check that we have 2 captures, one per vm1
	checkCaptures(2)

	// generate some packet and wait for seeing them, to be sure that the capture is started
	waitForFirstFlows := func() error {
		t.Logf("Wait for first flows...")

		retry = func() error {
			if flows, err = gh.GetFlows("G.Flows().Has('LayersPath', 'Ethernet/IPv4/ICMPv4')"); err != nil {
				return err
			}

			if len(flows) != 2 {
				return errors.New("Should get at least one flow")
			}
			return nil
		}
		if err = common.Retry(retry, 10, time.Second); err != nil {
			return err
		}
		return nil
	}

	firstFlowsFnc := func() error {
		setupCmds = []helper.Cmd{
			{fmt.Sprintf("%s ping agent-1-vm1 agent-2-vm1 -c 1", scale), false},
		}
		helper.ExecCmds(t, setupCmds...)

		return waitForFirstFlows()
	}
	if err = common.Retry(firstFlowsFnc, 10, time.Second); err != nil {
		helper.ExecCmds(t, tearDownCmds...)
		t.Fatalf(err.Error())
	}

	// generate some packet, do not check because connectivity is not ensured
	for i := 0; i != 30; i++ {
		setupCmds = []helper.Cmd{
			{fmt.Sprintf("%s ping agent-1-vm1 agent-2-vm1 -c 1", scale), false},
		}
		helper.ExecCmds(t, setupCmds...)
	}

	// 60 flows expected as we have two captures
	checkICMPv4Flows(60 + 2)

	// increase the agent number
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s start 2 4 2", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if we have now 4 hosts
	checkHostNodes(4)

	// check that we have 4 captures, one per vm1
	checkCaptures(4)

	// kill the last agent
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s stop-agent 4", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if we have now 3 hosts
	checkHostNodes(3)

	// switch back to the first analyzer
	os.Setenv("SKYDIVE_ANALYZERS", "localhost:8082")

	// test if we have now 3 hosts
	checkHostNodes(3)

	// destroy the second analyzer
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s stop-analyzer 2", scale), false},
		{"sleep 5", false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if the remaining analyzer have a correct graph
	checkHostNodes(3)

	// generate more icmp traffic
	for i := 0; i != 30; i++ {
		setupCmds = []helper.Cmd{
			{fmt.Sprintf("%s ping agent-3-vm1 agent-1-vm1 -c 1", scale), false},
		}
		helper.ExecCmds(t, setupCmds...)
	}

	// 4*30 expected because the gremlin expression matches all the eth0
	checkICMPv4Flows(120 + 2)

	// iperf test  10 sec, 1Mbits/s
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s iperf agent-3-vm1 agent-1-vm1", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)
	checkIPerfFlows(2)

	// delete the capture to check that all captures will be delete at the agent side
	client.Delete("capture", capture.ID())
	checkCaptures(0)

	// delete an agent
	setupCmds = []helper.Cmd{
		{fmt.Sprintf("%s stop-agent 1", scale), false},
	}
	helper.ExecCmds(t, setupCmds...)

	// test if we have now 2 hosts
	checkHostNodes(2)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"encoding/json"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
)

var _ = ginkgo.Describe("[sig-node] PodTemplates", func() {
	f := framework.NewDefaultFramework("podtemplate")
	/*
	   Release : v1.19
	   Testname: PodTemplate lifecycle
	   Description: Attempt to create a PodTemplate. Patch the created PodTemplate. Fetching the PodTemplate MUST reflect changes.
	          By fetching all the PodTemplates via a Label selector it MUST find the PodTemplate by it's static label and updated value. The PodTemplate must be deleted.
	*/
	framework.ConformanceIt("should run the lifecycle of PodTemplates", func() {
		testNamespaceName := f.Namespace.Name
		podTemplateName := "nginx-pod-template-" + string(uuid.NewUUID())

		// get a list of PodTemplates (in all namespaces to hit endpoint)
		podTemplateList, err := f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list all PodTemplates")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "unable to find templates")

		// create a PodTemplate
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Create(context.TODO(), &v1.PodTemplate{
			ObjectMeta: metav1.ObjectMeta{
				Name: podTemplateName,
				Labels: map[string]string{
					"podtemplate-static": "true",
				},
			},
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{Name: "nginx", Image: "nginx"},
					},
				},
			},
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err, "failed to create PodTemplate")

		// get template
		podTemplateRead, err := f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get created PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Name, podTemplateName)

		// patch template
		podTemplatePatch, err := json.Marshal(map[string]interface{}{
			"metadata": map[string]interface{}{
				"labels": map[string]string{
					"podtemplate": "patched",
				},
			},
		})
		framework.ExpectNoError(err, "failed to marshal patch data")
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Patch(context.TODO(), podTemplateName, types.StrategicMergePatchType, []byte(podTemplatePatch), metav1.PatchOptions{})
		framework.ExpectNoError(err, "failed to patch PodTemplate")

		// get template (ensure label is there)
		podTemplateRead, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Labels["podtemplate"], "patched", "failed to patch template, new label not found")

		// delete the PodTemplate
		err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Delete(context.TODO(), podTemplateName, metav1.DeleteOptions{})
		framework.ExpectNoError(err, "failed to delete PodTemplate")

		// list the PodTemplates
		podTemplateList, err = f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list PodTemplate")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "PodTemplate list returned items, failed to delete PodTemplate")
	})

	ginkgo.It("should delete a collection of pod templates", func() {
		podTemplateNames := []string{"test-podtemplate-1", "test-podtemplate-2", "test-podtemplate-3"}

		ginkgo.By("Create set of pod templates")
		// create a set of pod templates in test namespace
		for _, podTemplateName := range podTemplateNames {
			_, err := f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).Create(context.TODO(), &v1.PodTemplate{
				ObjectMeta: metav1.ObjectMeta{
					Name:   podTemplateName,
					Labels: map[string]string{"podtemplate-set": "true"},
				},
				Template: v1.PodTemplateSpec{
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							{Name: "token-test", Image: imageutils.GetE2EImage(imageutils.Agnhost)},
						},
					},
				},
			}, metav1.CreateOptions{})
			framework.ExpectNoError(err, "failed to create pod template")
			framework.Logf("created %v", podTemplateName)
		}

		ginkgo.By("get a list of pod templates with a label in the current namespace")
		// get a list of pod templates
		podTemplateList, err := f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-set=true",
		})
		framework.ExpectNoError(err, "failed to get a list of pod templates")

		framework.ExpectEqual(len(podTemplateList.Items), len(podTemplateNames), "looking for expected number of pod templates")

		ginkgo.By("delete collection of pod templates")
		// delete collection

		framework.Logf("requesting DeleteCollection of pod templates")
		err = f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{
			LabelSelector: "podtemplate-set=true"})
		framework.ExpectNoError(err, "failed to delete all pod templates")

		ginkgo.By("get a list of pod templates with a label in the current namespace")
		// get list of pod templates
		podTemplateList, err = f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-set=true",
		})
		framework.ExpectNoError(err, "failed to get a list of pod templates")

		framework.ExpectEqual(len(podTemplateList.Items), 0, "pod templates should all be deleted")

	})

})
<commit_msg>Use polling while deleting the collection of pod templates<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"encoding/json"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/kubernetes/test/e2e/framework"
	imageutils "k8s.io/kubernetes/test/utils/image"
	"time"

	"github.com/onsi/ginkgo"
)

const (
	podTemplateRetryPeriod  = 1 * time.Second
	podTemplateRetryTimeout = 1 * time.Minute
)

var _ = ginkgo.Describe("[sig-node] PodTemplates", func() {
	f := framework.NewDefaultFramework("podtemplate")
	/*
	   Release : v1.19
	   Testname: PodTemplate lifecycle
	   Description: Attempt to create a PodTemplate. Patch the created PodTemplate. Fetching the PodTemplate MUST reflect changes.
	          By fetching all the PodTemplates via a Label selector it MUST find the PodTemplate by it's static label and updated value. The PodTemplate must be deleted.
	*/
	framework.ConformanceIt("should run the lifecycle of PodTemplates", func() {
		testNamespaceName := f.Namespace.Name
		podTemplateName := "nginx-pod-template-" + string(uuid.NewUUID())

		// get a list of PodTemplates (in all namespaces to hit endpoint)
		podTemplateList, err := f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list all PodTemplates")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "unable to find templates")

		// create a PodTemplate
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Create(context.TODO(), &v1.PodTemplate{
			ObjectMeta: metav1.ObjectMeta{
				Name: podTemplateName,
				Labels: map[string]string{
					"podtemplate-static": "true",
				},
			},
			Template: v1.PodTemplateSpec{
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{Name: "nginx", Image: "nginx"},
					},
				},
			},
		}, metav1.CreateOptions{})
		framework.ExpectNoError(err, "failed to create PodTemplate")

		// get template
		podTemplateRead, err := f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get created PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Name, podTemplateName)

		// patch template
		podTemplatePatch, err := json.Marshal(map[string]interface{}{
			"metadata": map[string]interface{}{
				"labels": map[string]string{
					"podtemplate": "patched",
				},
			},
		})
		framework.ExpectNoError(err, "failed to marshal patch data")
		_, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Patch(context.TODO(), podTemplateName, types.StrategicMergePatchType, []byte(podTemplatePatch), metav1.PatchOptions{})
		framework.ExpectNoError(err, "failed to patch PodTemplate")

		// get template (ensure label is there)
		podTemplateRead, err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Get(context.TODO(), podTemplateName, metav1.GetOptions{})
		framework.ExpectNoError(err, "failed to get PodTemplate")
		framework.ExpectEqual(podTemplateRead.ObjectMeta.Labels["podtemplate"], "patched", "failed to patch template, new label not found")

		// delete the PodTemplate
		err = f.ClientSet.CoreV1().PodTemplates(testNamespaceName).Delete(context.TODO(), podTemplateName, metav1.DeleteOptions{})
		framework.ExpectNoError(err, "failed to delete PodTemplate")

		// list the PodTemplates
		podTemplateList, err = f.ClientSet.CoreV1().PodTemplates("").List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-static=true",
		})
		framework.ExpectNoError(err, "failed to list PodTemplate")
		framework.ExpectEqual(len(podTemplateList.Items), 0, "PodTemplate list returned items, failed to delete PodTemplate")
	})

	ginkgo.It("should delete a collection of pod templates", func() {
		podTemplateNames := []string{"test-podtemplate-1", "test-podtemplate-2", "test-podtemplate-3"}

		ginkgo.By("Create set of pod templates")
		// create a set of pod templates in test namespace
		for _, podTemplateName := range podTemplateNames {
			_, err := f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).Create(context.TODO(), &v1.PodTemplate{
				ObjectMeta: metav1.ObjectMeta{
					Name:   podTemplateName,
					Labels: map[string]string{"podtemplate-set": "true"},
				},
				Template: v1.PodTemplateSpec{
					Spec: v1.PodSpec{
						Containers: []v1.Container{
							{Name: "token-test", Image: imageutils.GetE2EImage(imageutils.Agnhost)},
						},
					},
				},
			}, metav1.CreateOptions{})
			framework.ExpectNoError(err, "failed to create pod template")
			framework.Logf("created %v", podTemplateName)
		}

		ginkgo.By("get a list of pod templates with a label in the current namespace")
		// get a list of pod templates
		podTemplateList, err := f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-set=true",
		})
		framework.ExpectNoError(err, "failed to get a list of pod templates")

		framework.ExpectEqual(len(podTemplateList.Items), len(podTemplateNames), "looking for expected number of pod templates")

		ginkgo.By("delete collection of pod templates")
		// confirm that delete collection does remove all pod templates

		err = wait.PollImmediate(podTemplateRetryPeriod, podTemplateRetryTimeout, deletePodTemplateCollection(f, "podtemplate-set=true"))
		framework.ExpectNoError(err, "failed to delete collection")

		ginkgo.By("get a list of pod templates with a label in the current namespace")
		// get list of pod templates
		podTemplateList, err = f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{
			LabelSelector: "podtemplate-set=true",
		})
		framework.ExpectNoError(err, "failed to get a list of pod templates")

		framework.ExpectEqual(len(podTemplateList.Items), 0, "pod templates should all be deleted")

	})

})

func deletePodTemplateCollection(f *framework.Framework, label string) func() (bool, error) {
	return func() (bool, error) {
		var err error

		framework.Logf("requesting DeleteCollection of pod templates")

		err = f.ClientSet.CoreV1().PodTemplates(f.Namespace.Name).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{
			LabelSelector: label})

		if err != nil {
			return false, err
		} else {
			return true, nil
		}

	}
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"io/ioutil"

	"github.com/sirupsen/logrus"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/argoproj/argo-cd/errors"
	. "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	"github.com/argoproj/argo-cd/test/e2e/fixture"
	"github.com/argoproj/argo-cd/util/json"
)

// this implements the "when" part of given/when/then
//
// none of the func implement error checks, and that is complete intended, you should check for errors
// using the Then()
type Actions struct {
	context      *Context
	lastOutput   string
	lastError    error
	ignoreErrors bool
}

func (a *Actions) IgnoreErrors() *Actions {
	a.ignoreErrors = true
	return a
}
func (a *Actions) DoNotIgnoreErrors() *Actions {
	a.ignoreErrors = false
	return a
}

func (a *Actions) DoNotIgnoreErrors() *Actions {
	a.ignoreErrors = false
	return a
}

func (a *Actions) PatchFile(file string, jsonPath string) *Actions {
	a.context.t.Helper()
	fixture.Patch(a.context.path+"/"+file, jsonPath)
	return a
}

func (a *Actions) DeleteFile(file string) *Actions {
	a.context.t.Helper()
	fixture.Delete(a.context.path + "/" + file)
	return a
}

func (a *Actions) AddFile(fileName, fileContents string) *Actions {
	a.context.t.Helper()
	fixture.AddFile(a.context.path+"/"+fileName, fileContents)
	return a
}

func (a *Actions) CreateFromFile(handler func(app *Application)) *Actions {
	a.context.t.Helper()
	app := &Application{
		ObjectMeta: v1.ObjectMeta{
			Name: a.context.name,
		},
		Spec: ApplicationSpec{
			Project: a.context.project,
			Source: ApplicationSource{
				RepoURL: fixture.RepoURL(a.context.repoURLType),
				Path:    a.context.path,
			},
			Destination: ApplicationDestination{
				Server:    a.context.destServer,
				Namespace: fixture.DeploymentNamespace(),
			},
		},
	}
	if a.context.env != "" {
		app.Spec.Source.Ksonnet = &ApplicationSourceKsonnet{
			Environment: a.context.env,
		}
	}
	if a.context.namePrefix != "" {
		app.Spec.Source.Kustomize = &ApplicationSourceKustomize{
			NamePrefix: a.context.namePrefix,
		}
	}
	if a.context.configManagementPlugin != "" {
		app.Spec.Source.Plugin = &ApplicationSourcePlugin{
			Name: a.context.configManagementPlugin,
		}
	}

	if len(a.context.jsonnetTLAStr) > 0 || len(a.context.parameters) > 0 {
		logrus.Fatal("Application parameters or json tlas are not supported")
	}

	handler(app)
	data := json.MustMarshal(app)
	tmpFile, err := ioutil.TempFile("", "")
	errors.CheckError(err)
	_, err = tmpFile.Write(data)
	errors.CheckError(err)

	a.runCli("app", "create", "-f", tmpFile.Name())
	return a
}

func (a *Actions) Create() *Actions {
	a.context.t.Helper()
	args := []string{
		"app", "create", a.context.name,
		"--repo", fixture.RepoURL(a.context.repoURLType),
		"--path", a.context.path,
		"--dest-server", a.context.destServer,
		"--dest-namespace", fixture.DeploymentNamespace(),
	}

	if a.context.env != "" {
		args = append(args, "--env", a.context.env)
	}

	for _, parameter := range a.context.parameters {
		args = append(args, "--parameter", parameter)
	}

	args = append(args, "--project", a.context.project)

	for _, jsonnetTLAParameter := range a.context.jsonnetTLAStr {
		args = append(args, "--jsonnet-tla-str", jsonnetTLAParameter)
	}

	for _, jsonnetTLAParameter := range a.context.jsonnetTLACode {
		args = append(args, "--jsonnet-tla-code", jsonnetTLAParameter)
	}

	if a.context.namePrefix != "" {
		args = append(args, "--nameprefix", a.context.namePrefix)
	}

	if a.context.configManagementPlugin != "" {
		args = append(args, "--config-management-plugin", a.context.configManagementPlugin)
	}

	a.runCli(args...)

	return a
}

func (a *Actions) Declarative(filename string) *Actions {
	a.context.t.Helper()
	return a.DeclarativeWithCustomRepo(filename, fixture.RepoURL(a.context.repoURLType))
}

func (a *Actions) DeclarativeWithCustomRepo(filename string, repoURL string) *Actions {
	a.context.t.Helper()
	values := map[string]interface{}{
		"ArgoCDNamespace":     fixture.ArgoCDNamespace,
		"DeploymentNamespace": fixture.DeploymentNamespace(),
		"Name":                a.context.name,
		"Path":                a.context.path,
		"Project":             a.context.project,
		"RepoURL":             repoURL,
	}
	a.lastOutput, a.lastError = fixture.Declarative(filename, values)
	a.verifyAction()
	return a
}

func (a *Actions) PatchApp(patch string) *Actions {
	a.context.t.Helper()
	a.runCli("app", "patch", a.context.name, "--patch", patch)
	return a
}

func (a *Actions) AppSet(flags ...string) *Actions {
	a.context.t.Helper()
	args := []string{"app", "set", a.context.name}
	args = append(args, flags...)
	a.runCli(args...)
	return a
}

func (a *Actions) Sync() *Actions {
	a.context.t.Helper()
	args := []string{"app", "sync", a.context.name, "--timeout", fmt.Sprintf("%v", a.context.timeout)}

	if a.context.async {
		args = append(args, "--async")
	}

	if a.context.prune {
		args = append(args, "--prune")
	}

	if a.context.resource != "" {
		args = append(args, "--resource", a.context.resource)
	}

	if a.context.localPath != "" {
		args = append(args, "--local", a.context.localPath)
	}

	if a.context.force {
		args = append(args, "--force")
	}

	a.runCli(args...)

	return a
}

func (a *Actions) TerminateOp() *Actions {
	a.context.t.Helper()
	a.runCli("app", "terminate-op", a.context.name)
	return a
}

func (a *Actions) Refresh(refreshType RefreshType) *Actions {
	a.context.t.Helper()
	flag := map[RefreshType]string{
		RefreshTypeNormal: "--refresh",
		RefreshTypeHard:   "--hard-refresh",
	}[refreshType]

	a.runCli("app", "get", a.context.name, flag)

	return a
}

func (a *Actions) Delete(cascade bool) *Actions {
	a.context.t.Helper()
	a.runCli("app", "delete", a.context.name, fmt.Sprintf("--cascade=%v", cascade))
	return a
}

func (a *Actions) And(block func()) *Actions {
	a.context.t.Helper()
	block()
	return a
}

func (a *Actions) Then() *Consequences {
	a.context.t.Helper()
	return &Consequences{a.context, a}
}

func (a *Actions) runCli(args ...string) {
	a.context.t.Helper()
	a.lastOutput, a.lastError = fixture.RunCli(args...)
	a.verifyAction()
}

func (a *Actions) verifyAction() {
	a.context.t.Helper()
	if !a.ignoreErrors {
		a.Then().Expect(Success(""))
	}
}
<commit_msg>Remove duplicated DoNotIgnoreErrors method (#2196)<commit_after>package app

import (
	"fmt"
	"io/ioutil"

	"github.com/sirupsen/logrus"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/argoproj/argo-cd/errors"
	. "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
	"github.com/argoproj/argo-cd/test/e2e/fixture"
	"github.com/argoproj/argo-cd/util/json"
)

// this implements the "when" part of given/when/then
//
// none of the func implement error checks, and that is complete intended, you should check for errors
// using the Then()
type Actions struct {
	context      *Context
	lastOutput   string
	lastError    error
	ignoreErrors bool
}

func (a *Actions) IgnoreErrors() *Actions {
	a.ignoreErrors = true
	return a
}

func (a *Actions) DoNotIgnoreErrors() *Actions {
	a.ignoreErrors = false
	return a
}

func (a *Actions) PatchFile(file string, jsonPath string) *Actions {
	a.context.t.Helper()
	fixture.Patch(a.context.path+"/"+file, jsonPath)
	return a
}

func (a *Actions) DeleteFile(file string) *Actions {
	a.context.t.Helper()
	fixture.Delete(a.context.path + "/" + file)
	return a
}

func (a *Actions) AddFile(fileName, fileContents string) *Actions {
	a.context.t.Helper()
	fixture.AddFile(a.context.path+"/"+fileName, fileContents)
	return a
}

func (a *Actions) CreateFromFile(handler func(app *Application)) *Actions {
	a.context.t.Helper()
	app := &Application{
		ObjectMeta: v1.ObjectMeta{
			Name: a.context.name,
		},
		Spec: ApplicationSpec{
			Project: a.context.project,
			Source: ApplicationSource{
				RepoURL: fixture.RepoURL(a.context.repoURLType),
				Path:    a.context.path,
			},
			Destination: ApplicationDestination{
				Server:    a.context.destServer,
				Namespace: fixture.DeploymentNamespace(),
			},
		},
	}
	if a.context.env != "" {
		app.Spec.Source.Ksonnet = &ApplicationSourceKsonnet{
			Environment: a.context.env,
		}
	}
	if a.context.namePrefix != "" {
		app.Spec.Source.Kustomize = &ApplicationSourceKustomize{
			NamePrefix: a.context.namePrefix,
		}
	}
	if a.context.configManagementPlugin != "" {
		app.Spec.Source.Plugin = &ApplicationSourcePlugin{
			Name: a.context.configManagementPlugin,
		}
	}

	if len(a.context.jsonnetTLAStr) > 0 || len(a.context.parameters) > 0 {
		logrus.Fatal("Application parameters or json tlas are not supported")
	}

	handler(app)
	data := json.MustMarshal(app)
	tmpFile, err := ioutil.TempFile("", "")
	errors.CheckError(err)
	_, err = tmpFile.Write(data)
	errors.CheckError(err)

	a.runCli("app", "create", "-f", tmpFile.Name())
	return a
}

func (a *Actions) Create() *Actions {
	a.context.t.Helper()
	args := []string{
		"app", "create", a.context.name,
		"--repo", fixture.RepoURL(a.context.repoURLType),
		"--path", a.context.path,
		"--dest-server", a.context.destServer,
		"--dest-namespace", fixture.DeploymentNamespace(),
	}

	if a.context.env != "" {
		args = append(args, "--env", a.context.env)
	}

	for _, parameter := range a.context.parameters {
		args = append(args, "--parameter", parameter)
	}

	args = append(args, "--project", a.context.project)

	for _, jsonnetTLAParameter := range a.context.jsonnetTLAStr {
		args = append(args, "--jsonnet-tla-str", jsonnetTLAParameter)
	}

	for _, jsonnetTLAParameter := range a.context.jsonnetTLACode {
		args = append(args, "--jsonnet-tla-code", jsonnetTLAParameter)
	}

	if a.context.namePrefix != "" {
		args = append(args, "--nameprefix", a.context.namePrefix)
	}

	if a.context.configManagementPlugin != "" {
		args = append(args, "--config-management-plugin", a.context.configManagementPlugin)
	}

	a.runCli(args...)

	return a
}

func (a *Actions) Declarative(filename string) *Actions {
	a.context.t.Helper()
	return a.DeclarativeWithCustomRepo(filename, fixture.RepoURL(a.context.repoURLType))
}

func (a *Actions) DeclarativeWithCustomRepo(filename string, repoURL string) *Actions {
	a.context.t.Helper()
	values := map[string]interface{}{
		"ArgoCDNamespace":     fixture.ArgoCDNamespace,
		"DeploymentNamespace": fixture.DeploymentNamespace(),
		"Name":                a.context.name,
		"Path":                a.context.path,
		"Project":             a.context.project,
		"RepoURL":             repoURL,
	}
	a.lastOutput, a.lastError = fixture.Declarative(filename, values)
	a.verifyAction()
	return a
}

func (a *Actions) PatchApp(patch string) *Actions {
	a.context.t.Helper()
	a.runCli("app", "patch", a.context.name, "--patch", patch)
	return a
}

func (a *Actions) AppSet(flags ...string) *Actions {
	a.context.t.Helper()
	args := []string{"app", "set", a.context.name}
	args = append(args, flags...)
	a.runCli(args...)
	return a
}

func (a *Actions) Sync() *Actions {
	a.context.t.Helper()
	args := []string{"app", "sync", a.context.name, "--timeout", fmt.Sprintf("%v", a.context.timeout)}

	if a.context.async {
		args = append(args, "--async")
	}

	if a.context.prune {
		args = append(args, "--prune")
	}

	if a.context.resource != "" {
		args = append(args, "--resource", a.context.resource)
	}

	if a.context.localPath != "" {
		args = append(args, "--local", a.context.localPath)
	}

	if a.context.force {
		args = append(args, "--force")
	}

	a.runCli(args...)

	return a
}

func (a *Actions) TerminateOp() *Actions {
	a.context.t.Helper()
	a.runCli("app", "terminate-op", a.context.name)
	return a
}

func (a *Actions) Refresh(refreshType RefreshType) *Actions {
	a.context.t.Helper()
	flag := map[RefreshType]string{
		RefreshTypeNormal: "--refresh",
		RefreshTypeHard:   "--hard-refresh",
	}[refreshType]

	a.runCli("app", "get", a.context.name, flag)

	return a
}

func (a *Actions) Delete(cascade bool) *Actions {
	a.context.t.Helper()
	a.runCli("app", "delete", a.context.name, fmt.Sprintf("--cascade=%v", cascade))
	return a
}

func (a *Actions) And(block func()) *Actions {
	a.context.t.Helper()
	block()
	return a
}

func (a *Actions) Then() *Consequences {
	a.context.t.Helper()
	return &Consequences{a.context, a}
}

func (a *Actions) runCli(args ...string) {
	a.context.t.Helper()
	a.lastOutput, a.lastError = fixture.RunCli(args...)
	a.verifyAction()
}

func (a *Actions) verifyAction() {
	a.context.t.Helper()
	if !a.ignoreErrors {
		a.Then().Expect(Success(""))
	}
}
<|endoftext|>


<commit_before>package gorest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	ContentType     = "Content-Type"
	JsonContentType = "application/json"
	TextContentType = "application/text"
	XmlContentType  = "application/xml"
	timeOutSeconds  = 10
)

type Rest struct {
	httpClient  http.Client
	baseurl     string
	verb        string
	headers     http.Header
	uriparams   []string
	url         string
	queryvalues url.Values
	payload     io.Reader
}

func New() *Rest {
	return &Rest{
		httpClient:  http.Client{Timeout: time.Second * timeOutSeconds},
		headers:     make(map[string][]string),
		queryvalues: make(url.Values),
	}
}

func (client *Rest) Get() *Rest {
	client.verb = "GET"
	return client
}

func (client *Rest) Post(payload interface{}) *Rest {
	client.verb = "POST"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Put(payload interface{}) *Rest {
	client.verb = "PUT"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Delete() *Rest {
	client.verb = "DELETE"
	return client
}

func (client *Rest) Patch(payload interface{}) *Rest {
	client.verb = "PATCH"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Head() *Rest {
	client.verb = "HEAD"
	return client
}

func (client *Rest) Option() *Rest {
	client.verb = "OPTIONS"
	return client
}

func (client *Rest) Path(param string) *Rest {
	if client.baseurl == "" {
		panic("BASE URL Not Present")
	} else if strings.Contains(param, "/") {
		panic("Incorrect PATH Param")
	}
	client.uriparams = append(client.uriparams, param)
	return client
}

func (client *Rest) Base(baseurl string) *Rest {
	v, err := url.Parse(baseurl)
	if err != nil {
		panic("Url is incorrect")
	}
	client.baseurl = v.String()
	return client
}

func (client *Rest) Header(key, value string) *Rest {
	client.headers.Add(key, value)
	return client
}

func (client *Rest) addPayload(payload interface{}) {
	var b []byte
	b, _ = json.Marshal(payload)
	client.payload = bytes.NewBuffer(b)

}

func (client *Rest) Query(options ...interface{}) *Rest {
	if len(options) > 0 {
		qry, ok := options[0].(map[string]string)
		if ok {
			for k, v := range qry {
				client.queryvalues.Set(k, v)
			}
		}
	}
	return client
}

func (client *Rest) Request() (*http.Request, error) {
	//Add all URI params to baseurl
	if !strings.HasSuffix(client.baseurl, "/") {
		client.baseurl = client.baseurl + "/"
	}

	client.url = client.baseurl
	params := strings.Join(client.uriparams, "/")
	client.url = client.baseurl + params
	v, err := url.Parse(client.url)
	if err != nil {
		panic("Complete URL is not correct")
	}

	client.url = v.String()

	//Adding Query String
	var requrl url.URL
	requrl.Path = client.url
	requrl.RawQuery = client.queryvalues.Encode()
	fmt.Println(requrl.String())

	req, err := http.NewRequest(client.verb, requrl.String(), client.payload)
	if err != nil {
		panic("Request Object is not proper")
	}

	//Add headers to the Request
	for key, values := range client.headers {
		for _, value := range values {
			req.Header.Add(key, value)
		}

	}
	return req, err
}

func (client *Rest) ResponseBodyString(response *http.Response) string {
	responsedata, _ := ioutil.ReadAll(response.Body)
	return string(responsedata)
}

func (client *Rest) Send(req *http.Request, successM, failureM interface{}) (*http.Response, error) {
	response, err := client.httpClient.Do(req)

	if err != nil {
		panic("Send request Failed")
	}
	return response, err
}
<commit_msg>introducing COPY HTTP Verb<commit_after>package gorest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	ContentType     = "Content-Type"
	JsonContentType = "application/json"
	TextContentType = "application/text"
	XmlContentType  = "application/xml"
	timeOutSeconds  = 10
)

type Rest struct {
	httpClient  http.Client
	baseurl     string
	verb        string
	headers     http.Header
	uriparams   []string
	url         string
	queryvalues url.Values
	payload     io.Reader
}

func New() *Rest {
	return &Rest{
		httpClient:  http.Client{Timeout: time.Second * timeOutSeconds},
		headers:     make(map[string][]string),
		queryvalues: make(url.Values),
	}
}

func (client *Rest) Get() *Rest {
	client.verb = "GET"
	return client
}

func (client *Rest) Post(payload interface{}) *Rest {
	client.verb = "POST"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Put(payload interface{}) *Rest {
	client.verb = "PUT"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Delete(payload interface{}) *Rest {
	client.verb = "DELETE"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Patch(payload interface{}) *Rest {
	client.verb = "PATCH"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Head() *Rest {
	client.verb = "HEAD"
	return client
}

func (client *Rest) Option(payload interface{}) *Rest {
	client.verb = "OPTIONS"
	if payload != nil {
		client.addPayload(payload)
	}
	return client
}

func (client *Rest) Copy() *Rest {
	client.verb = "COPY"
	return client
}

func (client *Rest) Path(param string) *Rest {
	if client.baseurl == "" {
		panic("BASE URL Not Present")
	} else if strings.Contains(param, "/") {
		panic("Incorrect PATH Param")
	}
	client.uriparams = append(client.uriparams, param)
	return client
}

func (client *Rest) Base(baseurl string) *Rest {
	v, err := url.Parse(baseurl)
	if err != nil {
		panic("Url is incorrect")
	}
	client.baseurl = v.String()
	return client
}

func (client *Rest) Header(key, value string) *Rest {
	client.headers.Add(key, value)
	return client
}

func (client *Rest) addPayload(payload interface{}) {
	var b []byte
	b, _ = json.Marshal(payload)
	client.payload = bytes.NewBuffer(b)

}

func (client *Rest) Query(options ...interface{}) *Rest {
	if len(options) > 0 {
		qry, ok := options[0].(map[string]string)
		if ok {
			for k, v := range qry {
				client.queryvalues.Set(k, v)
			}
		}
	}
	return client
}

func (client *Rest) Request() (*http.Request, error) {
	//Add all URI params to baseurl
	if !strings.HasSuffix(client.baseurl, "/") {
		client.baseurl = client.baseurl + "/"
	}

	client.url = client.baseurl
	params := strings.Join(client.uriparams, "/")
	client.url = client.baseurl + params
	v, err := url.Parse(client.url)
	if err != nil {
		panic("Complete URL is not correct")
	}

	client.url = v.String()

	//Adding Query String
	var requrl url.URL
	requrl.Path = client.url
	requrl.RawQuery = client.queryvalues.Encode()
	fmt.Println(requrl.String())

	req, err := http.NewRequest(client.verb, requrl.String(), client.payload)
	if err != nil {
		panic("Request Object is not proper")
	}

	//Add headers to the Request
	for key, values := range client.headers {
		for _, value := range values {
			req.Header.Add(key, value)
		}

	}
	return req, err
}

func (client *Rest) ResponseBodyString(response *http.Response) string {
	responsedata, _ := ioutil.ReadAll(response.Body)
	return string(responsedata)
}

func (client *Rest) Send(req *http.Request, successM, failureM interface{}) (*http.Response, error) {
	response, err := client.httpClient.Do(req)

	if err != nil {
		panic("Send request Failed")
	}
	return response, err
}
<|endoftext|>


<commit_before>// Copyright 2015 The http2amqp Authors. All rights reserved.  Use of this
// source code is governed by a MIT-style license that can be found in the
// LICENSE file.

package queries_service_test

import (
	"encoding/json"
	"github.com/aleasoluciones/http2amqp/queries_service/mocks"
	"github.com/aleasoluciones/simpleamqp"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/stretchr/testify/mock"
	"time"
)

const (
	A_TOPIC = "a-topic"
)

var _ = Describe("Queries service", func() {
	It("query returns response when a response for the query has been received", func() {
		// eventPublisher.publish A_TPIC, values
		// waits for a respons
		amqpResponses := make(chan simpleamqp.AmqpMessage)
		amqpConsumer := new(mocks.AMQPConsumer)
		amqpConsumer.On("Receive", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(amqpResponses)

		go func() {
			time.Sleep(1 * time.Second)
			amqpResponses <- newAmqpResponse([]string{"foo", "bar"})
		}()

		queriesService := NewQueriesService(amqpConsumer)
		response := queriesService.Query(A_TOPIC, map[string]interface{}{"q": "foo"})

		Expect(response.Content).To(ConsistOf("foo", "bar"))
	})
})

func NewQueriesService(amqpConsumer simpleamqp.AMQPConsumer) QueriesService {
	return &queriesServiceT{amqpConsumer: amqpConsumer}
}

type QueriesService interface {
	Query(topic string, criteria map[string]interface{}) queryResponse
}

type queriesServiceT struct {
	amqpConsumer simpleamqp.AMQPConsumer
}

func (service *queriesServiceT) Query(topic string, criteria map[string]interface{}) queryResponse {
	amqpResponses := service.amqpConsumer.Receive("foo exchange", []string{"foo routing key"}, " no se", simpleamqp.QueueOptions{}, 10)
	amqpResponse := <-amqpResponses
	var content interface{}
	_ = json.Unmarshal([]byte(amqpResponse.Body), &content)
	return queryResponse{Content: content}
}

type queryResponse struct {
	Content interface{}
}

func newAmqpResponse(response interface{}) simpleamqp.AmqpMessage {
	serialized, _ := json.Marshal(response)
	return simpleamqp.AmqpMessage{Body: string(serialized)}
}
<commit_msg>Test timeout error<commit_after>// Copyright 2015 The http2amqp Authors. All rights reserved.  Use of this
// source code is governed by a MIT-style license that can be found in the
// LICENSE file.

package queries_service_test

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/aleasoluciones/http2amqp/queries_service/mocks"
	"github.com/aleasoluciones/simpleamqp"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/stretchr/testify/mock"
)

const (
	A_TOPIC         = "a-topic"
	A_QUERY_TIMEOUT = 10 * time.Millisecond
)

var _ = Describe("Queries service", func() {
	Describe("Query", func() {
		var (
			amqpResponses  chan simpleamqp.AmqpMessage
			queriesService QueriesService
		)

		BeforeEach(func() {
			amqpResponses = make(chan simpleamqp.AmqpMessage)
			amqpConsumer := new(mocks.AMQPConsumer)
			amqpConsumer.On("Receive", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(amqpResponses)

			queriesService = NewQueriesService(amqpConsumer, A_QUERY_TIMEOUT)
		})

		It("returns response when a response for the query has been received", func() {
			go func() {
				time.Sleep(A_QUERY_TIMEOUT / time.Duration(2))
				amqpResponses <- newAmqpResponse([]string{"foo", "bar"})
			}()

			response := queriesService.Query(A_TOPIC, map[string]interface{}{"q": "foo"})

			Expect(response.Error).NotTo(HaveOccurred())
			Expect(response.Content).To(ConsistOf("foo", "bar"))
		})

		It("returns timeout error when no response for the query has been received in time", func() {
			go func() {
				time.Sleep(A_QUERY_TIMEOUT * time.Duration(2))
				amqpResponses <- newAmqpResponse([]string{"foo", "bar"})
			}()

			response := queriesService.Query(A_TOPIC, map[string]interface{}{"q": "foo"})

			Expect(response.Error).To(MatchError("Timeout"))
			Expect(response.Content).To(BeNil())
		})
	})
})

func NewQueriesService(amqpConsumer simpleamqp.AMQPConsumer, timeout time.Duration) QueriesService {
	return &queriesServiceT{
		amqpConsumer: amqpConsumer,
		queryTimeout: timeout,
	}
}

type QueriesService interface {
	Query(topic string, criteria map[string]interface{}) queryResponse
}

type queriesServiceT struct {
	amqpConsumer simpleamqp.AMQPConsumer
	queryTimeout time.Duration
}

func (service *queriesServiceT) Query(topic string, criteria map[string]interface{}) queryResponse {
	var content interface{}
	var err error

	amqpResponses := service.amqpConsumer.Receive("foo exchange", []string{"foo routing key"}, " no se", simpleamqp.QueueOptions{}, 10)

	afterTimeoutTicker := time.NewTicker(service.queryTimeout)
	defer afterTimeoutTicker.Stop()
	afterTimeout := afterTimeoutTicker.C

	select {
	case amqpResponse := <-amqpResponses:
		_ = json.Unmarshal([]byte(amqpResponse.Body), &content)
	case <-afterTimeout:
		err = errors.New("Timeout")
	}

	return queryResponse{Content: content, Error: err}
}

type queryResponse struct {
	Content interface{}
	Error   error
}

func newAmqpResponse(response interface{}) simpleamqp.AmqpMessage {
	serialized, _ := json.Marshal(response)
	return simpleamqp.AmqpMessage{Body: string(serialized)}
}
<|endoftext|>