<commit_before>package kapacitor

import (
	"sync"

	client "github.com/influxdata/kapacitor/client/v1"
)

const ListTaskWorkers = 4

// ensure PaginatingKapaClient is a KapaClient
var _ KapaClient = &PaginatingKapaClient{}

// PaginatingKapaClient is a Kapacitor client that automatically navigates
// through Kapacitor's pagination to fetch all results
type PaginatingKapaClient struct {
	KapaClient
	FetchRate int // specifies the number of elements to fetch from Kapacitor at a time
}

// ListTasks lists all available tasks from Kapacitor, navigating pagination as
// it fetches them
func (p *PaginatingKapaClient) ListTasks(opts *client.ListTasksOptions) ([]client.Task, error) {
	// only trigger auto-pagination with Offset=0 and Limit=0
	if opts.Limit != 0 || opts.Offset != 0 {
		return p.KapaClient.ListTasks(opts)
	}

	allTasks := []client.Task{}

	optChan := make(chan client.ListTasksOptions)
	taskChan := make(chan []client.Task, ListTaskWorkers)
	done := make(chan struct{})

	var once sync.Once

	doneCloser := func() {
		close(done)
	}

	go func() {
		opts := &client.ListTasksOptions{
			// copy existing fields
			TaskOptions: opts.TaskOptions,
			Pattern:     opts.Pattern,
			Fields:      opts.Fields,

			// we take control of these two in the loop below
			Limit:  p.FetchRate,
			Offset: 0,
		}

		for {
			select {
			case <-done:
				close(optChan)
				return
			case optChan <- *opts:
				// nop
			}
			opts.Offset = p.FetchRate + opts.Offset
		}
	}()

	var wg sync.WaitGroup

	wg.Add(ListTaskWorkers)
	for i := 0; i < ListTaskWorkers; i++ {
		go func() {
			defer wg.Done()
			for opt := range optChan {
				resp, err := p.KapaClient.ListTasks(&opt)
				if err != nil {
					return
				}

				// break and stop all workers if we're done
				if len(resp) == 0 {
					once.Do(doneCloser)
					return
				}

				// handoff tasks to consumer
				taskChan <- resp
			}
		}()
	}

	var taskAsm sync.WaitGroup
	taskAsm.Add(1)
	go func() {
		for task := range taskChan {
			allTasks = append(allTasks, task...)
		}
		taskAsm.Done()
	}()

	wg.Wait()
	close(taskChan)
	taskAsm.Wait()

	return allTasks, nil
}
<commit_msg>Extract option generator and task fetcher<commit_after>package kapacitor

import (
	"sync"

	client "github.com/influxdata/kapacitor/client/v1"
)

const ListTaskWorkers = 4

// ensure PaginatingKapaClient is a KapaClient
var _ KapaClient = &PaginatingKapaClient{}

// PaginatingKapaClient is a Kapacitor client that automatically navigates
// through Kapacitor's pagination to fetch all results
type PaginatingKapaClient struct {
	KapaClient
	FetchRate int // specifies the number of elements to fetch from Kapacitor at a time
}

// ListTasks lists all available tasks from Kapacitor, navigating pagination as
// it fetches them
func (p *PaginatingKapaClient) ListTasks(opts *client.ListTasksOptions) ([]client.Task, error) {
	// only trigger auto-pagination with Offset=0 and Limit=0
	if opts.Limit != 0 || opts.Offset != 0 {
		return p.KapaClient.ListTasks(opts)
	}

	allTasks := []client.Task{}

	optChan := make(chan client.ListTasksOptions)
	taskChan := make(chan []client.Task, ListTaskWorkers)
	done := make(chan struct{})

	var once sync.Once

	go p.generateKapacitorOptions(optChan, *opts, done)

	var wg sync.WaitGroup

	wg.Add(ListTaskWorkers)
	for i := 0; i < ListTaskWorkers; i++ {
		go p.fetchFromKapacitor(optChan, &wg, &once, taskChan, done)
	}

	var taskAsm sync.WaitGroup
	taskAsm.Add(1)
	go func() {
		for task := range taskChan {
			allTasks = append(allTasks, task...)
		}
		taskAsm.Done()
	}()

	wg.Wait()
	close(taskChan)
	taskAsm.Wait()

	return allTasks, nil
}

// fetchFromKapacitor fetches a set of results from a kapacitor by reading a
// set of options from the provided optChan. Fetched tasks are pushed onto the
// provided taskChan
func (p *PaginatingKapaClient) fetchFromKapacitor(optChan chan client.ListTasksOptions, wg *sync.WaitGroup, closer *sync.Once, taskChan chan []client.Task, done chan struct{}) {
	defer wg.Done()
	for opt := range optChan {
		resp, err := p.KapaClient.ListTasks(&opt)
		if err != nil {
			return
		}

		// break and stop all workers if we're done
		if len(resp) == 0 {
			closer.Do(func() {
				close(done)
			})
			return
		}

		// handoff tasks to consumer
		taskChan <- resp
	}
}

// generateKapacitorOptions creates ListTasksOptions with incrementally greater
// Limit and Offset parameters, and inserts them into the provided optChan
func (p *PaginatingKapaClient) generateKapacitorOptions(optChan chan client.ListTasksOptions, opts client.ListTasksOptions, done chan struct{}) {
	// ensure Limit and Offset start from known quantities
	opts.Limit = p.FetchRate
	opts.Offset = 0

	for {
		select {
		case <-done:
			close(optChan)
			return
		case optChan <- opts:
			// nop
		}
		opts.Offset = p.FetchRate + opts.Offset
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ipv6

import (
	"net"
	"syscall"
)

// A Conn represents a network endpoint that uses IPv6 transport.
// It allows to set basic IP-level socket options such as traffic
// class and hop limit.
type Conn struct {
	genericOpt
}

type genericOpt struct {
	net.Conn
}

func (c *genericOpt) ok() bool { return c != nil && c.Conn != nil }

// PathMTU returns a path MTU value for the destination associated
// with the endpoint.
func (c *Conn) PathMTU() (int, error) {
	if !c.genericOpt.ok() {
		return 0, syscall.EINVAL
	}
	fd, err := c.genericOpt.sysfd()
	if err != nil {
		return 0, err
	}
	return ipv6PathMTU(fd)
}

// NewConn returns a new Conn.
func NewConn(c net.Conn) *Conn {
	return &Conn{
		genericOpt: genericOpt{Conn: c},
	}
}

// A PacketConn represents a packet network endpoint that uses IPv6
// transport.  It is used to control several IP-level socket options
// including IPv6 header manipulation.  It also provides datagram
// based network I/O methods specific to the IPv6 and higher layer
// protocols such as OSPF, GRE, and UDP.
type PacketConn struct {
	genericOpt
	dgramOpt
	payloadHandler
}

type dgramOpt struct {
	net.PacketConn
}

func (c *dgramOpt) ok() bool { return c != nil && c.PacketConn != nil }

// SetControlMessage allows to receive the per packet basis IP-level
// socket options.
func (c *PacketConn) SetControlMessage(cf ControlFlags, on bool) error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	fd, err := c.payloadHandler.sysfd()
	if err != nil {
		return err
	}
	return setControlMessage(fd, &c.payloadHandler.rawOpt, cf, on)
}

// Close closes the endpoint.
func (c *PacketConn) Close() error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	return c.payloadHandler.PacketConn.Close()
}

// NewPacketConn returns a new PacketConn using c as its underlying
// transport.
func NewPacketConn(c net.PacketConn) *PacketConn {
	return &PacketConn{
		genericOpt:     genericOpt{Conn: c.(net.Conn)},
		dgramOpt:       dgramOpt{PacketConn: c},
		payloadHandler: payloadHandler{PacketConn: c},
	}
}
<commit_msg>go.net/ipv6: add deadline methods to PacketConn<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ipv6

import (
	"net"
	"syscall"
	"time"
)

// A Conn represents a network endpoint that uses IPv6 transport.
// It allows to set basic IP-level socket options such as traffic
// class and hop limit.
type Conn struct {
	genericOpt
}

type genericOpt struct {
	net.Conn
}

func (c *genericOpt) ok() bool { return c != nil && c.Conn != nil }

// PathMTU returns a path MTU value for the destination associated
// with the endpoint.
func (c *Conn) PathMTU() (int, error) {
	if !c.genericOpt.ok() {
		return 0, syscall.EINVAL
	}
	fd, err := c.genericOpt.sysfd()
	if err != nil {
		return 0, err
	}
	return ipv6PathMTU(fd)
}

// NewConn returns a new Conn.
func NewConn(c net.Conn) *Conn {
	return &Conn{
		genericOpt: genericOpt{Conn: c},
	}
}

// A PacketConn represents a packet network endpoint that uses IPv6
// transport.  It is used to control several IP-level socket options
// including IPv6 header manipulation.  It also provides datagram
// based network I/O methods specific to the IPv6 and higher layer
// protocols such as OSPF, GRE, and UDP.
type PacketConn struct {
	genericOpt
	dgramOpt
	payloadHandler
}

type dgramOpt struct {
	net.PacketConn
}

func (c *dgramOpt) ok() bool { return c != nil && c.PacketConn != nil }

// SetControlMessage allows to receive the per packet basis IP-level
// socket options.
func (c *PacketConn) SetControlMessage(cf ControlFlags, on bool) error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	fd, err := c.payloadHandler.sysfd()
	if err != nil {
		return err
	}
	return setControlMessage(fd, &c.payloadHandler.rawOpt, cf, on)
}

// SetDeadline sets the read and write deadlines associated with the
// endpoint.
func (c *PacketConn) SetDeadline(t time.Time) error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	return c.payloadHandler.SetDeadline(t)
}

// SetReadDeadline sets the read deadline associated with the
// endpoint.
func (c *PacketConn) SetReadDeadline(t time.Time) error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	return c.payloadHandler.SetReadDeadline(t)
}

// SetWriteDeadline sets the write deadline associated with the
// endpoint.
func (c *PacketConn) SetWriteDeadline(t time.Time) error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	return c.payloadHandler.SetWriteDeadline(t)
}

// Close closes the endpoint.
func (c *PacketConn) Close() error {
	if !c.payloadHandler.ok() {
		return syscall.EINVAL
	}
	return c.payloadHandler.Close()
}

// NewPacketConn returns a new PacketConn using c as its underlying
// transport.
func NewPacketConn(c net.PacketConn) *PacketConn {
	return &PacketConn{
		genericOpt:     genericOpt{Conn: c.(net.Conn)},
		dgramOpt:       dgramOpt{PacketConn: c},
		payloadHandler: payloadHandler{PacketConn: c},
	}
}
<|endoftext|>


<commit_before>/*

	stub is a library that helps generate stub code for new games

*/
package stub

import (
	"errors"
	"os"
)

type Options struct {
	//The name of the game
	Name string
}

//FileContents is the generated contents of the files to later write to the
//filesystem.
type FileContents map[string][]byte

//Generate generates FileContents for the given set of options.
func Generate(opt *Options) (FileContents, error) {
	return nil, errors.New("Not yet implemented")
}

//Save saves the given FileContents to the filesystem, creating any implied
//directories. Dir is the prefix to join with each path in FileContents; "" is
//fine. Will error if any of the files to create already exist. After saving,
//runs `go generate`.
func (f *FileContents) Save(dir string) error {
	return errors.New("Not yet implemented")
}

//InteractiveOptions renders an interactve prompt at out, in to generate an
//Options from the user. If in or out are nil, StdIn or StdOut will be used
//implicitly.
func InteractiveOptions(in, out *os.File) *Options {

	if in == nil {
		in = os.Stdin
	}

	if out == nil {
		out = os.Stdout
	}

	return nil
}
<commit_msg>Actually test for collisions in FileContents and error if so. Part of #663.<commit_after>/*

	stub is a library that helps generate stub code for new games

*/
package stub

import (
	"errors"
	"os"
	"path/filepath"
)

type Options struct {
	//The name of the game
	Name string
}

//FileContents is the generated contents of the files to later write to the
//filesystem.
type FileContents map[string][]byte

//Generate generates FileContents for the given set of options.
func Generate(opt *Options) (FileContents, error) {
	return nil, errors.New("Not yet implemented")
}

//Save saves the given FileContents to the filesystem, creating any implied
//directories. Dir is the prefix to join with each path in FileContents; "" is
//fine. Will error if any of the files to create already exist. After saving,
//runs `go generate`.
func (f FileContents) Save(dir string) error {

	for name := range f {
		path := filepath.Join(dir, name)
		if _, err := os.Stat(path); err == nil {
			return errors.New(name + " already existed; save aborted")
		}
	}

	return errors.New("Not yet implemented")
}

//InteractiveOptions renders an interactve prompt at out, in to generate an
//Options from the user. If in or out are nil, StdIn or StdOut will be used
//implicitly.
func InteractiveOptions(in, out *os.File) *Options {

	if in == nil {
		in = os.Stdin
	}

	if out == nil {
		out = os.Stdout
	}

	return nil
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	contour "github.com/mohae/contour"
	"github.com/mohae/magicnum"
	"github.com/pierrec/lz4"
)

// C is the handler for compression
func C(parms []string) (msg string, err error) {
	if len(parms) == 0 {
		return "", errors.New("compress: no compression algorithm specified")
	}
	if len(parms) == 1 {
		return "", errors.New("compress: no file specified")
	}
	switch parms[0] {
	case "lz4":
		msg, err = clz4(parms[1:])
	default:
		return "", fmt.Errorf("compress: %s is not a supported compression algorithm", parms[0])
	}
	return msg, err
}

// cOutFile creates the output filename for compression. The output filename's
// separator is normalized to the os.Separator. An error will occur if either
// the filename or the extension is an empty string.
//
// The filename will be appended with the received extension.
//
// If an output directory was specified, this will replace the file's
// directory, if it has one; otherwise it will become the file's directory.
func cOutFile(fname string, format magicnum.Format) (string, error) {
	if fname == "" {
		return "", errors.New("unable to create compression output filename: no filename received")
	}
	dir, fname := filepath.Split(fname)
	// see if there is an output dir specified; if so override the current dir info
	odir := contour.GetString(OutputDir)
	if odir != "" {
		dir = odir
	}
	// add the extension to the filename
	return filepath.Join(filepath.FromSlash(dir), fmt.Sprintf("%s%s", fname, format.Ext())), nil
}

// clz4 compresses using lz4 compression
func clz4(files []string) (string, error) {
	var errMsg string
	var errCnt int
	for _, file := range files {
		// open the file
		srcF, err := os.Open(file)
		if err != nil {
			return "", err
		}
		fname, err := cOutFile(file, magicnum.LZ4)
		if err != nil {
			srcF.Close()
			return "", err
		}
		// create the output file
		dstF, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0755)
		if err != nil {
			srcF.Close()
			return "", err
		}
		// create the lz4 writer
		lzw := lz4.NewWriter(dstF)
		_, err = io.Copy(lzw, srcF)
		if err != nil {
			// errors get counted and aggregated
			errMsg += fmt.Sprintf("\n%s", err)
			errCnt++
		}
		lzw.Close()
		srcF.Close()
		dstF.Close()
	}
	if errCnt > 0 {
		return fmt.Sprintf("%d files were processed\n%d were successfully compressed using lz4\n%d had errors\n", len(files), len(files)-errCnt, errCnt), fmt.Errorf("lz4 compression error(s): %s", errMsg)
	}
	if len(files) == 1 {
		return fmt.Sprintf("%s was successfully compressed using lz4", files[0]), nil
	}
	return fmt.Sprintf("%d files were proceseed\n^d were successfully compressed using lz4\n", len(files), len(files)-errCnt), nil

}
<commit_msg>magicnum -> magicnum/mcompress<commit_after>package app

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"

	contour "github.com/mohae/contour"
	magicnum "github.com/mohae/magicnum/mcompress"
	"github.com/pierrec/lz4"
)

// C is the handler for compression
func C(parms []string) (msg string, err error) {
	if len(parms) == 0 {
		return "", errors.New("compress: no compression algorithm specified")
	}
	if len(parms) == 1 {
		return "", errors.New("compress: no file specified")
	}
	switch parms[0] {
	case "lz4":
		msg, err = clz4(parms[1:])
	default:
		return "", fmt.Errorf("compress: %s is not a supported compression algorithm", parms[0])
	}
	return msg, err
}

// cOutFile creates the output filename for compression. The output filename's
// separator is normalized to the os.Separator. An error will occur if either
// the filename or the extension is an empty string.
//
// The filename will be appended with the received extension.
//
// If an output directory was specified, this will replace the file's
// directory, if it has one; otherwise it will become the file's directory.
func cOutFile(fname string, format magicnum.Format) (string, error) {
	if fname == "" {
		return "", errors.New("unable to create compression output filename: no filename received")
	}
	dir, fname := filepath.Split(fname)
	// see if there is an output dir specified; if so override the current dir info
	odir := contour.GetString(OutputDir)
	if odir != "" {
		dir = odir
	}
	// add the extension to the filename
	return filepath.Join(filepath.FromSlash(dir), fmt.Sprintf("%s%s", fname, format.Ext())), nil
}

// clz4 compresses using lz4 compression
func clz4(files []string) (string, error) {
	var errMsg string
	var errCnt int
	for _, file := range files {
		// open the file
		srcF, err := os.Open(file)
		if err != nil {
			return "", err
		}
		fname, err := cOutFile(file, magicnum.LZ4)
		if err != nil {
			srcF.Close()
			return "", err
		}
		// create the output file
		dstF, err := os.OpenFile(fname, os.O_CREATE|os.O_WRONLY, 0755)
		if err != nil {
			srcF.Close()
			return "", err
		}
		// create the lz4 writer
		lzw := lz4.NewWriter(dstF)
		_, err = io.Copy(lzw, srcF)
		if err != nil {
			// errors get counted and aggregated
			errMsg += fmt.Sprintf("\n%s", err)
			errCnt++
		}
		lzw.Close()
		srcF.Close()
		dstF.Close()
	}
	if errCnt > 0 {
		return fmt.Sprintf("%d files were processed\n%d were successfully compressed using lz4\n%d had errors\n", len(files), len(files)-errCnt, errCnt), fmt.Errorf("lz4 compression error(s): %s", errMsg)
	}
	if len(files) == 1 {
		return fmt.Sprintf("%s was successfully compressed using lz4", files[0]), nil
	}
	return fmt.Sprintf("%d files were proceseed\n^d were successfully compressed using lz4\n", len(files), len(files)-errCnt), nil

}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithm"

	"github.com/golang/glog"
)

var (
	_ = SystemModeler(&FakeModeler{})
	_ = SystemModeler(&SimpleModeler{})
)

// ExtendedPodLister: SimpleModeler needs to be able to check for a pod's
// existance in addition to listing the pods.
type ExtendedPodLister interface {
	algorithm.PodLister
	Exists(pod *api.Pod) (bool, error)
}

// actionLocker implements lockedAction (so the fake and SimpleModeler can both
// use it)
type actionLocker struct {
	sync.Mutex
}

// LockedAction serializes calls of whatever is passed as 'do'.
func (a *actionLocker) LockedAction(do func()) {
	a.Lock()
	defer a.Unlock()
	do()
}

// FakeModeler implements the SystemModeler interface.
type FakeModeler struct {
	AssumePodFunc      func(pod *api.Pod)
	ForgetPodFunc      func(pod *api.Pod)
	ForgetPodByKeyFunc func(key string)
	actionLocker
}

// AssumePod calls the function variable if it is not nil.
func (f *FakeModeler) AssumePod(pod *api.Pod) {
	if f.AssumePodFunc != nil {
		f.AssumePodFunc(pod)
	}
}

// ForgetPod calls the function variable if it is not nil.
func (f *FakeModeler) ForgetPod(pod *api.Pod) {
	if f.ForgetPodFunc != nil {
		f.ForgetPodFunc(pod)
	}
}

// ForgetPodByKey calls the function variable if it is not nil.
func (f *FakeModeler) ForgetPodByKey(key string) {
	if f.ForgetPodFunc != nil {
		f.ForgetPodByKeyFunc(key)
	}
}

// SimpleModeler implements the SystemModeler interface with a timed pod cache.
type SimpleModeler struct {
	queuedPods    ExtendedPodLister
	scheduledPods ExtendedPodLister

	// assumedPods holds the pods that we think we've scheduled, but that
	// haven't yet shown up in the scheduledPods variable.
	// TODO: periodically clear this.
	assumedPods *cache.StoreToPodLister

	actionLocker
}

// NewSimpleModeler returns a new SimpleModeler.
//   queuedPods: a PodLister that will return pods that have not been scheduled yet.
//   scheduledPods: a PodLister that will return pods that we know for sure have been scheduled.
func NewSimpleModeler(queuedPods, scheduledPods ExtendedPodLister) *SimpleModeler {
	return &SimpleModeler{
		queuedPods:    queuedPods,
		scheduledPods: scheduledPods,
		assumedPods: &cache.StoreToPodLister{
			cache.NewTTLStore(cache.MetaNamespaceKeyFunc, 30*time.Second),
		},
	}
}

func (s *SimpleModeler) AssumePod(pod *api.Pod) {
	s.assumedPods.Add(pod)
}

func (s *SimpleModeler) ForgetPod(pod *api.Pod) {
	s.assumedPods.Delete(pod)
}

func (s *SimpleModeler) ForgetPodByKey(key string) {
	s.assumedPods.Delete(cache.ExplicitKey(key))
}

// Extract names for readable logging.
func podNames(pods []*api.Pod) []string {
	out := make([]string, len(pods))
	for i := range pods {
		out[i] = fmt.Sprintf("'%v/%v (%v)'", pods[i].Namespace, pods[i].Name, pods[i].UID)
	}
	return out
}

func (s *SimpleModeler) listPods(selector labels.Selector) (pods []*api.Pod, err error) {
	assumed, err := s.assumedPods.List(selector)
	if err != nil {
		return nil, err
	}
	// Since the assumed list will be short, just check every one.
	// Goal here is to stop making assumptions about a pod once it shows
	// up in one of these other lists.
	for _, pod := range assumed {
		qExist, err := s.queuedPods.Exists(pod)
		if err != nil {
			return nil, err
		}
		if qExist {
			s.assumedPods.Store.Delete(pod)
			continue
		}
		sExist, err := s.scheduledPods.Exists(pod)
		if err != nil {
			return nil, err
		}
		if sExist {
			s.assumedPods.Store.Delete(pod)
			continue
		}
	}

	scheduled, err := s.scheduledPods.List(selector)
	if err != nil {
		return nil, err
	}
	// Listing purges the ttl cache and re-gets, in case we deleted any entries.
	assumed, err = s.assumedPods.List(selector)
	if err != nil {
		return nil, err
	}
	if len(assumed) == 0 {
		return scheduled, nil
	}
	glog.V(2).Infof(
		"listing pods: [%v] assumed to exist in addition to %v known pods.",
		strings.Join(podNames(assumed), ","),
		len(scheduled),
	)
	return append(scheduled, assumed...), nil
}

// PodLister returns a PodLister that will list pods that we think we have scheduled in
// addition to pods that we know have been scheduled.
func (s *SimpleModeler) PodLister() algorithm.PodLister {
	return simpleModelerPods{s}
}

// simpleModelerPods is an adaptor so that SimpleModeler can be a PodLister.
type simpleModelerPods struct {
	simpleModeler *SimpleModeler
}

// List returns pods known and assumed to exist.
func (s simpleModelerPods) List(selector labels.Selector) (pods []*api.Pod, err error) {
	return s.simpleModeler.listPods(selector)
}
<commit_msg>Fix scheduler race<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package scheduler

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/cache"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/plugin/pkg/scheduler/algorithm"

	"github.com/golang/glog"
)

var (
	_ = SystemModeler(&FakeModeler{})
	_ = SystemModeler(&SimpleModeler{})
)

// ExtendedPodLister: SimpleModeler needs to be able to check for a pod's
// existance in addition to listing the pods.
type ExtendedPodLister interface {
	algorithm.PodLister
	Exists(pod *api.Pod) (bool, error)
}

// actionLocker implements lockedAction (so the fake and SimpleModeler can both
// use it)
type actionLocker struct {
	sync.Mutex
}

// LockedAction serializes calls of whatever is passed as 'do'.
func (a *actionLocker) LockedAction(do func()) {
	a.Lock()
	defer a.Unlock()
	do()
}

// FakeModeler implements the SystemModeler interface.
type FakeModeler struct {
	AssumePodFunc      func(pod *api.Pod)
	ForgetPodFunc      func(pod *api.Pod)
	ForgetPodByKeyFunc func(key string)
	actionLocker
}

// AssumePod calls the function variable if it is not nil.
func (f *FakeModeler) AssumePod(pod *api.Pod) {
	if f.AssumePodFunc != nil {
		f.AssumePodFunc(pod)
	}
}

// ForgetPod calls the function variable if it is not nil.
func (f *FakeModeler) ForgetPod(pod *api.Pod) {
	if f.ForgetPodFunc != nil {
		f.ForgetPodFunc(pod)
	}
}

// ForgetPodByKey calls the function variable if it is not nil.
func (f *FakeModeler) ForgetPodByKey(key string) {
	if f.ForgetPodFunc != nil {
		f.ForgetPodByKeyFunc(key)
	}
}

// SimpleModeler implements the SystemModeler interface with a timed pod cache.
type SimpleModeler struct {
	queuedPods    ExtendedPodLister
	scheduledPods ExtendedPodLister

	// assumedPods holds the pods that we think we've scheduled, but that
	// haven't yet shown up in the scheduledPods variable.
	// TODO: periodically clear this.
	assumedPods *cache.StoreToPodLister

	actionLocker
}

// NewSimpleModeler returns a new SimpleModeler.
//   queuedPods: a PodLister that will return pods that have not been scheduled yet.
//   scheduledPods: a PodLister that will return pods that we know for sure have been scheduled.
func NewSimpleModeler(queuedPods, scheduledPods ExtendedPodLister) *SimpleModeler {
	return &SimpleModeler{
		queuedPods:    queuedPods,
		scheduledPods: scheduledPods,
		assumedPods: &cache.StoreToPodLister{
			cache.NewTTLStore(cache.MetaNamespaceKeyFunc, 30*time.Second),
		},
	}
}

func (s *SimpleModeler) AssumePod(pod *api.Pod) {
	s.assumedPods.Add(pod)
}

func (s *SimpleModeler) ForgetPod(pod *api.Pod) {
	s.assumedPods.Delete(pod)
}

func (s *SimpleModeler) ForgetPodByKey(key string) {
	s.assumedPods.Delete(cache.ExplicitKey(key))
}

// Extract names for readable logging.
func podNames(pods []*api.Pod) []string {
	out := make([]string, len(pods))
	for i := range pods {
		out[i] = fmt.Sprintf("'%v/%v (%v)'", pods[i].Namespace, pods[i].Name, pods[i].UID)
	}
	return out
}

func (s *SimpleModeler) listPods(selector labels.Selector) (pods []*api.Pod, err error) {
	assumed, err := s.assumedPods.List(selector)
	if err != nil {
		return nil, err
	}
	// Since the assumed list will be short, just check every one.
	// Goal here is to stop making assumptions about a pod once it shows
	// up in one of these other lists.
	for _, pod := range assumed {
		qExist, err := s.queuedPods.Exists(pod)
		if err != nil {
			return nil, err
		}
		if qExist {
			s.assumedPods.Store.Delete(pod)
			continue
		}
		sExist, err := s.scheduledPods.Exists(pod)
		if err != nil {
			return nil, err
		}
		if sExist {
			s.assumedPods.Store.Delete(pod)
			continue
		}
	}

	scheduled, err := s.scheduledPods.List(selector)
	if err != nil {
		return nil, err
	}
	// Listing purges the ttl cache and re-gets, in case we deleted any entries.
	assumed, err = s.assumedPods.List(selector)
	if err != nil {
		return nil, err
	}
	if len(assumed) == 0 {
		return scheduled, nil
	}
	glog.V(2).Infof(
		"listing pods: [%v] assumed to exist in addition to %v known pods.",
		strings.Join(podNames(assumed), ","),
		len(scheduled),
	)
	return append(scheduled, assumed...), nil
}

// PodLister returns a PodLister that will list pods that we think we have scheduled in
// addition to pods that we know have been scheduled.
func (s *SimpleModeler) PodLister() algorithm.PodLister {
	return simpleModelerPods{s}
}

// simpleModelerPods is an adaptor so that SimpleModeler can be a PodLister.
type simpleModelerPods struct {
	simpleModeler *SimpleModeler
}

// List returns pods known and assumed to exist.
func (s simpleModelerPods) List(selector labels.Selector) (pods []*api.Pod, err error) {
	s.simpleModeler.LockedAction(
		func() { pods, err = s.simpleModeler.listPods(selector) })
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"time"

	"github.com/docker/docker/pkg/integration/checker"
	"github.com/docker/docker/pkg/stringid"
	"github.com/go-check/check"
)

func (s *DockerSuite) TestImagesEnsureImageIsListed(c *check.C) {
	testRequires(c, DaemonIsLinux)
	out, _ := dockerCmd(c, "images")
	if !strings.Contains(out, "busybox") {
		c.Fatal("images should've listed busybox")
	}
}

func (s *DockerSuite) TestImagesEnsureImageWithTagIsListed(c *check.C) {
	testRequires(c, DaemonIsLinux)
	_, err := buildImage("imagewithtag:v1",
		`FROM scratch
		MAINTAINER dockerio1`, true)
	c.Assert(err, check.IsNil)

	_, err = buildImage("imagewithtag:v2",
		`FROM scratch
		MAINTAINER dockerio1`, true)
	c.Assert(err, check.IsNil)

	out, _ := dockerCmd(c, "images", "imagewithtag:v1")

	if !strings.Contains(out, "imagewithtag") || !strings.Contains(out, "v1") || strings.Contains(out, "v2") {
		c.Fatal("images should've listed imagewithtag:v1 and not imagewithtag:v2")
	}

	out, _ = dockerCmd(c, "images", "imagewithtag")

	if !strings.Contains(out, "imagewithtag") || !strings.Contains(out, "v1") || !strings.Contains(out, "v2") {
		c.Fatal("images should've listed imagewithtag:v1 and imagewithtag:v2")
	}
}

func (s *DockerSuite) TestImagesEnsureImageWithBadTagIsNotListed(c *check.C) {
	out, _ := dockerCmd(c, "images", "busybox:nonexistent")

	if strings.Contains(out, "busybox") {
		c.Fatal("images should not have listed busybox")
	}
}

func (s *DockerSuite) TestImagesOrderedByCreationDate(c *check.C) {
	testRequires(c, DaemonIsLinux)
	id1, err := buildImage("order:test_a",
		`FROM scratch
		MAINTAINER dockerio1`, true)
	if err != nil {
		c.Fatal(err)
	}
	time.Sleep(1 * time.Second)
	id2, err := buildImage("order:test_c",
		`FROM scratch
		MAINTAINER dockerio2`, true)
	if err != nil {
		c.Fatal(err)
	}
	time.Sleep(1 * time.Second)
	id3, err := buildImage("order:test_b",
		`FROM scratch
		MAINTAINER dockerio3`, true)
	if err != nil {
		c.Fatal(err)
	}

	out, _ := dockerCmd(c, "images", "-q", "--no-trunc")
	imgs := strings.Split(out, "\n")
	if imgs[0] != id3 {
		c.Fatalf("First image must be %s, got %s", id3, imgs[0])
	}
	if imgs[1] != id2 {
		c.Fatalf("Second image must be %s, got %s", id2, imgs[1])
	}
	if imgs[2] != id1 {
		c.Fatalf("Third image must be %s, got %s", id1, imgs[2])
	}
}

func (s *DockerSuite) TestImagesErrorWithInvalidFilterNameTest(c *check.C) {
	out, _, err := dockerCmdWithError("images", "-f", "FOO=123")
	if err == nil || !strings.Contains(out, "Invalid filter") {
		c.Fatalf("error should occur when listing images with invalid filter name FOO, %s", out)
	}
}

func (s *DockerSuite) TestImagesFilterLabel(c *check.C) {
	testRequires(c, DaemonIsLinux)
	imageName1 := "images_filter_test1"
	imageName2 := "images_filter_test2"
	imageName3 := "images_filter_test3"
	image1ID, err := buildImage(imageName1,
		`FROM scratch
		 LABEL match me`, true)
	c.Assert(err, check.IsNil)

	image2ID, err := buildImage(imageName2,
		`FROM scratch
		 LABEL match="me too"`, true)
	c.Assert(err, check.IsNil)

	image3ID, err := buildImage(imageName3,
		`FROM scratch
		 LABEL nomatch me`, true)
	c.Assert(err, check.IsNil)

	out, _ := dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=match")
	out = strings.TrimSpace(out)
	c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image1ID))
	c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image2ID))
	c.Assert(out, check.Not(check.Matches), fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image3ID))

	out, _ = dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=match=me too")
	out = strings.TrimSpace(out)
	c.Assert(out, check.Equals, image2ID)
}

// Regression : #15659
func (s *DockerSuite) TestImagesFilterLabelWithCommit(c *check.C) {
	// Create a container
	dockerCmd(c, "run", "--name", "bar", "busybox", "/bin/sh")
	// Commit with labels "using changes"
	out, _ := dockerCmd(c, "commit", "-c", "LABEL foo.version=1.0.0-1", "-c", "LABEL foo.name=bar", "-c", "LABEL foo.author=starlord", "bar", "bar:1.0.0-1")
	imageID := strings.TrimSpace(out)

	out, _ = dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=foo.version=1.0.0-1")
	out = strings.TrimSpace(out)
	c.Assert(out, check.Equals, imageID)
}

func (s *DockerSuite) TestImagesFilterSpaceTrimCase(c *check.C) {
	testRequires(c, DaemonIsLinux)
	imageName := "images_filter_test"
	buildImage(imageName,
		`FROM scratch
		 RUN touch /test/foo
		 RUN touch /test/bar
		 RUN touch /test/baz`, true)

	filters := []string{
		"dangling=true",
		"Dangling=true",
		" dangling=true",
		"dangling=true ",
		"dangling = true",
	}

	imageListings := make([][]string, 5, 5)
	for idx, filter := range filters {
		out, _ := dockerCmd(c, "images", "-q", "-f", filter)
		listing := strings.Split(out, "\n")
		sort.Strings(listing)
		imageListings[idx] = listing
	}

	for idx, listing := range imageListings {
		if idx < 4 && !reflect.DeepEqual(listing, imageListings[idx+1]) {
			for idx, errListing := range imageListings {
				fmt.Printf("out %d", idx)
				for _, image := range errListing {
					fmt.Print(image)
				}
				fmt.Print("")
			}
			c.Fatalf("All output must be the same")
		}
	}
}

func (s *DockerSuite) TestImagesEnsureDanglingImageOnlyListedOnce(c *check.C) {
	testRequires(c, DaemonIsLinux)
	// create container 1
	out, _ := dockerCmd(c, "run", "-d", "busybox", "true")
	containerID1 := strings.TrimSpace(out)

	// tag as foobox
	out, _ = dockerCmd(c, "commit", containerID1, "foobox")
	imageID := stringid.TruncateID(strings.TrimSpace(out))

	// overwrite the tag, making the previous image dangling
	dockerCmd(c, "tag", "-f", "busybox", "foobox")

	out, _ = dockerCmd(c, "images", "-q", "-f", "dangling=true")
	if e, a := 1, strings.Count(out, imageID); e != a {
		c.Fatalf("expected 1 dangling image, got %d: %s", a, out)
	}
}

func (s *DockerSuite) TestImagesWithIncorrectFilter(c *check.C) {
	out, _, err := dockerCmdWithError("images", "-f", "dangling=invalid")
	c.Assert(err, check.NotNil)
	c.Assert(out, checker.Contains, "Invalid filter")
}

func (s *DockerSuite) TestImagesEnsureOnlyHeadsImagesShown(c *check.C) {
	testRequires(c, DaemonIsLinux)

	dockerfile := `
        FROM scratch
        MAINTAINER docker
        ENV foo bar`

	head, out, err := buildImageWithOut("scratch-image", dockerfile, false)
	c.Assert(err, check.IsNil)

	// this is just the output of docker build
	// we're interested in getting the image id of the MAINTAINER instruction
	// and that's located at output, line 5, from 7 to end
	split := strings.Split(out, "\n")
	intermediate := strings.TrimSpace(split[5][7:])

	out, _ = dockerCmd(c, "images")
	if strings.Contains(out, intermediate) {
		c.Fatalf("images shouldn't show non-heads images, got %s in %s", intermediate, out)
	}
	if !strings.Contains(out, head[:12]) {
		c.Fatalf("images should contain final built images, want %s in out, got %s", head[:12], out)
	}
}

func (s *DockerSuite) TestImagesEnsureImagesFromScratchShown(c *check.C) {
	testRequires(c, DaemonIsLinux)

	dockerfile := `
        FROM scratch
        MAINTAINER docker`

	id, _, err := buildImageWithOut("scratch-image", dockerfile, false)
	c.Assert(err, check.IsNil)

	out, _ := dockerCmd(c, "images")
	if !strings.Contains(out, id[:12]) {
		c.Fatalf("images should contain images built from scratch (e.g. %s), got %s", id[:12], out)
	}
}
<commit_msg>Use of checkers on docker_cli_images_test.go<commit_after>package main

import (
	"fmt"
	"reflect"
	"sort"
	"strings"
	"time"

	"github.com/docker/docker/pkg/integration/checker"
	"github.com/docker/docker/pkg/stringid"
	"github.com/go-check/check"
)

func (s *DockerSuite) TestImagesEnsureImageIsListed(c *check.C) {
	testRequires(c, DaemonIsLinux)
	imagesOut, _ := dockerCmd(c, "images")
	c.Assert(imagesOut, checker.Contains, "busybox")
}

func (s *DockerSuite) TestImagesEnsureImageWithTagIsListed(c *check.C) {
	testRequires(c, DaemonIsLinux)
	_, err := buildImage("imagewithtag:v1",
		`FROM scratch
		MAINTAINER dockerio1`, true)
	c.Assert(err, check.IsNil)

	_, err = buildImage("imagewithtag:v2",
		`FROM scratch
		MAINTAINER dockerio1`, true)
	c.Assert(err, check.IsNil)

	imagesOut, _ := dockerCmd(c, "images", "imagewithtag:v1")
	c.Assert(imagesOut, checker.Contains, "imagewithtag")
	c.Assert(imagesOut, checker.Contains, "v1")
	c.Assert(imagesOut, checker.Not(checker.Contains), "v2")

	imagesOut, _ = dockerCmd(c, "images", "imagewithtag")
	c.Assert(imagesOut, checker.Contains, "imagewithtag")
	c.Assert(imagesOut, checker.Contains, "v1")
	c.Assert(imagesOut, checker.Contains, "v2")
}

func (s *DockerSuite) TestImagesEnsureImageWithBadTagIsNotListed(c *check.C) {
	imagesOut, _ := dockerCmd(c, "images", "busybox:nonexistent")
	c.Assert(imagesOut, checker.Not(checker.Contains), "busybox")
}

func (s *DockerSuite) TestImagesOrderedByCreationDate(c *check.C) {
	testRequires(c, DaemonIsLinux)
	id1, err := buildImage("order:test_a",
		`FROM scratch
		MAINTAINER dockerio1`, true)
	c.Assert(err, checker.IsNil)
	time.Sleep(1 * time.Second)
	id2, err := buildImage("order:test_c",
		`FROM scratch
		MAINTAINER dockerio2`, true)
	c.Assert(err, checker.IsNil)
	time.Sleep(1 * time.Second)
	id3, err := buildImage("order:test_b",
		`FROM scratch
		MAINTAINER dockerio3`, true)
	c.Assert(err, checker.IsNil)

	out, _ := dockerCmd(c, "images", "-q", "--no-trunc")
	imgs := strings.Split(out, "\n")
	c.Assert(imgs[0], checker.Equals, id3, check.Commentf("First image must be %s, got %s", id3, imgs[0]))
	c.Assert(imgs[1], checker.Equals, id2, check.Commentf("First image must be %s, got %s", id2, imgs[1]))
	c.Assert(imgs[2], checker.Equals, id1, check.Commentf("First image must be %s, got %s", id1, imgs[2]))
}

func (s *DockerSuite) TestImagesErrorWithInvalidFilterNameTest(c *check.C) {
	out, _, err := dockerCmdWithError("images", "-f", "FOO=123")
	c.Assert(err, checker.NotNil)
	c.Assert(out, checker.Contains, "Invalid filter")
}

func (s *DockerSuite) TestImagesFilterLabel(c *check.C) {
	testRequires(c, DaemonIsLinux)
	imageName1 := "images_filter_test1"
	imageName2 := "images_filter_test2"
	imageName3 := "images_filter_test3"
	image1ID, err := buildImage(imageName1,
		`FROM scratch
		 LABEL match me`, true)
	c.Assert(err, check.IsNil)

	image2ID, err := buildImage(imageName2,
		`FROM scratch
		 LABEL match="me too"`, true)
	c.Assert(err, check.IsNil)

	image3ID, err := buildImage(imageName3,
		`FROM scratch
		 LABEL nomatch me`, true)
	c.Assert(err, check.IsNil)

	out, _ := dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=match")
	out = strings.TrimSpace(out)
	c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image1ID))
	c.Assert(out, check.Matches, fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image2ID))
	c.Assert(out, check.Not(check.Matches), fmt.Sprintf("[\\s\\w]*%s[\\s\\w]*", image3ID))

	out, _ = dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=match=me too")
	out = strings.TrimSpace(out)
	c.Assert(out, check.Equals, image2ID)
}

// Regression : #15659
func (s *DockerSuite) TestImagesFilterLabelWithCommit(c *check.C) {
	// Create a container
	dockerCmd(c, "run", "--name", "bar", "busybox", "/bin/sh")
	// Commit with labels "using changes"
	out, _ := dockerCmd(c, "commit", "-c", "LABEL foo.version=1.0.0-1", "-c", "LABEL foo.name=bar", "-c", "LABEL foo.author=starlord", "bar", "bar:1.0.0-1")
	imageID := strings.TrimSpace(out)

	out, _ = dockerCmd(c, "images", "--no-trunc", "-q", "-f", "label=foo.version=1.0.0-1")
	out = strings.TrimSpace(out)
	c.Assert(out, check.Equals, imageID)
}

func (s *DockerSuite) TestImagesFilterSpaceTrimCase(c *check.C) {
	testRequires(c, DaemonIsLinux)
	imageName := "images_filter_test"
	buildImage(imageName,
		`FROM scratch
		 RUN touch /test/foo
		 RUN touch /test/bar
		 RUN touch /test/baz`, true)

	filters := []string{
		"dangling=true",
		"Dangling=true",
		" dangling=true",
		"dangling=true ",
		"dangling = true",
	}

	imageListings := make([][]string, 5, 5)
	for idx, filter := range filters {
		out, _ := dockerCmd(c, "images", "-q", "-f", filter)
		listing := strings.Split(out, "\n")
		sort.Strings(listing)
		imageListings[idx] = listing
	}

	for idx, listing := range imageListings {
		if idx < 4 && !reflect.DeepEqual(listing, imageListings[idx+1]) {
			for idx, errListing := range imageListings {
				fmt.Printf("out %d", idx)
				for _, image := range errListing {
					fmt.Print(image)
				}
				fmt.Print("")
			}
			c.Fatalf("All output must be the same")
		}
	}
}

func (s *DockerSuite) TestImagesEnsureDanglingImageOnlyListedOnce(c *check.C) {
	testRequires(c, DaemonIsLinux)
	// create container 1
	out, _ := dockerCmd(c, "run", "-d", "busybox", "true")
	containerID1 := strings.TrimSpace(out)

	// tag as foobox
	out, _ = dockerCmd(c, "commit", containerID1, "foobox")
	imageID := stringid.TruncateID(strings.TrimSpace(out))

	// overwrite the tag, making the previous image dangling
	dockerCmd(c, "tag", "-f", "busybox", "foobox")

	out, _ = dockerCmd(c, "images", "-q", "-f", "dangling=true")
	// Exect one dangling image
	c.Assert(strings.Count(out, imageID), checker.Equals, 1)
}

func (s *DockerSuite) TestImagesWithIncorrectFilter(c *check.C) {
	out, _, err := dockerCmdWithError("images", "-f", "dangling=invalid")
	c.Assert(err, check.NotNil)
	c.Assert(out, checker.Contains, "Invalid filter")
}

func (s *DockerSuite) TestImagesEnsureOnlyHeadsImagesShown(c *check.C) {
	testRequires(c, DaemonIsLinux)

	dockerfile := `
        FROM scratch
        MAINTAINER docker
        ENV foo bar`

	head, out, err := buildImageWithOut("scratch-image", dockerfile, false)
	c.Assert(err, check.IsNil)

	// this is just the output of docker build
	// we're interested in getting the image id of the MAINTAINER instruction
	// and that's located at output, line 5, from 7 to end
	split := strings.Split(out, "\n")
	intermediate := strings.TrimSpace(split[5][7:])

	out, _ = dockerCmd(c, "images")
	// images shouldn't show non-heads images
	c.Assert(out, checker.Not(checker.Contains), intermediate)
	// images should contain final built images
	c.Assert(out, checker.Contains, head[:12])
}

func (s *DockerSuite) TestImagesEnsureImagesFromScratchShown(c *check.C) {
	testRequires(c, DaemonIsLinux)

	dockerfile := `
        FROM scratch
        MAINTAINER docker`

	id, _, err := buildImageWithOut("scratch-image", dockerfile, false)
	c.Assert(err, check.IsNil)

	out, _ := dockerCmd(c, "images")
	// images should contain images built from scratch
	c.Assert(out, checker.Contains, id[:12])
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"

	"github.com/google/shenzhen-go/parts"
	"github.com/google/shenzhen-go/source"
)

// Part abstracts the implementation of a node. Concrete implementations should be
// able to be marshalled to and unmarshalled from JSON sensibly.
type Part interface {
	// AssociateEditor associates a template called "part_view" with the given template.
	AssociateEditor(*template.Template) error

	// Channels returns any channels used. Anything returned that is not a channel is ignored.
	Channels() (read, written source.StringSet)

	// Clone returns a copy of this part.
	Clone() interface{}

	// Impl returns Go source code implementing the part.
	// The head is executed, then the body is executed (# Multiplicity
	// instances of the body concurrently), then the tail (once the body/bodies
	// are finished).
	//
	// This allows cleanly closing channels for nodes with Multiplicity > 1.
	// The tail is deferred so that the body can use "return" and it is still
	// executed.
	Impl() (head, body, tail string)

	// Imports returns any extra import lines needed for the Part.
	Imports() []string

	// RenameChannel informs the part that a channel has been renamed.
	RenameChannel(from, to string)

	// TypeKey returns the "type" of part.
	TypeKey() string

	// Update sets fields in the part based on info in the given Request.
	Update(*http.Request) error
}

// PartFactory creates a part.
type PartFactory func() Part

// PartFactories translates part type strings into part factories.
var PartFactories = map[string]PartFactory{
	"Code":   func() Part { return new(parts.Code) },
	"Filter": func() Part { return new(parts.Filter) },
	// "Multiplexer": func() Part { return new(parts.Multiplexer) },
	"TextFileReader": func() Part { return new(parts.TextFileReader) },
}

// Node models a goroutine. It can be marshalled and unmarshalled to JSON sensibly.
type Node struct {
	Part

	Name         string
	Multiplicity uint
	Wait         bool
}

// ChannelsRead returns the channels read from by this node. It is a convenience
// function for the templates, which can't do multiple returns.
func (n *Node) ChannelsRead() []string {
	r, _ := n.Part.Channels()
	return r.Slice()
}

// ChannelsWritten returns the channels written to by this node. It is a convenience
// function for the templates, which can't do multiple returns.
func (n *Node) ChannelsWritten() []string {
	_, w := n.Part.Channels()
	return w.Slice()
}

// Copy returns a copy of this node, but with an empty name and a clone of the Part.
func (n *Node) Copy() *Node {
	return &Node{
		Name:         "",
		Multiplicity: n.Multiplicity,
		Wait:         n.Wait,
		Part:         n.Part.Clone().(Part),
	}
}

// ImplHead returns the Head part of the implementation.
func (n *Node) ImplHead() string {
	h, _, _ := n.Part.Impl()
	return h
}

// ImplBody returns the Body part of the implementation.
func (n *Node) ImplBody() string {
	_, b, _ := n.Part.Impl()
	return b
}

// ImplTail returns the Tail part of the implementation.
func (n *Node) ImplTail() string {
	_, _, t := n.Part.Impl()
	return t
}

func (n *Node) String() string { return n.Name }

type jsonNode struct {
	Name         string          `json:"name"`
	Wait         bool            `json:"wait"`
	Multiplicity uint            `json:"multiplicity"`
	Part         json.RawMessage `json:"part"`
	PartType     string          `json:"part_type"`
}

// MarshalJSON encodes the node and part as JSON.
func (n *Node) MarshalJSON() ([]byte, error) {
	p, err := json.Marshal(n.Part)
	if err != nil {
		return nil, err
	}
	if n.Multiplicity < 1 {
		n.Multiplicity = 1
	}
	return json.Marshal(&jsonNode{
		Part:         p,
		PartType:     n.Part.TypeKey(),
		Name:         n.Name,
		Wait:         n.Wait,
		Multiplicity: n.Multiplicity,
	})
}

// UnmarshalJSON decodes the node and part as JSON.
func (n *Node) UnmarshalJSON(j []byte) error {
	var mp jsonNode
	if err := json.Unmarshal(j, &mp); err != nil {
		return err
	}
	pf, ok := PartFactories[mp.PartType]
	if !ok {
		return fmt.Errorf("unknown part type %q", mp.PartType)
	}
	p := pf()
	if err := json.Unmarshal(mp.Part, p); err != nil {
		return err
	}
	if mp.Multiplicity < 1 {
		mp.Multiplicity = 1
	}
	n.Name = mp.Name
	n.Wait = mp.Wait
	n.Multiplicity = mp.Multiplicity
	n.Part = p
	return p.Update(nil)
}
<commit_msg>That Update is now unnecessary<commit_after>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"

	"github.com/google/shenzhen-go/parts"
	"github.com/google/shenzhen-go/source"
)

// Part abstracts the implementation of a node. Concrete implementations should be
// able to be marshalled to and unmarshalled from JSON sensibly.
type Part interface {
	// AssociateEditor associates a template called "part_view" with the given template.
	AssociateEditor(*template.Template) error

	// Channels returns any channels used. Anything returned that is not a channel is ignored.
	Channels() (read, written source.StringSet)

	// Clone returns a copy of this part.
	Clone() interface{}

	// Impl returns Go source code implementing the part.
	// The head is executed, then the body is executed (# Multiplicity
	// instances of the body concurrently), then the tail (once the body/bodies
	// are finished).
	//
	// This allows cleanly closing channels for nodes with Multiplicity > 1.
	// The tail is deferred so that the body can use "return" and it is still
	// executed.
	Impl() (head, body, tail string)

	// Imports returns any extra import lines needed for the Part.
	Imports() []string

	// RenameChannel informs the part that a channel has been renamed.
	RenameChannel(from, to string)

	// TypeKey returns the "type" of part.
	TypeKey() string

	// Update sets fields in the part based on info in the given Request.
	Update(*http.Request) error
}

// PartFactory creates a part.
type PartFactory func() Part

// PartFactories translates part type strings into part factories.
var PartFactories = map[string]PartFactory{
	"Code":   func() Part { return new(parts.Code) },
	"Filter": func() Part { return new(parts.Filter) },
	// "Multiplexer": func() Part { return new(parts.Multiplexer) },
	"TextFileReader": func() Part { return new(parts.TextFileReader) },
}

// Node models a goroutine. It can be marshalled and unmarshalled to JSON sensibly.
type Node struct {
	Part

	Name         string
	Multiplicity uint
	Wait         bool
}

// ChannelsRead returns the channels read from by this node. It is a convenience
// function for the templates, which can't do multiple returns.
func (n *Node) ChannelsRead() []string {
	r, _ := n.Part.Channels()
	return r.Slice()
}

// ChannelsWritten returns the channels written to by this node. It is a convenience
// function for the templates, which can't do multiple returns.
func (n *Node) ChannelsWritten() []string {
	_, w := n.Part.Channels()
	return w.Slice()
}

// Copy returns a copy of this node, but with an empty name and a clone of the Part.
func (n *Node) Copy() *Node {
	return &Node{
		Name:         "",
		Multiplicity: n.Multiplicity,
		Wait:         n.Wait,
		Part:         n.Part.Clone().(Part),
	}
}

// ImplHead returns the Head part of the implementation.
func (n *Node) ImplHead() string {
	h, _, _ := n.Part.Impl()
	return h
}

// ImplBody returns the Body part of the implementation.
func (n *Node) ImplBody() string {
	_, b, _ := n.Part.Impl()
	return b
}

// ImplTail returns the Tail part of the implementation.
func (n *Node) ImplTail() string {
	_, _, t := n.Part.Impl()
	return t
}

func (n *Node) String() string { return n.Name }

type jsonNode struct {
	Name         string          `json:"name"`
	Wait         bool            `json:"wait"`
	Multiplicity uint            `json:"multiplicity"`
	Part         json.RawMessage `json:"part"`
	PartType     string          `json:"part_type"`
}

// MarshalJSON encodes the node and part as JSON.
func (n *Node) MarshalJSON() ([]byte, error) {
	p, err := json.Marshal(n.Part)
	if err != nil {
		return nil, err
	}
	if n.Multiplicity < 1 {
		n.Multiplicity = 1
	}
	return json.Marshal(&jsonNode{
		Part:         p,
		PartType:     n.Part.TypeKey(),
		Name:         n.Name,
		Wait:         n.Wait,
		Multiplicity: n.Multiplicity,
	})
}

// UnmarshalJSON decodes the node and part as JSON.
func (n *Node) UnmarshalJSON(j []byte) error {
	var mp jsonNode
	if err := json.Unmarshal(j, &mp); err != nil {
		return err
	}
	pf, ok := PartFactories[mp.PartType]
	if !ok {
		return fmt.Errorf("unknown part type %q", mp.PartType)
	}
	p := pf()
	if err := json.Unmarshal(mp.Part, p); err != nil {
		return err
	}
	if mp.Multiplicity < 1 {
		mp.Multiplicity = 1
	}
	n.Name = mp.Name
	n.Wait = mp.Wait
	n.Multiplicity = mp.Multiplicity
	n.Part = p
	return nil
}
<|endoftext|>


<commit_before>package queue_test

import (
	"fmt"
	"testing"
	"time"

	"github.com/slok/ragnarok/client/util/queue"
	"github.com/stretchr/testify/assert"
)

func TestSimpleQueueBlocking(t *testing.T) {
	tests := []struct {
		name string
		item string
	}{
		{
			name: "Getting an item on an empty queue should block the caller until a push operation is made on that queue.",
			item: "this is a blocking test.",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)
			q := queue.NewSimpleQueue()

			// Caller1 pop in a goroutine.
			c1 := make(chan interface{})
			go func() {
				res, _ := q.Pop()
				c1 <- res
			}()

			// Caller2 pop in a goroutine.
			c2 := make(chan interface{})
			go func() {
				res, _ := q.Pop()
				c2 <- res
			}()

			// Wait until we have items on the queue.
			select {
			case <-time.After(10 * time.Millisecond):
			case <-c1:
				assert.Fail("pop should get blocked, it didn't")
			case <-c2:
				assert.Fail("pop should get blocked, it didn't")
			}

			// Add an item & wait until we have items on the queue.
			var gotItem1, gotItem2 interface{}
			err := q.Push(test.item)
			assert.NoError(err)
			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("pop shouldn't get blocked, we had push an item")
			case gotItem1 = <-c1:
			case gotItem2 = <-c2:
			}

			// Add a 2nd item & wait until we have items on the queue.
			err = q.Push(test.item)
			assert.NoError(err)

			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("pop shouldn't get blocked, we had push an item")
			case gotItem1 = <-c1:
			case gotItem2 = <-c2:
			}

			// At this moment both should have the items.
			assert.Equal(test.item, gotItem1.(string))
			assert.Equal(test.item, gotItem2.(string))
		})
	}
}

func TestSimpleQueueShutDown(t *testing.T) {
	tests := []struct {
		name string
	}{
		{
			name: "Shutting down the queue when there are blocking pops should unblock everyon.",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)
			q := queue.NewSimpleQueue()

			// Caller1 pop in a goroutine.
			c1 := make(chan bool)
			go func() {
				_, sd := q.Pop()
				c1 <- sd
			}()

			// Caller2 pop in a goroutine.
			c2 := make(chan bool)
			go func() {
				_, sd := q.Pop()
				c2 <- sd
			}()

			// Wait until we have items on the queue.
			select {
			case <-time.After(10 * time.Millisecond):
			case <-c1:
				assert.Fail("pop should get blocked, it didn't")
			case <-c2:
				assert.Fail("pop should get blocked, it didn't")
			}

			// Shut down the queue.
			err := q.ShutDown()
			assert.NoError(err)
			assert.True(q.IsShutDown())

			// Check if had a result
			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("shut down should had unblock the pops, it didn't")
			case shutDown := <-c1:
				assert.True(shutDown) // check if unlock sent us shutDown.
			}

			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("shut down should had unblock the pops, it didn't")
			case shutDown := <-c2:
				assert.True(shutDown) // check if unlock sent us shutDown.
			}

			// Pushing on a shut down queue should return error.
			assert.Error(q.Push("test"))

			// Shutting down an already shut donw queue should error.
			assert.Error(q.ShutDown())

			// Pop on a shutdown queue should return shut down flag.
			_, sd := q.Pop()
			assert.True(sd)
		})
	}
}

func TestSimpleQueueMultiplePushAndPop(t *testing.T) {
	tests := []struct {
		name string
	}{
		{
			name: "Pushing and poping multiple times should not block the pops and get the items in order.",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)
			times := 100
			q := queue.NewSimpleQueue()
			defer q.ShutDown() // Clean.

			// Items to queue.
			for i := 0; i < times; i++ {
				item := fmt.Sprintf("this is a test %d", i)
				assert.NoError(q.Push(item))
			}
			fmt.Println(q.Len())
			c := make(chan interface{})
			pop := func() {
				res, _ := q.Pop()
				c <- res
			}

			// Dequeue items.
			for i := 0; i < times; i++ {
				go pop()
				// Check if had a result.
				select {
				case <-time.After(10 * time.Millisecond):
					assert.Fail("shut down should had unblock the pops, it didn't")
				case res := <-c:
					expItem := fmt.Sprintf("this is a test %d", i)
					assert.Equal(expItem, res)
				}
			}

			// No items should block.
			go pop()
			select {
			case <-time.After(10 * time.Millisecond):
			case <-c:
				assert.Fail("pop should get blocked, it didn't")
			}
		})
	}
}
<commit_msg>Remove debugging code<commit_after>package queue_test

import (
	"fmt"
	"testing"
	"time"

	"github.com/slok/ragnarok/client/util/queue"
	"github.com/stretchr/testify/assert"
)

func TestSimpleQueueBlocking(t *testing.T) {
	tests := []struct {
		name string
		item string
	}{
		{
			name: "Getting an item on an empty queue should block the caller until a push operation is made on that queue.",
			item: "this is a blocking test.",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)
			q := queue.NewSimpleQueue()

			// Caller1 pop in a goroutine.
			c1 := make(chan interface{})
			go func() {
				res, _ := q.Pop()
				c1 <- res
			}()

			// Caller2 pop in a goroutine.
			c2 := make(chan interface{})
			go func() {
				res, _ := q.Pop()
				c2 <- res
			}()

			// Wait until we have items on the queue.
			select {
			case <-time.After(10 * time.Millisecond):
			case <-c1:
				assert.Fail("pop should get blocked, it didn't")
			case <-c2:
				assert.Fail("pop should get blocked, it didn't")
			}

			// Add an item & wait until we have items on the queue.
			var gotItem1, gotItem2 interface{}
			err := q.Push(test.item)
			assert.NoError(err)
			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("pop shouldn't get blocked, we had push an item")
			case gotItem1 = <-c1:
			case gotItem2 = <-c2:
			}

			// Add a 2nd item & wait until we have items on the queue.
			err = q.Push(test.item)
			assert.NoError(err)

			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("pop shouldn't get blocked, we had push an item")
			case gotItem1 = <-c1:
			case gotItem2 = <-c2:
			}

			// At this moment both should have the items.
			assert.Equal(test.item, gotItem1.(string))
			assert.Equal(test.item, gotItem2.(string))
		})
	}
}

func TestSimpleQueueShutDown(t *testing.T) {
	tests := []struct {
		name string
	}{
		{
			name: "Shutting down the queue when there are blocking pops should unblock everyon.",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)
			q := queue.NewSimpleQueue()

			// Caller1 pop in a goroutine.
			c1 := make(chan bool)
			go func() {
				_, sd := q.Pop()
				c1 <- sd
			}()

			// Caller2 pop in a goroutine.
			c2 := make(chan bool)
			go func() {
				_, sd := q.Pop()
				c2 <- sd
			}()

			// Wait until we have items on the queue.
			select {
			case <-time.After(10 * time.Millisecond):
			case <-c1:
				assert.Fail("pop should get blocked, it didn't")
			case <-c2:
				assert.Fail("pop should get blocked, it didn't")
			}

			// Shut down the queue.
			err := q.ShutDown()
			assert.NoError(err)
			assert.True(q.IsShutDown())

			// Check if had a result
			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("shut down should had unblock the pops, it didn't")
			case shutDown := <-c1:
				assert.True(shutDown) // check if unlock sent us shutDown.
			}

			select {
			case <-time.After(10 * time.Millisecond):
				assert.Fail("shut down should had unblock the pops, it didn't")
			case shutDown := <-c2:
				assert.True(shutDown) // check if unlock sent us shutDown.
			}

			// Pushing on a shut down queue should return error.
			assert.Error(q.Push("test"))

			// Shutting down an already shut donw queue should error.
			assert.Error(q.ShutDown())

			// Pop on a shutdown queue should return shut down flag.
			_, sd := q.Pop()
			assert.True(sd)
		})
	}
}

func TestSimpleQueueMultiplePushAndPop(t *testing.T) {
	tests := []struct {
		name string
	}{
		{
			name: "Pushing and poping multiple times should not block the pops and get the items in order.",
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			assert := assert.New(t)
			times := 100
			q := queue.NewSimpleQueue()
			defer q.ShutDown() // Clean.

			// Items to queue.
			for i := 0; i < times; i++ {
				item := fmt.Sprintf("this is a test %d", i)
				assert.NoError(q.Push(item))
			}
			c := make(chan interface{})
			pop := func() {
				res, _ := q.Pop()
				c <- res
			}

			// Dequeue items.
			for i := 0; i < times; i++ {
				go pop()
				// Check if had a result.
				select {
				case <-time.After(10 * time.Millisecond):
					assert.Fail("shut down should had unblock the pops, it didn't")
				case res := <-c:
					expItem := fmt.Sprintf("this is a test %d", i)
					assert.Equal(expItem, res)
				}
			}

			// No items should block.
			go pop()
			select {
			case <-time.After(10 * time.Millisecond):
			case <-c:
				assert.Fail("pop should get blocked, it didn't")
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
package main

import (
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
    "log"
    "time"
)

// Downloads an item from an S3 Bucket
//
// Usage:
//    go run s3_download.go BUCKET ITEM
func main() {
    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, err := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create S3 service client
    svc := s3.New(sess)

    req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
        Bucket: aws.String("myBucket"),
        Key:    aws.String("myKey"),
    })
    urlStr, err := req.Presign(15 * time.Minute)

    if err != nil {
        log.Println("Failed to sign request", err)
    }

    log.Println("The URL is", urlStr)
}
<commit_msg>Fixed erroneous comment<commit_after>/*
   Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.

   This file is licensed under the Apache License, Version 2.0 (the "License").
   You may not use this file except in compliance with the License. A copy of
   the License is located at

    http://aws.amazon.com/apache2.0/

   This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
   CONDITIONS OF ANY KIND, either express or implied. See the License for the
   specific language governing permissions and limitations under the License.
*/
package main

import (
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
    "log"
    "time"
)

// Downloads an item from an S3 Bucket
//
// Usage:
//    go run s3_download.go
func main() {
    // Initialize a session in us-west-2 that the SDK will use to load
    // credentials from the shared credentials file ~/.aws/credentials.
    sess, err := session.NewSession(&aws.Config{
        Region: aws.String("us-west-2")},
    )

    // Create S3 service client
    svc := s3.New(sess)

    req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
        Bucket: aws.String("myBucket"),
        Key:    aws.String("myKey"),
    })
    urlStr, err := req.Presign(15 * time.Minute)

    if err != nil {
        log.Println("Failed to sign request", err)
    }

    log.Println("The URL is", urlStr)
}
<|endoftext|>


<commit_before>package redis

import (
	"fmt"
	"testing"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	session *RedisSession
	prefix  string
)

func init() {
	var err error
	conf := &RedisConf{Server: "192.168.59.103:6379"}
	session, err = NewRedisSession(conf)
	if err != nil {
		panic(fmt.Sprintf("Could not start redis: %s", err))
	}
	prefix = "testing"
	// defer session.Close()
}

func TestPrefix(t *testing.T) {
	session.SetPrefix(prefix)
	key := session.AddPrefix("muppets")
	if key != "testing:muppets" {
		t.Errorf("Key must be formed correctly")
	}
}

func TestServerConnection(t *testing.T) {
	reply, err := session.Do("PING")
	if err != nil {
		panic(fmt.Sprintf("No response from redis: %s", err))
	}

	response, _ := session.String(reply)
	if response != "PONG" {
		t.Errorf("Wrong server response: %s", response)
	}
}

func TestKeyValue(t *testing.T) {
	err := session.Set("gonzo", "the great")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}

	response, err := session.Get("gonzo")
	if err != nil {
		t.Errorf("Could not get value of key ", err)
	}
	if response != "the great" {
		t.Error("Value did not match")
	}
}

func TestKeyIntValue(t *testing.T) {
	err := session.Set("bertsfavouritenumber", "6")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}
	response, err := session.GetInt("bertsfavouritenumber")
	if err != nil {
		t.Errorf("Could not get value of key: %s", err)
	}
	if response != 6 {
		t.Error("Value did not match: %s received", response)
	}

	err = session.Set("bertsfavouriteletter", "a")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}

	response, err = session.GetInt("bertsfavouriteletter")
	if err == nil {
		t.Errorf("Error was expected: %s", err)
	}
	if response != 0 {
		t.Error("0 was expected as response, but got %d", response)
	}

	session.Del("bertsfavouriteletter")
}

func TestDeleteKey(t *testing.T) {
	response, err := session.Del("gonzo")
	if err != nil {
		t.Errorf("Could not delete key: %s", err)
	}
	if response == 0 {
		t.Errorf("Key not found")
		return
	}
	if response != 1 {
		t.Errorf("Deleted key count is not correct")
	}
}

func TestIncrement(t *testing.T) {
	response, err := session.Incr("bertsfavouritenumber")
	if err != nil {
		t.Errorf("Could not increment value of the key: %s", err)
	}
	if response != 7 {
		t.Errorf("Could not increment value correctly")
	}
}

func TestExpire(t *testing.T) {
	err := session.Expire("bertsfavouritenumber", 100*time.Millisecond)
	if err != nil {
		t.Errorf("Could not set expire date of the key: %s", err)
	}
	time.Sleep(100 * time.Millisecond)
	_, err = session.Get("bertsfavouritenumber")
	if err != nil && err != redis.ErrNil {
		t.Errorf("Could not get value of the key: %s", err)
	}
}

func TestSetWithExpire(t *testing.T) {
	err := session.Setex("swedish", 1*time.Second, "chef")
	if err != nil {
		t.Errorf("Could not set key with expire date: %s", err)
	}
	time.Sleep(1 * time.Second)
	_, err = session.Get("swedish")
	if err != nil && err != redis.ErrNil {
		t.Errorf("Could not get value of the key: %s", err)
	}
}

func TestKeyExistence(t *testing.T) {
	response := session.Exists("oscar")
	if response {
		t.Errorf("Key must not exist")
	}
	err := session.Set("oscar", "the grouch")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}

	response = session.Exists("oscar")
	if !response {
		t.Errorf("Key must exist")
	}
	session.Del("oscar")
}

func TestPing(t *testing.T) {
	if err := session.Ping(); err != nil {
		t.Errorf("Server did not respond: %s", err)
	}
}

func TestAddSetMember(t *testing.T) {
	members := []interface{}{"janice", "floyd", "animal", "drteeth", "zoot"}
	response, err := session.AddSetMembers("electricmayhem", members...)
	if err != nil {
		t.Errorf("Could not add set members: %s", err)
	}
	if response != 5 {
		t.Errorf("Wrong set member return count: %d", response)
	}
}

func TestGetSetMember(t *testing.T) {
	members, err := session.GetSetMembers("electricmayhem")
	if err != nil {
		t.Errorf("Could not get set members: %s", err)
	}
	if len(members) != 5 {
		t.Errorf("Wrong set member count: %d", len(members))
	}
	found := false
	for i := range members {
		member, _ := session.String(members[i])
		if member == "janice" {
			found = true
			break
		}
	}

	if !found {
		t.Errorf("Member not found")
	}
}

func TestSetMemberCount(t *testing.T) {
	count, err := session.Scard("electricmayhem")
	if err != nil {
		t.Errorf("Could not get set member count: %s", err)
	}
	if count != 5 {
		t.Errorf("Wrong member count: %d", count)
	}
}

func TestRemoveSetMembers(t *testing.T) {
	response, err := session.RemoveSetMembers("electricmayhem", "janice")
	if err != nil {
		t.Errorf("Could not remove element from set: %s", err)
	}
	if response != 1 {
		t.Errorf("Wrong set member count: %d", response)
	}
	members, _ := session.GetSetMembers("electricmayhem")
	if len(members) != 4 {
		t.Errorf("Wrong remaining member count: %d", len(members))
	}
}

func TestIsSetMember(t *testing.T) {
	response, err := session.IsSetMember("electricmayhem", "statler")
	if err != nil {
		t.Errorf("Could not check member existence: %s", err)
	}
	if response != 0 {
		t.Errorf("Expected 0 but got %d", response)
	}

	response, err = session.IsSetMember("electricmayhem", "animal")
	if err != nil {
		t.Errorf("Could not check member existence: %s", err)
	}
	if response != 1 {
		t.Errorf("Expected 1 but got %d", response)
	}
}

func TestPopSetMember(t *testing.T) {
	response, err := session.PopSetMember("electricmayhem")
	if err != nil {
		t.Errorf("Could not pop element from set: %s", err)
	}
	members := []string{"floyd", "animal", "drteeth", "zoot"}
	found := false
	for i := range members {
		if members[i] == response {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("Popped element does not belong to the set: %s", response)
	}
}

func TestKeys(t *testing.T) {
	keys, err := session.Keys("*")
	if err != nil {
		t.Errorf("Could not fetch keys: %s", err)
	}

	if len(keys) != 1 {
		t.Errorf("Wrong key count: %d", len(keys))
		session.Del("electricmayhem")
		return
	}

	key, _ := session.String(keys[0])
	if key != "testing:electricmayhem" {
		t.Errorf("Wrong key: %s", key)
	}

	session.Del("electricmayhem")
}

func TestHashMultipleSet(t *testing.T) {
	item := map[string]interface{}{
		"janice":  "guitar",
		"floyd":   "bass",
		"drteeth": "keyboard",
		"animal":  "drums",
		"zoot":    "sax",
	}
	err := session.HashMultipleSet("mayhem", item)
	if err != nil {
		t.Errorf("Could create hash set: %s", err)
		return
	}
	defer session.Del("mayhem")

	length, err := session.GetHashLength("mayhem")
	if err != nil {
		t.Errorf("Could not get hash set length: %s", err)
		return
	}

	if length != len(item) {
		t.Errorf("Expected %d but got %d as hash length: ", len(item), length)
	}

	reply, err := session.GetHashMultipleSet("mayhem", "zoot")
	if err != nil {
		t.Errorf("Could not get hash set: %s", err)
		return
	}

	if len(reply) != 1 {
		t.Errorf("Wrong return value count: %d", len(reply))
		return
	}

	response, _ := session.String(reply[0])
	if response != "sax" {
		t.Errorf("Wrong hashset value of the element: %s", response)
	}

	value, err := session.GetHashSetField("mayhem", "zoot")
	if err != nil {
		t.Errorf("Could not get hash set field value: %s", err)
	}

	if value != "sax" {
		t.Errorf("Wrong hashset field value: %s", err)
	}

	result, err := session.HashSetIfNotExists("mayhem", "janice", "ukulele")
	if err != nil {
		t.Errorf("Could not set hash field: %s", err)
	}

	if result != false {
		t.Error("Expected false from hash set but got true")
	}

	result, err = session.HashSetIfNotExists("mayhem", "kermit", "frog")
	if err != nil {
		t.Errorf("Could not set hash field: %s", err)
	}

	if result != true {
		t.Error("Expected true from hash set but got false")
	}

	deleteCount, err := session.DeleteHashSetField("mayhem", "kermit")
	if err != nil {
		t.Errorf("Could not delete hash field: %s", err)
	}

	if deleteCount != 1 {
		t.Errorf("Expected 1 but got %d from hash set field deletion", deleteCount)
	}

	length, err = session.GetHashLength("mayhem")
	if err != nil {
		t.Errorf("Could not get hash set length: %s", err)
	}

	if length != len(item) {
		t.Errorf("Expected %d but got %d as hash length: ", len(item), length)
	}
}

func TestSortedSet(t *testing.T) {
	set1Key, set2Key, destination := "set1", "set2", "combined-set"

	_, err := session.SortedSetIncrBy(set1Key, 1, "item1")
	if err != nil {
		t.Errorf("Error creating set1", err)
		return
	}

	_, err = session.SortedSetIncrBy(set2Key, 1, "item2")
	if err != nil {
		t.Errorf("Error creating set2", err)

		session.Del(set1Key)
		return
	}

	keys := []string{set1Key, set2Key}
	weights := []interface{}{1, 1}

	reply, err := session.SortedSetsUnion(destination, keys, weights, "SUM")
	if err != nil {
		t.Errorf("Error creating combined sets", err)

		session.Del(set1Key)
		session.Del(set2Key)
		return
	}

	if reply < 2 {
		t.Errorf("Wrong number of elements added to combined set", err, reply)
	}

	score, err := session.SortedSetScore(destination, "item1")
	if err != nil {
		t.Errorf("Couldn't get score of item from sorted set: %s", err)
		return
	}

	if score != 1 {
		t.Errorf("Wrong number of elements added to combined set", err, reply)
	}

	_, err = session.SortedSetRem(destination, "item1")
	if err != nil {
		t.Errorf("Couldn't remove item from sorted set: %s", err)
		return
	}

	_, err = session.SortedSetScore(destination, "item1")
	if err == nil {
		t.Errorf("Didn't remove item from sorted set")
		return
	}

	err = session.SortedSetAddSingle(destination, "item1", 1)
	if err != nil {
		t.Errorf("Couldn't remove item from sorted set: %s", err)
		return
	}

	score, err = session.SortedSetScore(destination, "item1")
	if err != nil {
		t.Fatal(err)
	}

	if score != 1 {
		t.Errorf("Didn't set score for item in sorted set")
		return
	}

	session.Del(set1Key)
	session.Del(set2Key)
	session.Del(destination)
}
<commit_msg>go: update redis<commit_after>package redis

import (
	"fmt"
	"testing"
	"time"

	"github.com/garyburd/redigo/redis"
)

var (
	session *RedisSession
	prefix  string
)

func init() {
	var err error
	conf := &RedisConf{Server: "localhost:6379"}
	session, err = NewRedisSession(conf)
	if err != nil {
		panic(fmt.Sprintf("Could not start redis: %s", err))
	}
	prefix = "testing"
	// defer session.Close()
}

func TestPrefix(t *testing.T) {
	session.SetPrefix(prefix)
	key := session.AddPrefix("muppets")
	if key != "testing:muppets" {
		t.Errorf("Key must be formed correctly")
	}
}

func TestServerConnection(t *testing.T) {
	reply, err := session.Do("PING")
	if err != nil {
		panic(fmt.Sprintf("No response from redis: %s", err))
	}

	response, _ := session.String(reply)
	if response != "PONG" {
		t.Errorf("Wrong server response: %s", response)
	}
}

func TestKeyValue(t *testing.T) {
	err := session.Set("gonzo", "the great")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}

	response, err := session.Get("gonzo")
	if err != nil {
		t.Errorf("Could not get value of key ", err)
	}
	if response != "the great" {
		t.Error("Value did not match")
	}
}

func TestKeyIntValue(t *testing.T) {
	err := session.Set("bertsfavouritenumber", "6")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}
	response, err := session.GetInt("bertsfavouritenumber")
	if err != nil {
		t.Errorf("Could not get value of key: %s", err)
	}
	if response != 6 {
		t.Error("Value did not match: %s received", response)
	}

	err = session.Set("bertsfavouriteletter", "a")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}

	response, err = session.GetInt("bertsfavouriteletter")
	if err == nil {
		t.Errorf("Error was expected: %s", err)
	}
	if response != 0 {
		t.Error("0 was expected as response, but got %d", response)
	}

	session.Del("bertsfavouriteletter")
}

func TestDeleteKey(t *testing.T) {
	response, err := session.Del("gonzo")
	if err != nil {
		t.Errorf("Could not delete key: %s", err)
	}
	if response == 0 {
		t.Errorf("Key not found")
		return
	}
	if response != 1 {
		t.Errorf("Deleted key count is not correct")
	}
}

func TestIncrement(t *testing.T) {
	response, err := session.Incr("bertsfavouritenumber")
	if err != nil {
		t.Errorf("Could not increment value of the key: %s", err)
	}
	if response != 7 {
		t.Errorf("Could not increment value correctly")
	}
}

func TestExpire(t *testing.T) {
	err := session.Expire("bertsfavouritenumber", 100*time.Millisecond)
	if err != nil {
		t.Errorf("Could not set expire date of the key: %s", err)
	}
	time.Sleep(100 * time.Millisecond)
	_, err = session.Get("bertsfavouritenumber")
	if err != nil && err != redis.ErrNil {
		t.Errorf("Could not get value of the key: %s", err)
	}
}

func TestSetWithExpire(t *testing.T) {
	err := session.Setex("swedish", 1*time.Second, "chef")
	if err != nil {
		t.Errorf("Could not set key with expire date: %s", err)
	}
	time.Sleep(1 * time.Second)
	_, err = session.Get("swedish")
	if err != nil && err != redis.ErrNil {
		t.Errorf("Could not get value of the key: %s", err)
	}
}

func TestKeyExistence(t *testing.T) {
	response := session.Exists("oscar")
	if response {
		t.Errorf("Key must not exist")
	}
	err := session.Set("oscar", "the grouch")
	if err != nil {
		t.Errorf("Could not set value of key: %s", err)
	}

	response = session.Exists("oscar")
	if !response {
		t.Errorf("Key must exist")
	}
	session.Del("oscar")
}

func TestPing(t *testing.T) {
	if err := session.Ping(); err != nil {
		t.Errorf("Server did not respond: %s", err)
	}
}

func TestAddSetMember(t *testing.T) {
	members := []interface{}{"janice", "floyd", "animal", "drteeth", "zoot"}
	response, err := session.AddSetMembers("electricmayhem", members...)
	if err != nil {
		t.Errorf("Could not add set members: %s", err)
	}
	if response != 5 {
		t.Errorf("Wrong set member return count: %d", response)
	}
}

func TestGetSetMember(t *testing.T) {
	members, err := session.GetSetMembers("electricmayhem")
	if err != nil {
		t.Errorf("Could not get set members: %s", err)
	}
	if len(members) != 5 {
		t.Errorf("Wrong set member count: %d", len(members))
	}
	found := false
	for i := range members {
		member, _ := session.String(members[i])
		if member == "janice" {
			found = true
			break
		}
	}

	if !found {
		t.Errorf("Member not found")
	}
}

func TestSetMemberCount(t *testing.T) {
	count, err := session.Scard("electricmayhem")
	if err != nil {
		t.Errorf("Could not get set member count: %s", err)
	}
	if count != 5 {
		t.Errorf("Wrong member count: %d", count)
	}
}

func TestRemoveSetMembers(t *testing.T) {
	response, err := session.RemoveSetMembers("electricmayhem", "janice")
	if err != nil {
		t.Errorf("Could not remove element from set: %s", err)
	}
	if response != 1 {
		t.Errorf("Wrong set member count: %d", response)
	}
	members, _ := session.GetSetMembers("electricmayhem")
	if len(members) != 4 {
		t.Errorf("Wrong remaining member count: %d", len(members))
	}
}

func TestIsSetMember(t *testing.T) {
	response, err := session.IsSetMember("electricmayhem", "statler")
	if err != nil {
		t.Errorf("Could not check member existence: %s", err)
	}
	if response != 0 {
		t.Errorf("Expected 0 but got %d", response)
	}

	response, err = session.IsSetMember("electricmayhem", "animal")
	if err != nil {
		t.Errorf("Could not check member existence: %s", err)
	}
	if response != 1 {
		t.Errorf("Expected 1 but got %d", response)
	}
}

func TestPopSetMember(t *testing.T) {
	response, err := session.PopSetMember("electricmayhem")
	if err != nil {
		t.Errorf("Could not pop element from set: %s", err)
	}
	members := []string{"floyd", "animal", "drteeth", "zoot"}
	found := false
	for i := range members {
		if members[i] == response {
			found = true
			break
		}
	}
	if !found {
		t.Errorf("Popped element does not belong to the set: %s", response)
	}
}

func TestKeys(t *testing.T) {
	keys, err := session.Keys("*")
	if err != nil {
		t.Errorf("Could not fetch keys: %s", err)
	}

	if len(keys) != 1 {
		t.Errorf("Wrong key count: %d", len(keys))
		session.Del("electricmayhem")
		return
	}

	key, _ := session.String(keys[0])
	if key != "testing:electricmayhem" {
		t.Errorf("Wrong key: %s", key)
	}

	session.Del("electricmayhem")
}

func TestHashMultipleSet(t *testing.T) {
	item := map[string]interface{}{
		"janice":  "guitar",
		"floyd":   "bass",
		"drteeth": "keyboard",
		"animal":  "drums",
		"zoot":    "sax",
	}
	err := session.HashMultipleSet("mayhem", item)
	if err != nil {
		t.Errorf("Could create hash set: %s", err)
		return
	}
	defer session.Del("mayhem")

	length, err := session.GetHashLength("mayhem")
	if err != nil {
		t.Errorf("Could not get hash set length: %s", err)
		return
	}

	if length != len(item) {
		t.Errorf("Expected %d but got %d as hash length: ", len(item), length)
	}

	reply, err := session.GetHashMultipleSet("mayhem", "zoot")
	if err != nil {
		t.Errorf("Could not get hash set: %s", err)
		return
	}

	if len(reply) != 1 {
		t.Errorf("Wrong return value count: %d", len(reply))
		return
	}

	response, _ := session.String(reply[0])
	if response != "sax" {
		t.Errorf("Wrong hashset value of the element: %s", response)
	}

	value, err := session.GetHashSetField("mayhem", "zoot")
	if err != nil {
		t.Errorf("Could not get hash set field value: %s", err)
	}

	if value != "sax" {
		t.Errorf("Wrong hashset field value: %s", err)
	}

	result, err := session.HashSetIfNotExists("mayhem", "janice", "ukulele")
	if err != nil {
		t.Errorf("Could not set hash field: %s", err)
	}

	if result != false {
		t.Error("Expected false from hash set but got true")
	}

	result, err = session.HashSetIfNotExists("mayhem", "kermit", "frog")
	if err != nil {
		t.Errorf("Could not set hash field: %s", err)
	}

	if result != true {
		t.Error("Expected true from hash set but got false")
	}

	deleteCount, err := session.DeleteHashSetField("mayhem", "kermit")
	if err != nil {
		t.Errorf("Could not delete hash field: %s", err)
	}

	if deleteCount != 1 {
		t.Errorf("Expected 1 but got %d from hash set field deletion", deleteCount)
	}

	length, err = session.GetHashLength("mayhem")
	if err != nil {
		t.Errorf("Could not get hash set length: %s", err)
	}

	if length != len(item) {
		t.Errorf("Expected %d but got %d as hash length: ", len(item), length)
	}
}

func TestSortedSet(t *testing.T) {
	set1Key, set2Key, destination := "set1", "set2", "combined-set"

	_, err := session.SortedSetIncrBy(set1Key, 1, "item1")
	if err != nil {
		t.Errorf("Error creating set1", err)
		return
	}

	_, err = session.SortedSetIncrBy(set2Key, 1, "item2")
	if err != nil {
		t.Errorf("Error creating set2", err)

		session.Del(set1Key)
		return
	}

	keys := []string{set1Key, set2Key}
	weights := []interface{}{1, 1}

	reply, err := session.SortedSetsUnion(destination, keys, weights, "SUM")
	if err != nil {
		t.Errorf("Error creating combined sets", err)

		session.Del(set1Key)
		session.Del(set2Key)
		return
	}

	if reply < 2 {
		t.Errorf("Wrong number of elements added to combined set", err, reply)
	}

	score, err := session.SortedSetScore(destination, "item1")
	if err != nil {
		t.Errorf("Couldn't get score of item from sorted set: %s", err)
		return
	}

	if score != 1 {
		t.Errorf("Wrong number of elements added to combined set", err, reply)
	}

	_, err = session.SortedSetRem(destination, "item1")
	if err != nil {
		t.Errorf("Couldn't remove item from sorted set: %s", err)
		return
	}

	_, err = session.SortedSetScore(destination, "item1")
	if err == nil {
		t.Errorf("Didn't remove item from sorted set")
		return
	}

	err = session.SortedSetAddSingle(destination, "item1", 1)
	if err != nil {
		t.Errorf("Couldn't remove item from sorted set: %s", err)
		return
	}

	score, err = session.SortedSetScore(destination, "item1")
	if err != nil {
		t.Fatal(err)
	}

	if score != 1 {
		t.Errorf("Didn't set score for item in sorted set")
		return
	}

	session.Del(set1Key)
	session.Del(set2Key)
	session.Del(destination)
}
<|endoftext|>


<commit_before><commit_msg>Skip download if file exists on disk.<commit_after><|endoftext|>