<commit_before>// Copyright 2014 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

// Package models implements the models for an abstraction over the
// multiple data stores used by a BitTorrent tracker.
package models

import (
	"errors"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/chihaya/chihaya/config"
	"github.com/chihaya/chihaya/models/query"
	"github.com/julienschmidt/httprouter"
)

var (
	// ErrMalformedRequest is returned when an http.Request does no have the
	// required parameters to create a model.
	ErrMalformedRequest = errors.New("malformed request")
)

// Peer is a participant in a swarm.
type Peer struct {
	ID        string `json:"id"`
	UserID    uint64 `json:"user_id"`
	TorrentID uint64 `json:"torrent_id"`

	IP   net.IP `json:"ip"`
	Port uint64 `json:"port"`

	Uploaded     uint64 `json:"uploaded"`
	Downloaded   uint64 `json:"downloaded`
	Left         uint64 `json:"left"`
	LastAnnounce int64  `json:"last_announce"`
}

// NewPeer returns the Peer representation of an Announce. When provided nil
// for the announce parameter, it panics. When provided nil for the user or
// torrent parameter, it returns a Peer{UserID: 0} or Peer{TorrentID: 0}
// respectively.
func NewPeer(a *Announce, u *User, t *Torrent) *Peer {
	if a == nil {
		panic("models: announce cannot equal nil")
	}

	var userID uint64
	if u != nil {
		userID = u.ID
	}

	var torrentID uint64
	if t != nil {
		torrentID = t.ID
	}

	return &Peer{
		ID:           a.PeerID,
		UserID:       userID,
		TorrentID:    torrentID,
		IP:           a.IP,
		Port:         a.Port,
		Uploaded:     a.Uploaded,
		Downloaded:   a.Downloaded,
		Left:         a.Left,
		LastAnnounce: time.Now().Unix(),
	}
}

// Key returns the unique key used to look-up a peer in a swarm (i.e
// Torrent.Seeders & Torrent.Leechers).
func (p Peer) Key() string {
	return p.ID + ":" + strconv.FormatUint(p.UserID, 36)
}

// Torrent is a swarm for a given torrent file.
type Torrent struct {
	ID       uint64 `json:"id"`
	Infohash string `json:"infohash"`
	Active   bool   `json:"active"`

	Seeders  map[string]Peer `json:"seeders"`
	Leechers map[string]Peer `json:"leechers"`

	Snatches       uint64  `json:"snatches"`
	UpMultiplier   float64 `json:"up_multiplier"`
	DownMultiplier float64 `json:"down_multiplier"`
	LastAction     int64   `json:"last_action"`
}

// InSeederPool returns true if a peer is within a Torrent's pool of seeders.
func (t *Torrent) InSeederPool(p *Peer) (exists bool) {
	_, exists = t.Seeders[p.Key()]
	return
}

// InLeecherPool returns true if a peer is within a Torrent's pool of leechers.
func (t *Torrent) InLeecherPool(p *Peer) (exists bool) {
	_, exists = t.Leechers[p.Key()]
	return
}

// User is a registered user for private trackers.
type User struct {
	ID      uint64 `json:"id"`
	Passkey string `json:"passkey"`

	UpMultiplier   float64 `json:"up_multiplier"`
	DownMultiplier float64 `json:"down_multiplier"`
	Snatches       uint64  `json:"snatches"`
}

// Announce is an Announce by a Peer.
type Announce struct {
	Config  *config.Config `json:"config"`
	Request *http.Request  `json:"request"`

	Compact    bool   `json:"compact"`
	Downloaded uint64 `json:"downloaded"`
	Event      string `json:"event"`
	IP         net.IP `json:"ip"`
	Infohash   string `json:"infohash"`
	Left       uint64 `json:"left"`
	NumWant    int    `json:"numwant"`
	Passkey    string `json:"passkey"`
	PeerID     string `json:"peer_id"`
	Port       uint64 `json:"port"`
	Uploaded   uint64 `json:"uploaded"`
}

// NewAnnounce parses an HTTP request and generates an Announce.
func NewAnnounce(cfg *config.Config, r *http.Request, p httprouter.Params) (*Announce, error) {
	q, err := query.New(r.URL.RawQuery)
	if err != nil {
		return nil, err
	}

	compact := q.Params["compact"] != "0"
	event, _ := q.Params["event"]
	infohash, _ := q.Params["info_hash"]
	peerID, _ := q.Params["peer_id"]

	numWant := q.RequestedPeerCount(cfg.NumWantFallback)

	ip, ipErr := q.RequestedIP(r)
	port, portErr := q.Uint64("port")

	left, leftErr := q.Uint64("left")
	downloaded, downloadedErr := q.Uint64("downloaded")
	uploaded, uploadedErr := q.Uint64("uploaded")

	if downloadedErr != nil ||
		infohash == "" ||
		leftErr != nil ||
		peerID == "" ||
		portErr != nil ||
		uploadedErr != nil ||
		ipErr != nil {
		return nil, ErrMalformedRequest
	}

	return &Announce{
		Config:     cfg,
		Request:    r,
		Compact:    compact,
		Downloaded: downloaded,
		Event:      event,
		IP:         ip,
		Infohash:   infohash,
		Left:       left,
		NumWant:    numWant,
		Passkey:    p.ByName("passkey"),
		PeerID:     peerID,
		Port:       port,
		Uploaded:   uploaded,
	}, nil
}

// ClientID returns the part of a PeerID that identifies a Peer's client
// software.
func (a Announce) ClientID() (clientID string) {
	length := len(a.PeerID)
	if length >= 6 {
		if a.PeerID[0] == '-' {
			if length >= 7 {
				clientID = a.PeerID[1:7]
			}
		} else {
			clientID = a.PeerID[0:6]
		}
	}

	return
}

// AnnounceDelta contains the changes to a Peer's state. These changes are
// recorded by the backend driver.
type AnnounceDelta struct {
	Peer    *Peer
	Torrent *Torrent
	User    *User

	// Created is true if this announce created a new peer or changed an existing
	// peer's address
	Created bool
	// Snatched is true if this announce completed the download
	Snatched bool

	// Uploaded contains the raw upload delta for this announce, in bytes
	Uploaded uint64
	// Downloaded contains the raw download delta for this announce, in bytes
	Downloaded uint64
}

// NewAnnounceDelta calculates a Peer's download and upload deltas between
// Announces and generates an AnnounceDelta.
func NewAnnounceDelta(a *Announce, p *Peer, u *User, t *Torrent, created, snatched bool) *AnnounceDelta {
	var (
		rawDeltaUp   = p.Uploaded - a.Uploaded
		rawDeltaDown uint64
	)

	if !a.Config.Freeleech {
		rawDeltaDown = p.Downloaded - a.Downloaded
	}

	// Restarting a torrent may cause a delta to be negative.
	if rawDeltaUp < 0 {
		rawDeltaUp = 0
	}
	if rawDeltaDown < 0 {
		rawDeltaDown = 0
	}

	return &AnnounceDelta{
		Peer:    p,
		Torrent: t,
		User:    u,

		Created:  created,
		Snatched: snatched,

		Uploaded:   uint64(float64(rawDeltaUp) * u.UpMultiplier * t.UpMultiplier),
		Downloaded: uint64(float64(rawDeltaDown) * u.DownMultiplier * t.DownMultiplier),
	}
}

// Scrape is a Scrape by a Peer.
type Scrape struct {
	Config  *config.Config `json:"config"`
	Request *http.Request  `json:"request"`

	Passkey    string
	Infohashes []string
}

// NewScrape parses an HTTP request and generates a Scrape.
func NewScrape(cfg *config.Config, r *http.Request, p httprouter.Params) (*Scrape, error) {
	q, err := query.New(r.URL.RawQuery)
	if err != nil {
		return nil, err
	}

	if q.Infohashes == nil {
		if _, exists := q.Params["infohash"]; !exists {
			// There aren't any infohashes.
			return nil, ErrMalformedRequest
		}
		q.Infohashes = []string{q.Params["infohash"]}
	}

	return &Scrape{
		Config:  cfg,
		Request: r,

		Passkey:    p.ByName("passkey"),
		Infohashes: q.Infohashes,
	}, nil
}
<commit_msg>Add missing quotation mark in struct tag<commit_after>// Copyright 2014 The Chihaya Authors. All rights reserved.
// Use of this source code is governed by the BSD 2-Clause license,
// which can be found in the LICENSE file.

// Package models implements the models for an abstraction over the
// multiple data stores used by a BitTorrent tracker.
package models

import (
	"errors"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/chihaya/chihaya/config"
	"github.com/chihaya/chihaya/models/query"
	"github.com/julienschmidt/httprouter"
)

var (
	// ErrMalformedRequest is returned when an http.Request does no have the
	// required parameters to create a model.
	ErrMalformedRequest = errors.New("malformed request")
)

// Peer is a participant in a swarm.
type Peer struct {
	ID        string `json:"id"`
	UserID    uint64 `json:"user_id"`
	TorrentID uint64 `json:"torrent_id"`

	IP   net.IP `json:"ip"`
	Port uint64 `json:"port"`

	Uploaded     uint64 `json:"uploaded"`
	Downloaded   uint64 `json:"downloaded"`
	Left         uint64 `json:"left"`
	LastAnnounce int64  `json:"last_announce"`
}

// NewPeer returns the Peer representation of an Announce. When provided nil
// for the announce parameter, it panics. When provided nil for the user or
// torrent parameter, it returns a Peer{UserID: 0} or Peer{TorrentID: 0}
// respectively.
func NewPeer(a *Announce, u *User, t *Torrent) *Peer {
	if a == nil {
		panic("models: announce cannot equal nil")
	}

	var userID uint64
	if u != nil {
		userID = u.ID
	}

	var torrentID uint64
	if t != nil {
		torrentID = t.ID
	}

	return &Peer{
		ID:           a.PeerID,
		UserID:       userID,
		TorrentID:    torrentID,
		IP:           a.IP,
		Port:         a.Port,
		Uploaded:     a.Uploaded,
		Downloaded:   a.Downloaded,
		Left:         a.Left,
		LastAnnounce: time.Now().Unix(),
	}
}

// Key returns the unique key used to look-up a peer in a swarm (i.e
// Torrent.Seeders & Torrent.Leechers).
func (p Peer) Key() string {
	return p.ID + ":" + strconv.FormatUint(p.UserID, 36)
}

// Torrent is a swarm for a given torrent file.
type Torrent struct {
	ID       uint64 `json:"id"`
	Infohash string `json:"infohash"`
	Active   bool   `json:"active"`

	Seeders  map[string]Peer `json:"seeders"`
	Leechers map[string]Peer `json:"leechers"`

	Snatches       uint64  `json:"snatches"`
	UpMultiplier   float64 `json:"up_multiplier"`
	DownMultiplier float64 `json:"down_multiplier"`
	LastAction     int64   `json:"last_action"`
}

// InSeederPool returns true if a peer is within a Torrent's pool of seeders.
func (t *Torrent) InSeederPool(p *Peer) (exists bool) {
	_, exists = t.Seeders[p.Key()]
	return
}

// InLeecherPool returns true if a peer is within a Torrent's pool of leechers.
func (t *Torrent) InLeecherPool(p *Peer) (exists bool) {
	_, exists = t.Leechers[p.Key()]
	return
}

// User is a registered user for private trackers.
type User struct {
	ID      uint64 `json:"id"`
	Passkey string `json:"passkey"`

	UpMultiplier   float64 `json:"up_multiplier"`
	DownMultiplier float64 `json:"down_multiplier"`
	Snatches       uint64  `json:"snatches"`
}

// Announce is an Announce by a Peer.
type Announce struct {
	Config  *config.Config `json:"config"`
	Request *http.Request  `json:"request"`

	Compact    bool   `json:"compact"`
	Downloaded uint64 `json:"downloaded"`
	Event      string `json:"event"`
	IP         net.IP `json:"ip"`
	Infohash   string `json:"infohash"`
	Left       uint64 `json:"left"`
	NumWant    int    `json:"numwant"`
	Passkey    string `json:"passkey"`
	PeerID     string `json:"peer_id"`
	Port       uint64 `json:"port"`
	Uploaded   uint64 `json:"uploaded"`
}

// NewAnnounce parses an HTTP request and generates an Announce.
func NewAnnounce(cfg *config.Config, r *http.Request, p httprouter.Params) (*Announce, error) {
	q, err := query.New(r.URL.RawQuery)
	if err != nil {
		return nil, err
	}

	compact := q.Params["compact"] != "0"
	event, _ := q.Params["event"]
	infohash, _ := q.Params["info_hash"]
	peerID, _ := q.Params["peer_id"]

	numWant := q.RequestedPeerCount(cfg.NumWantFallback)

	ip, ipErr := q.RequestedIP(r)
	port, portErr := q.Uint64("port")

	left, leftErr := q.Uint64("left")
	downloaded, downloadedErr := q.Uint64("downloaded")
	uploaded, uploadedErr := q.Uint64("uploaded")

	if downloadedErr != nil ||
		infohash == "" ||
		leftErr != nil ||
		peerID == "" ||
		portErr != nil ||
		uploadedErr != nil ||
		ipErr != nil {
		return nil, ErrMalformedRequest
	}

	return &Announce{
		Config:     cfg,
		Request:    r,
		Compact:    compact,
		Downloaded: downloaded,
		Event:      event,
		IP:         ip,
		Infohash:   infohash,
		Left:       left,
		NumWant:    numWant,
		Passkey:    p.ByName("passkey"),
		PeerID:     peerID,
		Port:       port,
		Uploaded:   uploaded,
	}, nil
}

// ClientID returns the part of a PeerID that identifies a Peer's client
// software.
func (a Announce) ClientID() (clientID string) {
	length := len(a.PeerID)
	if length >= 6 {
		if a.PeerID[0] == '-' {
			if length >= 7 {
				clientID = a.PeerID[1:7]
			}
		} else {
			clientID = a.PeerID[0:6]
		}
	}

	return
}

// AnnounceDelta contains the changes to a Peer's state. These changes are
// recorded by the backend driver.
type AnnounceDelta struct {
	Peer    *Peer
	Torrent *Torrent
	User    *User

	// Created is true if this announce created a new peer or changed an existing
	// peer's address
	Created bool
	// Snatched is true if this announce completed the download
	Snatched bool

	// Uploaded contains the raw upload delta for this announce, in bytes
	Uploaded uint64
	// Downloaded contains the raw download delta for this announce, in bytes
	Downloaded uint64
}

// NewAnnounceDelta calculates a Peer's download and upload deltas between
// Announces and generates an AnnounceDelta.
func NewAnnounceDelta(a *Announce, p *Peer, u *User, t *Torrent, created, snatched bool) *AnnounceDelta {
	var (
		rawDeltaUp   = p.Uploaded - a.Uploaded
		rawDeltaDown uint64
	)

	if !a.Config.Freeleech {
		rawDeltaDown = p.Downloaded - a.Downloaded
	}

	// Restarting a torrent may cause a delta to be negative.
	if rawDeltaUp < 0 {
		rawDeltaUp = 0
	}
	if rawDeltaDown < 0 {
		rawDeltaDown = 0
	}

	return &AnnounceDelta{
		Peer:    p,
		Torrent: t,
		User:    u,

		Created:  created,
		Snatched: snatched,

		Uploaded:   uint64(float64(rawDeltaUp) * u.UpMultiplier * t.UpMultiplier),
		Downloaded: uint64(float64(rawDeltaDown) * u.DownMultiplier * t.DownMultiplier),
	}
}

// Scrape is a Scrape by a Peer.
type Scrape struct {
	Config  *config.Config `json:"config"`
	Request *http.Request  `json:"request"`

	Passkey    string
	Infohashes []string
}

// NewScrape parses an HTTP request and generates a Scrape.
func NewScrape(cfg *config.Config, r *http.Request, p httprouter.Params) (*Scrape, error) {
	q, err := query.New(r.URL.RawQuery)
	if err != nil {
		return nil, err
	}

	if q.Infohashes == nil {
		if _, exists := q.Params["infohash"]; !exists {
			// There aren't any infohashes.
			return nil, ErrMalformedRequest
		}
		q.Infohashes = []string{q.Params["infohash"]}
	}

	return &Scrape{
		Config:  cfg,
		Request: r,

		Passkey:    p.ByName("passkey"),
		Infohashes: q.Infohashes,
	}, nil
}
<|endoftext|>


<commit_before>package pachyderm

import (
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path"
	"strings"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pkg/require"
	"github.com/pachyderm/pachyderm/src/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/pkg/workload"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
	"google.golang.org/grpc"
)

const (
	NUMFILES = 25
)

func TestJob(t *testing.T) {
	t.Parallel()
	dataRepo := uniqueString("TestJob.data")
	pachClient := getPachClient(t)
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
	job, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		"",
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job,
		BlockOutput: true,
		BlockState:  true,
	}
	jobInfo, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	require.Equal(t, pps.JobState_JOB_STATE_SUCCESS.String(), jobInfo.State.String())
	commitInfo, err := pfsutil.InspectCommit(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id)
	require.NoError(t, err)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
}

func TestGrep(t *testing.T) {
	t.Parallel()
	dataRepo := uniqueString("TestGrep.data")
	pachClient := getPachClient(t)
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	for i := 0; i < 100; i++ {
		_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, fmt.Sprintf("file%d", i), 0, strings.NewReader("foo\nbar\nfizz\nbuzz\n"))
		require.NoError(t, err)
	}
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
	job1, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	job2, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		4,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job1,
		BlockOutput: true,
		BlockState:  true,
	}
	job1Info, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	inspectJobRequest.Job = job2
	job2Info, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	repo1Info, err := pfsutil.InspectRepo(pachClient, job1Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	repo2Info, err := pfsutil.InspectRepo(pachClient, job2Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	require.Equal(t, repo1Info.SizeBytes, repo2Info.SizeBytes)
}

func TestPipeline(t *testing.T) {
	t.Parallel()
	pachClient := getPachClient(t)
	// create repos
	dataRepo := uniqueString("TestPipeline.data")
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	// create pipeline
	pipelineName := uniqueString("pipeline")
	outRepo := pps.PipelineRepo(ppsutil.NewPipeline(pipelineName))
	require.NoError(t, ppsutil.CreatePipeline(
		pachClient,
		pipelineName,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		"",
		1,
		[]*pps.PipelineInput{{Repo: &pfs.Repo{Name: dataRepo}}},
	))
	// Do first commit to repo
	commit1, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit1.Id, "file", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit1.Id))
	listCommitRequest := &pfs.ListCommitRequest{
		Repo:       []*pfs.Repo{outRepo},
		CommitType: pfs.CommitType_COMMIT_TYPE_READ,
		Block:      true,
	}
	listCommitResponse, err := pachClient.ListCommit(
		context.Background(),
		listCommitRequest,
	)
	require.NoError(t, err)
	outCommits := listCommitResponse.CommitInfo
	require.Equal(t, 1, len(outCommits))
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	// Do second commit to repo
	commit2, err := pfsutil.StartCommit(pachClient, dataRepo, commit1.Id)
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit2.Id, "file", 0, strings.NewReader("bar\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit2.Id))
	listCommitRequest = &pfs.ListCommitRequest{
		Repo:       []*pfs.Repo{outRepo},
		FromCommit: []*pfs.Commit{outCommits[0].Commit},
		CommitType: pfs.CommitType_COMMIT_TYPE_READ,
		Block:      true,
	}
	listCommitResponse, err = pachClient.ListCommit(
		context.Background(),
		listCommitRequest,
	)
	require.NoError(t, err)
	outCommits = listCommitResponse.CommitInfo
	require.Equal(t, 1, len(outCommits))
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo\nbar\n", buffer.String())
}

func TestWorkload(t *testing.T) {
	t.Parallel()
	pachClient := getPachClient(t)
	seed := time.Now().UnixNano()
	require.NoError(t, workload.RunWorkload(pachClient, pachClient, rand.New(rand.NewSource(seed)), 100))
}

func TestSharding(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestSharding")
	pachClient := getPachClient(t)
	err := pfsutil.CreateRepo(pachClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pachClient, repo, "")
	require.NoError(t, err)
	var wg sync.WaitGroup
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			rand := rand.New(rand.NewSource(int64(i)))
			_, err = pfsutil.PutFile(pachClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, 1024*1024))
			require.NoError(t, err)
		}()
	}
	wg.Wait()
	err = pfsutil.FinishCommit(pachClient, repo, commit.Id)
	require.NoError(t, err)
	wg = sync.WaitGroup{}
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			var buffer1Shard bytes.Buffer
			var buffer4Shard bytes.Buffer
			shard := &pfs.Shard{FileModulus: 1, BlockModulus: 1}
			err := pfsutil.GetFile(pachClient, repo, commit.Id,
				fmt.Sprintf("file%d", i), 0, 0, shard, &buffer1Shard)
			require.NoError(t, err)
			shard.BlockModulus = 4
			for blockNumber := uint64(0); blockNumber < 4; blockNumber++ {
				shard.BlockNumber = blockNumber
				err := pfsutil.GetFile(pachClient, repo, commit.Id,
					fmt.Sprintf("file%d", i), 0, 0, shard, &buffer4Shard)
				require.NoError(t, err)
			}
			require.Equal(t, buffer1Shard.Len(), buffer4Shard.Len())
		}()
	}
	wg.Wait()
}

func getPachClient(t *testing.T) APIClient {
	pachAddr := os.Getenv("PACHD_PORT_650_TCP_ADDR")
	if pachAddr == "" {
		t.Error("PACHD_PORT_650_TCP_ADDR not set")
	}
	clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pachAddr), grpc.WithInsecure())
	require.NoError(t, err)
	return NewAPIClient(clientConn)
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<commit_msg>Make test passes.<commit_after>package pachyderm

import (
	"bytes"
	"fmt"
	"math/rand"
	"os"
	"path"
	"strings"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pkg/require"
	"github.com/pachyderm/pachyderm/src/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/pkg/workload"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
	"google.golang.org/grpc"
)

const (
	NUMFILES = 25
)

func TestJob(t *testing.T) {
	t.Parallel()
	dataRepo := uniqueString("TestJob.data")
	pachClient := getPachClient(t)
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, "file", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
	job, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		"",
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job,
		BlockOutput: true,
		BlockState:  true,
	}
	jobInfo, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	require.Equal(t, pps.JobState_JOB_STATE_SUCCESS.String(), jobInfo.State.String())
	commitInfo, err := pfsutil.InspectCommit(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id)
	require.NoError(t, err)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, commitInfo.CommitType)
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, jobInfo.OutputCommit.Repo.Name, jobInfo.OutputCommit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
}

func TestGrep(t *testing.T) {
	t.Parallel()
	dataRepo := uniqueString("TestGrep.data")
	pachClient := getPachClient(t)
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	commit, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	for i := 0; i < 100; i++ {
		_, err = pfsutil.PutFile(pachClient, dataRepo, commit.Id, fmt.Sprintf("file%d", i), 0, strings.NewReader("foo\nbar\nfizz\nbuzz\n"))
		require.NoError(t, err)
	}
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit.Id))
	job1, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		1,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	job2, err := ppsutil.CreateJob(
		pachClient,
		"",
		[]string{"bash"},
		fmt.Sprintf("grep foo /pfs/%s/* >/pfs/out/foo", dataRepo),
		4,
		[]*pps.JobInput{{Commit: commit}},
		"",
	)
	require.NoError(t, err)
	inspectJobRequest := &pps.InspectJobRequest{
		Job:         job1,
		BlockOutput: true,
		BlockState:  true,
	}
	job1Info, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	inspectJobRequest.Job = job2
	job2Info, err := pachClient.InspectJob(context.Background(), inspectJobRequest)
	require.NoError(t, err)
	repo1Info, err := pfsutil.InspectRepo(pachClient, job1Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	repo2Info, err := pfsutil.InspectRepo(pachClient, job2Info.OutputCommit.Repo.Name)
	require.NoError(t, err)
	require.Equal(t, repo1Info.SizeBytes, repo2Info.SizeBytes)
}

func TestPipeline(t *testing.T) {
	t.Parallel()
	pachClient := getPachClient(t)
	// create repos
	dataRepo := uniqueString("TestPipeline.data")
	require.NoError(t, pfsutil.CreateRepo(pachClient, dataRepo))
	// create pipeline
	pipelineName := uniqueString("pipeline")
	outRepo := pps.PipelineRepo(ppsutil.NewPipeline(pipelineName))
	require.NoError(t, ppsutil.CreatePipeline(
		pachClient,
		pipelineName,
		"",
		[]string{"cp", path.Join("/pfs", dataRepo, "file"), "/pfs/out/file"},
		"",
		1,
		[]*pps.PipelineInput{{Repo: &pfs.Repo{Name: dataRepo}}},
	))
	// Do first commit to repo
	commit1, err := pfsutil.StartCommit(pachClient, dataRepo, "")
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit1.Id, "file", 0, strings.NewReader("foo\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit1.Id))
	listCommitRequest := &pfs.ListCommitRequest{
		Repo:       []*pfs.Repo{outRepo},
		CommitType: pfs.CommitType_COMMIT_TYPE_READ,
		Block:      true,
	}
	listCommitResponse, err := pachClient.ListCommit(
		context.Background(),
		listCommitRequest,
	)
	require.NoError(t, err)
	outCommits := listCommitResponse.CommitInfo
	require.Equal(t, 1, len(outCommits))
	var buffer bytes.Buffer
	require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo\n", buffer.String())
	// Do second commit to repo
	commit2, err := pfsutil.StartCommit(pachClient, dataRepo, commit1.Id)
	require.NoError(t, err)
	_, err = pfsutil.PutFile(pachClient, dataRepo, commit2.Id, "file", 0, strings.NewReader("bar\n"))
	require.NoError(t, err)
	require.NoError(t, pfsutil.FinishCommit(pachClient, dataRepo, commit2.Id))
	listCommitRequest = &pfs.ListCommitRequest{
		Repo:       []*pfs.Repo{outRepo},
		FromCommit: []*pfs.Commit{outCommits[0].Commit},
		CommitType: pfs.CommitType_COMMIT_TYPE_READ,
		Block:      true,
	}
	listCommitResponse, err = pachClient.ListCommit(
		context.Background(),
		listCommitRequest,
	)
	require.NoError(t, err)
	outCommits = listCommitResponse.CommitInfo
	require.Equal(t, 1, len(outCommits))
	buffer = bytes.Buffer{}
	require.NoError(t, pfsutil.GetFile(pachClient, outRepo.Name, outCommits[0].Commit.Id, "file", 0, 0, nil, &buffer))
	require.Equal(t, "foo\nbar\n", buffer.String())
}

func TestWorkload(t *testing.T) {
	t.Parallel()
	pachClient := getPachClient(t)
	seed := time.Now().UnixNano()
	require.NoError(t, workload.RunWorkload(pachClient, pachClient, rand.New(rand.NewSource(seed)), 100))
}

func TestSharding(t *testing.T) {
	t.Parallel()
	repo := uniqueString("TestSharding")
	pachClient := getPachClient(t)
	err := pfsutil.CreateRepo(pachClient, repo)
	require.NoError(t, err)
	commit, err := pfsutil.StartCommit(pachClient, repo, "")
	require.NoError(t, err)
	var wg sync.WaitGroup
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			rand := rand.New(rand.NewSource(int64(i)))
			_, err = pfsutil.PutFile(pachClient, repo, commit.Id, fmt.Sprintf("file%d", i), 0, workload.NewReader(rand, 1024*1024))
			require.NoError(t, err)
		}()
	}
	wg.Wait()
	err = pfsutil.FinishCommit(pachClient, repo, commit.Id)
	require.NoError(t, err)
	wg = sync.WaitGroup{}
	for i := 0; i < NUMFILES; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			var buffer1Shard bytes.Buffer
			var buffer4Shard bytes.Buffer
			shard := &pfs.Shard{FileModulus: 1, BlockModulus: 1}
			err := pfsutil.GetFile(pachClient, repo, commit.Id,
				fmt.Sprintf("file%d", i), 0, 0, shard, &buffer1Shard)
			require.NoError(t, err)
			shard.BlockModulus = 4
			for blockNumber := uint64(0); blockNumber < 4; blockNumber++ {
				shard.BlockNumber = blockNumber
				err := pfsutil.GetFile(pachClient, repo, commit.Id,
					fmt.Sprintf("file%d", i), 0, 0, shard, &buffer4Shard)
				require.NoError(t, err)
			}
			require.Equal(t, buffer1Shard.Len(), buffer4Shard.Len())
		}()
	}
	wg.Wait()
}

func getPachClient(t *testing.T) *APIClient {
	pachAddr := os.Getenv("PACHD_PORT_650_TCP_ADDR")
	if pachAddr == "" {
		t.Error("PACHD_PORT_650_TCP_ADDR not set")
	}
	clientConn, err := grpc.Dial(fmt.Sprintf("%s:650", pachAddr), grpc.WithInsecure())
	require.NoError(t, err)
	return NewAPIClient(clientConn)
}

func uniqueString(prefix string) string {
	return prefix + "." + uuid.NewWithoutDashes()[0:12]
}
<|endoftext|>


<commit_before>package lib

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strings"

	"github.com/atinm/go-sonos"
	"github.com/atinm/go-sonos/didl"
	"github.com/atinm/go-sonos/ssdp"
	"github.com/atinm/go-sonos/upnp"
	"github.com/zmb3/spotify"
)

var (
	sonosDevices []*sonos.Sonos
	exit_chan    = make(chan bool)
)

// GetLocalInterfaceName returns the first interface name that has the non loopback local IPv4 addr of the host
func getLocalInterfaceName() string {
	list, err := net.Interfaces()
	if err != nil {
		panic(err)
	}

	for _, iface := range list {
		addrs, err := iface.Addrs()
		if err != nil {
			panic(err)
		}
		for _, addr := range addrs {
			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
				if ipnet.IP.To4() != nil {
					return iface.Name
				}
			}
		}
	}
	return ""
}

func HandleUpdate(w http.ResponseWriter, req *http.Request) {
	var update Update
	body, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()
	if err != nil {
		log.Print("[ERROR] ", err)
		return
	}
	if err = json.Unmarshal(body, &update); err != nil {
		log.Print("[ERROR] ", err)
		return
	}

	switch update.Type {
	case "transport-state":
		var s struct {
			Update
			TransportState
		}
		if err = json.Unmarshal(body, &s); err != nil {
			log.Print("[ERROR] ", err)
			return
		}

		bodyStr := fmt.Sprintf("%s", body)
		log.Print("[DEBUG] body: ", bodyStr)
		title := s.Player.State.CurrentTrack.Title
		artist := s.Player.State.CurrentTrack.Artist
		uri := s.Player.State.CurrentTrack.Uri
		roomName = s.Player.RoomName
		log.Println("[DEBUG]", title)
		log.Println("[DEBUG]", uri)
		// CurrentTrackURI: "x-sonos-spotify:spotify%3atrack%3a4Ro98RCK90oHqqSZUnTFq5?sid=12&flags=8224&sn=8"
		// "uri": "x-sonos-spotify:spotify%3atrack%3a6tF92PMv01Ug9Dh8Rmy6nH?sid=12&flags=8224&sn=8",
		if strings.HasPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a") {
			// extract the trackid from the uri
			trackID := strings.Split(strings.TrimPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a"), "?")[0]
			track, err = client.GetTrack(spotify.ID(trackID))
			if err != nil {
				log.Printf("[WARN] Could not get track info for '%s' by '%s', trackId: (%s): %v\n", title, artist, trackID, err)
				break
			}

			if Rules(track, roomName) {
				nextURL := fmt.Sprintf("http://localhost:5005/%s/next", roomName)
				response, err := http.Get(nextURL)
				if err != nil {
					log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
					return
				}
				defer response.Body.Close()
				log.Printf("[INFO] Skipped '%s' by '%s' playing on '%s'\n", title, artist, roomName)
			}
		}

	case "volume-change":
	case "mute-change":
	case "topology-change":
	default:
	}
}

func getTriggeredSonos(svc *upnp.Service) (sonos *sonos.Sonos) {
	for _, s := range sonosDevices {
		if s.AVTransport.Svc == svc {
			sonos = s
			break
		}
	}
	return
}

func handleAVTransportEvents(reactor upnp.Reactor, c chan bool) {
	for {
		select {
		case evt := <-reactor.Channel():
			switch evt.Type() {
			case upnp.AVTransport_EventType:
				var artist, title, album string

				b := evt.(upnp.AVTransportEvent)
				log.Printf("[DEBUG] TransportState: %v", b.LastChange.InstanceID.TransportState.Val)
				if b.LastChange.InstanceID.TransportState.Val != "PLAYING" {
					continue
				}
				uri := b.LastChange.InstanceID.CurrentTrackURI.Val
				log.Printf("[DEBUG] CurrentTrackURI: %v", b.LastChange.InstanceID.CurrentTrackURI.Val)
				log.Printf("[DEBUG] CurrentTrackMetadata: %s", b.LastChange.InstanceID.CurrentTrackMetaData.Val)
				if strings.HasPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a") {
					log.Print("Playing Spotify track")
					if b.LastChange.InstanceID.CurrentTrackMetaData.Val != "" {
						var doc didl.Lite
						err := xml.Unmarshal([]byte(b.LastChange.InstanceID.CurrentTrackMetaData.Val), &doc)
						if err != nil {
							log.Printf("[ERROR] Could not unmarshal %s: %v", b.LastChange.InstanceID.CurrentTrackMetaData.Val, err)
						}
						for _, item := range doc.Item {
							artist = item.Creator[0].Value
							album = item.Album[0].Value
							title = item.Title[0].Value
							log.Printf("[DEBUG] title: %s, artist: %s, album: %s", title, artist, album)
							break
						}
					}

					// extract the trackid from the uri
					trackID := strings.Split(strings.TrimPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a"), "?")[0]
					track, err := client.GetTrack(spotify.ID(trackID))
					if err != nil {
						log.Printf("[WARN] Could not get track info for trackId: (%s): %v\n", trackID, err)
						continue
					}
					player := getTriggeredSonos(b.Svc)
					if player == nil {
						log.Printf("[WARN] Could not skip track '%s' by '%s': (did not find player)", title, artist)
						continue
					}

					if Rules(track, player.Player.RoomName) {
						posInfo, err := player.GetPositionInfo(0)
						if nil != err {
							panic(err)
						}
						log.Printf("[DEBUG] Position.TrackURI: %s", posInfo.TrackURI)
						log.Printf("[DEBUG] Position.TrackDuration: %s", posInfo.TrackDuration)
						log.Printf("[DEBUG] Position.RelTime: %s", posInfo.RelTime)

						err = player.Next(0)
						if err != nil {
							log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
							continue
						}
						log.Printf("[INFO] Skipped '%s' by '%s' playing on '%s'\n", title, artist, player.Player.RoomName)
					}
				} else {
					log.Printf("Not playing Spotify track: %s", uri)
				}
			default:
				log.Panicf("[ERROR] Unexpected event %#v", evt)
			}
		}
	}
}

func SetupEvents(mgr ssdp.Manager) {
	// Startup and listen to events
	reactor := sonos.MakeReactor(port)
	go handleAVTransportEvents(reactor, exit_chan)
	sonosDevices = sonos.ConnectAll(mgr, reactor, sonos.SVC_AV_TRANSPORT)
	log.Printf("[DEBUG] Set up event handler for Sonos, found %d devices", len(sonosDevices))
	for _, player := range sonosDevices {
		log.Printf("Found %s", player.RoomName)
	}
}

func InitializeSonos() {
	mgr := ssdp.MakeManager()
	ifname := getLocalInterfaceName()
	log.Printf("Discovering devices over %s...", ifname)
	if err := mgr.Discover(ifname, port, false); nil != err {
		panic(err)
	} else {
		SetupEvents(mgr)
	}
}
<commit_msg>Add ability to find radio-friendly songs and skip to them<commit_after>package lib

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"

	"github.com/atinm/go-sonos"
	"github.com/atinm/go-sonos/didl"
	"github.com/atinm/go-sonos/ssdp"
	"github.com/atinm/go-sonos/upnp"
	"github.com/zmb3/spotify"
)

var (
	sonosDevices []*sonos.Sonos
	exit_chan    = make(chan bool)
)

// GetLocalInterfaceName returns the first interface name that has the non loopback local IPv4 addr of the host
func getLocalInterfaceName() string {
	list, err := net.Interfaces()
	if err != nil {
		panic(err)
	}

	for _, iface := range list {
		addrs, err := iface.Addrs()
		if err != nil {
			panic(err)
		}
		for _, addr := range addrs {
			if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
				if ipnet.IP.To4() != nil {
					return iface.Name
				}
			}
		}
	}
	return ""
}

func HandleUpdate(w http.ResponseWriter, req *http.Request) {
	var update Update
	body, err := ioutil.ReadAll(req.Body)
	defer req.Body.Close()
	if err != nil {
		log.Print("[ERROR] ", err)
		return
	}
	if err = json.Unmarshal(body, &update); err != nil {
		log.Print("[ERROR] ", err)
		return
	}

	switch update.Type {
	case "transport-state":
		var s struct {
			Update
			TransportState
		}
		if err = json.Unmarshal(body, &s); err != nil {
			log.Print("[ERROR] ", err)
			return
		}

		bodyStr := fmt.Sprintf("%s", body)
		log.Print("[DEBUG] body: ", bodyStr)
		title := s.Player.State.CurrentTrack.Title
		artist := s.Player.State.CurrentTrack.Artist
		uri := s.Player.State.CurrentTrack.Uri
		roomName = s.Player.RoomName
		log.Println("[DEBUG]", title)
		log.Println("[DEBUG]", uri)
		// CurrentTrackURI: "x-sonos-spotify:spotify%3atrack%3a4Ro98RCK90oHqqSZUnTFq5?sid=12&flags=8224&sn=8"
		// "uri": "x-sonos-spotify:spotify%3atrack%3a6tF92PMv01Ug9Dh8Rmy6nH?sid=12&flags=8224&sn=8",
		if strings.HasPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a") {
			// extract the trackid from the uri
			trackID := strings.Split(strings.TrimPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a"), "?")[0]
			track, err = client.GetTrack(spotify.ID(trackID))
			if err != nil {
				log.Printf("[WARN] Could not get track info for '%s' by '%s', trackId: (%s): %v\n", title, artist, trackID, err)
				break
			}

			if Rules(track, roomName) {
				nextURL := fmt.Sprintf("http://localhost:5005/%s/next", roomName)
				response, err := http.Get(nextURL)
				if err != nil {
					log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
					return
				}
				defer response.Body.Close()
				log.Printf("[INFO] Skipped '%s' by '%s' playing on '%s'\n", title, artist, roomName)
			}
		}

	case "volume-change":
	case "mute-change":
	case "topology-change":
	default:
	}
}

func getTriggeredSonos(svc *upnp.Service) (sonos *sonos.Sonos) {
	for _, s := range sonosDevices {
		if s.AVTransport.Svc == svc {
			sonos = s
			break
		}
	}
	return
}

func handleAVTransportEvents(reactor upnp.Reactor, c chan bool) {
	for {
		select {
		case evt := <-reactor.Channel():
			switch evt.Type() {
			case upnp.AVTransport_EventType:
				var artist, title, album string

				b := evt.(upnp.AVTransportEvent)
				log.Printf("[DEBUG] TransportState: %v", b.LastChange.InstanceID.TransportState.Val)
				if b.LastChange.InstanceID.TransportState.Val != "PLAYING" {
					continue
				}
				uri := b.LastChange.InstanceID.CurrentTrackURI.Val
				log.Printf("[DEBUG] CurrentTrackURI: %v", b.LastChange.InstanceID.CurrentTrackURI.Val)
				log.Printf("[DEBUG] CurrentTrackMetadata: %s", b.LastChange.InstanceID.CurrentTrackMetaData.Val)
				log.Printf("[DEBUG] CurrentTrack: %s", b.LastChange.InstanceID.CurrentTrack.Val)
				currentTrack, _ := strconv.Atoi(b.LastChange.InstanceID.CurrentTrack.Val)
				if strings.HasPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a") {
					log.Print("Playing Spotify track")
					if b.LastChange.InstanceID.CurrentTrackMetaData.Val != "" {
						var doc didl.Lite
						err := xml.Unmarshal([]byte(b.LastChange.InstanceID.CurrentTrackMetaData.Val), &doc)
						if err != nil {
							log.Printf("[ERROR] Could not unmarshal %s: %v", b.LastChange.InstanceID.CurrentTrackMetaData.Val, err)
							continue
						}
						for _, item := range doc.Item {
							for _, a := range item.Creator {
								artist = a.Value
							}
							for _, l := range item.Album {
								album = l.Value
							}
							for _, t := range item.Title {
								title = t.Value
							}
							log.Printf("[DEBUG] title: %s, artist: %s, album: %s", title, artist, album)
							break
						}
					}

					// extract the trackid from the uri
					parsedTrackURI := strings.Split(strings.TrimPrefix(uri, "x-sonos-spotify:spotify%3atrack%3a"), "?")
					trackID := parsedTrackURI[0]
					track, err := client.GetTrack(spotify.ID(trackID))
					if err != nil {
						log.Printf("[WARN] Could not get track info for trackId: (%s): %v\n", trackID, err)
						continue
					}
					player := getTriggeredSonos(b.Svc)
					if player == nil {
						log.Printf("[WARN] Could not skip track '%s' by '%s': (did not find player)", title, artist)
						continue
					}

					if Rules(track, player.Player.RoomName) {
						posInfo, err := player.GetPositionInfo(0)
						if nil != err {
							panic(err)
						}
						log.Printf("[DEBUG] Position.TrackURI: %s", posInfo.TrackURI)
						log.Printf("[DEBUG] Position.TrackDuration: %s", posInfo.TrackDuration)
						log.Printf("[DEBUG] Position.RelTime: %s", posInfo.RelTime)
						var query string
						query = fmt.Sprintf("title:%s artist:%s album:%s", title, artist, album)
						log.Printf("[DEBUG] Querying spotify for %s", query)
						// search for radio friendly track for artist, title, album, explicit=false instead
						if res, err := client.Search(query, spotify.SearchTypeTrack); err != nil || len(res.Tracks.Tracks) == 0 {
							log.Printf("Could not find '%s' by '%s' in search: %v", title, artist, err)
							player.Next(0)
							if err != nil {
								log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
								continue
							}
							log.Printf("[INFO] Skipped '%s' by '%s' playing on '%s'\n", title, artist, player.Player.RoomName)
						} else {
							var complete = false
							for _, track := range res.Tracks.Tracks {
								log.Print("Found tracks searching for alternates")
								if !track.Explicit {
									log.Print("Found radio-friendly track")
									uri := fmt.Sprintf("x-sonos-spotify:spotify%%3atrack%%3a%s?%s", track.ID, parsedTrackURI[1])
									//var uri = "x-sonos-spotify:spotify%3atrack%3a" + trackID
									var meta = "<DIDL-Lite xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:upnp=\"urn:schemas-upnp-org:metadata-1-0/upnp/\" xmlns:r=\"urn:schemas-rinconnetworks-com:metadata-1-0/\" xmlns=\"urn:schemas-upnp-org:metadata-1-0/DIDL-Lite/\"><item id=\"-1\" parentID=\"-1\" restricted=\"true\"><res protocolInfo=\"sonos.com-spotify:*:audio/x-spotify:*\" duration=\"" + posInfo.TrackDuration + "\">x-sonos-spotify:spotify%3atrack%3a" + trackID + "?sid=12&flags=8224&sn=8</res><r:streamContent></r:streamContent><r:radioShowMd></r:radioShowMd><upnp:albumArtURI>/getaa?s=1&u=x-sonos-spotify%3aspotify%253atrack%253a" + trackID + "%3fsid%3d12%26flags%3d8224%26sn%3d8</upnp:albumArtURI><dc:title>" + title + "</dc:title><upnp:class>object.item.audioItem.musicTrack</upnp:class><dc:creator>" + artist +
										"</dc:creator><upnp:album>" + album + "</upnp:album><r:tags>1</r:tags></item></DIDL-Lite>"
									req := upnp.AddURIToQueueIn{
										EnqueuedURI:                     uri,
										EnqueuedURIMetaData:             meta,
										DesiredFirstTrackNumberEnqueued: uint32(currentTrack + 1),
									}
									log.Printf("Queued: %s", meta)
									if _, err := player.AddURIToQueue(0 /*instanceId*/, &req); nil != err {
										log.Printf("[INFO] Couldn't add radio-friendly version '%s', '%s' of '%s' by '%s', skipped playing on '%s': %v\n",
											track.ID, track.URI, title, artist, player.Player.RoomName, err)
										err = player.Next(0)
										if err != nil {
											log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
											break
										}
										complete = true
									} else {
										err = player.Next(0)
										if err != nil {
											log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
											break
										}
										log.Printf("[INFO] Skipped to radio-friendly version '%s' of '%s' by '%s' playing on '%s'", uri, title, artist, player.Player.RoomName)
										complete = true
									}
									break
								}
							}

							if !complete {
								err = player.Next(0)
								if err != nil {
									log.Printf("[WARN] Could not skip track '%s' by '%s': %v", title, artist, err)
									continue
								}
								log.Printf("[INFO] Skipped '%s' by '%s' playing on '%s'\n", title, artist, player.Player.RoomName)
							}
						}
					}
				} else {
					log.Printf("Not playing Spotify track: %s", uri)
				}
			case upnp.ContentDirectory_EventType:

			default:
				log.Panicf("[ERROR] Unexpected event %#v", evt)
			}
		}
	}
}

func SetupEvents(mgr ssdp.Manager) {
	// Startup and listen to events
	reactor := sonos.MakeReactor(port)
	go handleAVTransportEvents(reactor, exit_chan)
	sonosDevices = sonos.ConnectAll(mgr, reactor, sonos.SVC_AV_TRANSPORT|sonos.SVC_CONTENT_DIRECTORY)
	log.Printf("[DEBUG] Set up event handler for Sonos, found %d devices", len(sonosDevices))
	for _, player := range sonosDevices {
		log.Printf("Found %s", player.RoomName)
	}
}

func InitializeSonos() {
	mgr := ssdp.MakeManager()
	ifname := getLocalInterfaceName()
	log.Printf("Discovering devices over %s...", ifname)
	if err := mgr.Discover(ifname, port, false); nil != err {
		panic(err)
	} else {
		SetupEvents(mgr)
	}
}
<|endoftext|>


<commit_before>package mpool

import (
	"errors"
)

var ErrorMessageOverflow = errors.New("message overflow")

// Message encapsulates the messages that we exchange back and forth.
type Message struct {
	Body    []byte
	bodyBuf []byte

	slabSize int
	offset   int
}

type slabClass struct {
	maxSize int
	ch      chan *Message
}

var messagePool = []slabClass{
	{maxSize: 256, ch: make(chan *Message, 20<<10)},     // 5MB   = 256 * 20K
	{maxSize: 1024, ch: make(chan *Message, 50<<10)},    // 50MB  = 1K * 50K
	{maxSize: 2 << 10, ch: make(chan *Message, 50<<10)}, // 100MB = 2K * 50K
	{maxSize: 8 << 10, ch: make(chan *Message, 1<<10)},  // 8MB   = 8K * 1K
	{maxSize: 64 << 10, ch: make(chan *Message, 1<<8)},  // 16MB  = 64K * 256
	{maxSize: 256 << 10, ch: make(chan *Message, 1<<7)}, // 32MB  = 256K * 128
}

// Free decrements the reference count on a message, and releases its
// resources if no further references remain.  While this is not
// strictly necessary thanks to GC, doing so allows for the resources to
// be recycled without engaging GC.  This can have rather substantial
// benefits for performance.
func (this *Message) Free() (recycled bool) {
	var ch chan *Message
	for _, slab := range messagePool {
		if this.slabSize == slab.maxSize {
			ch = slab.ch
			break
		}
	}

	select {
	case ch <- this:
	default:
		// this slab class pool is full, silently drop
	}

	return true
}

func (this *Message) Reset() {
	this.offset = 0
}

// WriteString is a costly function. Use it only when you know its underhood.
// Currently, kateway is not using this function.
func (this *Message) WriteString(s string) error {
	if len(s)+this.offset > cap(this.bodyBuf) {
		return ErrorMessageOverflow
	}

	this.Body = this.Body[0 : this.offset+len(s)]
	copy(this.Body[this.offset:], s)
	this.offset += len(s)
	return nil
}

func (this *Message) Bytes() []byte {
	return this.Body[0:]
}

// NewMessage is the supported way to obtain a new Message.  This makes
// use of a "slab allocator" which greatly reduces the load on the
// garbage collector.
func NewMessage(size int) *Message {
	var msg *Message
	var ch chan *Message
	for _, slabClass := range messagePool { // TODO improve perf
		if size <= slabClass.maxSize {
			ch = slabClass.ch
			size = slabClass.maxSize
			break
		}
	}

	select {
	case msg = <-ch:
	default:
		// message pool empty:
		// too busy or size greater than largest slab class
		msg = &Message{}
		msg.slabSize = size
		msg.bodyBuf = make([]byte, 0, msg.slabSize)
	}

	msg.Body = msg.bodyBuf
	return msg
}
<commit_msg>2KB~4KB message size is a common scenery<commit_after>package mpool

import (
	"errors"
)

var ErrorMessageOverflow = errors.New("message overflow")

// Message encapsulates the messages that we exchange back and forth.
type Message struct {
	Body    []byte
	bodyBuf []byte

	slabSize int
	offset   int
}

type slabClass struct {
	maxSize int
	ch      chan *Message
}

var messagePool = []slabClass{
	{maxSize: 256, ch: make(chan *Message, 20<<10)},     // 5MB   = 256 * 20K
	{maxSize: 1024, ch: make(chan *Message, 50<<10)},    // 50MB  = 1K * 50K
	{maxSize: 2 << 10, ch: make(chan *Message, 50<<10)}, // 100MB = 2K * 50K
	{maxSize: 4 << 10, ch: make(chan *Message, 50<<10)}, // 200MB = 4K * 50K
	{maxSize: 8 << 10, ch: make(chan *Message, 1<<10)},  // 8MB   = 8K * 1K
	{maxSize: 64 << 10, ch: make(chan *Message, 1<<8)},  // 16MB  = 64K * 256
	{maxSize: 256 << 10, ch: make(chan *Message, 1<<7)}, // 32MB  = 256K * 128
}

// Free decrements the reference count on a message, and releases its
// resources if no further references remain.  While this is not
// strictly necessary thanks to GC, doing so allows for the resources to
// be recycled without engaging GC.  This can have rather substantial
// benefits for performance.
func (this *Message) Free() (recycled bool) {
	var ch chan *Message
	for _, slab := range messagePool {
		if this.slabSize == slab.maxSize {
			ch = slab.ch
			break
		}
	}

	select {
	case ch <- this:
	default:
		// this slab class pool is full, silently drop
	}

	return true
}

func (this *Message) Reset() {
	this.offset = 0
}

// WriteString is a costly function. Use it only when you know its underhood.
// Currently, kateway is not using this function.
func (this *Message) WriteString(s string) error {
	if len(s)+this.offset > cap(this.bodyBuf) {
		return ErrorMessageOverflow
	}

	this.Body = this.Body[0 : this.offset+len(s)]
	copy(this.Body[this.offset:], s)
	this.offset += len(s)
	return nil
}

func (this *Message) Bytes() []byte {
	return this.Body[0:]
}

// NewMessage is the supported way to obtain a new Message.  This makes
// use of a "slab allocator" which greatly reduces the load on the
// garbage collector.
func NewMessage(size int) *Message {
	var msg *Message
	var ch chan *Message
	for _, slabClass := range messagePool { // TODO improve perf
		if size <= slabClass.maxSize {
			ch = slabClass.ch
			size = slabClass.maxSize
			break
		}
	}

	select {
	case msg = <-ch:
	default:
		// message pool empty:
		// too busy or size greater than largest slab class
		msg = &Message{}
		msg.slabSize = size
		msg.bodyBuf = make([]byte, 0, msg.slabSize)
	}

	msg.Body = msg.bodyBuf
	return msg
}
<|endoftext|>


<commit_before><commit_msg>chore(api): refactor post objects to remove timeout, make async<commit_after><|endoftext|>


<commit_before><commit_msg>fix issue with embedding complete<commit_after><|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"go.etcd.io/etcd/etcdutl/v3/snapshot"
	"go.etcd.io/etcd/pkg/v3/expect"
)

func TestCtlV3Snapshot(t *testing.T)        { testCtl(t, snapshotTest) }
func TestCtlV3SnapshotEtcdutl(t *testing.T) { testCtl(t, snapshotTest, withEtcdutl()) }

func snapshotTest(cx ctlCtx) {
	maintenanceInitKeys(cx)

	leaseID, err := ctlV3LeaseGrant(cx, 100)
	if err != nil {
		cx.t.Fatalf("snapshot: ctlV3LeaseGrant error (%v)", err)
	}
	if err = ctlV3Put(cx, "withlease", "withlease", leaseID); err != nil {
		cx.t.Fatalf("snapshot: ctlV3Put error (%v)", err)
	}

	fpath := filepath.Join(cx.t.TempDir(), "snapshot")
	defer os.RemoveAll(fpath)

	if err = ctlV3SnapshotSave(cx, fpath); err != nil {
		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
	}

	st, err := getSnapshotStatus(cx, fpath)
	if err != nil {
		cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err)
	}
	if st.Revision != 5 {
		cx.t.Fatalf("expected 4, got %d", st.Revision)
	}
	if st.TotalKey < 4 {
		cx.t.Fatalf("expected at least 4, got %d", st.TotalKey)
	}
}

func TestCtlV3SnapshotCorrupt(t *testing.T)        { testCtl(t, snapshotCorruptTest) }
func TestCtlV3SnapshotCorruptEtcdutl(t *testing.T) { testCtl(t, snapshotCorruptTest, withEtcdutl()) }

func snapshotCorruptTest(cx ctlCtx) {
	fpath := filepath.Join(cx.t.TempDir(), "snapshot")
	defer os.RemoveAll(fpath)

	if err := ctlV3SnapshotSave(cx, fpath); err != nil {
		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
	}

	// corrupt file
	f, oerr := os.OpenFile(fpath, os.O_WRONLY, 0)
	if oerr != nil {
		cx.t.Fatal(oerr)
	}
	if _, err := f.Write(make([]byte, 512)); err != nil {
		cx.t.Fatal(err)
	}
	f.Close()

	datadir := cx.t.TempDir()

	serr := spawnWithExpect(
		append(cx.PrefixArgsUtl(), "snapshot", "restore",
			"--data-dir", datadir,
			fpath),
		"expected sha256")

	if serr != nil {
		cx.t.Fatal(serr)
	}
}

// This test ensures that the snapshot status does not modify the snapshot file
func TestCtlV3SnapshotStatusBeforeRestore(t *testing.T) { testCtl(t, snapshotStatusBeforeRestoreTest) }
func TestCtlV3SnapshotStatusBeforeRestoreEtcdutl(t *testing.T) {
	testCtl(t, snapshotStatusBeforeRestoreTest, withEtcdutl())
}

func snapshotStatusBeforeRestoreTest(cx ctlCtx) {
	fpath := filepath.Join(cx.t.TempDir(), "snapshot")
	defer os.RemoveAll(fpath)

	if err := ctlV3SnapshotSave(cx, fpath); err != nil {
		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
	}

	// snapshot status on the fresh snapshot file
	_, err := getSnapshotStatus(cx, fpath)
	if err != nil {
		cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err)
	}

	dataDir := cx.t.TempDir()
	defer os.RemoveAll(dataDir)
	serr := spawnWithExpect(
		append(cx.PrefixArgsUtl(), "snapshot", "restore",
			"--data-dir", dataDir,
			fpath),
		"added member")
	if serr != nil {
		cx.t.Fatal(serr)
	}
}

func ctlV3SnapshotSave(cx ctlCtx, fpath string) error {
	cmdArgs := append(cx.PrefixArgs(), "snapshot", "save", fpath)
	return spawnWithExpect(cmdArgs, fmt.Sprintf("Snapshot saved at %s", fpath))
}

func getSnapshotStatus(cx ctlCtx, fpath string) (snapshot.Status, error) {
	cmdArgs := append(cx.PrefixArgsUtl(), "--write-out", "json", "snapshot", "status", fpath)

	proc, err := spawnCmd(cmdArgs)
	if err != nil {
		return snapshot.Status{}, err
	}
	var txt string
	txt, err = proc.Expect("totalKey")
	if err != nil {
		return snapshot.Status{}, err
	}
	if err = proc.Close(); err != nil {
		return snapshot.Status{}, err
	}

	resp := snapshot.Status{}
	dec := json.NewDecoder(strings.NewReader(txt))
	if err := dec.Decode(&resp); err == io.EOF {
		return snapshot.Status{}, err
	}
	return resp, nil
}

func TestIssue6361(t *testing.T)        { testIssue6361(t, false) }
func TestIssue6361etcdutl(t *testing.T) { testIssue6361(t, true) }

// TestIssue6361 ensures new member that starts with snapshot correctly
// syncs up with other members and serve correct data.
func testIssue6361(t *testing.T, etcdutl bool) {
	{
		// This tests is pretty flaky on semaphoreci as of 2021-01-10.
		// TODO: Remove when the flakiness source is identified.
		oldenv := os.Getenv("EXPECT_DEBUG")
		defer os.Setenv("EXPECT_DEBUG", oldenv)
		os.Setenv("EXPECT_DEBUG", "1")
	}

	BeforeTest(t)
	os.Setenv("ETCDCTL_API", "3")
	defer os.Unsetenv("ETCDCTL_API")

	epc, err := newEtcdProcessCluster(t, &etcdProcessClusterConfig{
		clusterSize:  1,
		initialToken: "new",
		keepDataDir:  true,
	})
	if err != nil {
		t.Fatalf("could not start etcd process cluster (%v)", err)
	}
	defer func() {
		if errC := epc.Close(); errC != nil {
			t.Fatalf("error closing etcd processes (%v)", errC)
		}
	}()

	dialTimeout := 10 * time.Second
	prefixArgs := []string{ctlBinPath, "--endpoints", strings.Join(epc.EndpointsV3(), ","), "--dial-timeout", dialTimeout.String()}

	t.Log("Writing some keys...")
	kvs := []kv{{"foo1", "val1"}, {"foo2", "val2"}, {"foo3", "val3"}}
	for i := range kvs {
		if err = spawnWithExpect(append(prefixArgs, "put", kvs[i].key, kvs[i].val), "OK"); err != nil {
			t.Fatal(err)
		}
	}

	fpath := filepath.Join(t.TempDir(), "test.snapshot")

	t.Log("etcdctl saving snapshot...")
	if err = spawnWithExpects(append(prefixArgs, "snapshot", "save", fpath),
		fmt.Sprintf("Snapshot saved at %s", fpath),
		"Server version 3.5.0",
	); err != nil {
		t.Fatal(err)
	}

	t.Log("Stopping the original server...")
	if err = epc.procs[0].Stop(); err != nil {
		t.Fatal(err)
	}

	newDataDir := filepath.Join(t.TempDir(), "test.data")

	uctlBinPath := ctlBinPath
	if etcdutl {
		uctlBinPath = utlBinPath
	}

	t.Log("etcdctl restoring the snapshot...")
	err = spawnWithExpect([]string{uctlBinPath, "snapshot", "restore", fpath, "--name", epc.procs[0].Config().name, "--initial-cluster", epc.procs[0].Config().initialCluster, "--initial-cluster-token", epc.procs[0].Config().initialToken, "--initial-advertise-peer-urls", epc.procs[0].Config().purl.String(), "--data-dir", newDataDir}, "added member")
	if err != nil {
		t.Fatal(err)
	}

	t.Log("(Re)starting the etcd member using the restored snapshot...")
	epc.procs[0].Config().dataDirPath = newDataDir
	for i := range epc.procs[0].Config().args {
		if epc.procs[0].Config().args[i] == "--data-dir" {
			epc.procs[0].Config().args[i+1] = newDataDir
		}
	}
	if err = epc.procs[0].Restart(); err != nil {
		t.Fatal(err)
	}

	t.Log("Ensuring the restored member has the correct data...")
	for i := range kvs {
		if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Adding new member into the cluster")
	clientURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+30)
	peerURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+31)
	err = spawnWithExpect(append(prefixArgs, "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)), " added to cluster ")
	if err != nil {
		t.Fatal(err)
	}

	newDataDir2 := t.TempDir()
	defer os.RemoveAll(newDataDir2)

	name2 := "infra2"
	initialCluster2 := epc.procs[0].Config().initialCluster + fmt.Sprintf(",%s=%s", name2, peerURL)

	t.Log("Starting the new member")
	// start the new member
	var nepc *expect.ExpectProcess
	nepc, err = spawnCmd([]string{epc.procs[0].Config().execPath, "--name", name2,
		"--listen-client-urls", clientURL, "--advertise-client-urls", clientURL,
		"--listen-peer-urls", peerURL, "--initial-advertise-peer-urls", peerURL,
		"--initial-cluster", initialCluster2, "--initial-cluster-state", "existing", "--data-dir", newDataDir2})
	if err != nil {
		t.Fatal(err)
	}
	if _, err = nepc.Expect("ready to serve client requests"); err != nil {
		t.Fatal(err)
	}

	prefixArgs = []string{ctlBinPath, "--endpoints", clientURL, "--dial-timeout", dialTimeout.String()}

	t.Log("Ensuring added member has data from incoming snapshot...")
	for i := range kvs {
		if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Stopping the second member")
	if err = nepc.Stop(); err != nil {
		t.Fatal(err)
	}
	t.Log("Test logic done")
}
<commit_msg>tests/e2e/ctl_v3_snapshot_test.go: Adjust version to 3.6.0<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"go.etcd.io/etcd/etcdutl/v3/snapshot"
	"go.etcd.io/etcd/pkg/v3/expect"
)

func TestCtlV3Snapshot(t *testing.T)        { testCtl(t, snapshotTest) }
func TestCtlV3SnapshotEtcdutl(t *testing.T) { testCtl(t, snapshotTest, withEtcdutl()) }

func snapshotTest(cx ctlCtx) {
	maintenanceInitKeys(cx)

	leaseID, err := ctlV3LeaseGrant(cx, 100)
	if err != nil {
		cx.t.Fatalf("snapshot: ctlV3LeaseGrant error (%v)", err)
	}
	if err = ctlV3Put(cx, "withlease", "withlease", leaseID); err != nil {
		cx.t.Fatalf("snapshot: ctlV3Put error (%v)", err)
	}

	fpath := filepath.Join(cx.t.TempDir(), "snapshot")
	defer os.RemoveAll(fpath)

	if err = ctlV3SnapshotSave(cx, fpath); err != nil {
		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
	}

	st, err := getSnapshotStatus(cx, fpath)
	if err != nil {
		cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err)
	}
	if st.Revision != 5 {
		cx.t.Fatalf("expected 4, got %d", st.Revision)
	}
	if st.TotalKey < 4 {
		cx.t.Fatalf("expected at least 4, got %d", st.TotalKey)
	}
}

func TestCtlV3SnapshotCorrupt(t *testing.T)        { testCtl(t, snapshotCorruptTest) }
func TestCtlV3SnapshotCorruptEtcdutl(t *testing.T) { testCtl(t, snapshotCorruptTest, withEtcdutl()) }

func snapshotCorruptTest(cx ctlCtx) {
	fpath := filepath.Join(cx.t.TempDir(), "snapshot")
	defer os.RemoveAll(fpath)

	if err := ctlV3SnapshotSave(cx, fpath); err != nil {
		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
	}

	// corrupt file
	f, oerr := os.OpenFile(fpath, os.O_WRONLY, 0)
	if oerr != nil {
		cx.t.Fatal(oerr)
	}
	if _, err := f.Write(make([]byte, 512)); err != nil {
		cx.t.Fatal(err)
	}
	f.Close()

	datadir := cx.t.TempDir()

	serr := spawnWithExpect(
		append(cx.PrefixArgsUtl(), "snapshot", "restore",
			"--data-dir", datadir,
			fpath),
		"expected sha256")

	if serr != nil {
		cx.t.Fatal(serr)
	}
}

// This test ensures that the snapshot status does not modify the snapshot file
func TestCtlV3SnapshotStatusBeforeRestore(t *testing.T) { testCtl(t, snapshotStatusBeforeRestoreTest) }
func TestCtlV3SnapshotStatusBeforeRestoreEtcdutl(t *testing.T) {
	testCtl(t, snapshotStatusBeforeRestoreTest, withEtcdutl())
}

func snapshotStatusBeforeRestoreTest(cx ctlCtx) {
	fpath := filepath.Join(cx.t.TempDir(), "snapshot")
	defer os.RemoveAll(fpath)

	if err := ctlV3SnapshotSave(cx, fpath); err != nil {
		cx.t.Fatalf("snapshotTest ctlV3SnapshotSave error (%v)", err)
	}

	// snapshot status on the fresh snapshot file
	_, err := getSnapshotStatus(cx, fpath)
	if err != nil {
		cx.t.Fatalf("snapshotTest getSnapshotStatus error (%v)", err)
	}

	dataDir := cx.t.TempDir()
	defer os.RemoveAll(dataDir)
	serr := spawnWithExpect(
		append(cx.PrefixArgsUtl(), "snapshot", "restore",
			"--data-dir", dataDir,
			fpath),
		"added member")
	if serr != nil {
		cx.t.Fatal(serr)
	}
}

func ctlV3SnapshotSave(cx ctlCtx, fpath string) error {
	cmdArgs := append(cx.PrefixArgs(), "snapshot", "save", fpath)
	return spawnWithExpect(cmdArgs, fmt.Sprintf("Snapshot saved at %s", fpath))
}

func getSnapshotStatus(cx ctlCtx, fpath string) (snapshot.Status, error) {
	cmdArgs := append(cx.PrefixArgsUtl(), "--write-out", "json", "snapshot", "status", fpath)

	proc, err := spawnCmd(cmdArgs)
	if err != nil {
		return snapshot.Status{}, err
	}
	var txt string
	txt, err = proc.Expect("totalKey")
	if err != nil {
		return snapshot.Status{}, err
	}
	if err = proc.Close(); err != nil {
		return snapshot.Status{}, err
	}

	resp := snapshot.Status{}
	dec := json.NewDecoder(strings.NewReader(txt))
	if err := dec.Decode(&resp); err == io.EOF {
		return snapshot.Status{}, err
	}
	return resp, nil
}

func TestIssue6361(t *testing.T)        { testIssue6361(t, false) }
func TestIssue6361etcdutl(t *testing.T) { testIssue6361(t, true) }

// TestIssue6361 ensures new member that starts with snapshot correctly
// syncs up with other members and serve correct data.
func testIssue6361(t *testing.T, etcdutl bool) {
	{
		// This tests is pretty flaky on semaphoreci as of 2021-01-10.
		// TODO: Remove when the flakiness source is identified.
		oldenv := os.Getenv("EXPECT_DEBUG")
		defer os.Setenv("EXPECT_DEBUG", oldenv)
		os.Setenv("EXPECT_DEBUG", "1")
	}

	BeforeTest(t)
	os.Setenv("ETCDCTL_API", "3")
	defer os.Unsetenv("ETCDCTL_API")

	epc, err := newEtcdProcessCluster(t, &etcdProcessClusterConfig{
		clusterSize:  1,
		initialToken: "new",
		keepDataDir:  true,
	})
	if err != nil {
		t.Fatalf("could not start etcd process cluster (%v)", err)
	}
	defer func() {
		if errC := epc.Close(); errC != nil {
			t.Fatalf("error closing etcd processes (%v)", errC)
		}
	}()

	dialTimeout := 10 * time.Second
	prefixArgs := []string{ctlBinPath, "--endpoints", strings.Join(epc.EndpointsV3(), ","), "--dial-timeout", dialTimeout.String()}

	t.Log("Writing some keys...")
	kvs := []kv{{"foo1", "val1"}, {"foo2", "val2"}, {"foo3", "val3"}}
	for i := range kvs {
		if err = spawnWithExpect(append(prefixArgs, "put", kvs[i].key, kvs[i].val), "OK"); err != nil {
			t.Fatal(err)
		}
	}

	fpath := filepath.Join(t.TempDir(), "test.snapshot")

	t.Log("etcdctl saving snapshot...")
	if err = spawnWithExpects(append(prefixArgs, "snapshot", "save", fpath),
		fmt.Sprintf("Snapshot saved at %s", fpath),
		"Server version 3.6.0",
	); err != nil {
		t.Fatal(err)
	}

	t.Log("Stopping the original server...")
	if err = epc.procs[0].Stop(); err != nil {
		t.Fatal(err)
	}

	newDataDir := filepath.Join(t.TempDir(), "test.data")

	uctlBinPath := ctlBinPath
	if etcdutl {
		uctlBinPath = utlBinPath
	}

	t.Log("etcdctl restoring the snapshot...")
	err = spawnWithExpect([]string{uctlBinPath, "snapshot", "restore", fpath, "--name", epc.procs[0].Config().name, "--initial-cluster", epc.procs[0].Config().initialCluster, "--initial-cluster-token", epc.procs[0].Config().initialToken, "--initial-advertise-peer-urls", epc.procs[0].Config().purl.String(), "--data-dir", newDataDir}, "added member")
	if err != nil {
		t.Fatal(err)
	}

	t.Log("(Re)starting the etcd member using the restored snapshot...")
	epc.procs[0].Config().dataDirPath = newDataDir
	for i := range epc.procs[0].Config().args {
		if epc.procs[0].Config().args[i] == "--data-dir" {
			epc.procs[0].Config().args[i+1] = newDataDir
		}
	}
	if err = epc.procs[0].Restart(); err != nil {
		t.Fatal(err)
	}

	t.Log("Ensuring the restored member has the correct data...")
	for i := range kvs {
		if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Adding new member into the cluster")
	clientURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+30)
	peerURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+31)
	err = spawnWithExpect(append(prefixArgs, "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL)), " added to cluster ")
	if err != nil {
		t.Fatal(err)
	}

	newDataDir2 := t.TempDir()
	defer os.RemoveAll(newDataDir2)

	name2 := "infra2"
	initialCluster2 := epc.procs[0].Config().initialCluster + fmt.Sprintf(",%s=%s", name2, peerURL)

	t.Log("Starting the new member")
	// start the new member
	var nepc *expect.ExpectProcess
	nepc, err = spawnCmd([]string{epc.procs[0].Config().execPath, "--name", name2,
		"--listen-client-urls", clientURL, "--advertise-client-urls", clientURL,
		"--listen-peer-urls", peerURL, "--initial-advertise-peer-urls", peerURL,
		"--initial-cluster", initialCluster2, "--initial-cluster-state", "existing", "--data-dir", newDataDir2})
	if err != nil {
		t.Fatal(err)
	}
	if _, err = nepc.Expect("ready to serve client requests"); err != nil {
		t.Fatal(err)
	}

	prefixArgs = []string{ctlBinPath, "--endpoints", clientURL, "--dial-timeout", dialTimeout.String()}

	t.Log("Ensuring added member has data from incoming snapshot...")
	for i := range kvs {
		if err = spawnWithExpect(append(prefixArgs, "get", kvs[i].key), kvs[i].val); err != nil {
			t.Fatal(err)
		}
	}

	t.Log("Stopping the second member")
	if err = nepc.Stop(); err != nil {
		t.Fatal(err)
	}
	t.Log("Test logic done")
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Rene Richard

This file is part of zmq-soundtouch.

zmq-soundtouch is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

zmq-soundtouch is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with zmq-soundtouch.  If not, see <http://www.gnu.org/licenses/>.
*/

package messaging

import (
	zmq "github.com/pebbe/zmq4"
	"github.com/redsofa/logger"
	"os"
	"strings"
)

const (
	ROUTER_URL        = "tcp://127.0.0.1:8000"
	CACHE_END_TOKEN   = "KTHXBYE"
	CACHE_START_TOKEN = "ICANHAZ?"
)

type dealer struct {
	ctx      *zmq.Context
	msgChan  chan string
	doneChan chan bool
	errChan  chan error
	client   *zmq.Socket
}

func NewDealer() *dealer {
	ctx, _ := zmq.NewContext()

	msgChan := make(chan string)
	doneChan := make(chan bool)
	errChan := make(chan error)

	client, err := ctx.NewSocket(zmq.DEALER)

	if err != nil {
		logger.Error.Println("Error openinng DEALER	socket", err)
		os.Exit(1)
	}

	return &dealer{ctx, msgChan, doneChan, errChan, client}
}

func (this *dealer) processMessages() {
	logger.Info.Println("Firing up processMessages loop")
	for {
		select {
		//We receive a message on the message channel
		case msg := <-this.msgChan:
			logger.Info.Println("Processing Message: " + msg)
		//We have an error on the error channel
		case err := <-this.errChan:
			logger.Error.Println("Error : ", err)
			return
		//We're done
		case <-this.doneChan:
			logger.Info.Println("Done Processing Messages")
			return
		}
	}
}

func (this *dealer) receiveMessages() {
	logger.Info.Println("Firing up receiveMessages loop")
	for {
		select {
		//We have an error
		case err := <-this.errChan:
			logger.Error.Println("Error :", err)
			this.errChan <- err // to notify processMessages()
			return

		//Read data from socket connection (loop)
		default:
			reply, err := this.client.Recv(0)

			if err != nil {
				this.errChan <- err
				return
			}
			if strings.Compare(reply, CACHE_END_TOKEN) == 0 {
				logger.Info.Println("Done reading cache")
				this.doneChan <- true //to notify processMessages()
				return
			} else {
				this.msgChan <- reply //will be picked up by processMessages()
			}
		}
	}
}

func (this *dealer) Start() {
	defer this.ctx.Term()

	logger.Info.Println("Starting Dealer ...")

	this.client.Connect(ROUTER_URL)
	defer this.client.Close()

	logger.Info.Println("Sending request for cache conntents")
	this.client.Send(CACHE_START_TOKEN, 0)

	go this.processMessages()
	this.receiveMessages()
}
<commit_msg>TODO on zmqdealer.go<commit_after>/*
Copyright 2016 Rene Richard

This file is part of zmq-soundtouch.

zmq-soundtouch is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

zmq-soundtouch is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with zmq-soundtouch.  If not, see <http://www.gnu.org/licenses/>.
*/

package messaging

import (
	zmq "github.com/pebbe/zmq4"
	"github.com/redsofa/logger"
	"os"
	"strings"
)

const (
	ROUTER_URL        = "tcp://127.0.0.1:8000"
	CACHE_END_TOKEN   = "KTHXBYE"
	CACHE_START_TOKEN = "ICANHAZ?"
)

type dealer struct {
	ctx      *zmq.Context
	msgChan  chan string
	doneChan chan bool
	errChan  chan error
	client   *zmq.Socket
}

func NewDealer() *dealer {
	ctx, _ := zmq.NewContext()

	msgChan := make(chan string)
	doneChan := make(chan bool)
	errChan := make(chan error)

	client, err := ctx.NewSocket(zmq.DEALER)

	if err != nil {
		logger.Error.Println("Error openinng DEALER	socket", err)
		os.Exit(1)
	}

	return &dealer{ctx, msgChan, doneChan, errChan, client}
}

func (this *dealer) processMessages() {
	logger.Info.Println("Firing up processMessages loop")
	for {
		select {
		//We receive a message on the message channel
		case msg := <-this.msgChan:
			logger.Info.Println("Processing Message: " + msg)
			//TODO : Relay cached messages over to WebSocket client. Should be method call on websocketclient

		//We have an error on the error channel
		case err := <-this.errChan:
			logger.Error.Println("Error : ", err)
			return
		//We're done
		case <-this.doneChan:
			logger.Info.Println("Done Processing Messages")
			return
		}
	}
}

func (this *dealer) receiveMessages() {
	logger.Info.Println("Firing up receiveMessages loop")
	for {
		select {
		//We have an error
		case err := <-this.errChan:
			logger.Error.Println("Error :", err)
			this.errChan <- err // to notify processMessages()
			return

		//Read data from socket connection (loop)
		default:
			reply, err := this.client.Recv(0)

			if err != nil {
				this.errChan <- err
				return
			}
			if strings.Compare(reply, CACHE_END_TOKEN) == 0 {
				logger.Info.Println("Done reading cache")
				this.doneChan <- true //to notify processMessages()
				return
			} else {
				this.msgChan <- reply //will be picked up by processMessages()
			}
		}
	}
}

func (this *dealer) Start() {
	defer this.ctx.Term()

	logger.Info.Println("Starting Dealer ...")

	this.client.Connect(ROUTER_URL)
	defer this.client.Close()

	logger.Info.Println("Sending request for cache conntents")
	this.client.Send(CACHE_START_TOKEN, 0)

	go this.processMessages()
	this.receiveMessages()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Delete a Replication Controller and all pods it spawned
func DeleteRC(c *client.Client, ns, name string) error {
	rc, err := c.ReplicationControllers(ns).Get(name)
	if err != nil {
		return fmt.Errorf("Failed to find replication controller %s in namespace %s: %v", name, ns, err)
	}

	rc.Spec.Replicas = 0

	if _, err := c.ReplicationControllers(ns).Update(rc); err != nil {
		return fmt.Errorf("Failed to resize replication controller %s to zero: %v", name, err)
	}

	if err := wait.Poll(time.Second, time.Minute*20, client.ControllerHasDesiredReplicas(c, rc)); err != nil {
		return fmt.Errorf("Error waiting for replication controller %s replicas to reach 0: %v", name, err)
	}

	// Delete the replication controller.
	if err := c.ReplicationControllers(ns).Delete(name); err != nil {
		return fmt.Errorf("Failed to delete replication controller %s: %v", name, err)
	}
	return nil
}

// Launch a Replication Controller and wait for all pods it spawns
// to become running
func RunRC(c *client.Client, name string, ns, image string, replicas int) {
	defer GinkgoRecover()

	var last int
	current := 0
	same := 0

	defer func() {
		By("Cleaning up the replication controller")
		err := DeleteRC(c, ns, name)
		Expect(err).NotTo(HaveOccurred())
	}()

	By(fmt.Sprintf("Creating replication controller %s", name))
	_, err := c.ReplicationControllers(ns).Create(&api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: api.ReplicationControllerSpec{
			Replicas: replicas,
			Selector: map[string]string{
				"name": name,
			},
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: map[string]string{"name": name},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  name,
							Image: image,
							Ports: []api.ContainerPort{{ContainerPort: 80}},
						},
					},
				},
			},
		},
	})
	Expect(err).NotTo(HaveOccurred())

	By(fmt.Sprintf("Making sure all %d replicas exist", replicas))
	label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
	pods, err := c.Pods(ns).List(label)
	Expect(err).NotTo(HaveOccurred())
	current = len(pods.Items)
	failCount := 5
	for same < failCount && current < replicas {
		Logf("Controller %s: Found %d pods out of %d", name, current, replicas)
		if last < current {
			same = 0
		} else if last == current {
			same++
		} else if current < last {
			Failf("Controller %s: Number of submitted pods dropped from %d to %d", last, current)
		}

		if same >= failCount {
			Logf("No pods submitted for the last %d checks", failCount)
		}

		last = current
		time.Sleep(5 * time.Second)
		pods, err = c.Pods(ns).List(label)
		Expect(err).NotTo(HaveOccurred())
		current = len(pods.Items)
	}
	Expect(current).To(Equal(replicas))
	Logf("Controller %s: Found %d pods out of %d", name, current, replicas)

	By("Waiting for each pod to be running")
	same = 0
	last = 0
	failCount = 6
	unknown := 0
	pending := 0
	current = 0
	for same < failCount && current < replicas {
		current = 0
		pending = 0
		unknown = 0
		time.Sleep(10 * time.Second)
		for _, pod := range pods.Items {
			p, err := c.Pods(ns).Get(pod.Name)
			Expect(err).NotTo(HaveOccurred())
			if p.Status.Phase == api.PodRunning {
				current++
			} else if p.Status.Phase == api.PodPending {
				pending++
			} else if p.Status.Phase == api.PodUnknown {
				unknown++
			}
		}
		Logf("Pod States: %d running, %d pending, %d unknown ", current, pending, unknown)
		if last < current {
			same = 0
		} else if last == current {
			same++
		} else if current < last {
			Failf("Number of running pods dropped from %d to %d", last, current)
		}
		if same >= failCount {
			Logf("No pods started for the last %d checks", failCount)
		}
		last = current
	}
	Expect(current).To(Equal(replicas))
}

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var ns string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List()
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		ns = api.NamespaceDefault
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			DeleteRC(c, ns, RCName)
		}
	})

	It("should allow starting 100 pods per node", func() {
		RCName = "my-hostname-density100-" + string(util.NewUUID())
		RunRC(c, RCName, ns, "dockerfile/nginx", 100*minionCount)
	})

	It("should have master components that can handle many short-lived pods", func() {
		threads := 5
		var wg sync.WaitGroup
		wg.Add(threads)
		for i := 0; i < threads; i++ {
			go func() {
				defer wg.Done()
				for i := 0; i < 10; i++ {
					name := "my-hostname-thrash-" + string(util.NewUUID())
					RunRC(c, name, ns, "kubernetes/pause", 10*minionCount)
				}
			}()
		}
		wg.Wait()
	})
})
<commit_msg>Improve density test to log unscheduled pods<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"sync"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait"

	"github.com/golang/glog"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Delete a Replication Controller and all pods it spawned
func DeleteRC(c *client.Client, ns, name string) error {
	rc, err := c.ReplicationControllers(ns).Get(name)
	if err != nil {
		return fmt.Errorf("Failed to find replication controller %s in namespace %s: %v", name, ns, err)
	}

	rc.Spec.Replicas = 0

	if _, err := c.ReplicationControllers(ns).Update(rc); err != nil {
		return fmt.Errorf("Failed to resize replication controller %s to zero: %v", name, err)
	}

	if err := wait.Poll(time.Second, time.Minute*20, client.ControllerHasDesiredReplicas(c, rc)); err != nil {
		return fmt.Errorf("Error waiting for replication controller %s replicas to reach 0: %v", name, err)
	}

	// Delete the replication controller.
	if err := c.ReplicationControllers(ns).Delete(name); err != nil {
		return fmt.Errorf("Failed to delete replication controller %s: %v", name, err)
	}
	return nil
}

// Launch a Replication Controller and wait for all pods it spawns
// to become running
func RunRC(c *client.Client, name string, ns, image string, replicas int) {
	defer GinkgoRecover()

	var last int
	current := 0
	same := 0

	defer func() {
		By("Cleaning up the replication controller")
		err := DeleteRC(c, ns, name)
		Expect(err).NotTo(HaveOccurred())
	}()

	By(fmt.Sprintf("Creating replication controller %s", name))
	_, err := c.ReplicationControllers(ns).Create(&api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Name: name,
		},
		Spec: api.ReplicationControllerSpec{
			Replicas: replicas,
			Selector: map[string]string{
				"name": name,
			},
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: map[string]string{"name": name},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  name,
							Image: image,
							Ports: []api.ContainerPort{{ContainerPort: 80}},
						},
					},
				},
			},
		},
	})
	Expect(err).NotTo(HaveOccurred())

	By(fmt.Sprintf("Making sure all %d replicas exist", replicas))
	label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
	pods, err := c.Pods(ns).List(label)
	Expect(err).NotTo(HaveOccurred())
	current = len(pods.Items)
	failCount := 5
	for same < failCount && current < replicas {
		glog.Infof("Controller %s: Found %d pods out of %d", name, current, replicas)
		if last < current {
			same = 0
		} else if last == current {
			same++
		} else if current < last {
			Failf("Controller %s: Number of submitted pods dropped from %d to %d", last, current)
		}

		if same >= failCount {
			glog.Infof("No pods submitted for the last %d checks", failCount)
		}

		last = current
		time.Sleep(5 * time.Second)
		pods, err = c.Pods(ns).List(label)
		Expect(err).NotTo(HaveOccurred())
		current = len(pods.Items)
	}
	Expect(current).To(Equal(replicas))
	glog.Infof("Controller %s: Found %d pods out of %d", name, current, replicas)

	By("Waiting for each pod to be running")
	same = 0
	last = 0
	failCount = 10
	current = 0
	for same < failCount && current < replicas {
		current = 0
		waiting := 0
		pending := 0
		unknown := 0
		time.Sleep(10 * time.Second)

		currentPods, listErr := c.Pods(ns).List(label)
		Expect(listErr).NotTo(HaveOccurred())
		if len(currentPods.Items) != len(pods.Items) {
			Failf("Number of reported pods changed: %d vs %d", len(currentPods.Items), len(pods.Items))
		}
		for _, p := range currentPods.Items {
			if p.Status.Phase == api.PodRunning {
				current++
			} else if p.Status.Phase == api.PodPending {
				if p.Status.Host == "" {
					waiting++
				} else {
					pending++
				}
			} else if p.Status.Phase == api.PodUnknown {
				unknown++
			}
		}
		glog.Infof("Pod States: %d running, %d pending, %d waiting, %d unknown ", current, pending, waiting, unknown)
		if last < current {
			same = 0
		} else if last == current {
			same++
		} else if current < last {
			Failf("Number of running pods dropped from %d to %d", last, current)
		}
		if same >= failCount {
			glog.Infof("No pods started for the last %d checks", failCount)
		}
		last = current
	}
	Expect(current).To(Equal(replicas))
}

// This test suite can take a long time to run, so by default it is added to
// the ginkgo.skip list (see driver.go).
// To run this suite you must explicitly ask for it by setting the
// -t/--test flag or ginkgo.focus flag.
var _ = Describe("Density", func() {
	var c *client.Client
	var minionCount int
	var RCName string
	var ns string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		minions, err := c.Nodes().List()
		expectNoError(err)
		minionCount = len(minions.Items)
		Expect(minionCount).NotTo(BeZero())
		ns = api.NamespaceDefault
	})

	AfterEach(func() {
		// Remove any remaining pods from this test if the
		// replication controller still exists and the replica count
		// isn't 0.  This means the controller wasn't cleaned up
		// during the test so clean it up here
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			DeleteRC(c, ns, RCName)
		}
	})

	It("should allow starting 100 pods per node", func() {
		RCName = "my-hostname-density100-" + string(util.NewUUID())
		RunRC(c, RCName, ns, "dockerfile/nginx", 100*minionCount)
	})

	It("should have master components that can handle many short-lived pods", func() {
		threads := 5
		var wg sync.WaitGroup
		wg.Add(threads)
		for i := 0; i < threads; i++ {
			go func() {
				defer wg.Done()
				for i := 0; i < 10; i++ {
					name := "my-hostname-thrash-" + string(util.NewUUID())
					RunRC(c, name, ns, "kubernetes/pause", 10*minionCount)
				}
			}()
		}
		wg.Wait()
	})
})
<|endoftext|>