<commit_before>package bitswap

import (
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"

	blocks "github.com/jbenet/go-ipfs/blocks"
	swarm "github.com/jbenet/go-ipfs/net/swarm"
	peer "github.com/jbenet/go-ipfs/peer"
	routing "github.com/jbenet/go-ipfs/routing"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	u "github.com/jbenet/go-ipfs/util"
)

// PartnerWantListMax is the bound for the number of keys we'll store per
// partner. These are usually taken from the top of the Partner's WantList
// advertisements. WantLists are sorted in terms of priority.
const PartnerWantListMax = 10

// KeySet is just a convenient alias for maps of keys, where we only care
// access/lookups.
type KeySet map[u.Key]struct{}

// BitSwap instances implement the bitswap protocol.
type BitSwap struct {
	// peer is the identity of this (local) node.
	peer *peer.Peer

	// net holds the connections to all peers.
	net     swarm.Network
	meschan *swarm.Chan

	// datastore is the local database // Ledgers of known
	datastore ds.Datastore

	// routing interface for communication
	routing *dht.IpfsDHT

	notifications *notifications

	// partners is a map of currently active bitswap relationships.
	// The Ledger has the peer.ID, and the peer connection works through net.
	// Ledgers of known relationships (active or inactive) stored in datastore.
	// Changes to the Ledger should be committed to the datastore.
	partners LedgerMap

	// haveList is the set of keys we have values for. a map for fast lookups.
	// haveList KeySet -- not needed. all values in datastore?

	// wantList is the set of keys we want values for. a map for fast lookups.
	wantList KeySet

	strategy StrategyFunc

	haltChan chan struct{}
}

// NewBitSwap creates a new BitSwap instance. It does not check its parameters.
func NewBitSwap(p *peer.Peer, net swarm.Network, d ds.Datastore, r routing.IpfsRouting) *BitSwap {
	bs := &BitSwap{
		peer:          p,
		net:           net,
		datastore:     d,
		partners:      LedgerMap{},
		wantList:      KeySet{},
		routing:       r.(*dht.IpfsDHT),
		meschan:       net.GetChannel(swarm.PBWrapper_BITSWAP),
		haltChan:      make(chan struct{}),
		notifications: newNotifications(),
	}

	go bs.handleMessages()
	return bs
}

// GetBlock attempts to retrieve a particular block from peers, within timeout.
func (bs *BitSwap) GetBlock(k u.Key, timeout time.Duration) (
	*blocks.Block, error) {
	u.DOut("Bitswap GetBlock: '%s'\n", k.Pretty())
	begin := time.Now()
	tleft := timeout - time.Now().Sub(begin)
	provs_ch := bs.routing.FindProvidersAsync(k, 20, timeout)

	blockChannel := make(chan *blocks.Block)
	after := time.After(tleft)

	// TODO: when the data is received, shut down this for loop ASAP
	go func() {
		for p := range provs_ch {
			go func(pr *peer.Peer) {
				blk, err := bs.getBlock(k, pr, tleft)
				if err != nil {
					u.PErr("getBlock returned: %v\n", err)
					return
				}
				select {
				case blockChannel <- blk:
				default:
				}
			}(p)
		}
	}()

	select {
	case block := <-blockChannel:
		close(blockChannel)
		return block, nil
	case <-after:
		return nil, u.ErrTimeout
	}
}

func (bs *BitSwap) getBlock(k u.Key, p *peer.Peer, timeout time.Duration) (*blocks.Block, error) {
	u.DOut("[%s] getBlock '%s' from [%s]\n", bs.peer.ID.Pretty(), k.Pretty(), p.ID.Pretty())

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	blockChannel := bs.notifications.Subscribe(ctx, k)

	message := newMessage()
	message.AppendWanted(k)
	bs.meschan.Outgoing <- message.ToSwarm(p)

	block, ok := <-blockChannel
	if !ok {
		u.PErr("getBlock for '%s' timed out.\n", k.Pretty())
		return nil, u.ErrTimeout
	}
	return block, nil
}

// HaveBlock announces the existance of a block to BitSwap, potentially sending
// it to peers (Partners) whose WantLists include it.
func (bs *BitSwap) HaveBlock(blk *blocks.Block) error {
	go func() {
		for _, ledger := range bs.partners {
			if ledger.WantListContains(blk.Key()) {
				//send block to node
				if ledger.ShouldSend() {
					bs.SendBlock(ledger.Partner, blk)
				}
			}
		}
	}()
	return bs.routing.Provide(blk.Key())
}

func (bs *BitSwap) SendBlock(p *peer.Peer, b *blocks.Block) {
	message := newMessage()
	message.AppendBlock(b)
	bs.meschan.Outgoing <- message.ToSwarm(p)
}

func (bs *BitSwap) handleMessages() {
	for {
		select {
		case mes := <-bs.meschan.Incoming:
			pmes := new(PBMessage)
			err := proto.Unmarshal(mes.Data, pmes)
			if err != nil {
				u.PErr("%v\n", err)
				continue
			}
			if pmes.Blocks != nil {
				for _, blkData := range pmes.Blocks {
					blk, err := blocks.NewBlock(blkData)
					if err != nil {
						u.PErr("%v\n", err)
						continue
					}
					go bs.blockReceive(mes.Peer, blk)
				}
			}

			if pmes.Wantlist != nil {
				for _, want := range pmes.Wantlist {
					go bs.peerWantsBlock(mes.Peer, want)
				}
			}
		case <-bs.haltChan:
			return
		}
	}
}

// peerWantsBlock will check if we have the block in question,
// and then if we do, check the ledger for whether or not we should send it.
func (bs *BitSwap) peerWantsBlock(p *peer.Peer, want string) {
	u.DOut("peer [%s] wants block [%s]\n", p.ID.Pretty(), u.Key(want).Pretty())
	ledger := bs.getLedger(p)

	dsk := ds.NewKey(want)
	blk_i, err := bs.datastore.Get(dsk)
	if err != nil {
		if err == ds.ErrNotFound {
			ledger.Wants(u.Key(want))
		}
		u.PErr("datastore get error: %v\n", err)
		return
	}

	blk, ok := blk_i.([]byte)
	if !ok {
		u.PErr("data conversion error.\n")
		return
	}

	if ledger.ShouldSend() {
		u.DOut("Sending block to peer.\n")
		bblk, err := blocks.NewBlock(blk)
		if err != nil {
			u.PErr("newBlock error: %v\n", err)
			return
		}
		bs.SendBlock(p, bblk)
		ledger.SentBytes(len(blk))
	} else {
		u.DOut("Decided not to send block.")
	}
}

func (bs *BitSwap) blockReceive(p *peer.Peer, blk *blocks.Block) {
	u.DOut("blockReceive: %s\n", blk.Key().Pretty())
	err := bs.datastore.Put(ds.NewKey(string(blk.Key())), blk.Data)
	if err != nil {
		u.PErr("blockReceive error: %v\n", err)
		return
	}

	bs.notifications.Publish(blk)

	ledger := bs.getLedger(p)
	ledger.ReceivedBytes(len(blk.Data))
}

func (bs *BitSwap) getLedger(p *peer.Peer) *Ledger {
	l, ok := bs.partners[p.Key()]
	if ok {
		return l
	}

	l = new(Ledger)
	l.Strategy = bs.strategy
	l.Partner = p
	bs.partners[p.Key()] = l
	return l
}

func (bs *BitSwap) SendWantList(wl KeySet) error {
	message := newMessage()
	for k, _ := range wl {
		message.AppendWanted(k)
	}

	// Lets just ping everybody all at once
	for _, ledger := range bs.partners {
		bs.meschan.Outgoing <- message.ToSwarm(ledger.Partner)
	}

	return nil
}

func (bs *BitSwap) Halt() {
	bs.haltChan <- struct{}{}
}

func (bs *BitSwap) SetStrategy(sf StrategyFunc) {
	bs.strategy = sf
	for _, ledger := range bs.partners {
		ledger.Strategy = sf
	}
}
<commit_msg>fix(bitswap:notifications) shutdown on bs.Halt()<commit_after>package bitswap

import (
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"

	blocks "github.com/jbenet/go-ipfs/blocks"
	swarm "github.com/jbenet/go-ipfs/net/swarm"
	peer "github.com/jbenet/go-ipfs/peer"
	routing "github.com/jbenet/go-ipfs/routing"
	dht "github.com/jbenet/go-ipfs/routing/dht"
	u "github.com/jbenet/go-ipfs/util"
)

// PartnerWantListMax is the bound for the number of keys we'll store per
// partner. These are usually taken from the top of the Partner's WantList
// advertisements. WantLists are sorted in terms of priority.
const PartnerWantListMax = 10

// KeySet is just a convenient alias for maps of keys, where we only care
// access/lookups.
type KeySet map[u.Key]struct{}

// BitSwap instances implement the bitswap protocol.
type BitSwap struct {
	// peer is the identity of this (local) node.
	peer *peer.Peer

	// net holds the connections to all peers.
	net     swarm.Network
	meschan *swarm.Chan

	// datastore is the local database // Ledgers of known
	datastore ds.Datastore

	// routing interface for communication
	routing *dht.IpfsDHT

	notifications *notifications

	// partners is a map of currently active bitswap relationships.
	// The Ledger has the peer.ID, and the peer connection works through net.
	// Ledgers of known relationships (active or inactive) stored in datastore.
	// Changes to the Ledger should be committed to the datastore.
	partners LedgerMap

	// haveList is the set of keys we have values for. a map for fast lookups.
	// haveList KeySet -- not needed. all values in datastore?

	// wantList is the set of keys we want values for. a map for fast lookups.
	wantList KeySet

	strategy StrategyFunc

	haltChan chan struct{}
}

// NewBitSwap creates a new BitSwap instance. It does not check its parameters.
func NewBitSwap(p *peer.Peer, net swarm.Network, d ds.Datastore, r routing.IpfsRouting) *BitSwap {
	bs := &BitSwap{
		peer:          p,
		net:           net,
		datastore:     d,
		partners:      LedgerMap{},
		wantList:      KeySet{},
		routing:       r.(*dht.IpfsDHT),
		meschan:       net.GetChannel(swarm.PBWrapper_BITSWAP),
		haltChan:      make(chan struct{}),
		notifications: newNotifications(),
	}

	go bs.handleMessages()
	return bs
}

// GetBlock attempts to retrieve a particular block from peers, within timeout.
func (bs *BitSwap) GetBlock(k u.Key, timeout time.Duration) (
	*blocks.Block, error) {
	u.DOut("Bitswap GetBlock: '%s'\n", k.Pretty())
	begin := time.Now()
	tleft := timeout - time.Now().Sub(begin)
	provs_ch := bs.routing.FindProvidersAsync(k, 20, timeout)

	blockChannel := make(chan *blocks.Block)
	after := time.After(tleft)

	// TODO: when the data is received, shut down this for loop ASAP
	go func() {
		for p := range provs_ch {
			go func(pr *peer.Peer) {
				blk, err := bs.getBlock(k, pr, tleft)
				if err != nil {
					u.PErr("getBlock returned: %v\n", err)
					return
				}
				select {
				case blockChannel <- blk:
				default:
				}
			}(p)
		}
	}()

	select {
	case block := <-blockChannel:
		close(blockChannel)
		return block, nil
	case <-after:
		return nil, u.ErrTimeout
	}
}

func (bs *BitSwap) getBlock(k u.Key, p *peer.Peer, timeout time.Duration) (*blocks.Block, error) {
	u.DOut("[%s] getBlock '%s' from [%s]\n", bs.peer.ID.Pretty(), k.Pretty(), p.ID.Pretty())

	ctx, _ := context.WithTimeout(context.Background(), timeout)
	blockChannel := bs.notifications.Subscribe(ctx, k)

	message := newMessage()
	message.AppendWanted(k)
	bs.meschan.Outgoing <- message.ToSwarm(p)

	block, ok := <-blockChannel
	if !ok {
		u.PErr("getBlock for '%s' timed out.\n", k.Pretty())
		return nil, u.ErrTimeout
	}
	return block, nil
}

// HaveBlock announces the existance of a block to BitSwap, potentially sending
// it to peers (Partners) whose WantLists include it.
func (bs *BitSwap) HaveBlock(blk *blocks.Block) error {
	go func() {
		for _, ledger := range bs.partners {
			if ledger.WantListContains(blk.Key()) {
				//send block to node
				if ledger.ShouldSend() {
					bs.SendBlock(ledger.Partner, blk)
				}
			}
		}
	}()
	return bs.routing.Provide(blk.Key())
}

func (bs *BitSwap) SendBlock(p *peer.Peer, b *blocks.Block) {
	message := newMessage()
	message.AppendBlock(b)
	bs.meschan.Outgoing <- message.ToSwarm(p)
}

func (bs *BitSwap) handleMessages() {
	for {
		select {
		case mes := <-bs.meschan.Incoming:
			pmes := new(PBMessage)
			err := proto.Unmarshal(mes.Data, pmes)
			if err != nil {
				u.PErr("%v\n", err)
				continue
			}
			if pmes.Blocks != nil {
				for _, blkData := range pmes.Blocks {
					blk, err := blocks.NewBlock(blkData)
					if err != nil {
						u.PErr("%v\n", err)
						continue
					}
					go bs.blockReceive(mes.Peer, blk)
				}
			}

			if pmes.Wantlist != nil {
				for _, want := range pmes.Wantlist {
					go bs.peerWantsBlock(mes.Peer, want)
				}
			}
		case <-bs.haltChan:
			bs.notifications.Shutdown()
			return
		}
	}
}

// peerWantsBlock will check if we have the block in question,
// and then if we do, check the ledger for whether or not we should send it.
func (bs *BitSwap) peerWantsBlock(p *peer.Peer, want string) {
	u.DOut("peer [%s] wants block [%s]\n", p.ID.Pretty(), u.Key(want).Pretty())
	ledger := bs.getLedger(p)

	dsk := ds.NewKey(want)
	blk_i, err := bs.datastore.Get(dsk)
	if err != nil {
		if err == ds.ErrNotFound {
			ledger.Wants(u.Key(want))
		}
		u.PErr("datastore get error: %v\n", err)
		return
	}

	blk, ok := blk_i.([]byte)
	if !ok {
		u.PErr("data conversion error.\n")
		return
	}

	if ledger.ShouldSend() {
		u.DOut("Sending block to peer.\n")
		bblk, err := blocks.NewBlock(blk)
		if err != nil {
			u.PErr("newBlock error: %v\n", err)
			return
		}
		bs.SendBlock(p, bblk)
		ledger.SentBytes(len(blk))
	} else {
		u.DOut("Decided not to send block.")
	}
}

func (bs *BitSwap) blockReceive(p *peer.Peer, blk *blocks.Block) {
	u.DOut("blockReceive: %s\n", blk.Key().Pretty())
	err := bs.datastore.Put(ds.NewKey(string(blk.Key())), blk.Data)
	if err != nil {
		u.PErr("blockReceive error: %v\n", err)
		return
	}

	bs.notifications.Publish(blk)

	ledger := bs.getLedger(p)
	ledger.ReceivedBytes(len(blk.Data))
}

func (bs *BitSwap) getLedger(p *peer.Peer) *Ledger {
	l, ok := bs.partners[p.Key()]
	if ok {
		return l
	}

	l = new(Ledger)
	l.Strategy = bs.strategy
	l.Partner = p
	bs.partners[p.Key()] = l
	return l
}

func (bs *BitSwap) SendWantList(wl KeySet) error {
	message := newMessage()
	for k, _ := range wl {
		message.AppendWanted(k)
	}

	// Lets just ping everybody all at once
	for _, ledger := range bs.partners {
		bs.meschan.Outgoing <- message.ToSwarm(ledger.Partner)
	}

	return nil
}

func (bs *BitSwap) Halt() {
	bs.haltChan <- struct{}{}
}

func (bs *BitSwap) SetStrategy(sf StrategyFunc) {
	bs.strategy = sf
	for _, ledger := range bs.partners {
		ledger.Strategy = sf
	}
}
<|endoftext|>


<commit_before>package sudoku

import (
	"math/rand"
)

type RankedObject interface {
	Rank() int
}

type FiniteQueue struct {
	min     int
	max     int
	objects []*finiteQueueList
}

type finiteQueueList struct {
	objects []RankedObject
	numNils int
	rank    int
}

const _REALLOCATE_PROPORTION = 0.20

type SyncedFiniteQueue struct {
	queue FiniteQueue
	In    chan RankedObject
	Out   chan RankedObject
	Exit  chan bool
}

type FiniteQueueGetter struct {
	queue         *FiniteQueue
	ignoreObjects map[RankedObject]bool
}

//Returns a new queue that will work for items with a rank as low as min or as high as max (inclusive)
func NewFiniteQueue(min int, max int) *FiniteQueue {
	result := FiniteQueue{min, max, make([]*finiteQueueList, max-min+1)}
	for i := 0; i < max-min+1; i++ {
		result.objects[i] = &finiteQueueList{make([]RankedObject, 1), 0, i + result.min}
	}
	return &result
}

func NewSyncedFiniteQueue(min int, max int) *SyncedFiniteQueue {
	result := &SyncedFiniteQueue{*NewFiniteQueue(min, max), make(chan RankedObject), make(chan RankedObject), make(chan bool, 1)}
	go result.workLoop()
	return result
}

func (self *finiteQueueList) trimNils() {
	if float64(self.numNils)/float64(len(self.objects)) > _REALLOCATE_PROPORTION {
		// fmt.Println("Reallocating...")
		// fmt.Println(self.numNils)
		// fmt.Println(self.objects)
		result := make([]RankedObject, len(self.objects)-self.numNils)
		targetIndex := 0
		for i := 0; i < len(self.objects); i++ {
			if self.objects[i] == nil {
				continue
			}
			result[targetIndex] = self.objects[i]
			targetIndex++
		}
		self.objects = result
		self.numNils = 0
	}
	for len(self.objects) > 0 && self.objects[0] == nil {
		self.objects = self.objects[1:]
		self.numNils--
	}

}

func (self *finiteQueueList) setNil(index int) {
	if self.objects[index] == nil {
		//We don't want to double-count nils.
		return
	}
	self.objects[index] = nil
	self.numNils++
}

func (self *finiteQueueList) getItem() RankedObject {
	self.trimNils()
	for len(self.objects) > 0 {
		index := rand.Intn(len(self.objects))
		obj := self.objects[index]
		//Mark this object as gathered.
		self.setNil(index)
		if obj != nil && obj.Rank() == self.rank {
			return obj
		}
		self.trimNils()
	}
	return nil
}

func (self *finiteQueueList) addItem(item RankedObject) {
	//Scrub the list for this item.
	for _, obj := range self.objects {
		//Structs will compare equal if all of their fields are the same.
		if item == obj {
			//It's already there, just return.
			return
		}
	}
	self.objects = append(self.objects, item)
}

func (self *SyncedFiniteQueue) workLoop() {
	for {
		firstItem := self.queue.Get()
		itemSent := false
		if firstItem == nil {
			//We can take in new things or accept an exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			}
		} else {
			//We can take in new things, send out smallest one, or exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			case self.Out <- firstItem:
				itemSent = true
			}
			//If we didn't send the item out, we need to put it back in.
			if !itemSent {
				self.queue.Insert(firstItem)
			}
		}
	}
}

func (self *FiniteQueue) NewGetter() *FiniteQueueGetter {
	return &FiniteQueueGetter{self, make(map[RankedObject]bool)}
}

func (self *FiniteQueue) Min() int {
	return self.min
}

func (self *FiniteQueue) Max() int {
	return self.max
}

func (self *FiniteQueue) Insert(obj RankedObject) {
	rank := obj.Rank()
	list, ok := self.getList(rank)
	if !ok {
		//Apparently rank wasn't legal.
		return
	}
	list.addItem(obj)
}

func (self *FiniteQueue) Get() RankedObject {
	return self.GetSmallerThan(self.max + 1)
}

func (self *FiniteQueue) GetSmallerThan(max int) RankedObject {
	return self.getSmallerThan(max, make(map[RankedObject]int))
}

func (self *FiniteQueue) getSmallerThan(max int, ignoredObjects map[RankedObject]int) RankedObject {
	//TOOD: actually respect ignoredObjects
	for i, list := range self.objects {
		if i+self.min >= max {
			return nil
		}
		result := list.getItem()
		if result != nil {
			return result
		}
	}
	return nil
}

func (self *FiniteQueue) legalRank(rank int) bool {
	return rank >= self.min && rank <= self.max
}

func (self *FiniteQueue) getList(rank int) (*finiteQueueList, bool) {
	if !self.legalRank(rank) {
		return nil, false
	}
	return self.objects[rank-self.min], true
}
<commit_msg>Add a currentList property in Getter.<commit_after>package sudoku

import (
	"math/rand"
)

type RankedObject interface {
	Rank() int
}

type FiniteQueue struct {
	min     int
	max     int
	objects []*finiteQueueList
}

type finiteQueueList struct {
	objects []RankedObject
	numNils int
	rank    int
}

const _REALLOCATE_PROPORTION = 0.20

type SyncedFiniteQueue struct {
	queue FiniteQueue
	In    chan RankedObject
	Out   chan RankedObject
	Exit  chan bool
}

type FiniteQueueGetter struct {
	queue         *FiniteQueue
	ignoreObjects map[RankedObject]bool
	currentList   *finiteQueueList
}

//Returns a new queue that will work for items with a rank as low as min or as high as max (inclusive)
func NewFiniteQueue(min int, max int) *FiniteQueue {
	result := FiniteQueue{min, max, make([]*finiteQueueList, max-min+1)}
	for i := 0; i < max-min+1; i++ {
		result.objects[i] = &finiteQueueList{make([]RankedObject, 1), 0, i + result.min}
	}
	return &result
}

func NewSyncedFiniteQueue(min int, max int) *SyncedFiniteQueue {
	result := &SyncedFiniteQueue{*NewFiniteQueue(min, max), make(chan RankedObject), make(chan RankedObject), make(chan bool, 1)}
	go result.workLoop()
	return result
}

func (self *finiteQueueList) trimNils() {
	if float64(self.numNils)/float64(len(self.objects)) > _REALLOCATE_PROPORTION {
		// fmt.Println("Reallocating...")
		// fmt.Println(self.numNils)
		// fmt.Println(self.objects)
		result := make([]RankedObject, len(self.objects)-self.numNils)
		targetIndex := 0
		for i := 0; i < len(self.objects); i++ {
			if self.objects[i] == nil {
				continue
			}
			result[targetIndex] = self.objects[i]
			targetIndex++
		}
		self.objects = result
		self.numNils = 0
	}
	for len(self.objects) > 0 && self.objects[0] == nil {
		self.objects = self.objects[1:]
		self.numNils--
	}

}

func (self *finiteQueueList) setNil(index int) {
	if self.objects[index] == nil {
		//We don't want to double-count nils.
		return
	}
	self.objects[index] = nil
	self.numNils++
}

func (self *finiteQueueList) getItem() RankedObject {
	self.trimNils()
	for len(self.objects) > 0 {
		index := rand.Intn(len(self.objects))
		obj := self.objects[index]
		//Mark this object as gathered.
		self.setNil(index)
		if obj != nil && obj.Rank() == self.rank {
			return obj
		}
		self.trimNils()
	}
	return nil
}

func (self *finiteQueueList) addItem(item RankedObject) {
	//Scrub the list for this item.
	for _, obj := range self.objects {
		//Structs will compare equal if all of their fields are the same.
		if item == obj {
			//It's already there, just return.
			return
		}
	}
	self.objects = append(self.objects, item)
}

func (self *SyncedFiniteQueue) workLoop() {
	for {
		firstItem := self.queue.Get()
		itemSent := false
		if firstItem == nil {
			//We can take in new things or accept an exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			}
		} else {
			//We can take in new things, send out smallest one, or exit.
			select {
			case <-self.Exit:
				return
			case incoming := <-self.In:
				self.queue.Insert(incoming)
			case self.Out <- firstItem:
				itemSent = true
			}
			//If we didn't send the item out, we need to put it back in.
			if !itemSent {
				self.queue.Insert(firstItem)
			}
		}
	}
}

func (self *FiniteQueue) NewGetter() *FiniteQueueGetter {
	list, _ := self.getList(self.min)
	return &FiniteQueueGetter{self, make(map[RankedObject]bool), list}
}

func (self *FiniteQueue) Min() int {
	return self.min
}

func (self *FiniteQueue) Max() int {
	return self.max
}

func (self *FiniteQueue) Insert(obj RankedObject) {
	rank := obj.Rank()
	list, ok := self.getList(rank)
	if !ok {
		//Apparently rank wasn't legal.
		return
	}
	list.addItem(obj)
}

func (self *FiniteQueue) Get() RankedObject {
	return self.GetSmallerThan(self.max + 1)
}

func (self *FiniteQueue) GetSmallerThan(max int) RankedObject {
	return self.getSmallerThan(max, make(map[RankedObject]int))
}

func (self *FiniteQueue) getSmallerThan(max int, ignoredObjects map[RankedObject]int) RankedObject {
	//TOOD: actually respect ignoredObjects
	for i, list := range self.objects {
		if i+self.min >= max {
			return nil
		}
		result := list.getItem()
		if result != nil {
			return result
		}
	}
	return nil
}

func (self *FiniteQueue) legalRank(rank int) bool {
	return rank >= self.min && rank <= self.max
}

func (self *FiniteQueue) getList(rank int) (*finiteQueueList, bool) {
	if !self.legalRank(rank) {
		return nil, false
	}
	return self.objects[rank-self.min], true
}
<|endoftext|>


<commit_before>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package githubwiki

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/nmiyake/pkg/dirs"
	"github.com/pkg/errors"

	"github.com/palantir/godel/layout"
)

type Params struct {
	DocsDir        string
	Repo           string
	AuthorName     string
	AuthorEmail    string
	CommitterName  string
	CommitterEmail string
	Msg            string
}

type GitTemplateParams struct {
	CommitID   string
	CommitTime time.Time
}

type commitUserParam struct {
	authorName     string
	authorEmail    string
	committerName  string
	committerEmail string
}

type userParam struct {
	envVar string
	format string
}

var (
	authorNameParam     = userParam{envVar: "GIT_AUTHOR_NAME", format: "an"}
	authorEmailParam    = userParam{envVar: "GIT_AUTHOR_EMAIL", format: "ae"}
	committerNameParam  = userParam{envVar: "GIT_COMMITTER_NAME", format: "cn"}
	committerEmailParam = userParam{envVar: "GIT_COMMITTER_EMAIL", format: "ce"}
)

type git string

func (g git) exec(args ...string) (string, error) {
	cmd := exec.Command("git", args...)
	cmd.Dir = string(g)
	output, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%v failed: %v", cmd.Args, err)
	}
	return strings.TrimSpace(string(output)), nil
}

func (g git) clone(p Params) error {
	_, err := g.exec("clone", p.Repo, string(g))
	return err
}

func (g git) commitAll(msg string, p commitUserParam) error {
	// add all files
	if _, err := g.exec("add", "."); err != nil {
		return err
	}

	// commit to current branch
	cmd := exec.Command("git", "commit", "-m", msg)
	cmd.Dir = string(g)
	env := os.Environ()
	// set environment variables for author and committer
	env = append(env, fmt.Sprintf("%v=%v", authorNameParam.envVar, p.authorName))
	env = append(env, fmt.Sprintf("%v=%v", authorEmailParam.envVar, p.authorEmail))
	env = append(env, fmt.Sprintf("%v=%v", committerNameParam.envVar, p.committerName))
	env = append(env, fmt.Sprintf("%v=%v", committerEmailParam.envVar, p.committerEmail))
	cmd.Env = env
	output, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "%v failed: %s", cmd.Args, string(output))
	}

	return nil
}

func (g git) commitID(branch string) (string, error) {
	return g.exec("rev-parse", branch)
}

func (g git) commitTime(branch string) (time.Time, error) {
	output, err := g.exec("show", "-s", "--format=%ct", branch)
	if err != nil {
		return time.Time{}, err
	}
	unixTime, err := strconv.ParseInt(output, 10, 64)
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "failed to parse %s as an int64", output)
	}
	return time.Unix(unixTime, 0), nil
}

func (g git) push() error {
	_, err := g.exec("push", "origin", "HEAD")
	return err
}

func (g git) valueOr(v string, p userParam) (string, error) {
	if v != "" {
		return v, nil
	}
	output, err := g.exec("--no-pager", "show", "-s", fmt.Sprintf("--format=%%%v", p.format), "HEAD")
	if err != nil {
		return "", err
	}
	return output, nil
}

func createGitTemplateParams(docsDir string) (GitTemplateParams, error) {
	g := git(docsDir)

	commitID, err := g.commitID("HEAD")
	if err != nil {
		return GitTemplateParams{}, err
	}

	commitTime, err := g.commitTime("HEAD")
	if err != nil {
		return GitTemplateParams{}, err
	}

	return GitTemplateParams{
		CommitID:   commitID,
		CommitTime: commitTime,
	}, nil
}

func SyncGitHubWiki(p Params, stdout io.Writer) error {
	if err := layout.VerifyDirExists(p.DocsDir); err != nil {
		return errors.Wrapf(err, "Docs directory %s does not exist", p.DocsDir)
	}

	// apply templating to commit message
	msg := p.Msg
	if gitTemplateParams, err := createGitTemplateParams(p.DocsDir); err != nil {
		fmt.Fprintf(stdout, "Failed to determine Git properties of documents directory %s: %v.\n", p.DocsDir, err)
		fmt.Fprintln(stdout, "Continuing with templating disabled. To fix this issue, ensure that the directory is in a Git repository.")
	} else if t, err := template.New("message").Parse(p.Msg); err != nil {
		fmt.Fprintf(stdout, "Failed to parse message %s as a template: %v. Using message as a string literal instead.\n", p.Msg, err)
	} else {
		buf := &bytes.Buffer{}
		if err := t.Execute(buf, gitTemplateParams); err != nil {
			fmt.Fprintf(stdout, "Failed to execute template %s: %v. Using message as a string literal instead.\n", p.Msg, err)
		} else {
			msg = buf.String()
		}
	}

	tmpCloneDir, cleanup, err := dirs.TempDir("", "")
	defer cleanup()
	if err != nil {
		return errors.Wrapf(err, "Failed to create temporary directory")
	}

	g := git(tmpCloneDir)

	if err := g.clone(p); err != nil {
		return err
	}

	// update contents of cloned directory to match docs directory (ignoring .git directory)
	if modified, err := layout.SyncDir(p.DocsDir, tmpCloneDir, []string{".git"}); err != nil {
		return errors.Wrapf(err, "Failed to sync contents of repo %s to docs directory %s", tmpCloneDir, p.DocsDir)
	} else if !modified {
		// nothing to do if cloned repo is identical to docs directory
		return nil
	}

	authorName, err := g.valueOr(p.AuthorName, authorNameParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get authorName")
	}
	authorEmail, err := g.valueOr(p.AuthorEmail, authorEmailParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get authorEmail")
	}
	committerName, err := g.valueOr(p.CommitterName, committerNameParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get committerName")
	}
	committerEmail, err := g.valueOr(p.CommitterEmail, committerEmailParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get committerEmail")
	}

	if err := g.commitAll(msg, commitUserParam{
		authorName:     authorName,
		authorEmail:    authorEmail,
		committerName:  committerName,
		committerEmail: committerEmail,
	}); err != nil {
		return err
	}

	fmt.Fprintf(stdout, "Pushing content of %s to %s...\n", p.DocsDir, p.Repo)
	if err := g.push(); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add more detailed output for github-wiki task (#206)<commit_after>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package githubwiki

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/nmiyake/pkg/dirs"
	"github.com/pkg/errors"

	"github.com/palantir/godel/layout"
)

type Params struct {
	DocsDir        string
	Repo           string
	AuthorName     string
	AuthorEmail    string
	CommitterName  string
	CommitterEmail string
	Msg            string
}

type GitTemplateParams struct {
	CommitID   string
	CommitTime time.Time
}

type commitUserParam struct {
	authorName     string
	authorEmail    string
	committerName  string
	committerEmail string
}

type userParam struct {
	envVar string
	format string
}

var (
	authorNameParam     = userParam{envVar: "GIT_AUTHOR_NAME", format: "an"}
	authorEmailParam    = userParam{envVar: "GIT_AUTHOR_EMAIL", format: "ae"}
	committerNameParam  = userParam{envVar: "GIT_COMMITTER_NAME", format: "cn"}
	committerEmailParam = userParam{envVar: "GIT_COMMITTER_EMAIL", format: "ce"}
)

type git string

func (g git) exec(args ...string) (string, error) {
	cmd := exec.Command("git", args...)
	cmd.Dir = string(g)
	output, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("%v failed\nError: %v\nOutput: %s", cmd.Args, err, string(output))
	}
	return strings.TrimSpace(string(output)), nil
}

func (g git) clone(p Params) error {
	_, err := g.exec("clone", p.Repo, string(g))
	return err
}

func (g git) commitAll(msg string, p commitUserParam) error {
	// add all files
	if _, err := g.exec("add", "."); err != nil {
		return err
	}

	// commit to current branch
	cmd := exec.Command("git", "commit", "-m", msg)
	cmd.Dir = string(g)
	env := os.Environ()
	// set environment variables for author and committer
	env = append(env, fmt.Sprintf("%v=%v", authorNameParam.envVar, p.authorName))
	env = append(env, fmt.Sprintf("%v=%v", authorEmailParam.envVar, p.authorEmail))
	env = append(env, fmt.Sprintf("%v=%v", committerNameParam.envVar, p.committerName))
	env = append(env, fmt.Sprintf("%v=%v", committerEmailParam.envVar, p.committerEmail))
	cmd.Env = env
	output, err := cmd.CombinedOutput()
	if err != nil {
		return errors.Wrapf(err, "%v failed: %s", cmd.Args, string(output))
	}

	return nil
}

func (g git) commitID(branch string) (string, error) {
	return g.exec("rev-parse", branch)
}

func (g git) commitTime(branch string) (time.Time, error) {
	output, err := g.exec("show", "-s", "--format=%ct", branch)
	if err != nil {
		return time.Time{}, err
	}
	unixTime, err := strconv.ParseInt(output, 10, 64)
	if err != nil {
		return time.Time{}, errors.Wrapf(err, "failed to parse %s as an int64", output)
	}
	return time.Unix(unixTime, 0), nil
}

func (g git) push() error {
	_, err := g.exec("push", "origin", "HEAD")
	return err
}

func (g git) valueOr(v string, p userParam) (string, error) {
	if v != "" {
		return v, nil
	}
	output, err := g.exec("--no-pager", "show", "-s", fmt.Sprintf("--format=%%%v", p.format), "HEAD")
	if err != nil {
		return "", err
	}
	return output, nil
}

func createGitTemplateParams(docsDir string) (GitTemplateParams, error) {
	g := git(docsDir)

	commitID, err := g.commitID("HEAD")
	if err != nil {
		return GitTemplateParams{}, err
	}

	commitTime, err := g.commitTime("HEAD")
	if err != nil {
		return GitTemplateParams{}, err
	}

	return GitTemplateParams{
		CommitID:   commitID,
		CommitTime: commitTime,
	}, nil
}

func SyncGitHubWiki(p Params, stdout io.Writer) error {
	if err := layout.VerifyDirExists(p.DocsDir); err != nil {
		return errors.Wrapf(err, "Docs directory %s does not exist", p.DocsDir)
	}

	// apply templating to commit message
	msg := p.Msg
	if gitTemplateParams, err := createGitTemplateParams(p.DocsDir); err != nil {
		fmt.Fprintf(stdout, "Failed to determine Git properties of documents directory %s: %v.\n", p.DocsDir, err)
		fmt.Fprintln(stdout, "Continuing with templating disabled. To fix this issue, ensure that the directory is in a Git repository.")
	} else if t, err := template.New("message").Parse(p.Msg); err != nil {
		fmt.Fprintf(stdout, "Failed to parse message %s as a template: %v. Using message as a string literal instead.\n", p.Msg, err)
	} else {
		buf := &bytes.Buffer{}
		if err := t.Execute(buf, gitTemplateParams); err != nil {
			fmt.Fprintf(stdout, "Failed to execute template %s: %v. Using message as a string literal instead.\n", p.Msg, err)
		} else {
			msg = buf.String()
		}
	}

	tmpCloneDir, cleanup, err := dirs.TempDir("", "")
	defer cleanup()
	if err != nil {
		return errors.Wrapf(err, "Failed to create temporary directory")
	}

	g := git(tmpCloneDir)

	if err := g.clone(p); err != nil {
		return err
	}

	// update contents of cloned directory to match docs directory (ignoring .git directory)
	if modified, err := layout.SyncDir(p.DocsDir, tmpCloneDir, []string{".git"}); err != nil {
		return errors.Wrapf(err, "Failed to sync contents of repo %s to docs directory %s", tmpCloneDir, p.DocsDir)
	} else if !modified {
		// nothing to do if cloned repo is identical to docs directory
		return nil
	}

	authorName, err := g.valueOr(p.AuthorName, authorNameParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get authorName")
	}
	authorEmail, err := g.valueOr(p.AuthorEmail, authorEmailParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get authorEmail")
	}
	committerName, err := g.valueOr(p.CommitterName, committerNameParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get committerName")
	}
	committerEmail, err := g.valueOr(p.CommitterEmail, committerEmailParam)
	if err != nil {
		return errors.Wrapf(err, "Failed to get committerEmail")
	}

	if err := g.commitAll(msg, commitUserParam{
		authorName:     authorName,
		authorEmail:    authorEmail,
		committerName:  committerName,
		committerEmail: committerEmail,
	}); err != nil {
		return err
	}

	fmt.Fprintf(stdout, "Pushing content of %s to %s...\n", p.DocsDir, p.Repo)
	if err := g.push(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package compute

import (
	"encoding/xml"
	"errors"
  "fmt"
		log "github.com/Sirupsen/logrus"
			"github.com/megamsys/libgo/cmd"
	"github.com/megamsys/opennebula-go/api"
	"github.com/megamsys/opennebula-go/template"
	"github.com/megamsys/opennebula-go/virtualmachine"
)

var ErrNoVM = errors.New("no vm found, Did you launch them ?")

const (
	TEMPLATE_INSTANTIATE = "one.template.instantiate"
	ONE_VM_ACTION        = "one.vm.action"
	ONE_DISK_SNAPSHOT    = "one.vm.disksaveas"
	DELETE               = "delete"
	REBOOT               = "reboot"
	POWEROFF             = "poweroff"
	RESUME               = "resume"
	ASSEMBLY_ID          = "assembly_id"
	ASSEMBLIES_ID        = "assemblies_id"
	ACCOUNTS_ID          = "accounts_id"
	PLATFORM_ID          = "platform_id"
	SSH_PUBLIC_KEY       = "SSH_PUBLIC_KEY"
)

type VirtualMachine struct {
	Name         string
	TemplateName string
	TemplateId   int
	Image        string
	ContextMap   map[string]string
	Cpu          string
	VCpu         string
	Memory       string
	HDD          string
	Region       string
	ClusterId    string
	Vnets        map[string]string
	T            *api.Rpc
}

// Creates a new VirtualMachine
func (v *VirtualMachine) Create() ([]interface{}, error) {
	templateObj := template.TemplateReqs{TemplateName: v.TemplateName, T: v.T}
	defer v.T.Client.Close()

	XMLtemplate, err := templateObj.Get()
	if err != nil {
		return nil, err
	}

	XMLtemplate[0].Template.Cpu = v.Cpu
	XMLtemplate[0].Template.VCpu = v.VCpu
	XMLtemplate[0].Template.Memory = v.Memory
	XMLtemplate[0].Template.Disk.Image = v.Image
	XMLtemplate[0].Template.Disk.Size = v.HDD
	XMLtemplate[0].Template.Context.Accounts_id = v.ContextMap[ACCOUNTS_ID]
	XMLtemplate[0].Template.Context.Platform_id = v.ContextMap[PLATFORM_ID]
	XMLtemplate[0].Template.Context.Assembly_id = v.ContextMap[ASSEMBLY_ID]
	XMLtemplate[0].Template.Context.Assemblies_id = v.ContextMap[ASSEMBLIES_ID]
	XMLtemplate[0].Template.Context.SSH_Public_key = v.ContextMap[SSH_PUBLIC_KEY]

	XMLtemplate[0].Template.Sched_requirments = "CLUSTER_ID=\""+ v.ClusterId +"\""
	XMLtemplate[0].Template.Nic = XMLtemplate[0].Template.Nic[:0]
	for _, v := range v.Vnets {
		net := &template.NIC{Network: v,Network_uname: "oneadmin"}
		XMLtemplate[0].Template.Nic	= append(XMLtemplate[0].Template.Nic,net)
	}

	finalXML := template.UserTemplates{}
	finalXML.UserTemplate = XMLtemplate
	finalData, _ := xml.Marshal(finalXML.UserTemplate[0].Template)
	data := string(finalData)
	args := []interface{}{v.T.Key, finalXML.UserTemplate[0].Id, v.Name, false, data}
	res, err := v.T.Call(TEMPLATE_INSTANTIATE, args)
	if err != nil {
		return nil, err
	}
	return res, nil
}

/**
*
* REBoot a new virtualMachine
*
**/
func (v *VirtualMachine) Reboot() ([]interface{}, error) {
	uvm, err := listByName(v.Name, v.T)
	if err != nil {
		return nil, err
	}

	args := []interface{}{v.T.Key, REBOOT, uvm.Id}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
*
* POWEROFF a new virtualMachine
*
**/
func (v *VirtualMachine) Poweroff() ([]interface{}, error) {
	uvm, err := listByName(v.Name, v.T)
	if err != nil {
		return nil, err
	}
	args := []interface{}{v.T.Key, POWEROFF, uvm.Id}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
*
* Resume a new virtualMachine
*
**/
func (v *VirtualMachine) Resume() ([]interface{}, error) {
	uvm, err := listByName(v.Name, v.T)
	if err != nil {
		return nil, err
	}

	args := []interface{}{v.T.Key, RESUME, uvm.Id}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
 *
 * Deletes a new virtualMachine
 *
 **/
func (v *VirtualMachine) Delete() ([]interface{}, error) {
	uvm, err := listByName(v.Name, v.T)
	if err != nil {
		return nil, err
	}

	args := []interface{}{v.T.Key, DELETE, uvm.Id}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
 *
 * VM save as a new Image (DISK_SNAPSHOT)
 *
 **/
func (v *VirtualMachine) DiskSnap() ([]interface{}, error) {
	uvm, err := listByName(v.Name, v.T)
	if err != nil {
		return nil, err
	}
	args := []interface{}{v.T.Key,uvm.Id,0,v.Name,"",-1}
	res, err := v.T.Call(ONE_DISK_SNAPSHOT, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil
}

func listByName(name string, client *api.Rpc) (*virtualmachine.UserVM, error) {
	vms := virtualmachine.Query{VMName: name, T: client}

	svm, err := vms.GetByName()
	if err != nil {
		return nil, err
	}

	if len(svm) <= 0 {
		return nil, ErrNoVM
	}

	if svm[0] == nil {
		return nil, ErrNoVM
	}
	return svm[0], nil
}
<commit_msg>one.vm.actions by vm id<commit_after>package compute

import (
	"encoding/xml"
	"errors"

	"github.com/megamsys/opennebula-go/api"
	"github.com/megamsys/opennebula-go/template"
	"github.com/megamsys/opennebula-go/virtualmachine"
)

var ErrNoVM = errors.New("no vm found, Did you launch them ?")

const (
	TEMPLATE_INSTANTIATE = "one.template.instantiate"
	ONE_VM_ACTION        = "one.vm.action"
	ONE_DISK_SNAPSHOT    = "one.vm.disksaveas"
	DELETE               = "delete"
	REBOOT               = "reboot"
	POWEROFF             = "poweroff"
	RESUME               = "resume"
	ASSEMBLY_ID          = "assembly_id"
	ASSEMBLIES_ID        = "assemblies_id"
	ACCOUNTS_ID          = "accounts_id"
	PLATFORM_ID          = "platform_id"
	SSH_PUBLIC_KEY       = "SSH_PUBLIC_KEY"
)

type VirtualMachine struct {
	Name         string
	TemplateName string
	TemplateId   int
	Image        string
	ContextMap   map[string]string
	Cpu          string
	VCpu         string
	Memory       string
	HDD          string
	Region       string
	ClusterId    string
	VMId         int
	Vnets        map[string]string
	T            *api.Rpc
}

// Creates a new VirtualMachine
func (v *VirtualMachine) Create() ([]interface{}, error) {
	templateObj := template.TemplateReqs{TemplateName: v.TemplateName, T: v.T}
	defer v.T.Client.Close()

	XMLtemplate, err := templateObj.Get()
	if err != nil {
		return nil, err
	}

	XMLtemplate[0].Template.Cpu = v.Cpu
	XMLtemplate[0].Template.VCpu = v.VCpu
	XMLtemplate[0].Template.Memory = v.Memory
	XMLtemplate[0].Template.Disk.Image = v.Image
	XMLtemplate[0].Template.Disk.Size = v.HDD
	XMLtemplate[0].Template.Context.Accounts_id = v.ContextMap[ACCOUNTS_ID]
	XMLtemplate[0].Template.Context.Platform_id = v.ContextMap[PLATFORM_ID]
	XMLtemplate[0].Template.Context.Assembly_id = v.ContextMap[ASSEMBLY_ID]
	XMLtemplate[0].Template.Context.Assemblies_id = v.ContextMap[ASSEMBLIES_ID]
	XMLtemplate[0].Template.Context.SSH_Public_key = v.ContextMap[SSH_PUBLIC_KEY]

	XMLtemplate[0].Template.Sched_requirments = "CLUSTER_ID=\""+ v.ClusterId +"\""
	if len(v.Vnets) > 0 {
		XMLtemplate[0].Template.Nic = XMLtemplate[0].Template.Nic[:0]
		for _, v := range v.Vnets {
			net := &template.NIC{Network: v,Network_uname: "oneadmin"}
			XMLtemplate[0].Template.Nic	= append(XMLtemplate[0].Template.Nic,net)
		}
	}

	finalXML := template.UserTemplates{}
	finalXML.UserTemplate = XMLtemplate
	finalData, _ := xml.Marshal(finalXML.UserTemplate[0].Template)
	data := string(finalData)
	args := []interface{}{v.T.Key, finalXML.UserTemplate[0].Id, v.Name, false, data}
	res, err := v.T.Call(TEMPLATE_INSTANTIATE, args)
	if err != nil {
		return nil, err
	}
	return res, nil
}

/**
*
* REBoot a new virtualMachine
*
**/
func (v *VirtualMachine) Reboot() ([]interface{}, error) {

	args := []interface{}{v.T.Key, REBOOT, v.VMId}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
*
* POWEROFF a new virtualMachine
*
**/
func (v *VirtualMachine) Poweroff() ([]interface{}, error) {

	args := []interface{}{v.T.Key, POWEROFF, v.VMId}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
*
* Resume a new virtualMachine
*
**/
func (v *VirtualMachine) Resume() ([]interface{}, error) {

	args := []interface{}{v.T.Key, RESUME, v.VMId}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
 *
 * Deletes a new virtualMachine
 *
 **/
func (v *VirtualMachine) Delete() ([]interface{}, error) {

	args := []interface{}{v.T.Key, DELETE, v.VMId}
	res, err := v.T.Call(ONE_VM_ACTION, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil

}

/**
 *
 * VM save as a new Image (DISK_SNAPSHOT)
 *
 **/
func (v *VirtualMachine) DiskSnap() ([]interface{}, error) {

	args := []interface{}{v.T.Key,v.VMId,0,v.Name,"",-1}
	res, err := v.T.Call(ONE_DISK_SNAPSHOT, args)
	//close connection
	defer v.T.Client.Close()
	if err != nil {
		return nil, err
	}

	return res, nil
}

func listByName(name string, client *api.Rpc) (*virtualmachine.UserVM, error) {
	vms := virtualmachine.Query{VMName: name, T: client}

	svm, err := vms.GetByName()
	if err != nil {
		return nil, err
	}

	if len(svm) <= 0 {
		return nil, ErrNoVM
	}

	if svm[0] == nil {
		return nil, ErrNoVM
	}
	return svm[0], nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package commands

import (
	"github.com/juju/cmd"
	"github.com/juju/errors"

	"github.com/juju/juju/api/controller"
	"github.com/juju/juju/cmd/modelcmd"
)

func newMigrateCommand() cmd.Command {
	return modelcmd.WrapController(&migrateCommand{})
}

// migrateCommand initiates a model migration.
type migrateCommand struct {
	modelcmd.ControllerCommandBase
	api migrateAPI

	model            string
	targetController string
}

type migrateAPI interface {
	InitiateModelMigration(spec controller.ModelMigrationSpec) (string, error)
}

const migrateDoc = `
migrate begins the migration of a model from its current controller to
a new controller. This is useful for load balancing when a controller
is too busy, or as a way to upgrade a model's controller to a newer
Juju version. Once complete, the model's machine and and unit agents
will be connected to the new controller. The model will no longer be
available at the source controller.

Note that only hosted models can be migrated. Controller models can
not be migrated.

If the migration fails for some reason, the model be returned to its
original state with the model being managed by the original
controller.

In order to start a migration, the target controller must be in the
juju client's local configuration cache. See the juju "login" command
for details of how to do this.

This command only starts a model migration - it does not wait for its
completion. The progress of a migration can be tracked using the
"status" command and by consulting the logs.

See also:
    login
    controllers
    status
`

// Info implements cmd.Command.
func (c *migrateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "migrate",
		Args:    "<model-name> <target-controller-name>",
		Purpose: "migrate a hosted model to another controller",
		Doc:     migrateDoc,
	}
}

// Init implements cmd.Command.
func (c *migrateCommand) Init(args []string) error {
	if len(args) < 1 {
		return errors.New("model not specified")
	}
	if len(args) < 2 {
		return errors.New("target controller not specified")
	}
	if len(args) > 2 {
		return errors.New("too many arguments specified")
	}

	c.model = args[0]
	c.targetController = args[1]
	return nil
}

func (c *migrateCommand) getMigrationSpec() (*controller.ModelMigrationSpec, error) {
	store := c.ClientStore()

	modelUUIDs, err := c.ModelUUIDs([]string{c.model})
	if err != nil {
		return nil, err
	}
	modelUUID := modelUUIDs[0]

	controllerInfo, err := store.ControllerByName(c.targetController)
	if err != nil {
		return nil, err
	}

	accountInfo, err := store.AccountDetails(c.targetController)
	if err != nil {
		return nil, err
	}

	return &controller.ModelMigrationSpec{
		ModelUUID:            modelUUID,
		TargetControllerUUID: controllerInfo.ControllerUUID,
		TargetAddrs:          controllerInfo.APIEndpoints,
		TargetCACert:         controllerInfo.CACert,
		TargetUser:           accountInfo.User,
		TargetPassword:       accountInfo.Password,
	}, nil
}

// Run implements cmd.Command.
func (c *migrateCommand) Run(ctx *cmd.Context) error {
	spec, err := c.getMigrationSpec()
	if err != nil {
		return err
	}
	api, err := c.getAPI()
	if err != nil {
		return err
	}
	id, err := api.InitiateModelMigration(*spec)
	if err != nil {
		return err
	}
	ctx.Infof("Migration started with ID %q", id)
	return nil
}

func (c *migrateCommand) getAPI() (migrateAPI, error) {
	if c.api != nil {
		return c.api, nil
	}
	return c.NewControllerAPIClient()
}
<commit_msg>cmd/juju/commands: Fix help summary formatting for migrate command<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package commands

import (
	"github.com/juju/cmd"
	"github.com/juju/errors"

	"github.com/juju/juju/api/controller"
	"github.com/juju/juju/cmd/modelcmd"
)

func newMigrateCommand() cmd.Command {
	return modelcmd.WrapController(&migrateCommand{})
}

// migrateCommand initiates a model migration.
type migrateCommand struct {
	modelcmd.ControllerCommandBase
	api migrateAPI

	model            string
	targetController string
}

type migrateAPI interface {
	InitiateModelMigration(spec controller.ModelMigrationSpec) (string, error)
}

const migrateDoc = `
migrate begins the migration of a model from its current controller to
a new controller. This is useful for load balancing when a controller
is too busy, or as a way to upgrade a model's controller to a newer
Juju version. Once complete, the model's machine and and unit agents
will be connected to the new controller. The model will no longer be
available at the source controller.

Note that only hosted models can be migrated. Controller models can
not be migrated.

If the migration fails for some reason, the model be returned to its
original state with the model being managed by the original
controller.

In order to start a migration, the target controller must be in the
juju client's local configuration cache. See the juju "login" command
for details of how to do this.

This command only starts a model migration - it does not wait for its
completion. The progress of a migration can be tracked using the
"status" command and by consulting the logs.

See also:
    login
    controllers
    status
`

// Info implements cmd.Command.
func (c *migrateCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "migrate",
		Args:    "<model-name> <target-controller-name>",
		Purpose: "Migrate a hosted model to another controller.",
		Doc:     migrateDoc,
	}
}

// Init implements cmd.Command.
func (c *migrateCommand) Init(args []string) error {
	if len(args) < 1 {
		return errors.New("model not specified")
	}
	if len(args) < 2 {
		return errors.New("target controller not specified")
	}
	if len(args) > 2 {
		return errors.New("too many arguments specified")
	}

	c.model = args[0]
	c.targetController = args[1]
	return nil
}

func (c *migrateCommand) getMigrationSpec() (*controller.ModelMigrationSpec, error) {
	store := c.ClientStore()

	modelUUIDs, err := c.ModelUUIDs([]string{c.model})
	if err != nil {
		return nil, err
	}
	modelUUID := modelUUIDs[0]

	controllerInfo, err := store.ControllerByName(c.targetController)
	if err != nil {
		return nil, err
	}

	accountInfo, err := store.AccountDetails(c.targetController)
	if err != nil {
		return nil, err
	}

	return &controller.ModelMigrationSpec{
		ModelUUID:            modelUUID,
		TargetControllerUUID: controllerInfo.ControllerUUID,
		TargetAddrs:          controllerInfo.APIEndpoints,
		TargetCACert:         controllerInfo.CACert,
		TargetUser:           accountInfo.User,
		TargetPassword:       accountInfo.Password,
	}, nil
}

// Run implements cmd.Command.
func (c *migrateCommand) Run(ctx *cmd.Context) error {
	spec, err := c.getMigrationSpec()
	if err != nil {
		return err
	}
	api, err := c.getAPI()
	if err != nil {
		return err
	}
	id, err := api.InitiateModelMigration(*spec)
	if err != nil {
		return err
	}
	ctx.Infof("Migration started with ID %q", id)
	return nil
}

func (c *migrateCommand) getAPI() (migrateAPI, error) {
	if c.api != nil {
		return c.api, nil
	}
	return c.NewControllerAPIClient()
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"net"
	"net/http"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

func startSwitchboard(args ...string) *gexec.Session {
	command := exec.Command(switchboardBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("started"))
	return session
}

func startBackend(args ...string) *gexec.Session {
	command := exec.Command(dummyBackendBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("Backend listening on"))
	return session
}

func startHealthcheck(args ...string) *gexec.Session {
	command := exec.Command(dummyHealthcheckBinPath, args...)
	session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session).Should(gbytes.Say("Healthcheck listening on"))
	return session
}

func sendData(conn net.Conn, data string) (string, error) {
	conn.Write([]byte(data))
	buffer := make([]byte, 1024)
	_, err := conn.Read(buffer)
	if err != nil {
		return "", err
	} else {
		return string(buffer), nil
	}
}

var _ = Describe("Switchboard", func() {
	var (
		backendSession      *gexec.Session
		backendSession2     *gexec.Session
		healthcheckSession  *gexec.Session
		healthcheckSession2 *gexec.Session
		proxySession        *gexec.Session
	)

	BeforeEach(func() {
		backendSession = startBackend(
			fmt.Sprintf("-port=%d", backendPort),
		)

		backendSession2 = startBackend(
			fmt.Sprintf("-port=%d", backendPort2),
		)

		healthcheckSession = startHealthcheck(
			fmt.Sprintf("-port=%d", dummyHealthcheckPort),
		)

		healthcheckSession2 = startHealthcheck(
			fmt.Sprintf("-port=%d", dummyHealthcheckPort2),
		)

		proxySession = startSwitchboard(
			fmt.Sprintf("-config=%s", proxyConfigFile),
			fmt.Sprintf("-pidFile=%s", pidFile),
		)
	})

	AfterEach(func() {
		proxySession.Terminate()
		healthcheckSession2.Terminate()
		healthcheckSession.Terminate()
		backendSession2.Terminate()
		backendSession.Terminate()
	})

	Context("when there are multiple concurrent clients", func() {
		var conn1, conn2, conn3 net.Conn
		var data1, data2, data3 string

		It("proxies all the connections to the backend", func() {
			done1 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done1)

				var err error
				Eventually(func() error {
					conn1, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data1, err = sendData(conn1, "test1")
				Expect(err).ToNot(HaveOccurred())
			}()

			done2 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done2)

				var err error
				Eventually(func() error {
					conn2, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data2, err = sendData(conn2, "test2")
				Expect(err).ToNot(HaveOccurred())
			}()

			done3 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done3)

				var err error
				Eventually(func() error {
					conn3, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data3, err = sendData(conn3, "test3")
				Expect(err).ToNot(HaveOccurred())
			}()

			<-done1
			<-done2
			<-done3

			Expect(data1).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test1", backendPort)))
			Expect(data2).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test2", backendPort)))
			Expect(data3).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test3", backendPort)))
		})
	})

	Context("when other clients disconnect", func() {
		var conn net.Conn
		var connToDisconnect net.Conn

		It("maintains a long-lived connection when other clients disconnect", func() {
			Eventually(func() error {
				var err error
				conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			Eventually(func() error {
				var err error
				connToDisconnect, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			dataBeforeDisconnect, err := sendData(conn, "data before disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataBeforeDisconnect).Should(ContainSubstring("data before disconnect"))

			connToDisconnect.Close()

			dataAfterDisconnect, err := sendData(conn, "data after disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataAfterDisconnect).Should(ContainSubstring("data after disconnect"))
		})
	})

	Context("when the healthcheck succeeds", func() {
		var client net.Conn

		It("checks health again after the specified interval", func() {
			Eventually(func() error {
				var err error
				client, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			buffer := make([]byte, 1024)

			client.Write([]byte("data around first healthcheck"))
			n, err := client.Read(buffer)
			Expect(err).ToNot(HaveOccurred())
			Expect(string(buffer[:n])).Should(ContainSubstring("data around first healthcheck"))

			Consistently(func() error {
				client.Write([]byte("data around subsequent healthcheck"))
				_, err = client.Read(buffer)
				return err
			}, 3*time.Second).ShouldNot(HaveOccurred())
		})
	})

	Context("when the cluster is down", func() {
		Context("when the healthcheck reports a 503", func() {
			It("disconnects client connections", func() {
				var conn net.Conn
				Eventually(func() error {
					var err error
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				dataWhileHealthy, err := sendData(conn, "data while healthy")
				Expect(err).ToNot(HaveOccurred())
				Expect(dataWhileHealthy).Should(ContainSubstring("data while healthy"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/set503", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/", dummyHealthcheckPort))
				Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable))
				Expect(httpErr).NotTo(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "data when unhealthy")
					return err
				}, 2*time.Second).Should(HaveOccurred())
			})
		})

		Context("when a backend goes down", func() {
			var conn net.Conn
			var data string

			BeforeEach(func() {
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err := sendData(conn, "data before hang")
				Expect(err).ShouldNot(HaveOccurred())
				Expect(data).Should(ContainSubstring("data before hang"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("disconnects existing client connections", func(done Done) {
				defer close(done)

				Eventually(func() error {
					_, err := sendData(conn, "data after hang")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())
			}, 5)

			It("proxies new connections to another backend", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var err error
				Eventually(func() error {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err = sendData(conn, "test")
				Expect(err).ToNot(HaveOccurred())
				Expect(data).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test", backendPort2)))
			}, 5)
		})

		Context("when all backends are down", func() {
			BeforeEach(func() {
				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort2))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("rejects any new connections that are attempted", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var conn net.Conn
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}, 1*time.Second).ShouldNot(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "write that should fail")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())

			}, 20)
		})
	})
})
<commit_msg>Use ginkgomon to run external processes in main_test<commit_after>package main_test

import (
	"fmt"
	"net"
	"net/http"
	"os/exec"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

func startSwitchboard(args ...string) ifrit.Process {
	command := exec.Command(switchboardBinPath, args...)
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("switchboard"),
		StartCheck: "started",
	})
	return ginkgomon.Invoke(runner)
}

func startBackend(port uint) ifrit.Process {
	command := exec.Command(dummyBackendBinPath, fmt.Sprintf("-port=%d", port))
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("fake-backend:%d", port),
		StartCheck: "Backend listening on",
	})
	return ginkgomon.Invoke(runner)
}

func startHealthcheck(port uint) ifrit.Process {
	command := exec.Command(dummyHealthcheckBinPath, fmt.Sprintf("-port=%d", port))
	runner := ginkgomon.New(ginkgomon.Config{
		Command:    command,
		Name:       fmt.Sprintf("fake-healthcheck:%d", port),
		StartCheck: "Healthcheck listening on",
	})
	return ginkgomon.Invoke(runner)
}

func sendData(conn net.Conn, data string) (string, error) {
	conn.Write([]byte(data))
	buffer := make([]byte, 1024)
	_, err := conn.Read(buffer)
	if err != nil {
		return "", err
	} else {
		return string(buffer), nil
	}
}

var _ = Describe("Switchboard", func() {
	var (
		backendProcess      ifrit.Process
		backendProcess2     ifrit.Process
		healthcheckProcess  ifrit.Process
		healthcheckProcess2 ifrit.Process
		switchboardProcess  ifrit.Process
	)

	BeforeEach(func() {
		backendProcess = startBackend(backendPort)
		backendProcess2 = startBackend(backendPort2)

		healthcheckProcess = startHealthcheck(dummyHealthcheckPort)
		healthcheckProcess2 = startHealthcheck(dummyHealthcheckPort2)

		switchboardProcess = startSwitchboard(
			fmt.Sprintf("-config=%s", proxyConfigFile),
			fmt.Sprintf("-pidFile=%s", pidFile),
		)
	})

	AfterEach(func() {
		ginkgomon.Kill(switchboardProcess)
		ginkgomon.Kill(healthcheckProcess2)
		ginkgomon.Kill(healthcheckProcess)
		ginkgomon.Kill(backendProcess2)
		ginkgomon.Kill(backendProcess)
	})

	Context("when there are multiple concurrent clients", func() {
		var conn1, conn2, conn3 net.Conn
		var data1, data2, data3 string

		It("proxies all the connections to the backend", func() {
			done1 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done1)

				var err error
				Eventually(func() error {
					conn1, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data1, err = sendData(conn1, "test1")
				Expect(err).ToNot(HaveOccurred())
			}()

			done2 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done2)

				var err error
				Eventually(func() error {
					conn2, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data2, err = sendData(conn2, "test2")
				Expect(err).ToNot(HaveOccurred())
			}()

			done3 := make(chan interface{})
			go func() {
				defer GinkgoRecover()
				defer close(done3)

				var err error
				Eventually(func() error {
					conn3, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data3, err = sendData(conn3, "test3")
				Expect(err).ToNot(HaveOccurred())
			}()

			<-done1
			<-done2
			<-done3

			Expect(data1).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test1", backendPort)))
			Expect(data2).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test2", backendPort)))
			Expect(data3).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test3", backendPort)))
		})
	})

	Context("when other clients disconnect", func() {
		var conn net.Conn
		var connToDisconnect net.Conn

		It("maintains a long-lived connection when other clients disconnect", func() {
			Eventually(func() error {
				var err error
				conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			Eventually(func() error {
				var err error
				connToDisconnect, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			dataBeforeDisconnect, err := sendData(conn, "data before disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataBeforeDisconnect).Should(ContainSubstring("data before disconnect"))

			connToDisconnect.Close()

			dataAfterDisconnect, err := sendData(conn, "data after disconnect")
			Expect(err).ToNot(HaveOccurred())
			Expect(dataAfterDisconnect).Should(ContainSubstring("data after disconnect"))
		})
	})

	Context("when the healthcheck succeeds", func() {
		var client net.Conn

		It("checks health again after the specified interval", func() {
			Eventually(func() error {
				var err error
				client, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
				return err
			}).ShouldNot(HaveOccurred())

			buffer := make([]byte, 1024)

			client.Write([]byte("data around first healthcheck"))
			n, err := client.Read(buffer)
			Expect(err).ToNot(HaveOccurred())
			Expect(string(buffer[:n])).Should(ContainSubstring("data around first healthcheck"))

			Consistently(func() error {
				client.Write([]byte("data around subsequent healthcheck"))
				_, err = client.Read(buffer)
				return err
			}, 3*time.Second).ShouldNot(HaveOccurred())
		})
	})

	Context("when the cluster is down", func() {
		Context("when the healthcheck reports a 503", func() {
			It("disconnects client connections", func() {
				var conn net.Conn
				Eventually(func() error {
					var err error
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				dataWhileHealthy, err := sendData(conn, "data while healthy")
				Expect(err).ToNot(HaveOccurred())
				Expect(dataWhileHealthy).Should(ContainSubstring("data while healthy"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/set503", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/", dummyHealthcheckPort))
				Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable))
				Expect(httpErr).NotTo(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "data when unhealthy")
					return err
				}, 2*time.Second).Should(HaveOccurred())
			})
		})

		Context("when a backend goes down", func() {
			var conn net.Conn
			var data string

			BeforeEach(func() {
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err := sendData(conn, "data before hang")
				Expect(err).ShouldNot(HaveOccurred())
				Expect(data).Should(ContainSubstring("data before hang"))

				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("disconnects existing client connections", func(done Done) {
				defer close(done)

				Eventually(func() error {
					_, err := sendData(conn, "data after hang")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())
			}, 5)

			It("proxies new connections to another backend", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var err error
				Eventually(func() error {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}).ShouldNot(HaveOccurred())

				data, err = sendData(conn, "test")
				Expect(err).ToNot(HaveOccurred())
				Expect(data).Should(ContainSubstring(fmt.Sprintf("Echo from port %d: test", backendPort2)))
			}, 5)
		})

		Context("when all backends are down", func() {
			BeforeEach(func() {
				resp, httpErr := http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				resp, httpErr = http.Get(fmt.Sprintf("http://localhost:%d/setHang", dummyHealthcheckPort2))
				Expect(httpErr).NotTo(HaveOccurred())
				Expect(resp.StatusCode).To(Equal(http.StatusOK))
			})

			It("rejects any new connections that are attempted", func(done Done) {
				defer close(done)

				time.Sleep(3 * proxyConfig.HealthcheckTimeout()) // wait for failover

				var conn net.Conn
				Eventually(func() (err error) {
					conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%d", switchboardPort))
					return err
				}, 1*time.Second).ShouldNot(HaveOccurred())

				Eventually(func() error {
					_, err := sendData(conn, "write that should fail")
					return err
				}, proxyConfig.HealthcheckTimeout()*4).Should(HaveOccurred())

			}, 20)
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if prepareData(b...)[0] != true {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && ((*b[0]).IsRed() || (*b[0]).IsThanYesterday())
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if !prepareData(b...)[0] {
		return false
	}
	var price = b[0].GetPriceList()
	var volume = b[0].GetVolumeList()
	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-46:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-46:]) < 0.25)
}

func prepareData(b ...*twse.Data) []bool {
	var result []bool
	var mindata int
	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					result[i] = false
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
}
<commit_msg>Fixed lost data check in check_list.<commit_after>package main

import (
	"github.com/toomore/gogrs/twse"
	"github.com/toomore/gogrs/utils"
)

type checkGroup interface {
	String() string
	CheckFunc(...*twse.Data) bool
	Mindata() int
}

type check01 struct{}

func (check01) String() string {
	return "MA 3 > 6 > 18"
}

func (check01) Mindata() int {
	return 18
}

func (check01) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if !prepareData(b...)[0] {
		return false
	}
	var ma3 = b[0].MA(3)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma3)); !ok || days == 0 {
		return false
	}
	var ma6 = b[0].MA(6)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma6)); !ok || days == 0 {
		return false
	}
	var ma18 = b[0].MA(18)
	if days, ok := utils.CountCountineFloat64(utils.DeltaFloat64(ma18)); !ok || days == 0 {
		return false
	}
	//log.Println(ma3[len(ma3)-1], ma6[len(ma6)-1], ma18[len(ma18)-1])
	if ma3[len(ma3)-1] > ma6[len(ma6)-1] && ma6[len(ma6)-1] > ma18[len(ma18)-1] {
		return true
	}
	return false
}

type check02 struct{}

func (check02) String() string {
	return "量大於前三天 K 線收紅"
}

func (check02) Mindata() int {
	return 4
}

func (check02) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if !prepareData(b...)[0] {
		return false
	}
	return utils.ThanSumPastUint64((*b[0]).GetVolumeList(), 3, true) && ((*b[0]).IsRed() || (*b[0]).IsThanYesterday())
}

type check03 struct{}

func (check03) String() string {
	return "量或價走平 45 天"
}

func (check03) Mindata() int {
	return 45
}

func (check03) CheckFunc(b ...*twse.Data) bool {
	defer wg.Done()
	if !prepareData(b...)[0] {
		return false
	}
	var price = b[0].GetPriceList()
	var volume = b[0].GetVolumeList()
	return price[len(price)-1] > 10 &&
		(utils.SD(price[len(price)-46:]) < 0.25 ||
			utils.SDUint64(volume[len(volume)-46:]) < 0.25)
}

func prepareData(b ...*twse.Data) []bool {
	var result []bool
	var mindata int
	for i := range ckList {
		if ckList[i].Mindata() > mindata {
			mindata = ckList[i].Mindata()
		}
	}

	for i := range b {
		result = make([]bool, len(b))
		b[i].Get()
		if b[i].Len() < mindata {
			start := b[i].Len()
			for {
				b[i].PlusData()
				if b[i].Len() > mindata {
					result[i] = true
					break
				}
				if b[i].Len() == start {
					result[i] = false
					break
				}
				start = b[i].Len()
			}
			if b[i].Len() < mindata {
				result[i] = false
			}
		} else {
			result[i] = true
		}
	}
	return result
}

func init() {
	ckList.Add(checkGroup(check01{}))
	ckList.Add(checkGroup(check02{}))
	ckList.Add(checkGroup(check03{}))
}
<|endoftext|>


<commit_before>package helpers

import (
	. "github.com/francoishill/golang-web-dry/errors/checkerror"
	"github.com/francoishill/golang-web-dry/osutils"
	"io/ioutil"
	"strings"
)

func replaceTabsWithSpaces(s string) string {
	return strings.Replace(s, "\t", "    ", -1)
}

func InjectContentIntoFilePlaceholder(logger Logger, filePath, placeholder, content string) {
	finalContent := content
	globalSnippetIndentSpaces := ""
	if osutils.FileExists(filePath) {
		logger.Debug("Found existing file '%s'. Replacing the placeholders.", filePath)

		fileContent, err := ioutil.ReadFile(filePath)
		CheckError(err)

		isBeginPlaceholder := false

		beginText := "{{BEGIN " + placeholder + "}}"
		endText := "{{END " + placeholder + "}}"

		origLines := strings.Split(string(fileContent), "\n")
		finalLines := []string{}
		for _, originalLine := range origLines {
			trimmedLine := strings.TrimSpace(originalLine)
			lineWithTabsReplacedBySpaces := replaceTabsWithSpaces(originalLine)

			if strings.Contains(trimmedLine, endText) {
				isBeginPlaceholder = false
				finalLines = append(finalLines, globalSnippetIndentSpaces+trimmedLine)
				continue
			}

			if strings.Contains(trimmedLine, beginText) {
				isBeginPlaceholder = true

				//Indent all the code with the same indentation as the 'begin' placeholder line
				prefixSpacesCount := len(lineWithTabsReplacedBySpaces) - len(strings.TrimLeft(lineWithTabsReplacedBySpaces, " "))
				globalSnippetIndentSpaces = strings.Repeat(" ", prefixSpacesCount)

				finalLines = append(finalLines, globalSnippetIndentSpaces+trimmedLine)

				contentLines := strings.Split(content, "\n")
				for index, _ := range contentLines {
					contentLines[index] = globalSnippetIndentSpaces + replaceTabsWithSpaces(contentLines[index])
				}
				finalLines = append(finalLines, strings.Join(contentLines, "\n"))

				logger.Debug("Appended content after line containing '%s'", beginText)

				continue
			}

			if isBeginPlaceholder {
				//Skip when between the placeholder tags
				continue
			}

			//These are the code lines "outside" of the placeholder block, so keep the lines untouched
			finalLines = append(finalLines, originalLine)
		}

		finalContent = strings.Join(finalLines, "\n")
	}
	err := ioutil.WriteFile(filePath, []byte(finalContent), 0600)
	CheckError(err)
}
<commit_msg>Log ERROR if no placeholder found in file<commit_after>package helpers

import (
	. "github.com/francoishill/golang-web-dry/errors/checkerror"
	"github.com/francoishill/golang-web-dry/osutils"
	"io/ioutil"
	"strings"
)

func replaceTabsWithSpaces(s string) string {
	return strings.Replace(s, "\t", "    ", -1)
}

func InjectContentIntoFilePlaceholder(logger Logger, filePath, placeholder, content string) {
	finalContent := content
	globalSnippetIndentSpaces := ""
	if osutils.FileExists(filePath) {
		logger.Debug("Found existing file '%s'. Replacing the placeholders.", filePath)

		fileContent, err := ioutil.ReadFile(filePath)
		CheckError(err)

		foundPlaceholder := false
		isBeginPlaceholder := false

		beginText := "{{BEGIN " + placeholder + "}}"
		endText := "{{END " + placeholder + "}}"

		origLines := strings.Split(string(fileContent), "\n")
		finalLines := []string{}
		for _, originalLine := range origLines {
			trimmedLine := strings.TrimSpace(originalLine)
			lineWithTabsReplacedBySpaces := replaceTabsWithSpaces(originalLine)

			if strings.Contains(trimmedLine, endText) {
				isBeginPlaceholder = false
				finalLines = append(finalLines, globalSnippetIndentSpaces+trimmedLine)
				continue
			}

			if strings.Contains(trimmedLine, beginText) {
				isBeginPlaceholder = true
				foundPlaceholder = true

				//Indent all the code with the same indentation as the 'begin' placeholder line
				prefixSpacesCount := len(lineWithTabsReplacedBySpaces) - len(strings.TrimLeft(lineWithTabsReplacedBySpaces, " "))
				globalSnippetIndentSpaces = strings.Repeat(" ", prefixSpacesCount)

				finalLines = append(finalLines, globalSnippetIndentSpaces+trimmedLine)

				contentLines := strings.Split(content, "\n")
				for index, _ := range contentLines {
					contentLines[index] = globalSnippetIndentSpaces + replaceTabsWithSpaces(contentLines[index])
				}
				finalLines = append(finalLines, strings.Join(contentLines, "\n"))

				logger.Debug("Appended content after line containing '%s'", beginText)

				continue
			}

			if isBeginPlaceholder {
				//Skip when between the placeholder tags
				continue
			}

			//These are the code lines "outside" of the placeholder block, so keep the lines untouched
			finalLines = append(finalLines, originalLine)
		}

		finalContent = strings.Join(finalLines, "\n")

		if !foundPlaceholder {
			logger.Error("Placeholder '%s' not found in '%s'", placeholder, filePath)
		}
	}
	err := ioutil.WriteFile(filePath, []byte(finalContent), 0600)
	CheckError(err)
}
<|endoftext|>


<commit_before>// Program webp-manual-test checks that the Go WEBP library's decodings match
// the C WEBP library's.
package main // import "golang.org/x/image/cmd/webp-manual-test"

import (
	"bytes"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/image/webp"
	"golang.org/x/image/webp/nycbcra"
)

var (
	dwebp = flag.String("dwebp", "/usr/bin/dwebp", "path to the dwebp program "+
		"installed from https://developers.google.com/speed/webp/download")
	testdata = flag.String("testdata", "", "path to the libwebp-test-data directory "+
		"checked out from https://chromium.googlesource.com/webm/libwebp-test-data")
)

func main() {
	flag.Parse()
	if *dwebp == "" {
		flag.Usage()
		log.Fatal("dwebp flag was not specified")
	}
	if _, err := os.Stat(*dwebp); err != nil {
		flag.Usage()
		log.Fatalf("could not find dwebp program at %q", *dwebp)
	}
	if *testdata == "" {
		flag.Usage()
		log.Fatal("testdata flag was not specified")
	}

	f, err := os.Open(*testdata)
	if err != nil {
		log.Fatalf("Open: %v", err)
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		log.Fatalf("Readdirnames: %v", err)
	}
	sort.Strings(names)

	nFail, nPass := 0, 0
	for _, name := range names {
		if !strings.HasSuffix(name, "webp") {
			continue
		}
		if err := test(name); err != nil {
			fmt.Printf("FAIL\t%s\t%v\n", name, err)
			nFail++
		} else {
			fmt.Printf("PASS\t%s\n", name)
			nPass++
		}
	}
	fmt.Printf("%d PASS, %d FAIL, %d TOTAL\n", nPass, nFail, nPass+nFail)
	if nFail != 0 {
		os.Exit(1)
	}
}

// test tests a single WEBP image.
func test(name string) error {
	filename := filepath.Join(*testdata, name)
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("Open: %v", err)
	}
	defer f.Close()

	gotImage, err := webp.Decode(f)
	if err != nil {
		return fmt.Errorf("Decode: %v", err)
	}
	format, encode := "-pgm", encodePGM
	if _, lossless := gotImage.(*image.NRGBA); lossless {
		format, encode = "-pam", encodePAM
	}
	got, err := encode(gotImage)
	if err != nil {
		return fmt.Errorf("encode: %v", err)
	}

	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	c := exec.Command(*dwebp, filename, format, "-o", "/dev/stdout")
	c.Stdout = stdout
	c.Stderr = stderr
	if err := c.Run(); err != nil {
		os.Stderr.Write(stderr.Bytes())
		return fmt.Errorf("executing dwebp: %v", err)
	}
	want := stdout.Bytes()

	if len(got) != len(want) {
		return fmt.Errorf("encodings have different length: got %d, want %d", len(got), len(want))
	}
	for i, g := range got {
		if w := want[i]; g != w {
			return fmt.Errorf("encodings differ at position 0x%x: got 0x%02x, want 0x%02x", i, g, w)
		}
	}
	return nil
}

// encodePAM encodes gotImage in the PAM format.
func encodePAM(gotImage image.Image) ([]byte, error) {
	m, ok := gotImage.(*image.NRGBA)
	if !ok {
		return nil, fmt.Errorf("lossless image did not decode to an *image.NRGBA")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P7\nWIDTH %d\nHEIGHT %d\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n", w, h)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.PixOffset(b.Min.X, y)
		buf.Write(m.Pix[o : o+4*w])
	}
	return buf.Bytes(), nil
}

// encodePGM encodes gotImage in the PGM format in the IMC4 layout.
func encodePGM(gotImage image.Image) ([]byte, error) {
	var (
		m  *image.YCbCr
		ma *nycbcra.Image
	)
	switch g := gotImage.(type) {
	case *image.YCbCr:
		m = g
	case *nycbcra.Image:
		m = &g.YCbCr
		ma = g
	default:
		return nil, fmt.Errorf("lossy image did not decode to an *image.YCbCr")
	}
	if m.SubsampleRatio != image.YCbCrSubsampleRatio420 {
		return nil, fmt.Errorf("lossy image did not decode to a 4:2:0 YCbCr")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	w2, h2 := (w+1)/2, (h+1)/2
	outW, outH := 2*w2, h+h2
	if ma != nil {
		outH += h
	}
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P5\n%d %d\n255\n", outW, outH)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.YOffset(b.Min.X, y)
		buf.Write(m.Y[o : o+w])
		if w&1 != 0 {
			buf.WriteByte(0x00)
		}
	}
	for y := b.Min.Y; y < b.Max.Y; y += 2 {
		o := m.COffset(b.Min.X, y)
		buf.Write(m.Cb[o : o+w2])
		buf.Write(m.Cr[o : o+w2])
	}
	if ma != nil {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			o := ma.AOffset(b.Min.X, y)
			buf.Write(ma.A[o : o+w])
			if w&1 != 0 {
				buf.WriteByte(0x00)
			}
		}
	}
	return buf.Bytes(), nil
}

// dump can be useful for debugging.
func dump(w io.Writer, b []byte) {
	h := hex.Dumper(w)
	h.Write(b)
	h.Close()
}
<commit_msg>cmd/webp-manual-test: add a "build ignore" tag, so that "go install golang.org/x/image/..." doesn't install the manual testing program.<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore
//
// This build tag means that "go install golang.org/x/image/..." doesn't
// install this manual test. Use "go run main.go" to explicitly run it.

// Program webp-manual-test checks that the Go WEBP library's decodings match
// the C WEBP library's.
package main // import "golang.org/x/image/cmd/webp-manual-test"

import (
	"bytes"
	"encoding/hex"
	"flag"
	"fmt"
	"image"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"

	"golang.org/x/image/webp"
	"golang.org/x/image/webp/nycbcra"
)

var (
	dwebp = flag.String("dwebp", "/usr/bin/dwebp", "path to the dwebp program "+
		"installed from https://developers.google.com/speed/webp/download")
	testdata = flag.String("testdata", "", "path to the libwebp-test-data directory "+
		"checked out from https://chromium.googlesource.com/webm/libwebp-test-data")
)

func main() {
	flag.Parse()
	if *dwebp == "" {
		flag.Usage()
		log.Fatal("dwebp flag was not specified")
	}
	if _, err := os.Stat(*dwebp); err != nil {
		flag.Usage()
		log.Fatalf("could not find dwebp program at %q", *dwebp)
	}
	if *testdata == "" {
		flag.Usage()
		log.Fatal("testdata flag was not specified")
	}

	f, err := os.Open(*testdata)
	if err != nil {
		log.Fatalf("Open: %v", err)
	}
	defer f.Close()
	names, err := f.Readdirnames(-1)
	if err != nil {
		log.Fatalf("Readdirnames: %v", err)
	}
	sort.Strings(names)

	nFail, nPass := 0, 0
	for _, name := range names {
		if !strings.HasSuffix(name, "webp") {
			continue
		}
		if err := test(name); err != nil {
			fmt.Printf("FAIL\t%s\t%v\n", name, err)
			nFail++
		} else {
			fmt.Printf("PASS\t%s\n", name)
			nPass++
		}
	}
	fmt.Printf("%d PASS, %d FAIL, %d TOTAL\n", nPass, nFail, nPass+nFail)
	if nFail != 0 {
		os.Exit(1)
	}
}

// test tests a single WEBP image.
func test(name string) error {
	filename := filepath.Join(*testdata, name)
	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("Open: %v", err)
	}
	defer f.Close()

	gotImage, err := webp.Decode(f)
	if err != nil {
		return fmt.Errorf("Decode: %v", err)
	}
	format, encode := "-pgm", encodePGM
	if _, lossless := gotImage.(*image.NRGBA); lossless {
		format, encode = "-pam", encodePAM
	}
	got, err := encode(gotImage)
	if err != nil {
		return fmt.Errorf("encode: %v", err)
	}

	stdout := new(bytes.Buffer)
	stderr := new(bytes.Buffer)
	c := exec.Command(*dwebp, filename, format, "-o", "/dev/stdout")
	c.Stdout = stdout
	c.Stderr = stderr
	if err := c.Run(); err != nil {
		os.Stderr.Write(stderr.Bytes())
		return fmt.Errorf("executing dwebp: %v", err)
	}
	want := stdout.Bytes()

	if len(got) != len(want) {
		return fmt.Errorf("encodings have different length: got %d, want %d", len(got), len(want))
	}
	for i, g := range got {
		if w := want[i]; g != w {
			return fmt.Errorf("encodings differ at position 0x%x: got 0x%02x, want 0x%02x", i, g, w)
		}
	}
	return nil
}

// encodePAM encodes gotImage in the PAM format.
func encodePAM(gotImage image.Image) ([]byte, error) {
	m, ok := gotImage.(*image.NRGBA)
	if !ok {
		return nil, fmt.Errorf("lossless image did not decode to an *image.NRGBA")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P7\nWIDTH %d\nHEIGHT %d\nDEPTH 4\nMAXVAL 255\nTUPLTYPE RGB_ALPHA\nENDHDR\n", w, h)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.PixOffset(b.Min.X, y)
		buf.Write(m.Pix[o : o+4*w])
	}
	return buf.Bytes(), nil
}

// encodePGM encodes gotImage in the PGM format in the IMC4 layout.
func encodePGM(gotImage image.Image) ([]byte, error) {
	var (
		m  *image.YCbCr
		ma *nycbcra.Image
	)
	switch g := gotImage.(type) {
	case *image.YCbCr:
		m = g
	case *nycbcra.Image:
		m = &g.YCbCr
		ma = g
	default:
		return nil, fmt.Errorf("lossy image did not decode to an *image.YCbCr")
	}
	if m.SubsampleRatio != image.YCbCrSubsampleRatio420 {
		return nil, fmt.Errorf("lossy image did not decode to a 4:2:0 YCbCr")
	}
	b := m.Bounds()
	w, h := b.Dx(), b.Dy()
	w2, h2 := (w+1)/2, (h+1)/2
	outW, outH := 2*w2, h+h2
	if ma != nil {
		outH += h
	}
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "P5\n%d %d\n255\n", outW, outH)
	for y := b.Min.Y; y < b.Max.Y; y++ {
		o := m.YOffset(b.Min.X, y)
		buf.Write(m.Y[o : o+w])
		if w&1 != 0 {
			buf.WriteByte(0x00)
		}
	}
	for y := b.Min.Y; y < b.Max.Y; y += 2 {
		o := m.COffset(b.Min.X, y)
		buf.Write(m.Cb[o : o+w2])
		buf.Write(m.Cr[o : o+w2])
	}
	if ma != nil {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			o := ma.AOffset(b.Min.X, y)
			buf.Write(ma.A[o : o+w])
			if w&1 != 0 {
				buf.WriteByte(0x00)
			}
		}
	}
	return buf.Bytes(), nil
}

// dump can be useful for debugging.
func dump(w io.Writer, b []byte) {
	h := hex.Dumper(w)
	h.Write(b)
	h.Close()
}
<|endoftext|>


<commit_before>// Copyright 2014 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command ql is a utility to explore a database, prototype a schema or test
// drive a query, etc.
//
// Installation:
//
//	$ go get github.com/cznic/ql/ql
//
// Usage:
//
//	ql [-db name] [-schema regexp] [-tables regexp] [-fld] statement_list
//
// Options:
//
//	-db name	Name of the database to use. Defaults to "ql.db".
//			If the DB file does not exists it is created automatically.
//
//	-schema re	If re != "" show the CREATE statements of matching tables and exit.
//
//	-tables re	If re != "" show the matching table names and exit.
//
//	-fld		First row of a query result set will show field names.
//
//	statement_list	QL statements to execute.
//			If no non flag arguments are present, ql reads from stdin.
//			The list is wrapped into an automatic transaction.
//
//	-t		Report and measure time to execute, including creating/opening and closing the DB.
//
// Example:
//
//	$ ql 'create table t (i int, s string)'
//	$ ql << EOF
//	> insert into t values
//	> (1, "a"),
//	> (2, "b"),
//	> (3, "c"),
//	> EOF
//	$ ql 'select * from t'
//	3, "c"
//	2, "b"
//	1, "a"
//	$ ql -fld 'select * from t where i != 2 order by s'
//	"i", "s"
//	1, "a"
//	3, "c"
//	$
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/cznic/ql"
)

func str(data []interface{}) string {
	a := make([]string, len(data))
	for i, v := range data {
		switch x := v.(type) {
		case string:
			a[i] = fmt.Sprintf("%q", x)
		default:
			a[i] = fmt.Sprint(x)
		}
	}
	return strings.Join(a, ", ")
}

func main() {
	if err := do(); err != nil {
		log.Fatal(err)
	}
}

func do() (err error) {
	oDB := flag.String("db", "ql.db", "The DB file to open. It'll be created if missing.")
	oFlds := flag.Bool("fld", false, "Show recordset's field names.")
	oSchema := flag.String("schema", "", "If non empty, show the CREATE statements of matching tables and exit.")
	oTables := flag.String("tables", "", "If non empty, list matching table names and exit.")
	oTime := flag.Bool("t", false, "Measure and report time to execute the statement(s) including DB create/open/close.")
	flag.Parse()

	t0 := time.Now()
	if *oTime {
		defer func() {
			fmt.Fprintf(os.Stderr, "%s\n", time.Since(t0))
		}()
	}

	db, err := ql.OpenFile(*oDB, &ql.Options{CanCreate: true})
	if err != nil {
		return err
	}

	defer func() {
		ec := db.Close()
		switch {
		case ec != nil && err != nil:
			log.Println(ec)
		case ec != nil:
			err = ec
		}
	}()

	if pat := *oSchema; pat != "" {
		re, err := regexp.Compile(pat)
		if err != nil {
			return err
		}

		nfo, err := db.Info()
		if err != nil {
			return err
		}

		r := []string{}
		for _, ti := range nfo.Tables {
			if !re.MatchString(ti.Name) {
				continue
			}

			a := []string{}
			for _, ci := range ti.Columns {
				a = append(a, fmt.Sprintf("%s %s", ci.Name, ci.Type))
			}
			r = append(r, fmt.Sprintf("CREATE TABLE %s (%s);", ti.Name, strings.Join(a, ", ")))
		}
		sort.Strings(r)
		if len(r) != 0 {
			fmt.Println(strings.Join(r, "\n"))
		}
		return nil
	}

	if pat := *oTables; pat != "" {
		re, err := regexp.Compile(pat)
		if err != nil {
			return err
		}

		nfo, err := db.Info()
		if err != nil {
			return err
		}

		r := []string{}
		for _, ti := range nfo.Tables {
			if !re.MatchString(ti.Name) {
				continue
			}

			r = append(r, ti.Name)
		}
		sort.Strings(r)
		if len(r) != 0 {
			fmt.Println(strings.Join(r, "\n"))
		}
		return nil
	}

	var src string
	switch n := flag.NArg(); n {
	case 0:
		b, err := ioutil.ReadAll(bufio.NewReader(os.Stdin))
		if err != nil {
			return err
		}

		src = string(b)
	default:
		a := make([]string, n)
		for i := range a {
			a[i] = flag.Arg(i)
		}
		src = strings.Join(a, " ")
	}

	src = "BEGIN TRANSACTION; " + src + "; COMMIT;"
	l, err := ql.Compile(src)
	if err != nil {
		log.Println(src)
		return err
	}

	rs, i, err := db.Execute(ql.NewRWCtx(), l)
	if err != nil {
		a := strings.Split(strings.TrimSpace(fmt.Sprint(l)), "\n")
		return fmt.Errorf("%v: %s", err, a[i])
	}

	if len(rs) == 0 {
		return
	}

	switch {
	case l.IsExplainStmt():
		return rs[len(rs)-1].Do(*oFlds, func(data []interface{}) (bool, error) {
			fmt.Println(data[0])
			return true, nil
		})
	default:
		return rs[len(rs)-1].Do(*oFlds, func(data []interface{}) (bool, error) {
			fmt.Println(str(data))
			return true, nil
		})
	}
}
<commit_msg>improve flags<commit_after>// Copyright 2014 The ql Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command ql is a utility to explore a database, prototype a schema or test
// drive a query, etc.
//
// Installation:
//
//	$ go get github.com/cznic/ql/ql
//
// Usage:
//
//	ql [-db name] [-schema regexp] [-tables regexp] [-fld] statement_list
//
// Options:
//
//	-db name	Name of the database to use. Defaults to "ql.db".
//			If the DB file does not exists it is created automatically.
//
//	-schema re	If re != "" show the CREATE statements of matching tables and exit.
//
//	-tables re	If re != "" show the matching table names and exit.
//
//	-fld		First row of a query result set will show field names.
//
//	statement_list	QL statements to execute.
//			If no non flag arguments are present, ql reads from stdin.
//			The list is wrapped into an automatic transaction.
//
//	-t		Report and measure time to execute, including creating/opening and closing the DB.
//
// Example:
//
//	$ ql 'create table t (i int, s string)'
//	$ ql << EOF
//	> insert into t values
//	> (1, "a"),
//	> (2, "b"),
//	> (3, "c"),
//	> EOF
//	$ ql 'select * from t'
//	3, "c"
//	2, "b"
//	1, "a"
//	$ ql -fld 'select * from t where i != 2 order by s'
//	"i", "s"
//	1, "a"
//	3, "c"
//	$
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	"github.com/cznic/ql"
)

func str(data []interface{}) string {
	a := make([]string, len(data))
	for i, v := range data {
		switch x := v.(type) {
		case string:
			a[i] = fmt.Sprintf("%q", x)
		default:
			a[i] = fmt.Sprint(x)
		}
	}
	return strings.Join(a, ", ")
}

func main() {
	if err := do(); err != nil {
		log.Fatal(err)
	}
}

type config struct {
	db     string
	flds   bool
	schema string
	tables string
	time   bool
	help   bool
}

func (c *config) parse() {
	c.db = *flag.String("db", "ql.db", "The DB file to open. It'll be created if missing.")
	c.flds = *flag.Bool("fld", false, "Show recordset's field names.")
	c.schema = *flag.String("schema", "", "If non empty, show the CREATE statements of matching tables and exit.")
	c.tables = *flag.String("tables", "", "If non empty, list matching table names and exit.")
	c.time = *flag.Bool("t", false, "Measure and report time to execute the statement(s) including DB create/open/close.")
	c.help = *flag.Bool("h", false, "Shows this help text.")
	flag.Parse()
}

func do() (err error) {
	cfg := &config{}
	cfg.parse()
	if flag.NArg() == 0 || cfg.help {
		flag.PrintDefaults()
		return nil
	}
	t0 := time.Now()
	if cfg.time {
		defer func() {
			fmt.Fprintf(os.Stderr, "%s\n", time.Since(t0))
		}()
	}

	db, err := ql.OpenFile(cfg.db, &ql.Options{CanCreate: true})
	if err != nil {
		return err
	}

	defer func() {
		ec := db.Close()
		switch {
		case ec != nil && err != nil:
			log.Println(ec)
		case ec != nil:
			err = ec
		}
	}()

	if pat := cfg.schema; pat != "" {
		re, err := regexp.Compile(pat)
		if err != nil {
			return err
		}

		nfo, err := db.Info()
		if err != nil {
			return err
		}

		r := []string{}
		for _, ti := range nfo.Tables {
			if !re.MatchString(ti.Name) {
				continue
			}

			a := []string{}
			for _, ci := range ti.Columns {
				a = append(a, fmt.Sprintf("%s %s", ci.Name, ci.Type))
			}
			r = append(r, fmt.Sprintf("CREATE TABLE %s (%s);", ti.Name, strings.Join(a, ", ")))
		}
		sort.Strings(r)
		if len(r) != 0 {
			fmt.Println(strings.Join(r, "\n"))
		}
		return nil
	}

	if pat := cfg.tables; pat != "" {
		re, err := regexp.Compile(pat)
		if err != nil {
			return err
		}

		nfo, err := db.Info()
		if err != nil {
			return err
		}

		r := []string{}
		for _, ti := range nfo.Tables {
			if !re.MatchString(ti.Name) {
				continue
			}

			r = append(r, ti.Name)
		}
		sort.Strings(r)
		if len(r) != 0 {
			fmt.Println(strings.Join(r, "\n"))
		}
		return nil
	}

	var src string
	switch n := flag.NArg(); n {
	case 0:
		b, err := ioutil.ReadAll(bufio.NewReader(os.Stdin))
		if err != nil {
			return err
		}

		src = string(b)
	default:
		a := make([]string, n)
		for i := range a {
			a[i] = flag.Arg(i)
		}
		src = strings.Join(a, " ")
	}

	src = "BEGIN TRANSACTION; " + src + "; COMMIT;"
	l, err := ql.Compile(src)
	if err != nil {
		log.Println(src)
		return err
	}

	rs, i, err := db.Execute(ql.NewRWCtx(), l)
	if err != nil {
		a := strings.Split(strings.TrimSpace(fmt.Sprint(l)), "\n")
		return fmt.Errorf("%v: %s", err, a[i])
	}

	if len(rs) == 0 {
		return
	}

	switch {
	case l.IsExplainStmt():
		return rs[len(rs)-1].Do(cfg.flds, func(data []interface{}) (bool, error) {
			fmt.Println(data[0])
			return true, nil
		})
	default:
		return rs[len(rs)-1].Do(cfg.flds, func(data []interface{}) (bool, error) {
			fmt.Println(str(data))
			return true, nil
		})
	}
}
<|endoftext|>