<commit_before>package pollyanna

import (
	"encoding/xml"
	"errors"
	"fmt"
	"strings"
)

var baseDOM = `<div class="pollyanna"><div class="wrap">%s</div></div>`
var shardDOM = `<div class="shard-wrap"><div class="shard"></div></div>`
var baseCSS = `.pollyanna{position:absolute;width:100%;height:100%;top:0;left:0}.wrap{width:800;height:600px;top:5%;left:5%;position:absolute}.shard-wrap,.shard-wrap .shard,.shard-wrap .shard::before{width:100%;height:100%;position:absolute}.shard-wrap{z-index:2}.shard-wrap .shard{background-color:#fff}.shard-wrap .shard::before{content:"";background:rgba(255,255,255,0);top:0;left:0}%s`
var shardCSS = `.shard-wrap:nth-child(%d) .shard{clip-path: polygon(%s);background-color:%s;}`

// Svg is the top level <svg/> node
type Svg struct {
	XMLName  xml.Name  `xml:"svg"`
	Version  string    `xml:"version,attr"`
	Width    string    `xml:"width,attr"`
	Height   string    `xml:"height,attr"`
	ViewBox  string    `xml:"viewBox,attr"`
	Polygons []Polygon `xml:"polygon"`
}

// Polygon is a single <polygon/> node
type Polygon struct {
	Fill      string `xml:"fill,attr"`
	RawPoints string `xml:"points,attr"`
}

// Output represents the associated HTML Dom and CSS markup for the SVG image
type Output struct {
	HTML string
	CSS  string
}

type cssNode struct {
	selector string
	rules    []cssRule
}

type cssRule struct {
	property string
	value    string
}

// ParseSVG will parse the incoming SVG document bytes
func ParseSVG(bytes []byte) (Svg, error) {
	var svg Svg

	err := xml.Unmarshal(bytes, &svg)
	if err != nil {
		return svg, err
	}

	if 0 == len(svg.Polygons) {
		return svg, errors.New("No <polygon/> nodes were found in the SVG data.")
	}

	return svg, nil
}

// GenerateOutput converts the raw SVG data into HTML DOM nodes and associated CSS rules
func (s Svg) GenerateOutput() (Output, error) {
	var output Output
	output.HTML = fmt.Sprintf(baseDOM, strings.Repeat(shardDOM, len(s.Polygons)))
	output.CSS = baseCSS + s.cssShardChildren()
	return output, nil
}

func (s Svg) cssShardChildren() string {
	css := make([]string, len(s.Polygons))

	for i, p := range s.Polygons {
		css[i] = fmt.Sprintf(shardCSS, i+1, p.FormattedCSSPolygonPoints(), p.Fill)
	}

	return strings.Join(css, ``)
}

func (p Polygon) FormattedCSSPolygonPoints() string {
	return cssPolygonBuilder(p.Points()[0], p.Points()[1:len(p.Points())])
}

func (p Polygon) Points() [][]string {
	var points [][]string
	pointsStr := strings.Split(strings.Trim(p.RawPoints, ` `), ` `)

	for _, pointStr := range pointsStr {
		points = append(points, strings.Split(pointStr, `,`))
	}

	return points
}

func (s Svg) String() string {
	return fmt.Sprintf("SVG: version - %s, width - %s, height - %s, box - %s", s.Version, s.Width, s.Height, s.ViewBox)
}

func (p Polygon) String() string {
	return fmt.Sprintf("Polygon: fill - %s, points - %s", p.Fill, p.RawPoints)
}

func cssPolygonBuilder(points []string, rest [][]string) string {
	if len(points) == 0 {
		return ``
	}

	if len(rest) == 0 {
		return cssFormatSinglePoint(points)
	}

	fmtString := "%s, %s"
	if len(rest) == 1 {
		return fmt.Sprintf(fmtString, cssFormatSinglePoint(points), cssFormatSinglePoint(rest[0]))
	}

	return fmt.Sprintf(fmtString, cssFormatSinglePoint(points), cssPolygonBuilder(rest[0], rest[1:len(rest)]))
}

func cssFormatSinglePoint(p []string) string {
	return fmt.Sprintf("%spx %spx", p[0], p[1])
}
<commit_msg>Need to use -webkit prefix currently<commit_after>package pollyanna

import (
	"encoding/xml"
	"errors"
	"fmt"
	"strings"
)

var baseDOM = `<div class="pollyanna"><div class="wrap">%s</div></div>`
var shardDOM = `<div class="shard-wrap"><div class="shard"></div></div>`
var baseCSS = `.pollyanna{position:absolute;width:100%;height:100%;top:0;left:0}.wrap{width:800;height:600px;top:5%;left:5%;position:absolute}.shard-wrap,.shard-wrap .shard,.shard-wrap .shard::before{width:100%;height:100%;position:absolute}.shard-wrap{z-index:2}.shard-wrap .shard{background-color:#fff}.shard-wrap .shard::before{content:"";background:rgba(255,255,255,0);top:0;left:0}%s`
var shardCSS = `.shard-wrap:nth-child(%d) .shard{-webkit-clip-path: polygon(%s);background-color:%s;}`

// Svg is the top level <svg/> node
type Svg struct {
	XMLName  xml.Name  `xml:"svg"`
	Version  string    `xml:"version,attr"`
	Width    string    `xml:"width,attr"`
	Height   string    `xml:"height,attr"`
	ViewBox  string    `xml:"viewBox,attr"`
	Polygons []Polygon `xml:"polygon"`
}

// Polygon is a single <polygon/> node
type Polygon struct {
	Fill      string `xml:"fill,attr"`
	RawPoints string `xml:"points,attr"`
}

// Output represents the associated HTML Dom and CSS markup for the SVG image
type Output struct {
	HTML string
	CSS  string
}

type cssNode struct {
	selector string
	rules    []cssRule
}

type cssRule struct {
	property string
	value    string
}

// ParseSVG will parse the incoming SVG document bytes
func ParseSVG(bytes []byte) (Svg, error) {
	var svg Svg

	err := xml.Unmarshal(bytes, &svg)
	if err != nil {
		return svg, err
	}

	if 0 == len(svg.Polygons) {
		return svg, errors.New("No <polygon/> nodes were found in the SVG data.")
	}

	return svg, nil
}

// GenerateOutput converts the raw SVG data into HTML DOM nodes and associated CSS rules
func (s Svg) GenerateOutput() (Output, error) {
	var output Output
	output.HTML = fmt.Sprintf(baseDOM, strings.Repeat(shardDOM, len(s.Polygons)))
	output.CSS = baseCSS + s.cssShardChildren()
	return output, nil
}

func (s Svg) cssShardChildren() string {
	css := make([]string, len(s.Polygons))

	for i, p := range s.Polygons {
		css[i] = fmt.Sprintf(shardCSS, i+1, p.FormattedCSSPolygonPoints(), p.Fill)
	}

	return strings.Join(css, ``)
}

func (p Polygon) FormattedCSSPolygonPoints() string {
	return cssPolygonBuilder(p.Points()[0], p.Points()[1:len(p.Points())])
}

func (p Polygon) Points() [][]string {
	var points [][]string
	pointsStr := strings.Split(strings.Trim(p.RawPoints, ` `), ` `)

	for _, pointStr := range pointsStr {
		points = append(points, strings.Split(pointStr, `,`))
	}

	return points
}

func (s Svg) String() string {
	return fmt.Sprintf("SVG: version - %s, width - %s, height - %s, box - %s", s.Version, s.Width, s.Height, s.ViewBox)
}

func (p Polygon) String() string {
	return fmt.Sprintf("Polygon: fill - %s, points - %s", p.Fill, p.RawPoints)
}

func cssPolygonBuilder(points []string, rest [][]string) string {
	if len(points) == 0 {
		return ``
	}

	if len(rest) == 0 {
		return cssFormatSinglePoint(points)
	}

	fmtString := "%s, %s"
	if len(rest) == 1 {
		return fmt.Sprintf(fmtString, cssFormatSinglePoint(points), cssFormatSinglePoint(rest[0]))
	}

	return fmt.Sprintf(fmtString, cssFormatSinglePoint(points), cssPolygonBuilder(rest[0], rest[1:len(rest)]))
}

func cssFormatSinglePoint(p []string) string {
	return fmt.Sprintf("%spx %spx", p[0], p[1])
}
<|endoftext|>


<commit_before>package optmem

import (
	"bytes"
	"errors"
	"net"
	"runtime"
	"time"

	"github.com/chihaya/chihaya"
	"github.com/chihaya/chihaya/pkg/stopper"
	"github.com/chihaya/chihaya/server/store"
)

func init() {
	store.RegisterPeerStoreDriver("optmem", &peerStoreDriver{})
}

// ErrInvalidIP is returned if a peer with an invalid IP was specified.
var ErrInvalidIP = errors.New("invalid IP")

type peerStoreDriver struct{}

func (p *peerStoreDriver) New(storecfg *store.DriverConfig) (store.PeerStore, error) {
	cfg, err := newPeerStoreConfig(storecfg)
	if err != nil {
		return nil, err
	}

	gcInterval, err := time.ParseDuration(cfg.GCInterval)
	if err != nil {
		return nil, err
	}

	gcCutoff, err := time.ParseDuration(cfg.GCCutoff)
	if err != nil {
		return nil, err
	}

	ps := &peerStore{
		shards: newShardContainer(cfg.ShardCountBits),
		closed: make(chan struct{}),
	}

	go func() {
		for {
			select {
			case <-ps.closed:
				return
			case <-time.After(gcInterval):
				cutoffTime := time.Now().Add(gcCutoff * -1)
				logln("collecting garbage. Cutoff time: " + cutoffTime.String())
				err := ps.CollectGarbage(cutoffTime)
				if err != nil {
					logln("failed to collect garbage: " + err.Error())
				} else {
					logln("finished collecting garbage")
				}
			}
		}
	}()

	return ps, nil
}

type peerStore struct {
	shards *shardContainer
	closed chan struct{}
}

func (s *peerStore) collectGarbage(cutoff time.Time) {
	internalCutoff := uint16(cutoff.Unix())
	maxDiff := uint16(time.Now().Unix() - cutoff.Unix())
	logf("running GC. internal cutoff: %d, maxDiff: %d\n", internalCutoff, maxDiff)

	for i := 0; i < len(s.shards.shards); i++ {
		deltaTorrents := 0
		shard := s.shards.lockShard(i)

		for ih, s := range shard.swarms {
			gc := s.peers.collectGarbage(internalCutoff, maxDiff)
			if gc {
				s.peers.rebalanceBuckets()
			}
			if s.peers.numPeers == 0 {
				delete(shard.swarms, ih)
				deltaTorrents--
			}
		}

		s.shards.unlockShard(i, deltaTorrents)
		runtime.Gosched()
	}
}

func (s *peerStore) CollectGarbage(cutoff time.Time) error {
	s.collectGarbage(cutoff)
	return nil
}

func (s *peerStore) PutSeeder(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagSeeder)
	peer.setPeerTime(uint16(time.Now().Unix()))
	var created bool
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer func() {
		if created {
			s.shards.unlockShardByHash(ih, 1)
		} else {
			s.shards.unlockShardByHash(ih, 0)
		}
	}()

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		created = true
		pl = swarm{peers: newPeerList()}
		shard.swarms[ih] = pl
	}

	pl.peers.putPeer(peer)
	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) DeleteSeeder(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagSeeder)
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return store.ErrResourceDoesNotExist
	}
	found := pl.peers.removePeer(peer)
	if !found {
		return store.ErrResourceDoesNotExist
	}

	if pl.peers.numPeers == 0 {
		delete(shard.swarms, ih)
		return nil
	}

	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) PutLeecher(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagLeecher)
	peer.setPeerTime(uint16(time.Now().Unix()))
	var created bool
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer func() {
		if created {
			s.shards.unlockShardByHash(ih, 1)
		} else {
			s.shards.unlockShardByHash(ih, 0)
		}
	}()

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		created = true
		pl = swarm{peers: newPeerList()}
		shard.swarms[ih] = pl
	}

	pl.peers.putPeer(peer)
	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) DeleteLeecher(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagLeecher)
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return store.ErrResourceDoesNotExist
	}
	found := pl.peers.removePeer(peer)
	if !found {
		return store.ErrResourceDoesNotExist
	}

	if pl.peers.numPeers == 0 {
		delete(shard.swarms, ih)
		return nil
	}

	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) GraduateLeecher(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	// we can just overwrite any leecher we already have, so
	return s.PutSeeder(infoHash, p)
}

func (s *peerStore) AnnouncePeers(infoHash chihaya.InfoHash, seeder bool, numWant int, peer4, peer6 chihaya.Peer) (peers4, peers6 []chihaya.Peer, err error) {
	peer := &peer{}
	if peer4.IP != nil && (len(peer4.IP) == 4 || len(peer4.IP) == 16) {
		peer.setIP(peer4.IP.To16())
		peer.setPort(peer4.Port)
	} else if peer6.IP != nil && len(peer6.IP) == 16 {
		peer.setIP(peer6.IP.To16())
		peer.setPort(peer6.Port)
	} else {
		return nil, nil, ErrInvalidIP
	}
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return nil, nil, store.ErrResourceDoesNotExist
	}

	peers := pl.peers.getAnnouncePeers(numWant, seeder, peer)

	for _, p := range peers {
		if bytes.Equal(p.data[:12], v4InV6Prefix) {
			peers4 = append(peers4, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		} else {
			peers6 = append(peers6, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		}
	}

	return
}

func (s *peerStore) NumSeeders(infoHash chihaya.InfoHash) int {
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return 0
	}
	return pl.peers.numSeeders
}

func (s *peerStore) NumLeechers(infoHash chihaya.InfoHash) int {
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return 0
	}
	return pl.peers.numPeers - pl.peers.numSeeders
}

var v4InV6Prefix = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}

func (s *peerStore) GetSeeders(infoHash chihaya.InfoHash) (peers4, peers6 []chihaya.Peer, err error) {
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return nil, nil, store.ErrResourceDoesNotExist
	}

	peers := pl.peers.getAllSeeders()

	for _, p := range peers {
		if bytes.Equal(p.data[:12], v4InV6Prefix) {
			peers4 = append(peers4, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		} else {
			peers6 = append(peers6, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		}
	}

	return
}

func (s *peerStore) GetLeechers(infoHash chihaya.InfoHash) (peers4, peers6 []chihaya.Peer, err error) {
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return nil, nil, store.ErrResourceDoesNotExist
	}

	peers := pl.peers.getAllLeechers()

	for _, p := range peers {
		if bytes.Equal(p.data[:12], v4InV6Prefix) {
			peers4 = append(peers4, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		} else {
			peers6 = append(peers6, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		}
	}

	return
}

func (s *peerStore) Stop() <-chan error {
	return stopper.AlreadyStopped
}

// NumSwarms returns the total number of swarms tracked by the PeerStore.
// This is the same as the amount of infohashes tracked.
// Runs in constant time.
func (s *peerStore) NumSwarms() uint64 {
	return s.shards.getTorrentCount()
}

// NumTotalSeeders returns the total number of seeders tracked by the PeerStore.
// Runs in linear time in regards to the number of swarms tracked.
func (s *peerStore) NumTotalSeeders() uint64 {
	n := uint64(0)

	for i := 0; i < len(s.shards.shards); i++ {
		shard := s.shards.lockShard(i)
		for _, s := range shard.swarms {
			n += uint64(s.peers.numSeeders)
		}
		s.shards.unlockShard(i, 0)
		runtime.Gosched()
	}

	return n
}

// NumTotalLeechers returns the total number of leechers tracked by the
// PeerStore.
// Runs in linear time in regards to the number of swarms tracked.
func (s *peerStore) NumTotalLeechers() uint64 {
	n := uint64(0)

	for i := 0; i < len(s.shards.shards); i++ {
		shard := s.shards.lockShard(i)
		for _, s := range shard.swarms {
			n += uint64(s.peers.numPeers - s.peers.numSeeders)
		}
		s.shards.unlockShard(i, 0)
		runtime.Gosched()
	}

	return n
}

// NumTotalPeers returns the total number of peers tracked by the PeerStore.
// Runs in linear time in regards to the number of swarms tracked.
func (s *peerStore) NumTotalPeers() uint64 {
	n := uint64(0)

	for i := 0; i < len(s.shards.shards); i++ {
		shard := s.shards.lockShard(i)
		for _, s := range shard.swarms {
			n += uint64(s.peers.numPeers)
		}
		s.shards.unlockShard(i, 0)
		runtime.Gosched()
	}

	return n
}
<commit_msg>add stopping<commit_after>package optmem

import (
	"bytes"
	"errors"
	"net"
	"runtime"
	"time"

	"github.com/chihaya/chihaya"
	"github.com/chihaya/chihaya/pkg/stopper"
	"github.com/chihaya/chihaya/server/store"
)

func init() {
	store.RegisterPeerStoreDriver("optmem", &peerStoreDriver{})
}

// ErrInvalidIP is returned if a peer with an invalid IP was specified.
var ErrInvalidIP = errors.New("invalid IP")

type peerStoreDriver struct{}

func (p *peerStoreDriver) New(storecfg *store.DriverConfig) (store.PeerStore, error) {
	cfg, err := newPeerStoreConfig(storecfg)
	if err != nil {
		return nil, err
	}

	gcInterval, err := time.ParseDuration(cfg.GCInterval)
	if err != nil {
		return nil, err
	}

	gcCutoff, err := time.ParseDuration(cfg.GCCutoff)
	if err != nil {
		return nil, err
	}

	ps := &peerStore{
		shards: newShardContainer(cfg.ShardCountBits),
		closed: make(chan struct{}),
	}

	go func() {
		for {
			select {
			case <-ps.closed:
				return
			case <-time.After(gcInterval):
				cutoffTime := time.Now().Add(gcCutoff * -1)
				logln("collecting garbage. Cutoff time: " + cutoffTime.String())
				err := ps.CollectGarbage(cutoffTime)
				if err != nil {
					logln("failed to collect garbage: " + err.Error())
				} else {
					logln("finished collecting garbage")
				}
			}
		}
	}()

	return ps, nil
}

type peerStore struct {
	shards *shardContainer
	closed chan struct{}
}

func (s *peerStore) collectGarbage(cutoff time.Time) {
	internalCutoff := uint16(cutoff.Unix())
	maxDiff := uint16(time.Now().Unix() - cutoff.Unix())
	logf("running GC. internal cutoff: %d, maxDiff: %d\n", internalCutoff, maxDiff)

	for i := 0; i < len(s.shards.shards); i++ {
		deltaTorrents := 0
		shard := s.shards.lockShard(i)

		for ih, s := range shard.swarms {
			gc := s.peers.collectGarbage(internalCutoff, maxDiff)
			if gc {
				s.peers.rebalanceBuckets()
			}
			if s.peers.numPeers == 0 {
				delete(shard.swarms, ih)
				deltaTorrents--
			}
		}

		s.shards.unlockShard(i, deltaTorrents)
		runtime.Gosched()
	}
}

func (s *peerStore) CollectGarbage(cutoff time.Time) error {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	s.collectGarbage(cutoff)
	return nil
}

func (s *peerStore) PutSeeder(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagSeeder)
	peer.setPeerTime(uint16(time.Now().Unix()))
	var created bool
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer func() {
		if created {
			s.shards.unlockShardByHash(ih, 1)
		} else {
			s.shards.unlockShardByHash(ih, 0)
		}
	}()

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		created = true
		pl = swarm{peers: newPeerList()}
		shard.swarms[ih] = pl
	}

	pl.peers.putPeer(peer)
	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) DeleteSeeder(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagSeeder)
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return store.ErrResourceDoesNotExist
	}
	found := pl.peers.removePeer(peer)
	if !found {
		return store.ErrResourceDoesNotExist
	}

	if pl.peers.numPeers == 0 {
		delete(shard.swarms, ih)
		return nil
	}

	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) PutLeecher(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagLeecher)
	peer.setPeerTime(uint16(time.Now().Unix()))
	var created bool
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer func() {
		if created {
			s.shards.unlockShardByHash(ih, 1)
		} else {
			s.shards.unlockShardByHash(ih, 0)
		}
	}()

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		created = true
		pl = swarm{peers: newPeerList()}
		shard.swarms[ih] = pl
	}

	pl.peers.putPeer(peer)
	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) DeleteLeecher(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	peer := &peer{}
	v6 := p.IP.To16()
	if v6 == nil {
		return ErrInvalidIP
	}
	peer.setIP(v6)
	peer.setPort(p.Port)
	peer.setPeerFlag(peerFlagLeecher)
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return store.ErrResourceDoesNotExist
	}
	found := pl.peers.removePeer(peer)
	if !found {
		return store.ErrResourceDoesNotExist
	}

	if pl.peers.numPeers == 0 {
		delete(shard.swarms, ih)
		return nil
	}

	pl.peers.rebalanceBuckets()

	return nil
}

func (s *peerStore) GraduateLeecher(infoHash chihaya.InfoHash, p chihaya.Peer) error {
	// we can just overwrite any leecher we already have, so
	return s.PutSeeder(infoHash, p)
}

func (s *peerStore) AnnouncePeers(infoHash chihaya.InfoHash, seeder bool, numWant int, peer4, peer6 chihaya.Peer) (peers4, peers6 []chihaya.Peer, err error) {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	peer := &peer{}
	if peer4.IP != nil && (len(peer4.IP) == 4 || len(peer4.IP) == 16) {
		peer.setIP(peer4.IP.To16())
		peer.setPort(peer4.Port)
	} else if peer6.IP != nil && len(peer6.IP) == 16 {
		peer.setIP(peer6.IP.To16())
		peer.setPort(peer6.Port)
	} else {
		return nil, nil, ErrInvalidIP
	}
	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return nil, nil, store.ErrResourceDoesNotExist
	}

	peers := pl.peers.getAnnouncePeers(numWant, seeder, peer)

	for _, p := range peers {
		if bytes.Equal(p.data[:12], v4InV6Prefix) {
			peers4 = append(peers4, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		} else {
			peers6 = append(peers6, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		}
	}

	return
}

func (s *peerStore) NumSeeders(infoHash chihaya.InfoHash) int {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return 0
	}
	return pl.peers.numSeeders
}

func (s *peerStore) NumLeechers(infoHash chihaya.InfoHash) int {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return 0
	}
	return pl.peers.numPeers - pl.peers.numSeeders
}

var v4InV6Prefix = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff}

func (s *peerStore) GetSeeders(infoHash chihaya.InfoHash) (peers4, peers6 []chihaya.Peer, err error) {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return nil, nil, store.ErrResourceDoesNotExist
	}

	peers := pl.peers.getAllSeeders()

	for _, p := range peers {
		if bytes.Equal(p.data[:12], v4InV6Prefix) {
			peers4 = append(peers4, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		} else {
			peers6 = append(peers6, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		}
	}

	return
}

func (s *peerStore) GetLeechers(infoHash chihaya.InfoHash) (peers4, peers6 []chihaya.Peer, err error) {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	ih := infohash(infoHash)

	shard := s.shards.lockShardByHash(ih)
	defer s.shards.unlockShardByHash(ih, 0)

	var pl swarm
	var ok bool
	if pl, ok = shard.swarms[ih]; !ok {
		return nil, nil, store.ErrResourceDoesNotExist
	}

	peers := pl.peers.getAllLeechers()

	for _, p := range peers {
		if bytes.Equal(p.data[:12], v4InV6Prefix) {
			peers4 = append(peers4, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		} else {
			peers6 = append(peers6, chihaya.Peer{IP: net.IP(p.ip()), Port: p.port()})
		}
	}

	return
}

func (s *peerStore) Stop() <-chan error {
	select {
	case <-s.closed:
		return stopper.AlreadyStopped
	default:
	}
	close(s.closed)
	return stopper.AlreadyStopped
}

// NumSwarms returns the total number of swarms tracked by the PeerStore.
// This is the same as the amount of infohashes tracked.
// Runs in constant time.
func (s *peerStore) NumSwarms() uint64 {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	return s.shards.getTorrentCount()
}

// NumTotalSeeders returns the total number of seeders tracked by the PeerStore.
// Runs in linear time in regards to the number of swarms tracked.
func (s *peerStore) NumTotalSeeders() uint64 {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	n := uint64(0)

	for i := 0; i < len(s.shards.shards); i++ {
		shard := s.shards.lockShard(i)
		for _, s := range shard.swarms {
			n += uint64(s.peers.numSeeders)
		}
		s.shards.unlockShard(i, 0)
		runtime.Gosched()
	}

	return n
}

// NumTotalLeechers returns the total number of leechers tracked by the
// PeerStore.
// Runs in linear time in regards to the number of swarms tracked.
func (s *peerStore) NumTotalLeechers() uint64 {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	n := uint64(0)

	for i := 0; i < len(s.shards.shards); i++ {
		shard := s.shards.lockShard(i)
		for _, s := range shard.swarms {
			n += uint64(s.peers.numPeers - s.peers.numSeeders)
		}
		s.shards.unlockShard(i, 0)
		runtime.Gosched()
	}

	return n
}

// NumTotalPeers returns the total number of peers tracked by the PeerStore.
// Runs in linear time in regards to the number of swarms tracked.
func (s *peerStore) NumTotalPeers() uint64 {
	select {
	case <-s.closed:
		panic("attempted to interact with closed store")
	default:
	}

	n := uint64(0)

	for i := 0; i < len(s.shards.shards); i++ {
		shard := s.shards.lockShard(i)
		for _, s := range shard.swarms {
			n += uint64(s.peers.numPeers)
		}
		s.shards.unlockShard(i, 0)
		runtime.Gosched()
	}

	return n
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	projecthelpers "github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

func doProfileUpdate(d *Daemon, project, name string, id int64, profile *api.Profile, req api.ProfilePut) error {
	// Check project limits.
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return projecthelpers.AllowProfileUpdate(tx, project, name, req)
	})
	if err != nil {
		return err
	}

	// Sanity checks
	err = instance.ValidConfig(d.os, req.Config, true, false)
	if err != nil {
		return err
	}

	// At this point we don't know the instance type, so just use instancetype.Any type for validation.
	err = instance.ValidDevices(d.State(), d.cluster, instancetype.Any, deviceConfig.NewDevices(req.Devices), false)
	if err != nil {
		return err
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query instances associated with profile '%s'", name)
	}

	// Check if the root device is supposed to be changed or removed.
	oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := shared.GetRootDiskDevice(profile.Devices)
	_, newProfileRootDiskDevice, _ := shared.GetRootDiskDevice(req.Devices)
	if len(containers) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) {
		// Check for containers using the device
		for _, container := range containers {
			// Check if the device is locally overridden
			k, v, _ := shared.GetRootDiskDevice(container.Devices.CloneNative())
			if k != "" && v["pool"] != "" {
				continue
			}

			// Check what profile the device comes from
			profiles := container.Profiles
			for i := len(profiles) - 1; i >= 0; i-- {
				_, profile, err := d.cluster.GetProfile(projecthelpers.Default, profiles[i])
				if err != nil {
					return err
				}

				// Check if we find a match for the device
				_, ok := profile.Devices[oldProfileRootDiskDeviceKey]
				if ok {
					// Found the profile
					if profiles[i] == name {
						// If it's the current profile, then we can't modify that root device
						return fmt.Errorf("At least one instance relies on this profile's root disk device")
					} else {
						// If it's not, then move on to the next container
						break
					}
				}
			}
		}
	}

	// Update the database
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return tx.UpdateProfile(project, name, db.Profile{
			Project:     project,
			Name:        name,
			Description: req.Description,
			Config:      req.Config,
			Devices:     req.Devices,
		})
	})
	if err != nil {
		return err
	}

	// Update all the containers on this node using the profile. Must be
	// done after db.TxCommit due to DB lock.
	nodeName := ""
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}
	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, profile.ProfilePut, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Like doProfileUpdate but does not update the database, since it was already
// updated by doProfileUpdate itself, called on the notifying node.
func doProfileUpdateCluster(d *Daemon, project, name string, old api.ProfilePut) error {
	nodeName := ""
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "Failed to query local node name")
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "Failed to query instances associated with profile '%s'", name)
	}

	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, old, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Profile update of a single container.
func doProfileUpdateContainer(d *Daemon, name string, old api.ProfilePut, nodeName string, args db.InstanceArgs) error {
	if args.Node != "" && args.Node != nodeName {
		// No-op, this container does not belong to this node.
		return nil
	}

	profiles, err := d.cluster.GetProfiles(args.Project, args.Profiles)
	if err != nil {
		return err
	}

	for i, profileName := range args.Profiles {
		if profileName == name {
			// Overwrite the new config from the database with the old config and devices.
			profiles[i].Config = old.Config
			profiles[i].Devices = old.Devices
			break
		}
	}

	// Load the instance using the old profile config.
	inst, err := instance.Load(d.State(), args, profiles)
	if err != nil {
		return err
	}

	// Update will internally load the new profile configs and detect the changes to apply.
	return inst.Update(db.InstanceArgs{
		Architecture: inst.Architecture(),
		Config:       inst.LocalConfig(),
		Description:  inst.Description(),
		Devices:      inst.LocalDevices(),
		Ephemeral:    inst.IsEphemeral(),
		Profiles:     inst.Profiles(),
		Project:      inst.Project(),
		Type:         inst.Type(),
		Snapshot:     inst.IsSnapshot(),
	}, true)
}

// Query the db for information about containers associated with the given
// profile.
func getProfileContainersInfo(cluster *db.Cluster, project, profile string) ([]db.InstanceArgs, error) {
	// Query the db for information about containers associated with the
	// given profile.
	names, err := cluster.GetInstancesWithProfile(project, profile)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to query instances with profile '%s'", profile)
	}

	containers := []db.InstanceArgs{}
	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		for ctProject, ctNames := range names {
			for _, ctName := range ctNames {
				container, err := tx.GetInstance(ctProject, ctName)
				if err != nil {
					return err
				}

				containers = append(containers, db.InstanceToArgs(container))
			}
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to fetch instances")
	}

	return containers, nil
}
<commit_msg>lxd/profiles/utils: Renames project arg to projectName in doProfileUpdate<commit_after>package main

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	projecthelpers "github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/pkg/errors"
)

func doProfileUpdate(d *Daemon, projectName string, name string, id int64, profile *api.Profile, req api.ProfilePut) error {
	// Check project limits.
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return projecthelpers.AllowProfileUpdate(tx, projectName, name, req)
	})
	if err != nil {
		return err
	}

	// Sanity checks
	err = instance.ValidConfig(d.os, req.Config, true, false)
	if err != nil {
		return err
	}

	// At this point we don't know the instance type, so just use instancetype.Any type for validation.
	err = instance.ValidDevices(d.State(), d.cluster, projectName, instancetype.Any, deviceConfig.NewDevices(req.Devices), false)
	if err != nil {
		return err
	}

	containers, err := getProfileContainersInfo(d.cluster, projectName, name)
	if err != nil {
		return errors.Wrapf(err, "failed to query instances associated with profile '%s'", name)
	}

	// Check if the root device is supposed to be changed or removed.
	oldProfileRootDiskDeviceKey, oldProfileRootDiskDevice, _ := shared.GetRootDiskDevice(profile.Devices)
	_, newProfileRootDiskDevice, _ := shared.GetRootDiskDevice(req.Devices)
	if len(containers) > 0 && oldProfileRootDiskDevice["pool"] != "" && newProfileRootDiskDevice["pool"] == "" || (oldProfileRootDiskDevice["pool"] != newProfileRootDiskDevice["pool"]) {
		// Check for containers using the device
		for _, container := range containers {
			// Check if the device is locally overridden
			k, v, _ := shared.GetRootDiskDevice(container.Devices.CloneNative())
			if k != "" && v["pool"] != "" {
				continue
			}

			// Check what profile the device comes from
			profiles := container.Profiles
			for i := len(profiles) - 1; i >= 0; i-- {
				_, profile, err := d.cluster.GetProfile(projecthelpers.Default, profiles[i])
				if err != nil {
					return err
				}

				// Check if we find a match for the device
				_, ok := profile.Devices[oldProfileRootDiskDeviceKey]
				if ok {
					// Found the profile
					if profiles[i] == name {
						// If it's the current profile, then we can't modify that root device
						return fmt.Errorf("At least one instance relies on this profile's root disk device")
					} else {
						// If it's not, then move on to the next container
						break
					}
				}
			}
		}
	}

	// Update the database
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return tx.UpdateProfile(projectName, name, db.Profile{
			Project:     projectName,
			Name:        name,
			Description: req.Description,
			Config:      req.Config,
			Devices:     req.Devices,
		})
	})
	if err != nil {
		return err
	}

	// Update all the containers on this node using the profile. Must be
	// done after db.TxCommit due to DB lock.
	nodeName := ""
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "failed to query local node name")
	}
	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, profile.ProfilePut, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Like doProfileUpdate but does not update the database, since it was already
// updated by doProfileUpdate itself, called on the notifying node.
func doProfileUpdateCluster(d *Daemon, project, name string, old api.ProfilePut) error {
	nodeName := ""
	err := d.cluster.Transaction(func(tx *db.ClusterTx) error {
		var err error
		nodeName, err = tx.GetLocalNodeName()
		return err
	})
	if err != nil {
		return errors.Wrap(err, "Failed to query local node name")
	}

	containers, err := getProfileContainersInfo(d.cluster, project, name)
	if err != nil {
		return errors.Wrapf(err, "Failed to query instances associated with profile '%s'", name)
	}

	failures := map[string]error{}
	for _, args := range containers {
		err := doProfileUpdateContainer(d, name, old, nodeName, args)
		if err != nil {
			failures[args.Name] = err
		}
	}

	if len(failures) != 0 {
		msg := "The following containers failed to update (profile change still saved):\n"
		for cname, err := range failures {
			msg += fmt.Sprintf(" - %s: %s\n", cname, err)
		}
		return fmt.Errorf("%s", msg)
	}

	return nil
}

// Profile update of a single container.
func doProfileUpdateContainer(d *Daemon, name string, old api.ProfilePut, nodeName string, args db.InstanceArgs) error {
	if args.Node != "" && args.Node != nodeName {
		// No-op, this container does not belong to this node.
		return nil
	}

	profiles, err := d.cluster.GetProfiles(args.Project, args.Profiles)
	if err != nil {
		return err
	}

	for i, profileName := range args.Profiles {
		if profileName == name {
			// Overwrite the new config from the database with the old config and devices.
			profiles[i].Config = old.Config
			profiles[i].Devices = old.Devices
			break
		}
	}

	// Load the instance using the old profile config.
	inst, err := instance.Load(d.State(), args, profiles)
	if err != nil {
		return err
	}

	// Update will internally load the new profile configs and detect the changes to apply.
	return inst.Update(db.InstanceArgs{
		Architecture: inst.Architecture(),
		Config:       inst.LocalConfig(),
		Description:  inst.Description(),
		Devices:      inst.LocalDevices(),
		Ephemeral:    inst.IsEphemeral(),
		Profiles:     inst.Profiles(),
		Project:      inst.Project(),
		Type:         inst.Type(),
		Snapshot:     inst.IsSnapshot(),
	}, true)
}

// Query the db for information about containers associated with the given
// profile.
func getProfileContainersInfo(cluster *db.Cluster, project, profile string) ([]db.InstanceArgs, error) {
	// Query the db for information about containers associated with the
	// given profile.
	names, err := cluster.GetInstancesWithProfile(project, profile)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to query instances with profile '%s'", profile)
	}

	containers := []db.InstanceArgs{}
	err = cluster.Transaction(func(tx *db.ClusterTx) error {
		for ctProject, ctNames := range names {
			for _, ctName := range ctNames {
				container, err := tx.GetInstance(ctProject, ctName)
				if err != nil {
					return err
				}

				containers = append(containers, db.InstanceToArgs(container))
			}
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to fetch instances")
	}

	return containers, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"github.com/ghodss/yaml"

	"github.com/kubernetes/deployment-manager/expandybird/expander"
	"github.com/kubernetes/deployment-manager/manager/manager"
	"github.com/kubernetes/deployment-manager/registry"

	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	stdin             = flag.Bool("stdin", false, "Reads a configuration from the standard input")
	properties        = flag.String("properties", "", "Properties to use when deploying a template (e.g., --properties k1=v1,k2=v2)")
	template_registry = flag.String("registry", "kubernetes/deployment-manager/templates", "Github based template registry (owner/repo[/path])")
	service           = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager")
	binary            = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary")
)

var commands = []string{
	"expand \t\t\t Expands the supplied configuration(s)",
	"deploy \t\t\t Deploys the named template or the supplied configuration(s)",
	"list \t\t\t Lists the deployments in the cluster",
	"get \t\t\t Retrieves the supplied deployment",
	"delete \t\t\t Deletes the supplied deployment",
	"update \t\t\t Updates a deployment using the supplied configuration(s)",
	"deployed-types \t\t Lists the types deployed in the cluster",
	"deployed-instances \t Lists the instances of the named type deployed in the cluster",
	"templates \t\t Lists the templates in a given template registry",
	"describe \t\t Describes the named template in a given template registry",
}

var usage = func() {
	message := "Usage: %s [<flags>] <command> (<template-name> | <deployment-name> | (<configuration> [<import1>...<importN>]))\n"
	fmt.Fprintf(os.Stderr, message, os.Args[0])
	fmt.Fprintln(os.Stderr, "Commands:")
	for _, command := range commands {
		fmt.Fprintln(os.Stderr, command)
	}

	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "Flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func getGitRegistry() *registry.GithubRegistry {
	s := strings.Split(*template_registry, "/")
	if len(s) < 2 {
		log.Fatalf("invalid template registry: %s", *template_registry)
	}

	var path = ""
	if len(s) > 2 {
		path = strings.Join(s[2:], "/")
	}

	return registry.NewGithubRegistry(s[0], s[1], path)
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		fmt.Fprintln(os.Stderr, "No command supplied")
		usage()
	}

	if *stdin {
		fmt.Printf("reading from stdin is not yet implemented")
		os.Exit(0)
	}

	command := args[0]
	switch command {
	case "templates":
		git := getGitRegistry()
		templates, err := git.List()
		if err != nil {
			log.Fatalf("Cannot list %v", err)
		}

		fmt.Printf("Templates:\n")
		for _, t := range templates {
			fmt.Printf("%s:%s\n", t.Name, t.Version)
			downloadURL, err := git.GetURL(t)
			if err != nil {
				log.Printf("Failed to get download URL for template %s:%s", t.Name, t.Version)
			}

			fmt.Printf("\tdownload URL: %s\n", downloadURL)
		}
	case "describe":
		describeType(args)
	case "expand":
		template := loadTemplate(args)
		callService("expand", "POST", "expand configuration", marshalTemplate(template))
	case "deploy":
		template := loadTemplate(args)
		action := fmt.Sprintf("deploy configuration named %s", template.Name)
		callService("deployments", "POST", action, marshalTemplate(template))
	case "list":
		callService("deployments", "GET", "list deployments", nil)
	case "get":
		if len(args) < 2 {
			fmt.Fprintln(os.Stderr, "No deployment name supplied")
			usage()
		}

		path := fmt.Sprintf("deployments/%s", args[1])
		action := fmt.Sprintf("get deployment named %s", args[1])
		callService(path, "GET", action, nil)
	case "delete":
		if len(args) < 2 {
			fmt.Fprintln(os.Stderr, "No deployment name supplied")
			usage()
		}

		path := fmt.Sprintf("deployments/%s", args[1])
		action := fmt.Sprintf("delete deployment named %s", args[1])
		callService(path, "DELETE", action, nil)
	case "update":
		template := loadTemplate(args)
		path := fmt.Sprintf("deployments/%s", template.Name)
		action := fmt.Sprintf("delete deployment named %s", template.Name)
		callService(path, "PUT", action, marshalTemplate(template))
	case "deployed-types":
		callService("types", "GET", "list deployed types", nil)
	case "deployed-instances":
		if len(args) < 2 {
			fmt.Fprintln(os.Stderr, "No type name supplied")
			usage()
		}

		path := fmt.Sprintf("types/%s/instances", url.QueryEscape(args[1]))
		action := fmt.Sprintf("list deployed instances of type %s", args[1])
		callService(path, "GET", action, nil)
	default:
		usage()
	}
}

func callService(path, method, action string, reader io.ReadCloser) {
	u := fmt.Sprintf("%s/%s", *service, path)

	resp := callHttp(u, method, action, reader)
	var prettyJSON bytes.Buffer
	if err := json.Indent(&prettyJSON, []byte(resp), "", "  "); err != nil {
		log.Fatalf("Failed to parse JSON response from service: %s", resp)
	}
	fmt.Println(prettyJSON.String())
}

func callHttp(path, method, action string, reader io.ReadCloser) string {
	request, err := http.NewRequest(method, path, reader)
	request.Header.Add("Content-Type", "application/json")
	response, err := http.DefaultClient.Do(request)
	if err != nil {
		log.Fatalf("cannot %s: %s\n", action, err)
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Fatalf("cannot %s: %s\n", action, err)
	}

	if response.StatusCode < http.StatusOK ||
		response.StatusCode >= http.StatusMultipleChoices {
		message := fmt.Sprintf("status code: %d status: %s : %s", response.StatusCode, response.Status, body)
		log.Fatalf("cannot %s: %s\n", action, message)
	}

	return string(body)
}

// describeType prints the schema for a type specified by either a
// template URL or a fully qualified registry type name (e.g.,
// <type-name>:<version>)
func describeType(args []string) {
	if len(args) != 2 {
		fmt.Fprintln(os.Stderr, "No type name or URL supplied")
		usage()
	}

	var tUrl string

	if strings.HasPrefix(args[1], "http://") || strings.HasPrefix(args[1], "https://") {
		// User can pass raw URL to template.
		tUrl = args[1]
	} else {
		// User can pass registry type.
		t := getRegistryType(args[1])
		if t == nil {
			log.Fatalf("Invalid type name, must be in the form \"<type-name>:<version>\": %s", args[1])
		}

		git := getGitRegistry()
		url, err := git.GetURL(*t)
		if err != nil {
			log.Fatalf("Failed to fetch type information for %s: %s", args[1], err)
		}
		tUrl = url
	}

	schemaUrl := tUrl + ".schema"
	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil))
}

func loadTemplate(args []string) *expander.Template {
	var template *expander.Template
	var err error
	if len(args) < 2 {
		fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied")
		usage()
	}

	if len(args) < 3 {
		if t := getRegistryType(args[1]); t != nil {
			template = buildTemplateFromType(*t)
		} else {
			template, err = expander.NewTemplateFromRootTemplate(args[1])
		}
	} else {
		template, err = expander.NewTemplateFromFileNames(args[1], args[2:])
	}

	if err != nil {
		log.Fatalf("cannot create configuration from supplied arguments: %s\n", err)
	}

	return template
}

// TODO: needs better validation that this is actually a registry type.
func getRegistryType(fullType string) *registry.Type {
	tList := strings.Split(fullType, ":")
	if len(tList) != 2 {
		return nil
	}

	return &registry.Type{
		Name:    tList[0],
		Version: tList[1],
	}
}

func buildTemplateFromType(t registry.Type) *expander.Template {
	git := getGitRegistry()
	downloadURL, err := git.GetURL(t)
	if err != nil {
		log.Fatalf("Failed to get download URL for type %s:%s\n%s\n", t.Name, t.Version, err)
	}

	props := make(map[string]interface{})
	if *properties != "" {
		plist := strings.Split(*properties, ",")
		for _, p := range plist {
			ppair := strings.Split(p, "=")
			if len(ppair) != 2 {
				log.Fatalf("--properties must be in the form \"p1=v1,p2=v2,...\": %s\n", p)
			}

			// support ints
			// TODO: needs to support other types.
			i, err := strconv.Atoi(ppair[1])
			if err != nil {
				props[ppair[0]] = ppair[1]
			} else {
				props[ppair[0]] = i
			}
		}
	}

	name := fmt.Sprintf("%s-%s", t.Name, t.Version)
	config := manager.Configuration{Resources: []*manager.Resource{&manager.Resource{
		Name:       name,
		Type:       downloadURL,
		Properties: props,
	}}}

	y, err := yaml.Marshal(config)
	if err != nil {
		log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config)
	}

	return &expander.Template{
		Name:    name,
		Content: string(y),
		// No imports, as this is a single type from repository.
	}
}

func marshalTemplate(template *expander.Template) io.ReadCloser {
	j, err := json.Marshal(template)
	if err != nil {
		log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err)
	}

	return ioutil.NopCloser(bytes.NewReader(j))
}

func getRandomName() string {
	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
}
<commit_msg>Add flag for timeout for talking to service<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"github.com/ghodss/yaml"

	"github.com/kubernetes/deployment-manager/expandybird/expander"
	"github.com/kubernetes/deployment-manager/manager/manager"
	"github.com/kubernetes/deployment-manager/registry"

	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	stdin             = flag.Bool("stdin", false, "Reads a configuration from the standard input")
	properties        = flag.String("properties", "", "Properties to use when deploying a template (e.g., --properties k1=v1,k2=v2)")
	template_registry = flag.String("registry", "kubernetes/deployment-manager/templates", "Github based template registry (owner/repo[/path])")
	service           = flag.String("service", "http://localhost:8001/api/v1/proxy/namespaces/dm/services/manager-service:manager", "URL for deployment manager")
	binary            = flag.String("binary", "../expandybird/expansion/expansion.py", "Path to template expansion binary")
	timeout           = flag.Int("timeout", 10, "Time in seconds to wait for response")
)

var commands = []string{
	"expand \t\t\t Expands the supplied configuration(s)",
	"deploy \t\t\t Deploys the named template or the supplied configuration(s)",
	"list \t\t\t Lists the deployments in the cluster",
	"get \t\t\t Retrieves the supplied deployment",
	"delete \t\t\t Deletes the supplied deployment",
	"update \t\t\t Updates a deployment using the supplied configuration(s)",
	"deployed-types \t\t Lists the types deployed in the cluster",
	"deployed-instances \t Lists the instances of the named type deployed in the cluster",
	"templates \t\t Lists the templates in a given template registry",
	"describe \t\t Describes the named template in a given template registry",
}

var usage = func() {
	message := "Usage: %s [<flags>] <command> (<template-name> | <deployment-name> | (<configuration> [<import1>...<importN>]))\n"
	fmt.Fprintf(os.Stderr, message, os.Args[0])
	fmt.Fprintln(os.Stderr, "Commands:")
	for _, command := range commands {
		fmt.Fprintln(os.Stderr, command)
	}

	fmt.Fprintln(os.Stderr)
	fmt.Fprintln(os.Stderr, "Flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr)
	os.Exit(1)
}

func getGitRegistry() *registry.GithubRegistry {
	s := strings.Split(*template_registry, "/")
	if len(s) < 2 {
		log.Fatalf("invalid template registry: %s", *template_registry)
	}

	var path = ""
	if len(s) > 2 {
		path = strings.Join(s[2:], "/")
	}

	return registry.NewGithubRegistry(s[0], s[1], path)
}

func main() {
	flag.Parse()
	args := flag.Args()
	if len(args) < 1 {
		fmt.Fprintln(os.Stderr, "No command supplied")
		usage()
	}

	if *stdin {
		fmt.Printf("reading from stdin is not yet implemented")
		os.Exit(0)
	}

	command := args[0]
	switch command {
	case "templates":
		git := getGitRegistry()
		templates, err := git.List()
		if err != nil {
			log.Fatalf("Cannot list %v", err)
		}

		fmt.Printf("Templates:\n")
		for _, t := range templates {
			fmt.Printf("%s:%s\n", t.Name, t.Version)
			downloadURL, err := git.GetURL(t)
			if err != nil {
				log.Printf("Failed to get download URL for template %s:%s", t.Name, t.Version)
			}

			fmt.Printf("\tdownload URL: %s\n", downloadURL)
		}
	case "describe":
		describeType(args)
	case "expand":
		template := loadTemplate(args)
		callService("expand", "POST", "expand configuration", marshalTemplate(template))
	case "deploy":
		template := loadTemplate(args)
		action := fmt.Sprintf("deploy configuration named %s", template.Name)
		callService("deployments", "POST", action, marshalTemplate(template))
	case "list":
		callService("deployments", "GET", "list deployments", nil)
	case "get":
		if len(args) < 2 {
			fmt.Fprintln(os.Stderr, "No deployment name supplied")
			usage()
		}

		path := fmt.Sprintf("deployments/%s", args[1])
		action := fmt.Sprintf("get deployment named %s", args[1])
		callService(path, "GET", action, nil)
	case "delete":
		if len(args) < 2 {
			fmt.Fprintln(os.Stderr, "No deployment name supplied")
			usage()
		}

		path := fmt.Sprintf("deployments/%s", args[1])
		action := fmt.Sprintf("delete deployment named %s", args[1])
		callService(path, "DELETE", action, nil)
	case "update":
		template := loadTemplate(args)
		path := fmt.Sprintf("deployments/%s", template.Name)
		action := fmt.Sprintf("delete deployment named %s", template.Name)
		callService(path, "PUT", action, marshalTemplate(template))
	case "deployed-types":
		callService("types", "GET", "list deployed types", nil)
	case "deployed-instances":
		if len(args) < 2 {
			fmt.Fprintln(os.Stderr, "No type name supplied")
			usage()
		}

		path := fmt.Sprintf("types/%s/instances", url.QueryEscape(args[1]))
		action := fmt.Sprintf("list deployed instances of type %s", args[1])
		callService(path, "GET", action, nil)
	default:
		usage()
	}
}

func callService(path, method, action string, reader io.ReadCloser) {
	u := fmt.Sprintf("%s/%s", *service, path)

	resp := callHttp(u, method, action, reader)
	var prettyJSON bytes.Buffer
	if err := json.Indent(&prettyJSON, []byte(resp), "", "  "); err != nil {
		log.Fatalf("Failed to parse JSON response from service: %s", resp)
	}
	fmt.Println(prettyJSON.String())
}

func callHttp(path, method, action string, reader io.ReadCloser) string {
	request, err := http.NewRequest(method, path, reader)
	request.Header.Add("Content-Type", "application/json")

	client := http.Client{
		Timeout: time.Duration(time.Duration(*timeout) * time.Second),
	}

	response, err := client.Do(request)
	if err != nil {
		log.Fatalf("cannot %s: %s\n", action, err)
	}

	defer response.Body.Close()
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Fatalf("cannot %s: %s\n", action, err)
	}

	if response.StatusCode < http.StatusOK ||
		response.StatusCode >= http.StatusMultipleChoices {
		message := fmt.Sprintf("status code: %d status: %s : %s", response.StatusCode, response.Status, body)
		log.Fatalf("cannot %s: %s\n", action, message)
	}

	return string(body)
}

// describeType prints the schema for a type specified by either a
// template URL or a fully qualified registry type name (e.g.,
// <type-name>:<version>)
func describeType(args []string) {
	if len(args) != 2 {
		fmt.Fprintln(os.Stderr, "No type name or URL supplied")
		usage()
	}

	var tUrl string

	if strings.HasPrefix(args[1], "http://") || strings.HasPrefix(args[1], "https://") {
		// User can pass raw URL to template.
		tUrl = args[1]
	} else {
		// User can pass registry type.
		t := getRegistryType(args[1])
		if t == nil {
			log.Fatalf("Invalid type name, must be in the form \"<type-name>:<version>\": %s", args[1])
		}

		git := getGitRegistry()
		url, err := git.GetURL(*t)
		if err != nil {
			log.Fatalf("Failed to fetch type information for %s: %s", args[1], err)
		}
		tUrl = url
	}

	schemaUrl := tUrl + ".schema"
	fmt.Println(callHttp(schemaUrl, "GET", "get schema for type ("+tUrl+")", nil))
}

func loadTemplate(args []string) *expander.Template {
	var template *expander.Template
	var err error
	if len(args) < 2 {
		fmt.Fprintln(os.Stderr, "No template name or configuration(s) supplied")
		usage()
	}

	if len(args) < 3 {
		if t := getRegistryType(args[1]); t != nil {
			template = buildTemplateFromType(*t)
		} else {
			template, err = expander.NewTemplateFromRootTemplate(args[1])
		}
	} else {
		template, err = expander.NewTemplateFromFileNames(args[1], args[2:])
	}

	if err != nil {
		log.Fatalf("cannot create configuration from supplied arguments: %s\n", err)
	}

	return template
}

// TODO: needs better validation that this is actually a registry type.
func getRegistryType(fullType string) *registry.Type {
	tList := strings.Split(fullType, ":")
	if len(tList) != 2 {
		return nil
	}

	return &registry.Type{
		Name:    tList[0],
		Version: tList[1],
	}
}

func buildTemplateFromType(t registry.Type) *expander.Template {
	git := getGitRegistry()
	downloadURL, err := git.GetURL(t)
	if err != nil {
		log.Fatalf("Failed to get download URL for type %s:%s\n%s\n", t.Name, t.Version, err)
	}

	props := make(map[string]interface{})
	if *properties != "" {
		plist := strings.Split(*properties, ",")
		for _, p := range plist {
			ppair := strings.Split(p, "=")
			if len(ppair) != 2 {
				log.Fatalf("--properties must be in the form \"p1=v1,p2=v2,...\": %s\n", p)
			}

			// support ints
			// TODO: needs to support other types.
			i, err := strconv.Atoi(ppair[1])
			if err != nil {
				props[ppair[0]] = ppair[1]
			} else {
				props[ppair[0]] = i
			}
		}
	}

	name := fmt.Sprintf("%s-%s", t.Name, t.Version)
	config := manager.Configuration{Resources: []*manager.Resource{&manager.Resource{
		Name:       name,
		Type:       downloadURL,
		Properties: props,
	}}}

	y, err := yaml.Marshal(config)
	if err != nil {
		log.Fatalf("error: %s\ncannot create configuration for deployment: %v\n", err, config)
	}

	return &expander.Template{
		Name:    name,
		Content: string(y),
		// No imports, as this is a single type from repository.
	}
}

func marshalTemplate(template *expander.Template) io.ReadCloser {
	j, err := json.Marshal(template)
	if err != nil {
		log.Fatalf("cannot deploy configuration %s: %s\n", template.Name, err)
	}

	return ioutil.NopCloser(bytes.NewReader(j))
}

func getRandomName() string {
	return fmt.Sprintf("manifest-%d", time.Now().UTC().UnixNano())
}
<|endoftext|>


<commit_before>package gcache

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
)

var (
	oauth2TokenSource oauth2.TokenSource // The token is valid for 30 minutes.
)

// GetGDriveService returns the API service of Google Drive.
func GetGDriveService(
	r *http.Request,
) (
	*drive.Service,
	error,
) {
	n := 1
retry:
	service, err := drive.New(createGDriveClient(r))
	if err == nil {
		return service, nil
	}
	_, n, err = Triable(n, err)
	if err != nil {
		return nil, err
	}
	goto retry
}

func getGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	*drive.Service,
	error,
) {
	if field == "" {
		field = MinimumField
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return nil, nil, err
	}

retry:
	<-tokenBucketGDriveAPI
	fileList, err := service.Files.List().PageSize(1).Spaces("drive").Q(fmt.Sprintf("name='%s'", name)).Fields(field).Do()

	if err != nil {
		refreshToken, n, err = Triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			goto refresh
		}
		goto retry
	}

	if len(fileList.Files) <= 0 {
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	return fileList.Files[0], service, nil
}

// GetGDriveFile returns a file on Google Drive.
func GetGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, _, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// GetGDriveFileContent returns a file with content on Google Drive.
func GetGDriveFileContent(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	[]byte, // payload
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, service, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, nil, err
	}
	fileID := file.Id

	var payload []byte

	var refreshToken bool
	n := 1
retry:
	httpResponse, err := service.Files.Export(fileID, mimeTxt).Download()
	if IsFileNotExportableError(err) {
		<-tokenBucketGDriveAPI
		err = service.Files.Delete(fileID).Do()
		if err != nil {
			// pass
		}
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	if err != nil {
		refreshToken, n, err = Triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			service, err = GetGDriveService(r)
			if err != nil {
				return nil, nil, err
			}
		}
		goto retry
	}

	defer httpResponse.Body.Close()
	payload, err = ioutil.ReadAll(httpResponse.Body)
	if err != nil {
		return nil, nil, err
	}
	return file, payload, err
}

// Triable returns whether it can retry or not.
func Triable(
	retries int,
	err error,
) (
	bool, //refreshToken
	int, //retries
	error,
) {
	if err == nil {
		return false, retries, nil
	}
	if retries < 1 {
		retries = 1
	}
	switch {
	case IsInvalidSecurityTicket(err):
		oauth2TokenSource = nil
		return true, retries, nil
	case IsServerError(err):
		retries, err = sleeping(retries)
		if err != nil {
			return false, retries, err
		}
		return false, retries, nil
	}
	return false, retries, err
}

func sleeping(
	n int,
) (
	int,
	error,
) {
	if n > 16 {
		return 0, errors.New("Sleeping Timeout")
	}
	time.Sleep(time.Duration(n)*time.Second + time.Duration(random.Intn(1000))*time.Millisecond)
	return n * 2, nil
}
<commit_msg>refactor<commit_after>package gcache

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"golang.org/x/oauth2"
	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
)

var (
	oauth2TokenSource oauth2.TokenSource // The token is valid for 30 minutes.
)

// GetGDriveService returns the API service of Google Drive.
func GetGDriveService(
	r *http.Request,
) (
	*drive.Service,
	error,
) {
	n := 1
retry:
	service, err := drive.New(createGDriveClient(r))
	if err == nil {
		return service, nil
	}
	_, n, err = Triable(n, err)
	if err != nil {
		return nil, err
	}
	goto retry
}

func getGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	*drive.Service,
	error,
) {
	if field == "" {
		field = MinimumField
	}

	var refreshToken bool
	n := 1
refresh:
	service, err := GetGDriveService(r)
	if err != nil {
		return nil, nil, err
	}

retry:
	<-tokenBucketGDriveAPI
	fileList, err := service.Files.List().PageSize(1).Spaces("drive").Q(fmt.Sprintf("name='%s'", name)).Fields(field).Do()

	if err != nil {
		refreshToken, n, err = Triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			goto refresh
		}
		goto retry
	}

	if len(fileList.Files) <= 0 {
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	return fileList.Files[0], service, nil
}

// GetGDriveFile returns a file on Google Drive.
func GetGDriveFile(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, _, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, err
	}

	return file, nil
}

// GetGDriveFileContent returns a file with content on Google Drive.
func GetGDriveFileContent(
	r *http.Request,
	name string,
	field googleapi.Field,
) (
	*drive.File,
	[]byte, // payload
	error,
) {
	if field == "" {
		field = defaultField
	}

	file, service, err := getGDriveFile(r, name, field)
	if err != nil {
		return nil, nil, err
	}
	fileID := file.Id

	var payload []byte

	var refreshToken bool
	n := 1
retry:
	httpResponse, err := service.Files.Export(fileID, mimeTxt).Download()
	if IsFileNotExportableError(err) {
		<-tokenBucketGDriveAPI
		err = service.Files.Delete(fileID).Do()
		if err != nil {
			// pass
		}
		return nil, nil, &DriveFileDoesNotExistError{}
	}
	if err != nil {
		refreshToken, n, err = Triable(n, err)
		if err != nil {
			return nil, nil, err
		}
		if refreshToken {
			service, err = GetGDriveService(r)
			if err != nil {
				return nil, nil, err
			}
		}
		goto retry
	}

	defer httpResponse.Body.Close()
	payload, err = ioutil.ReadAll(httpResponse.Body)
	if err != nil {
		return nil, nil, err
	}
	return file, payload, err
}

// Triable returns whether it can retry or not.
func Triable(
	retries int,
	err error,
) (
	bool, //refreshToken
	int, //retries
	error,
) {
	if err == nil {
		return false, retries, nil
	}
	switch {
	case IsInvalidSecurityTicket(err):
		oauth2TokenSource = nil
		return true, retries, nil
	case IsServerError(err):
		retries, err = sleeping(retries)
		if err != nil {
			return false, retries, err
		}
		return false, retries, nil
	}
	return false, retries, err
}

func sleeping(
	n int,
) (
	int,
	error,
) {
	switch {
	case n > 16:
		return 0, errors.New("Sleeping Timeout")
	case n < 1:
		n = 1
	}
	time.Sleep(time.Duration(n)*time.Second + time.Duration(random.Intn(1000))*time.Millisecond)
	return n * 2, nil
}
<|endoftext|>


<commit_before>/*

	record is a package to open, read, and save game records stored in
	filesystem's format.

*/
package record

import (
	"encoding/json"
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const randomStringChars = "ABCDEF0123456789"

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

//Record is a record of moves, states, and game. Get a new one based on the
//contents of a file with New(). Instantiate directly if you'll set the
//starter state yourself.
type Record struct {
	Game  *boardgame.GameStorageRecord
	Moves []*boardgame.MoveStorageRecord
	//StatePatches are diffs from the state before. Get the actual state for a
	//version with State().
	StatePatches   []json.RawMessage
	states         []json.RawMessage
	expandedStates []map[string]interface{}
}

func New(filename string) (*Record, error) {
	data, err := ioutil.ReadFile(filename)

	if err != nil {
		return nil, errors.New("Couldn't read file: " + err.Error())
	}

	var result Record

	if err := json.Unmarshal(data, &result); err != nil {
		return nil, errors.New("Couldn't decode json: " + err.Error())
	}

	return &result, nil
}

//randomString returns a random string of the given length.
func randomString(length int) string {
	var result = ""

	for len(result) < length {
		result += string(randomStringChars[rand.Intn(len(randomStringChars))])
	}

	return result
}

func safeOvewritefile(path string, blob []byte) error {

	//Check for the easy case where the file doesn't exist yet
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return ioutil.WriteFile(path, blob, 0644)
	}

	dir, name := filepath.Split(path)

	ext := filepath.Ext(name)

	nameWithoutExt := strings.TrimSuffix(name, ext)

	tempFileName := filepath.Join(dir, nameWithoutExt+".TEMP."+randomString(6)+ext)

	if err := ioutil.WriteFile(tempFileName, blob, 0644); err != nil {
		return errors.New("Couldn't write temp file: " + err.Error())
	}

	if err := os.Remove(path); err != nil {
		return errors.New("Couldn't delete the original file: " + err.Error())
	}

	if err := os.Rename(tempFileName, path); err != nil {
		return errors.New("Couldn't rename the new file: " + err.Error())
	}

	return nil

}

//Save saves to the given path.
func (r *Record) Save(filename string) error {

	blob, err := json.MarshalIndent(r, "", "\t")

	if err != nil {
		return errors.New("Couldn't marshal blob: " + err.Error())
	}

	return safeOvewritefile(filename, blob)
}

//AddGameAndCurrentState adds the game, state, and move (if non-nil), ready
//for saving. Designed to be used in a SaveGameAndCurrentState method.
func (r *Record) AddGameAndCurrentState(game *boardgame.GameStorageRecord, state boardgame.StateStorageRecord, move *boardgame.MoveStorageRecord) error {

	r.Game = game

	if move != nil {
		r.Moves = append(r.Moves, move)
	}

	lastState, err := r.State(len(r.StatePatches) - 1)

	if err != nil {
		return errors.New("Couldn't fetch last state: " + err.Error())
	}

	differ := gojsondiff.New()

	patch, err := differ.Compare(lastState, state)

	if err != nil {
		return err
	}

	f := formatter.NewDeltaFormatter()

	js, err := f.FormatAsJson(patch)

	if err != nil {
		return errors.New("Couldn't format patch as json: " + err.Error())
	}

	formattedPatch, err := json.Marshal(js)

	if err != nil {
		return errors.New("Couldn't format patch json to byte: " + err.Error())
	}

	r.states = append(r.states, json.RawMessage(state))
	r.StatePatches = append(r.StatePatches, formattedPatch)

	return nil

}

//State is a method, not just a normal array, because we actually serialize to
//disk state patches, given that states are big objects and they rarely
//change. So when you request a State we have to derive what it is by applying
//all of the patches up in order.
func (r *Record) State(version int) (json.RawMessage, error) {

	if version < 0 {
		//The base object that version 0 is diffed against is the empty object
		return json.RawMessage(`{}`), nil
	}

	if len(r.states) > version {
		return r.states[version], nil
	}

	//Otherwise, derive forward, recursively.

	lastStateBlob, err := r.State(version - 1)

	if err != nil {
		//Don't decorate the error because it will likely stack
		return nil, err
	}

	unmarshaller := gojsondiff.NewUnmarshaller()

	patch, err := unmarshaller.UnmarshalBytes(r.StatePatches[version])

	if err != nil {
		return nil, err
	}

	differ := gojsondiff.New()

	var state map[string]interface{}

	if err := json.Unmarshal(lastStateBlob, &state); err != nil {
		return nil, errors.New("Couldn't unmarshal last blob: " + err.Error())
	}

	differ.ApplyPatch(state, patch)

	blob, err := json.MarshalIndent(state, "", "\t")

	if err != nil {
		return nil, errors.New("Couldn't marshal modified blob: " + err.Error())
	}

	r.states = append(r.states, blob)

	return blob, nil

}
<commit_msg>Get rid of unused cache in record. Part of #648.<commit_after>/*

	record is a package to open, read, and save game records stored in
	filesystem's format.

*/
package record

import (
	"encoding/json"
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const randomStringChars = "ABCDEF0123456789"

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

//Record is a record of moves, states, and game. Get a new one based on the
//contents of a file with New(). Instantiate directly if you'll set the
//starter state yourself.
type Record struct {
	Game  *boardgame.GameStorageRecord
	Moves []*boardgame.MoveStorageRecord
	//StatePatches are diffs from the state before. Get the actual state for a
	//version with State().
	StatePatches []json.RawMessage
	states       []json.RawMessage
}

func New(filename string) (*Record, error) {
	data, err := ioutil.ReadFile(filename)

	if err != nil {
		return nil, errors.New("Couldn't read file: " + err.Error())
	}

	var result Record

	if err := json.Unmarshal(data, &result); err != nil {
		return nil, errors.New("Couldn't decode json: " + err.Error())
	}

	return &result, nil
}

//randomString returns a random string of the given length.
func randomString(length int) string {
	var result = ""

	for len(result) < length {
		result += string(randomStringChars[rand.Intn(len(randomStringChars))])
	}

	return result
}

func safeOvewritefile(path string, blob []byte) error {

	//Check for the easy case where the file doesn't exist yet
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return ioutil.WriteFile(path, blob, 0644)
	}

	dir, name := filepath.Split(path)

	ext := filepath.Ext(name)

	nameWithoutExt := strings.TrimSuffix(name, ext)

	tempFileName := filepath.Join(dir, nameWithoutExt+".TEMP."+randomString(6)+ext)

	if err := ioutil.WriteFile(tempFileName, blob, 0644); err != nil {
		return errors.New("Couldn't write temp file: " + err.Error())
	}

	if err := os.Remove(path); err != nil {
		return errors.New("Couldn't delete the original file: " + err.Error())
	}

	if err := os.Rename(tempFileName, path); err != nil {
		return errors.New("Couldn't rename the new file: " + err.Error())
	}

	return nil

}

//Save saves to the given path.
func (r *Record) Save(filename string) error {

	blob, err := json.MarshalIndent(r, "", "\t")

	if err != nil {
		return errors.New("Couldn't marshal blob: " + err.Error())
	}

	return safeOvewritefile(filename, blob)
}

//AddGameAndCurrentState adds the game, state, and move (if non-nil), ready
//for saving. Designed to be used in a SaveGameAndCurrentState method.
func (r *Record) AddGameAndCurrentState(game *boardgame.GameStorageRecord, state boardgame.StateStorageRecord, move *boardgame.MoveStorageRecord) error {

	r.Game = game

	if move != nil {
		r.Moves = append(r.Moves, move)
	}

	lastState, err := r.State(len(r.StatePatches) - 1)

	if err != nil {
		return errors.New("Couldn't fetch last state: " + err.Error())
	}

	differ := gojsondiff.New()

	patch, err := differ.Compare(lastState, state)

	if err != nil {
		return err
	}

	f := formatter.NewDeltaFormatter()

	js, err := f.FormatAsJson(patch)

	if err != nil {
		return errors.New("Couldn't format patch as json: " + err.Error())
	}

	formattedPatch, err := json.Marshal(js)

	if err != nil {
		return errors.New("Couldn't format patch json to byte: " + err.Error())
	}

	r.states = append(r.states, json.RawMessage(state))
	r.StatePatches = append(r.StatePatches, formattedPatch)

	return nil

}

//State is a method, not just a normal array, because we actually serialize to
//disk state patches, given that states are big objects and they rarely
//change. So when you request a State we have to derive what it is by applying
//all of the patches up in order.
func (r *Record) State(version int) (json.RawMessage, error) {

	if version < 0 {
		//The base object that version 0 is diffed against is the empty object
		return json.RawMessage(`{}`), nil
	}

	if len(r.states) > version {
		return r.states[version], nil
	}

	//Otherwise, derive forward, recursively.

	lastStateBlob, err := r.State(version - 1)

	if err != nil {
		//Don't decorate the error because it will likely stack
		return nil, err
	}

	unmarshaller := gojsondiff.NewUnmarshaller()

	patch, err := unmarshaller.UnmarshalBytes(r.StatePatches[version])

	if err != nil {
		return nil, err
	}

	differ := gojsondiff.New()

	var state map[string]interface{}

	if err := json.Unmarshal(lastStateBlob, &state); err != nil {
		return nil, errors.New("Couldn't unmarshal last blob: " + err.Error())
	}

	differ.ApplyPatch(state, patch)

	blob, err := json.MarshalIndent(state, "", "\t")

	if err != nil {
		return nil, errors.New("Couldn't marshal modified blob: " + err.Error())
	}

	r.states = append(r.states, blob)

	return blob, nil

}
<|endoftext|>


<commit_before>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package proxy

import (
	"fmt"
	"net/http"

	"testing"

	"regexp"

	"github.com/openfaas/faas-cli/test"
)

type DeployProxyTest struct {
	title               string
	mockServerResponses []int
	replace             bool
	update              bool
	expectedOutput      string
}

var DeployProxyTests = []DeployProxyTest{
	{
		title:               "200_Deploy",
		mockServerResponses: []int{http.StatusOK, http.StatusOK},
		replace:             true,
		update:              false,
		expectedOutput:      `(?m:Deployed.)`,
	},
	{
		title:               "404_Deploy",
		mockServerResponses: []int{http.StatusOK, http.StatusNotFound},
		replace:             true,
		update:              false,
		expectedOutput:      `(?m:Unexpected status: 404)`,
	},
	{
		title:               "UpdateFailedDeployed",
		mockServerResponses: []int{http.StatusNotFound, http.StatusOK},
		replace:             false,
		update:              true,
		expectedOutput:      `(?m:Deployed.)`,
	},
}

func runDeployProxyTest(t *testing.T, deployTest DeployProxyTest) {
	s := test.MockHttpServerStatus(
		t,
		deployTest.mockServerResponses...,
	)
	defer s.Close()

	stdout := test.CaptureStdout(func() {
		DeployFunction(
			"fproces",
			s.URL,
			"function",
			"image",
			"language",
			deployTest.replace,
			nil,
			"network",
			[]string{},
			deployTest.update,
			[]string{},
			map[string]string{},
			FunctionResourceRequest{},
		)
	})

	r := regexp.MustCompile(deployTest.expectedOutput)
	if !r.MatchString(stdout) {
		t.Fatalf("Output not matched: %s", stdout)
	}
}

func Test_RunDeployProxyTests(t *testing.T) {
	for _, tst := range DeployProxyTests {
		t.Run(tst.title, func(t *testing.T) {
			runDeployProxyTest(t, tst)
		})
	}
}

func Test_DeployFunction_MissingURLPrefix(t *testing.T) {
	url := "127.0.0.1:8080"

	stdout := test.CaptureStdout(func() {
		DeployFunction(
			"fprocess",
			url,
			"function",
			"image",
			"language",
			false,
			nil,
			"network",
			[]string{},
			false,
			[]string{},
			map[string]string{},
			FunctionResourceRequest{},
		)
	})

	expectedErrMsg := "first path segment in URL cannot contain colon"
	r := regexp.MustCompile(fmt.Sprintf("(?m:%s)", expectedErrMsg))
	if !r.MatchString(stdout) {
		t.Fatalf("Want: %s\nGot: %s", expectedErrMsg, stdout)
	}
}
<commit_msg>Refactor proxy/deploy_test.go test cases<commit_after>// Copyright (c) Alex Ellis 2017. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

package proxy

import (
	"fmt"
	"net/http"

	"testing"

	"regexp"

	"github.com/openfaas/faas-cli/test"
)

type deployProxyTest struct {
	title               string
	mockServerResponses []int
	replace             bool
	update              bool
	expectedOutput      string
}

func runDeployProxyTest(t *testing.T, deployTest deployProxyTest) {
	s := test.MockHttpServerStatus(
		t,
		deployTest.mockServerResponses...,
	)
	defer s.Close()

	stdout := test.CaptureStdout(func() {
		DeployFunction(
			"fproces",
			s.URL,
			"function",
			"image",
			"language",
			deployTest.replace,
			nil,
			"network",
			[]string{},
			deployTest.update,
			[]string{},
			map[string]string{},
			FunctionResourceRequest{},
		)
	})

	r := regexp.MustCompile(deployTest.expectedOutput)
	if !r.MatchString(stdout) {
		t.Fatalf("Output not matched: %s", stdout)
	}
}

func Test_RunDeployProxyTests(t *testing.T) {
	var deployProxyTests = []deployProxyTest{
		{
			title:               "200_Deploy",
			mockServerResponses: []int{http.StatusOK, http.StatusOK},
			replace:             true,
			update:              false,
			expectedOutput:      `(?m:Deployed.)`,
		},
		{
			title:               "404_Deploy",
			mockServerResponses: []int{http.StatusOK, http.StatusNotFound},
			replace:             true,
			update:              false,
			expectedOutput:      `(?m:Unexpected status: 404)`,
		},
		{
			title:               "UpdateFailedDeployed",
			mockServerResponses: []int{http.StatusNotFound, http.StatusOK},
			replace:             false,
			update:              true,
			expectedOutput:      `(?m:Deployed.)`,
		},
	}
	for _, tst := range deployProxyTests {
		t.Run(tst.title, func(t *testing.T) {
			runDeployProxyTest(t, tst)
		})
	}
}

func Test_DeployFunction_MissingURLPrefix(t *testing.T) {
	url := "127.0.0.1:8080"

	stdout := test.CaptureStdout(func() {
		DeployFunction(
			"fprocess",
			url,
			"function",
			"image",
			"language",
			false,
			nil,
			"network",
			[]string{},
			false,
			[]string{},
			map[string]string{},
			FunctionResourceRequest{},
		)
	})

	expectedErrMsg := "first path segment in URL cannot contain colon"
	r := regexp.MustCompile(fmt.Sprintf("(?m:%s)", expectedErrMsg))
	if !r.MatchString(stdout) {
		t.Fatalf("Want: %s\nGot: %s", expectedErrMsg, stdout)
	}
}
<|endoftext|>


<commit_before>// +build freebsd,!cgo

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package console

//
// Implementing the functions below requires cgo support.  Non-cgo stubs
// versions are defined below to enable cross-compilation of source code
// that depends on these functions, but the resultant cross-compiled
// binaries cannot actually be used.  If the stub function(s) below are
// actually invoked they will display an error message and cause the
// calling process to exit.
//

func openpt() (*os.File, error) {
	panic("openpt() support requires cgo.")
}
<commit_msg>Add missing import in freebsd nocgo stub<commit_after>// +build freebsd,!cgo

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package console

import (
	"os"
)

//
// Implementing the functions below requires cgo support.  Non-cgo stubs
// versions are defined below to enable cross-compilation of source code
// that depends on these functions, but the resultant cross-compiled
// binaries cannot actually be used.  If the stub function(s) below are
// actually invoked they will display an error message and cause the
// calling process to exit.
//

func openpt() (*os.File, error) {
	panic("openpt() support requires cgo.")
}
<|endoftext|>


<commit_before>package lexer

type lookaheadPair struct {
	Lexeme Lexeme
	Error  error
}

// Lexer takes a stream of bytes and provides a stream of lexemes
type Lexer struct {
	Stream RuneStream

	lookahead []lookaheadPair
}

// EOF returns true if there is no more lookahead and if the
// byte stream has been consumed
func (l *Lexer) EOF() bool {
	return len(l.lookahead) == 0 && l.Stream.EOF()
}

// Get returns the very next lexeme from the byte stream
func (l *Lexer) Get(n int) ([]Lexeme, []error) {
	lexemes := make([]Lexeme, n)
	errors := make([]error, n)

	for i := 0; i < n; i++ {
		lexeme, err := l.Next()
		lexemes[i] = lexeme
		errors[i] = err
	}

	return lexemes, errors
}

// Peek does a lookahead at the very next lexeme
func (l *Lexer) Peek() (Lexeme, error) {
	lexemes, errors := l.Lookahead(1)
	return lexemes[0], errors[0]
}

// Lookahead peeks at the next n lexemes
func (l *Lexer) Lookahead(n int) ([]Lexeme, []error) {
	rem := n - len(l.lookahead)

	if rem > 0 {
		lexemes, errors := l.getN(rem)

		for i := 0; i < rem; i++ {
			l.lookahead = append(l.lookahead, lookaheadPair{
				Lexeme: lexemes[i],
				Error:  errors[i],
			})
		}
	}

	lookahead := l.lookahead[0:n]
	lexemes := make([]Lexeme, n)
	errors := make([]error, n)

	for i := 0; i < n; i++ {
		lexemes[i] = lookahead[i].Lexeme
		errors[i] = lookahead[i].Error
	}

	return lexemes, errors
}

// Next returns the very next lexeme from the byte stream
func (l *Lexer) Next() (Lexeme, error) {
	if len(l.lookahead) > 0 {
		pair := l.lookahead[0]
		l.lookahead = l.lookahead[1:]
		return pair.Lexeme, pair.Error
	}

	return l.lex()
}

func (l *Lexer) getN(n int) ([]Lexeme, []error) {
	lexemes := make([]Lexeme, n)
	errors := []error{}

	for i := 0; i < n; i++ {
		lexeme, err := l.lex()
		lexemes[i] = lexeme

		if err != nil {
			errors = append(errors, err)
		}
	}

	return lexemes, errors
}
<commit_msg>Fix lexer to not return a slice of nil errors<commit_after>package lexer

type lookaheadPair struct {
	Lexeme Lexeme
	Error  error
}

// Lexer takes a stream of bytes and provides a stream of lexemes
type Lexer struct {
	Stream RuneStream

	lookahead []lookaheadPair
}

// EOF returns true if there is no more lookahead and if the
// byte stream has been consumed
func (l *Lexer) EOF() bool {
	return len(l.lookahead) == 0 && l.Stream.EOF()
}

// Get returns the very next lexeme from the byte stream
func (l *Lexer) Get(n int) ([]Lexeme, []error) {
	lexemes := make([]Lexeme, n)
	errors := []error{}

	for i := 0; i < n; i++ {
		lexeme, err := l.Next()
		lexemes[i] = lexeme

		if err != nil {
			errors = append(errors, err)
		}
	}

	return lexemes, errors
}

// Peek does a lookahead at the very next lexeme
func (l *Lexer) Peek() (Lexeme, error) {
	lexemes, errors := l.Lookahead(1)

	var err error
	if len(errors) > 0 {
		err = errors[0]
	}

	return lexemes[0], err
}

// Lookahead peeks at the next n lexemes
func (l *Lexer) Lookahead(n int) ([]Lexeme, []error) {
	rem := n - len(l.lookahead)

	if rem > 0 {
		lexemes, errors := l.getN(rem)

		for i := 0; i < rem; i++ {
			l.lookahead = append(l.lookahead, lookaheadPair{
				Lexeme: lexemes[i],
				Error:  errors[i],
			})
		}
	}

	lookahead := l.lookahead[0:n]
	lexemes := make([]Lexeme, n)
	errors := []error{}

	for i := 0; i < n; i++ {
		lexemes[i] = lookahead[i].Lexeme

		if lookahead[i].Error != nil {
			errors = append(errors, lookahead[i].Error)
		}
	}

	return lexemes, errors
}

// Next returns the very next lexeme from the byte stream
func (l *Lexer) Next() (Lexeme, error) {
	if len(l.lookahead) > 0 {
		pair := l.lookahead[0]
		l.lookahead = l.lookahead[1:]
		return pair.Lexeme, pair.Error
	}

	return l.lex()
}

func (l *Lexer) getN(n int) ([]Lexeme, []error) {
	lexemes := make([]Lexeme, n)
	errors := make([]error, n)

	for i := 0; i < n; i++ {
		lexeme, err := l.lex()
		lexemes[i] = lexeme
		errors[i] = err
	}

	return lexemes, errors
}
<|endoftext|>