<commit_before>package swarm

import (
	"bytes"
	"net"
	"sync"
	"time"
	"xd/lib/bittorrent"
	"xd/lib/bittorrent/extensions"
	"xd/lib/common"
	"xd/lib/log"
	"xd/lib/metainfo"
	"xd/lib/network"
	"xd/lib/storage"
	"xd/lib/tracker"
)

// single torrent tracked in a swarm
type Torrent struct {
	Completed      func()
	Started        func()
	Stopped        func()
	netacces       sync.Mutex
	suspended      bool
	netContext     network.Network
	Trackers       map[string]tracker.Announcer
	announcers     map[string]*torrentAnnounce
	announceMtx    sync.Mutex
	announceTicker *time.Ticker
	id             common.PeerID
	st             storage.Torrent
	piece          chan pieceEvent
	obconns        map[string]*PeerConn
	ibconns        map[string]*PeerConn
	mtx            sync.Mutex
	pt             *pieceTracker
	defaultOpts    *extensions.ExtendedOptions
	closing        bool
}

func (t *Torrent) ObtainedNetwork(n network.Network) {
	t.netContext = n
	if t.suspended {
		t.suspended = false
		t.netacces.Unlock()
	}
}

// get our current network context
func (t *Torrent) Network() (n network.Network) {
	for t.suspended {
		time.Sleep(time.Millisecond)
	}
	t.netacces.Lock()
	n = t.netContext
	t.netacces.Unlock()
	return
}

// called when we lost network access abruptly
func (t *Torrent) LostNetwork() {
	if t.suspended {
		return
	}
	t.netacces.Lock()
	t.suspended = true
	t.netContext = nil
}

// implements io.Closer
func (t *Torrent) Close() error {
	if t.closing {
		return nil
	}
	chnl := t.piece
	t.piece = nil
	t.closing = true
	t.StopAnnouncing()
	t.VisitPeers(func(c *PeerConn) {
		c.Close()
	})
	for t.NumPeers() > 0 {
		time.Sleep(time.Millisecond)
	}
	close(chnl)
	return t.st.Flush()
}

func (t *Torrent) shouldAnnounce(name string) bool {
	return time.Now().After(t.nextAnnounceFor(name))
}

func (t *Torrent) nextAnnounceFor(name string) (tm time.Time) {
	t.announceMtx.Lock()
	a, ok := t.announcers[name]
	if ok {
		tm = a.next
	} else {
		tm = time.Now()
		t.announcers[name] = &torrentAnnounce{
			next:     tm,
			t:        t,
			announce: t.Trackers[name],
		}
	}
	t.announceMtx.Unlock()
	return tm
}

func newTorrent(st storage.Torrent) *Torrent {
	t := &Torrent{
		Trackers:    make(map[string]tracker.Announcer),
		announcers:  make(map[string]*torrentAnnounce),
		st:          st,
		piece:       make(chan pieceEvent),
		ibconns:     make(map[string]*PeerConn),
		obconns:     make(map[string]*PeerConn),
		defaultOpts: extensions.New(),
	}
	t.pt = createPieceTracker(st, t.getRarestPiece)
	t.pt.have = t.broadcastHave
	return t
}

func (t *Torrent) getRarestPiece(remote *bittorrent.Bitfield) (idx uint32) {
	var swarm []*bittorrent.Bitfield
	t.VisitPeers(func(c *PeerConn) {
		if c.bf != nil {
			swarm = append(swarm, c.bf)
		}
	})
	idx = remote.FindRarest(swarm)
	return
}

// NumPeers counts how many peers we have on this torrent
func (t *Torrent) NumPeers() (count uint) {
	t.VisitPeers(func(_ *PeerConn) {
		count++
	})
	return
}

// call a visitor on each open peer connection
func (t *Torrent) VisitPeers(v func(*PeerConn)) {
	var conns []*PeerConn
	t.mtx.Lock()
	for _, conn := range t.obconns {
		if conn != nil {
			conns = append(conns, conn)
		}
	}
	for _, conn := range t.ibconns {
		if conn != nil {
			conns = append(conns, conn)
		}
	}
	t.mtx.Unlock()
	for _, conn := range conns {
		v(conn)
	}
}

func (t *Torrent) GetStatus() TorrentStatus {
	name := t.Name()
	var peers []*PeerConnStats
	t.VisitPeers(func(c *PeerConn) {
		peers = append(peers, c.Stats())
	})
	log.Debugf("unlocked torrent mutex for %s", name)
	state := Downloading
	if t.Done() {
		state = Seeding
	}
	return TorrentStatus{
		Peers:    peers,
		Name:     name,
		State:    state,
		Infohash: t.MetaInfo().Infohash().Hex(),
	}

}

func (t *Torrent) Bitfield() *bittorrent.Bitfield {
	return t.st.Bitfield()
}

// start annoucing on all trackers
func (t *Torrent) StartAnnouncing() {
	ev := tracker.Started
	if t.Done() {
		ev = tracker.Completed
	}
	for name := range t.Trackers {
		t.nextAnnounceFor(name)
		t.announce(name, ev)
	}
	if t.announceTicker == nil {
		t.announceTicker = time.NewTicker(time.Second)
	}
	go t.pollAnnounce()
}

// stop annoucing on all trackers
func (t *Torrent) StopAnnouncing() {
	if t.announceTicker != nil {
		t.announceTicker.Stop()
	}
	for name := range t.Trackers {
		t.announce(name, tracker.Stopped)
	}
}

// poll announce ticker channel and issue announces
func (t *Torrent) pollAnnounce() {
	for {
		_, ok := <-t.announceTicker.C
		if !ok {
			// done
			return
		}
		ev := tracker.Nop
		if t.Done() {
			ev = tracker.Completed
		}
		for name := range t.Trackers {
			if t.shouldAnnounce(name) {
				go t.announce(name, ev)
			}
		}
	}
}

func (t *Torrent) announce(name string, ev tracker.Event) {
	t.announceMtx.Lock()
	log.Infof("announcing to %s", name)
	a := t.announcers[name]
	t.announceMtx.Unlock()
	err := a.tryAnnounce(ev)
	if err != nil {
		log.Warnf("announce to %s failed: %s", name, err)
	}
}

// add peers to torrent
func (t *Torrent) addPeers(peers []common.Peer) {
	for _, p := range peers {
		a, e := p.Resolve(t.Network())
		if e == nil {
			if a.String() == t.Network().Addr().String() {
				// don't connect to self or a duplicate
				continue
			}
			if t.HasOBConn(a) {
				continue
			}
			// no error resolving
			go t.PersistPeer(a, p.ID)
		} else {
			log.Warnf("failed to resolve peer %s", e.Error())
		}
	}
}

// persit a connection to a peer
func (t *Torrent) PersistPeer(a net.Addr, id common.PeerID) {

	triesLeft := 10
	for !t.Done() {
		if t.HasIBConn(a) {
			return
		}
		if !t.HasOBConn(a) {
			err := t.AddPeer(a, id)
			if err == nil {
				return
			} else {
				triesLeft--
			}
			if triesLeft <= 0 {
				return
			}
		} else {
			time.Sleep(time.Second)
		}
	}
}

func (t *Torrent) HasIBConn(a net.Addr) (has bool) {
	t.mtx.Lock()
	_, has = t.ibconns[a.String()]
	t.mtx.Unlock()
	return
}

func (t *Torrent) HasOBConn(a net.Addr) (has bool) {
	t.mtx.Lock()
	_, has = t.obconns[a.String()]
	t.mtx.Unlock()
	return
}

func (t *Torrent) addOBPeer(c *PeerConn) {
	t.mtx.Lock()
	t.obconns[c.c.RemoteAddr().String()] = c
	t.mtx.Unlock()
}

func (t *Torrent) removeOBConn(c *PeerConn) {
	t.mtx.Lock()
	delete(t.obconns, c.c.RemoteAddr().String())
	t.mtx.Unlock()
}

func (t *Torrent) addIBPeer(c *PeerConn) {
	t.mtx.Lock()
	t.ibconns[c.c.RemoteAddr().String()] = c
	t.mtx.Unlock()
	c.inbound = true
}

func (t *Torrent) removeIBConn(c *PeerConn) {
	t.mtx.Lock()
	delete(t.ibconns, c.c.RemoteAddr().String())
	t.mtx.Unlock()
}

// connect to a new peer for this swarm, blocks
func (t *Torrent) AddPeer(a net.Addr, id common.PeerID) error {
	if t.HasOBConn(a) {
		return nil
	}
	c, err := t.Network().Dial(a.Network(), a.String())
	if err == nil {
		// connected
		ih := t.st.Infohash()
		// build handshake
		h := new(bittorrent.Handshake)
		// enable bittorrent extensions
		h.Reserved.Set(bittorrent.Extension)
		copy(h.Infohash[:], ih[:])
		copy(h.PeerID[:], t.id[:])
		// send handshake
		err = h.Send(c)
		if err == nil {
			// get response to handshake
			err = h.Recv(c)
			if err == nil {
				if bytes.Equal(ih[:], h.Infohash[:]) {
					// infohashes match
					var opts *extensions.ExtendedOptions
					if h.Reserved.Has(bittorrent.Extension) {
						opts = t.defaultOpts.Copy()
					}
					pc := makePeerConn(c, t, h.PeerID, opts)
					t.addOBPeer(pc)
					pc.Send(t.Bitfield().ToWireMessage())
					pc.start()
					return nil
				} else {
					log.Warn("Infohash missmatch")
				}
			}
		}
		log.Debugf("didn't complete handshake with peer: %s", err)
		// bad thing happened
		c.Close()
	}
	log.Debugf("didn't connect to %s: %s", a, err)
	return err
}

func (t *Torrent) broadcastHave(idx uint32) {
	msg := common.NewHave(idx)
	log.Infof("%s got piece %d", t.Name(), idx)
	conns := make(map[string]*PeerConn)
	t.mtx.Lock()
	for k, conn := range t.ibconns {
		if conn != nil {
			conns[k] = conn
		}
	}
	for k, conn := range t.obconns {
		if conn != nil {
			conns[k] = conn
		}
	}
	t.mtx.Unlock()
	for _, conn := range conns {
		go conn.Send(msg)
	}
}

// get metainfo for this torrent
func (t *Torrent) MetaInfo() *metainfo.TorrentFile {
	return t.st.MetaInfo()
}

func (t *Torrent) Name() string {
	return t.MetaInfo().TorrentName()
}

// callback called when we get a new inbound peer
func (t *Torrent) onNewPeer(c *PeerConn) {
	a := c.c.RemoteAddr()
	if t.HasIBConn(a) {
		log.Infof("duplicate peer from %s", a)
		c.Close()
		return
	}
	log.Infof("New peer (%s) for %s", c.id.String(), t.st.Infohash().Hex())
	t.addIBPeer(c)
	c.Send(t.Bitfield().ToWireMessage())
	c.start()

}

// handle a piece request
func (t *Torrent) onPieceRequest(c *PeerConn, req *common.PieceRequest) {
	if t.piece != nil {
		t.piece <- pieceEvent{c, req}
	}
}

func (t *Torrent) Run() {
	go t.handlePieces()
	if t.Started != nil {
		t.Started()
	}
	for !t.Done() {
		time.Sleep(time.Minute)
	}
	if t.Completed != nil {
		t.Completed()
	}
}

func (t *Torrent) handlePieces() {
	log.Infof("%s running", t.Name())
	for {
		ev, ok := <-t.piece
		if !ok {
			log.Infof("%s torrent run exit", t.Name())
			// channel closed
			return
		}
		if ev.r != nil && ev.r.Length > 0 {
			log.Debugf("%s asked for piece %d %d-%d", ev.c.id.String(), ev.r.Index, ev.r.Begin, ev.r.Begin+ev.r.Length)
			// TODO: cache common pieces (?)
			err := t.st.VisitPiece(ev.r, func(p *common.PieceData) error {
				// have the piece, send it
				ev.c.Send(p.ToWireMessage())
				log.Debugf("%s queued piece %d %d-%d", ev.c.id.String(), ev.r.Index, ev.r.Begin, ev.r.Begin+ev.r.Length)
				return nil
			})
			if err != nil {
				ev.c.Close()
			}
		} else {
			log.Infof("%s asked for a zero length piece", ev.c.id.String())
			// TODO: should we close here?
			ev.c.Close()
		}

	}
}

func (t *Torrent) Done() bool {
	return t.Bitfield().Completed()
}
<commit_msg>announce initial in background<commit_after>package swarm

import (
	"bytes"
	"net"
	"sync"
	"time"
	"xd/lib/bittorrent"
	"xd/lib/bittorrent/extensions"
	"xd/lib/common"
	"xd/lib/log"
	"xd/lib/metainfo"
	"xd/lib/network"
	"xd/lib/storage"
	"xd/lib/tracker"
)

// single torrent tracked in a swarm
type Torrent struct {
	Completed      func()
	Started        func()
	Stopped        func()
	netacces       sync.Mutex
	suspended      bool
	netContext     network.Network
	Trackers       map[string]tracker.Announcer
	announcers     map[string]*torrentAnnounce
	announceMtx    sync.Mutex
	announceTicker *time.Ticker
	id             common.PeerID
	st             storage.Torrent
	piece          chan pieceEvent
	obconns        map[string]*PeerConn
	ibconns        map[string]*PeerConn
	mtx            sync.Mutex
	pt             *pieceTracker
	defaultOpts    *extensions.ExtendedOptions
	closing        bool
}

func (t *Torrent) ObtainedNetwork(n network.Network) {
	t.netContext = n
	if t.suspended {
		t.suspended = false
		t.netacces.Unlock()
	}
}

// get our current network context
func (t *Torrent) Network() (n network.Network) {
	for t.suspended {
		time.Sleep(time.Millisecond)
	}
	t.netacces.Lock()
	n = t.netContext
	t.netacces.Unlock()
	return
}

// called when we lost network access abruptly
func (t *Torrent) LostNetwork() {
	if t.suspended {
		return
	}
	t.netacces.Lock()
	t.suspended = true
	t.netContext = nil
}

// implements io.Closer
func (t *Torrent) Close() error {
	if t.closing {
		return nil
	}
	chnl := t.piece
	t.piece = nil
	t.closing = true
	t.StopAnnouncing()
	t.VisitPeers(func(c *PeerConn) {
		c.Close()
	})
	for t.NumPeers() > 0 {
		time.Sleep(time.Millisecond)
	}
	close(chnl)
	return t.st.Flush()
}

func (t *Torrent) shouldAnnounce(name string) bool {
	return time.Now().After(t.nextAnnounceFor(name))
}

func (t *Torrent) nextAnnounceFor(name string) (tm time.Time) {
	t.announceMtx.Lock()
	a, ok := t.announcers[name]
	if ok {
		tm = a.next
	} else {
		tm = time.Now()
		t.announcers[name] = &torrentAnnounce{
			next:     tm,
			t:        t,
			announce: t.Trackers[name],
		}
	}
	t.announceMtx.Unlock()
	return tm
}

func newTorrent(st storage.Torrent) *Torrent {
	t := &Torrent{
		Trackers:    make(map[string]tracker.Announcer),
		announcers:  make(map[string]*torrentAnnounce),
		st:          st,
		piece:       make(chan pieceEvent),
		ibconns:     make(map[string]*PeerConn),
		obconns:     make(map[string]*PeerConn),
		defaultOpts: extensions.New(),
	}
	t.pt = createPieceTracker(st, t.getRarestPiece)
	t.pt.have = t.broadcastHave
	return t
}

func (t *Torrent) getRarestPiece(remote *bittorrent.Bitfield) (idx uint32) {
	var swarm []*bittorrent.Bitfield
	t.VisitPeers(func(c *PeerConn) {
		if c.bf != nil {
			swarm = append(swarm, c.bf)
		}
	})
	idx = remote.FindRarest(swarm)
	return
}

// NumPeers counts how many peers we have on this torrent
func (t *Torrent) NumPeers() (count uint) {
	t.VisitPeers(func(_ *PeerConn) {
		count++
	})
	return
}

// call a visitor on each open peer connection
func (t *Torrent) VisitPeers(v func(*PeerConn)) {
	var conns []*PeerConn
	t.mtx.Lock()
	for _, conn := range t.obconns {
		if conn != nil {
			conns = append(conns, conn)
		}
	}
	for _, conn := range t.ibconns {
		if conn != nil {
			conns = append(conns, conn)
		}
	}
	t.mtx.Unlock()
	for _, conn := range conns {
		v(conn)
	}
}

func (t *Torrent) GetStatus() TorrentStatus {
	name := t.Name()
	var peers []*PeerConnStats
	t.VisitPeers(func(c *PeerConn) {
		peers = append(peers, c.Stats())
	})
	log.Debugf("unlocked torrent mutex for %s", name)
	state := Downloading
	if t.Done() {
		state = Seeding
	}
	return TorrentStatus{
		Peers:    peers,
		Name:     name,
		State:    state,
		Infohash: t.MetaInfo().Infohash().Hex(),
	}

}

func (t *Torrent) Bitfield() *bittorrent.Bitfield {
	return t.st.Bitfield()
}

// start annoucing on all trackers
func (t *Torrent) StartAnnouncing() {
	ev := tracker.Started
	if t.Done() {
		ev = tracker.Completed
	}
	for name := range t.Trackers {
		t.nextAnnounceFor(name)
		go t.announce(name, ev)
	}
	if t.announceTicker == nil {
		t.announceTicker = time.NewTicker(time.Second)
	}
	go t.pollAnnounce()
}

// stop annoucing on all trackers
func (t *Torrent) StopAnnouncing() {
	if t.announceTicker != nil {
		t.announceTicker.Stop()
	}
	for name := range t.Trackers {
		t.announce(name, tracker.Stopped)
	}
}

// poll announce ticker channel and issue announces
func (t *Torrent) pollAnnounce() {
	for {
		_, ok := <-t.announceTicker.C
		if !ok {
			// done
			return
		}
		ev := tracker.Nop
		if t.Done() {
			ev = tracker.Completed
		}
		for name := range t.Trackers {
			if t.shouldAnnounce(name) {
				go t.announce(name, ev)
			}
		}
	}
}

func (t *Torrent) announce(name string, ev tracker.Event) {
	t.announceMtx.Lock()
	log.Infof("announcing to %s", name)
	a := t.announcers[name]
	t.announceMtx.Unlock()
	err := a.tryAnnounce(ev)
	if err != nil {
		log.Warnf("announce to %s failed: %s", name, err)
	}
}

// add peers to torrent
func (t *Torrent) addPeers(peers []common.Peer) {
	for _, p := range peers {
		a, e := p.Resolve(t.Network())
		if e == nil {
			if a.String() == t.Network().Addr().String() {
				// don't connect to self or a duplicate
				continue
			}
			if t.HasOBConn(a) {
				continue
			}
			// no error resolving
			go t.PersistPeer(a, p.ID)
		} else {
			log.Warnf("failed to resolve peer %s", e.Error())
		}
	}
}

// persit a connection to a peer
func (t *Torrent) PersistPeer(a net.Addr, id common.PeerID) {

	triesLeft := 10
	for !t.Done() {
		if t.HasIBConn(a) {
			return
		}
		if !t.HasOBConn(a) {
			err := t.AddPeer(a, id)
			if err == nil {
				return
			} else {
				triesLeft--
			}
			if triesLeft <= 0 {
				return
			}
		} else {
			time.Sleep(time.Second)
		}
	}
}

func (t *Torrent) HasIBConn(a net.Addr) (has bool) {
	t.mtx.Lock()
	_, has = t.ibconns[a.String()]
	t.mtx.Unlock()
	return
}

func (t *Torrent) HasOBConn(a net.Addr) (has bool) {
	t.mtx.Lock()
	_, has = t.obconns[a.String()]
	t.mtx.Unlock()
	return
}

func (t *Torrent) addOBPeer(c *PeerConn) {
	t.mtx.Lock()
	t.obconns[c.c.RemoteAddr().String()] = c
	t.mtx.Unlock()
}

func (t *Torrent) removeOBConn(c *PeerConn) {
	t.mtx.Lock()
	delete(t.obconns, c.c.RemoteAddr().String())
	t.mtx.Unlock()
}

func (t *Torrent) addIBPeer(c *PeerConn) {
	t.mtx.Lock()
	t.ibconns[c.c.RemoteAddr().String()] = c
	t.mtx.Unlock()
	c.inbound = true
}

func (t *Torrent) removeIBConn(c *PeerConn) {
	t.mtx.Lock()
	delete(t.ibconns, c.c.RemoteAddr().String())
	t.mtx.Unlock()
}

// connect to a new peer for this swarm, blocks
func (t *Torrent) AddPeer(a net.Addr, id common.PeerID) error {
	if t.HasOBConn(a) {
		return nil
	}
	c, err := t.Network().Dial(a.Network(), a.String())
	if err == nil {
		// connected
		ih := t.st.Infohash()
		// build handshake
		h := new(bittorrent.Handshake)
		// enable bittorrent extensions
		h.Reserved.Set(bittorrent.Extension)
		copy(h.Infohash[:], ih[:])
		copy(h.PeerID[:], t.id[:])
		// send handshake
		err = h.Send(c)
		if err == nil {
			// get response to handshake
			err = h.Recv(c)
			if err == nil {
				if bytes.Equal(ih[:], h.Infohash[:]) {
					// infohashes match
					var opts *extensions.ExtendedOptions
					if h.Reserved.Has(bittorrent.Extension) {
						opts = t.defaultOpts.Copy()
					}
					pc := makePeerConn(c, t, h.PeerID, opts)
					t.addOBPeer(pc)
					pc.Send(t.Bitfield().ToWireMessage())
					pc.start()
					return nil
				} else {
					log.Warn("Infohash missmatch")
				}
			}
		}
		log.Debugf("didn't complete handshake with peer: %s", err)
		// bad thing happened
		c.Close()
	}
	log.Debugf("didn't connect to %s: %s", a, err)
	return err
}

func (t *Torrent) broadcastHave(idx uint32) {
	msg := common.NewHave(idx)
	log.Infof("%s got piece %d", t.Name(), idx)
	conns := make(map[string]*PeerConn)
	t.mtx.Lock()
	for k, conn := range t.ibconns {
		if conn != nil {
			conns[k] = conn
		}
	}
	for k, conn := range t.obconns {
		if conn != nil {
			conns[k] = conn
		}
	}
	t.mtx.Unlock()
	for _, conn := range conns {
		go conn.Send(msg)
	}
}

// get metainfo for this torrent
func (t *Torrent) MetaInfo() *metainfo.TorrentFile {
	return t.st.MetaInfo()
}

func (t *Torrent) Name() string {
	return t.MetaInfo().TorrentName()
}

// callback called when we get a new inbound peer
func (t *Torrent) onNewPeer(c *PeerConn) {
	a := c.c.RemoteAddr()
	if t.HasIBConn(a) {
		log.Infof("duplicate peer from %s", a)
		c.Close()
		return
	}
	log.Infof("New peer (%s) for %s", c.id.String(), t.st.Infohash().Hex())
	t.addIBPeer(c)
	c.Send(t.Bitfield().ToWireMessage())
	c.start()

}

// handle a piece request
func (t *Torrent) onPieceRequest(c *PeerConn, req *common.PieceRequest) {
	if t.piece != nil {
		t.piece <- pieceEvent{c, req}
	}
}

func (t *Torrent) Run() {
	go t.handlePieces()
	if t.Started != nil {
		t.Started()
	}
	for !t.Done() {
		time.Sleep(time.Minute)
	}
	if t.Completed != nil {
		t.Completed()
	}
}

func (t *Torrent) handlePieces() {
	log.Infof("%s running", t.Name())
	for {
		ev, ok := <-t.piece
		if !ok {
			log.Infof("%s torrent run exit", t.Name())
			// channel closed
			return
		}
		if ev.r != nil && ev.r.Length > 0 {
			log.Debugf("%s asked for piece %d %d-%d", ev.c.id.String(), ev.r.Index, ev.r.Begin, ev.r.Begin+ev.r.Length)
			// TODO: cache common pieces (?)
			err := t.st.VisitPiece(ev.r, func(p *common.PieceData) error {
				// have the piece, send it
				ev.c.Send(p.ToWireMessage())
				log.Debugf("%s queued piece %d %d-%d", ev.c.id.String(), ev.r.Index, ev.r.Begin, ev.r.Begin+ev.r.Length)
				return nil
			})
			if err != nil {
				ev.c.Close()
			}
		} else {
			log.Infof("%s asked for a zero length piece", ev.c.id.String())
			// TODO: should we close here?
			ev.c.Close()
		}

	}
}

func (t *Torrent) Done() bool {
	return t.Bitfield().Completed()
}
<|endoftext|>


<commit_before>// Copyright 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package classifier

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

// Match is the information about a single instance of a detected match.
type Match struct {
	Name            string
	Confidence      float64
	MatchType       string
	StartLine       int
	EndLine         int
	StartTokenIndex int
	EndTokenIndex   int
}

// Matches is a sortable slice of Match.
type Matches []*Match

// Swap two elements of Matches.
func (d Matches) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d Matches) Len() int      { return len(d) }
func (d Matches) Less(i, j int) bool {
	di, dj := d[i], d[j]
	// Return matches ordered by confidence
	if di.Confidence != dj.Confidence {
		return di.Confidence > dj.Confidence
	}
	// Licenses of same confidence are ordered by their appearance
	if di.StartTokenIndex != dj.StartTokenIndex {
		return di.StartTokenIndex < dj.StartTokenIndex
	}
	// Should never get here, but tiebreak based on the larger license.
	return di.EndTokenIndex > dj.EndTokenIndex
}

// Match reports instances of the supplied content in the corpus.
func (c *Corpus) Match(in string) Matches {
	id := c.createTargetIndexedDocument(in)

	firstPass := make(map[string]*indexedDocument)
	for l, d := range c.docs {
		sim := id.tokenSimilarity(d)
		if sim >= c.threshold {
			firstPass[l] = d
		}
	}

	if len(firstPass) == 0 {
		return nil
	}

	// Perform the expensive work of generating a searchset to look for token runs.
	id.generateSearchSet(c.q)

	var candidates Matches
	for l, d := range firstPass {
		matches := findPotentialMatches(d.s, id.s, c.threshold)
		for _, m := range matches {
			startIndex := m.TargetStart
			endIndex := m.TargetEnd
			conf, startOffset, endOffset := score(l, id, d, startIndex, endIndex)
			if conf >= c.threshold && (endIndex-startIndex-startOffset-endOffset) > 0 {
				candidates = append(candidates, &Match{
					Name:            licName(l),
					MatchType:       detectionType(l),
					Confidence:      conf,
					StartLine:       id.Tokens[startIndex+startOffset].Line,
					EndLine:         id.Tokens[endIndex-endOffset-1].Line,
					StartTokenIndex: id.Tokens[startIndex+startOffset].Index,
					EndTokenIndex:   id.Tokens[endIndex-endOffset-1].Index,
				})
			}

		}
	}
	sort.Sort(candidates)
	retain := make([]bool, len(candidates))
	for i, c := range candidates {
		// Filter out overlapping licenses based primarily on confidence. Since
		// the candidates slice is ordered by confidence, we look for overlaps and
		// decide if we retain the record c.

		// For each candidate, only add it to the report unless we have a
		// higher-quality hit that contains these lines. In the case of two
		// licenses having overlap, we consider 'token density' to break ties. If a
		// less confident match of a larger license has more matching tokens than a
		// perfect match of a smaller license, we want to keep that. This handles
		// licenses that include another license as a subtext. NPL contains MPL
		// as a concrete example.

		keep := true
		proposals := make(map[int]bool)
		for j, o := range candidates {
			if j == i {
				break
			}
			// Make sure to only check containment on licenses that are still in consideration at this point.
			if contains(c, o) && retain[j] {
				// The license here can override a previous detection, but that isn't sufficient to be kept
				// on its own. Consider the licenses Xnet, MPL-1.1 and NPL-1.1 in a file that just has MPL-1.1.
				// The confidence rating on NPL-1.1 will cause Xnet to not be retained, which is correct, but it
				// shouldn't be retained if the token confidence for MPL is higher than NPL since the NPL-specific
				// bits are missing.

				ctoks := float64(c.EndTokenIndex - c.StartTokenIndex)
				otoks := float64(o.EndTokenIndex - o.StartTokenIndex)
				cconf := ctoks * c.Confidence
				oconf := otoks * o.Confidence

				// If the two licenses are exactly the same confidence, that means we
				// have an ambiguous detect and should retain both, so the caller can
				// see and resolve the situation.
				if cconf > oconf {
					proposals[j] = false
				} else if oconf > cconf {
					keep = false
				}
			} else if overlaps(c, o) && retain[j] {
				keep = false
			}

		}
		if keep {
			retain[i] = true
			for p, v := range proposals {
				retain[p] = v
			}
		}
	}

	var out Matches
	for i, keep := range retain {
		if keep {
			out = append(out, candidates[i])
		}
	}
	return out
}

// Classifier provides methods for identifying open source licenses in text
// content.
type Classifier struct {
	Corpus *Corpus
}

// LoadLicenses adds the contents of the supplied directory to the corpus.
func (c *Corpus) LoadLicenses(dir string) error {
	var files []string
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if !strings.HasSuffix(path, "txt") {
			return nil
		}
		files = append(files, path)
		return nil
	})
	if err != nil {
		return err
	}

	for _, f := range files {
		_, name := path.Split(f)
		name = strings.Replace(name, ".txt", "", 1)
		b, err := ioutil.ReadFile(f)
		if err != nil {
			return err
		}

		content := trimExtraneousTrailingText(string(b))
		c.AddContent(name, content)
	}
	return nil
}

// Match finds matches within an unknown text.
func (c *Classifier) Match(in string) Matches {
	return c.Corpus.Match(in)
}

func detectionType(in string) string {
	if strings.Index(in, ".header") != -1 {
		return "Header"
	}
	return "License"
}

// licName produces the output name for a license, removing the internal structure
// of the filename in use.
func licName(in string) string {
	out := in
	if idx := strings.Index(in, ".txt"); idx != -1 {
		out = out[0:idx]
	}
	if idx := strings.Index(in, "_"); idx != -1 {
		out = out[0:idx]
	}
	if idx := strings.Index(in, ".header"); idx != -1 {
		out = out[0:idx]
	}
	return out
}

// contains returns true iff b is completely inside a
func contains(a, b *Match) bool {
	return a.StartLine <= b.StartLine && a.EndLine >= b.EndLine
}

// returns true iff b <= a <= c
func between(a, b, c int) bool {
	return b <= a && a <= c
}

// returns true iff the ranges covered by a and b overlap.
func overlaps(a, b *Match) bool {
	return between(a.StartLine, b.StartLine, b.EndLine) || between(a.EndLine, b.StartLine, b.EndLine)
}

// endOfLicenseText is text commonly associated with the end of a license. We
// can remove text that occurs after it.
var endOfLicenseText = []string{
	"END OF TERMS AND CONDITIONS",
}

// trimExtraneousTrailingText removes text after an obvious end of the license
// and does not include substantive text of the license.
func trimExtraneousTrailingText(s string) string {
	for _, e := range endOfLicenseText {
		if i := strings.LastIndex(s, e); i != -1 {
			return s[:i+len(e)]
		}
	}
	return s
}
<commit_msg>Export licName for use by callers.<commit_after>// Copyright 2020 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package classifier

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
)

// Match is the information about a single instance of a detected match.
type Match struct {
	Name            string
	Confidence      float64
	MatchType       string
	StartLine       int
	EndLine         int
	StartTokenIndex int
	EndTokenIndex   int
}

// Matches is a sortable slice of Match.
type Matches []*Match

// Swap two elements of Matches.
func (d Matches) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d Matches) Len() int      { return len(d) }
func (d Matches) Less(i, j int) bool {
	di, dj := d[i], d[j]
	// Return matches ordered by confidence
	if di.Confidence != dj.Confidence {
		return di.Confidence > dj.Confidence
	}
	// Licenses of same confidence are ordered by their appearance
	if di.StartTokenIndex != dj.StartTokenIndex {
		return di.StartTokenIndex < dj.StartTokenIndex
	}
	// Should never get here, but tiebreak based on the larger license.
	return di.EndTokenIndex > dj.EndTokenIndex
}

// Match reports instances of the supplied content in the corpus.
func (c *Corpus) Match(in string) Matches {
	id := c.createTargetIndexedDocument(in)

	firstPass := make(map[string]*indexedDocument)
	for l, d := range c.docs {
		sim := id.tokenSimilarity(d)
		if sim >= c.threshold {
			firstPass[l] = d
		}
	}

	if len(firstPass) == 0 {
		return nil
	}

	// Perform the expensive work of generating a searchset to look for token runs.
	id.generateSearchSet(c.q)

	var candidates Matches
	for l, d := range firstPass {
		matches := findPotentialMatches(d.s, id.s, c.threshold)
		for _, m := range matches {
			startIndex := m.TargetStart
			endIndex := m.TargetEnd
			conf, startOffset, endOffset := score(l, id, d, startIndex, endIndex)
			if conf >= c.threshold && (endIndex-startIndex-startOffset-endOffset) > 0 {
				candidates = append(candidates, &Match{
					Name:            LicenseName(l),
					MatchType:       detectionType(l),
					Confidence:      conf,
					StartLine:       id.Tokens[startIndex+startOffset].Line,
					EndLine:         id.Tokens[endIndex-endOffset-1].Line,
					StartTokenIndex: id.Tokens[startIndex+startOffset].Index,
					EndTokenIndex:   id.Tokens[endIndex-endOffset-1].Index,
				})
			}

		}
	}
	sort.Sort(candidates)
	retain := make([]bool, len(candidates))
	for i, c := range candidates {
		// Filter out overlapping licenses based primarily on confidence. Since
		// the candidates slice is ordered by confidence, we look for overlaps and
		// decide if we retain the record c.

		// For each candidate, only add it to the report unless we have a
		// higher-quality hit that contains these lines. In the case of two
		// licenses having overlap, we consider 'token density' to break ties. If a
		// less confident match of a larger license has more matching tokens than a
		// perfect match of a smaller license, we want to keep that. This handles
		// licenses that include another license as a subtext. NPL contains MPL
		// as a concrete example.

		keep := true
		proposals := make(map[int]bool)
		for j, o := range candidates {
			if j == i {
				break
			}
			// Make sure to only check containment on licenses that are still in consideration at this point.
			if contains(c, o) && retain[j] {
				// The license here can override a previous detection, but that isn't sufficient to be kept
				// on its own. Consider the licenses Xnet, MPL-1.1 and NPL-1.1 in a file that just has MPL-1.1.
				// The confidence rating on NPL-1.1 will cause Xnet to not be retained, which is correct, but it
				// shouldn't be retained if the token confidence for MPL is higher than NPL since the NPL-specific
				// bits are missing.

				ctoks := float64(c.EndTokenIndex - c.StartTokenIndex)
				otoks := float64(o.EndTokenIndex - o.StartTokenIndex)
				cconf := ctoks * c.Confidence
				oconf := otoks * o.Confidence

				// If the two licenses are exactly the same confidence, that means we
				// have an ambiguous detect and should retain both, so the caller can
				// see and resolve the situation.
				if cconf > oconf {
					proposals[j] = false
				} else if oconf > cconf {
					keep = false
				}
			} else if overlaps(c, o) && retain[j] {
				keep = false
			}

		}
		if keep {
			retain[i] = true
			for p, v := range proposals {
				retain[p] = v
			}
		}
	}

	var out Matches
	for i, keep := range retain {
		if keep {
			out = append(out, candidates[i])
		}
	}
	return out
}

// Classifier provides methods for identifying open source licenses in text
// content.
type Classifier struct {
	Corpus *Corpus
}

// LoadLicenses adds the contents of the supplied directory to the corpus.
func (c *Corpus) LoadLicenses(dir string) error {
	var files []string
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if !strings.HasSuffix(path, "txt") {
			return nil
		}
		files = append(files, path)
		return nil
	})
	if err != nil {
		return err
	}

	for _, f := range files {
		_, name := path.Split(f)
		name = strings.Replace(name, ".txt", "", 1)
		b, err := ioutil.ReadFile(f)
		if err != nil {
			return err
		}

		content := trimExtraneousTrailingText(string(b))
		c.AddContent(name, content)
	}
	return nil
}

// Match finds matches within an unknown text.
func (c *Classifier) Match(in string) Matches {
	return c.Corpus.Match(in)
}

func detectionType(in string) string {
	if strings.Index(in, ".header") != -1 {
		return "Header"
	}
	return "License"
}

// LicenseName produces the output name for a license, removing the internal structure
// of the filename in use.
func LicenseName(in string) string {
	out := in
	if idx := strings.Index(in, ".txt"); idx != -1 {
		out = out[0:idx]
	}
	if idx := strings.Index(in, "_"); idx != -1 {
		out = out[0:idx]
	}
	if idx := strings.Index(in, ".header"); idx != -1 {
		out = out[0:idx]
	}
	return out
}

// contains returns true iff b is completely inside a
func contains(a, b *Match) bool {
	return a.StartLine <= b.StartLine && a.EndLine >= b.EndLine
}

// returns true iff b <= a <= c
func between(a, b, c int) bool {
	return b <= a && a <= c
}

// returns true iff the ranges covered by a and b overlap.
func overlaps(a, b *Match) bool {
	return between(a.StartLine, b.StartLine, b.EndLine) || between(a.EndLine, b.StartLine, b.EndLine)
}

// endOfLicenseText is text commonly associated with the end of a license. We
// can remove text that occurs after it.
var endOfLicenseText = []string{
	"END OF TERMS AND CONDITIONS",
}

// trimExtraneousTrailingText removes text after an obvious end of the license
// and does not include substantive text of the license.
func trimExtraneousTrailingText(s string) string {
	for _, e := range endOfLicenseText {
		if i := strings.LastIndex(s, e); i != -1 {
			return s[:i+len(e)]
		}
	}
	return s
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/mitchellh/go-homedir"
)

var (
	ShortGitHubURI = regexp.MustCompile(`^[\w\-.]+/[\w\-.]+$`)
	GitHubURI      = regexp.MustCompile(`^github.com/[\w\-.]+/[\w\-.]+$`)
	BitbucketURI   = regexp.MustCompile(`^bitbucket.org/[\w\-.]+/[\w\-.]+$`)
)

func ToSourceURI(uri string) string {
	switch {
	case ShortGitHubURI.MatchString(uri):
		return "https://github.com/" + uri
	case GitHubURI.MatchString(uri):
		return "https://" + uri
	case BitbucketURI.MatchString(uri):
		return "https://" + uri
	default:
		return uri
	}
}

var (
	home, errInit = homedir.Dir()
	dotvim        = filepath.Join(home, ".vim")
)

func ToDestinationPath(uri, filetype string) string {
	name := filepath.Base(uri)
	if filetype == "" {
		return filepath.Join(dotvim, "bundle", name)
	}
	return filepath.Join(dotvim, "ftbundle", filetype, name)
}

func RunCommandOn(dir string, name string, arg ...string) error {
	if _, err := exec.LookPath(name); err != nil {
		return err
	}

	errBuf := bytes.NewBuffer(make([]byte, 0))
	c := exec.Command(name, arg...)
	c.Dir = dir
	c.Stderr = errBuf
	if err := c.Run(); err != nil {
		return fmt.Errorf("%s", strings.TrimSpace(errBuf.String()))
	}
	return nil
}

func ListPackages(filetype string) {
	var path string
	if filetype == "" {
		path = filepath.Join(dotvim, "bundle")
	} else {
		path = filepath.Join(dotvim, "ftbundle", filetype)
	}

	// Ignore err for filetype doesn't exist.
	files, _ := ioutil.ReadDir(path)
	for _, file := range files {
		fmt.Println(file.Name())
	}
}

type Package struct {
	srcURI  string
	dstPath string
}

func NewPackage(uri, filetype string) *Package {
	return &Package{
		srcURI:  ToSourceURI(uri),
		dstPath: ToDestinationPath(uri, filetype),
	}
}

func (p *Package) installed() bool {
	_, err := os.Stat(p.dstPath)
	return err == nil
}

func (p *Package) Install() error {
	if p.installed() {
		return nil
	}
	return RunCommandOn("", "git", "clone", p.srcURI, p.dstPath)
}

func (p *Package) Remove() error {
	if p.installed() {
		return os.RemoveAll(p.dstPath)
	}
	return nil
}

func (p *Package) Update() error {
	if !p.installed() {
		return p.Install()
	}
	return RunCommandOn(p.dstPath, "git", "pull")
}
<commit_msg>Rewrite the comment in ListPackages<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/mitchellh/go-homedir"
)

var (
	ShortGitHubURI = regexp.MustCompile(`^[\w\-.]+/[\w\-.]+$`)
	GitHubURI      = regexp.MustCompile(`^github.com/[\w\-.]+/[\w\-.]+$`)
	BitbucketURI   = regexp.MustCompile(`^bitbucket.org/[\w\-.]+/[\w\-.]+$`)
)

func ToSourceURI(uri string) string {
	switch {
	case ShortGitHubURI.MatchString(uri):
		return "https://github.com/" + uri
	case GitHubURI.MatchString(uri):
		return "https://" + uri
	case BitbucketURI.MatchString(uri):
		return "https://" + uri
	default:
		return uri
	}
}

var (
	home, errInit = homedir.Dir()
	dotvim        = filepath.Join(home, ".vim")
)

func ToDestinationPath(uri, filetype string) string {
	name := filepath.Base(uri)
	if filetype == "" {
		return filepath.Join(dotvim, "bundle", name)
	}
	return filepath.Join(dotvim, "ftbundle", filetype, name)
}

func RunCommandOn(dir string, name string, arg ...string) error {
	if _, err := exec.LookPath(name); err != nil {
		return err
	}

	errBuf := bytes.NewBuffer(make([]byte, 0))
	c := exec.Command(name, arg...)
	c.Dir = dir
	c.Stderr = errBuf
	if err := c.Run(); err != nil {
		return fmt.Errorf("%s", strings.TrimSpace(errBuf.String()))
	}
	return nil
}

func ListPackages(filetype string) {
	var path string
	if filetype == "" {
		path = filepath.Join(dotvim, "bundle")
	} else {
		path = filepath.Join(dotvim, "ftbundle", filetype)
	}

	// Ignore err to allow that the filetype doesn't exist
	files, _ := ioutil.ReadDir(path)
	for _, file := range files {
		fmt.Println(file.Name())
	}
}

type Package struct {
	srcURI  string
	dstPath string
}

func NewPackage(uri, filetype string) *Package {
	return &Package{
		srcURI:  ToSourceURI(uri),
		dstPath: ToDestinationPath(uri, filetype),
	}
}

func (p *Package) installed() bool {
	_, err := os.Stat(p.dstPath)
	return err == nil
}

func (p *Package) Install() error {
	if p.installed() {
		return nil
	}
	return RunCommandOn("", "git", "clone", p.srcURI, p.dstPath)
}

func (p *Package) Remove() error {
	if p.installed() {
		return os.RemoveAll(p.dstPath)
	}
	return nil
}

func (p *Package) Update() error {
	if !p.installed() {
		return p.Install()
	}
	return RunCommandOn(p.dstPath, "git", "pull")
}
<|endoftext|>


<commit_before>/**
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// define test package name
package cloudbuild_private_pool

import (
	"fmt"
	"testing"

	// import the blueprints test framework modules for testing and assertions
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
	// "github.com/gruntwork-io/terratest/modules/terraform"
	"github.com/stretchr/testify/assert"
)

// name the function as Test*
func TestCloudBuildPrivatePoolExample(t *testing.T) {
	// define constants for all required assertions in the test case

	// initialize Terraform test from the Blueprints test framework
	cloudBuildPrivatePoolT := tft.NewTFBlueprintTest(t)

	// define and write a custom verifier for this test case call the default verify for confirming no additional changes
	cloudBuildPrivatePoolT.DefineVerify(func(assert *assert.Assertions) {
		// perform default verification ensuring Terraform reports no additional changes on an applied blueprint
		cloudBuildPrivatePoolT.DefaultVerify(assert)

		// invoke the gcloud module in the Blueprints test framework to run a gcloud command that will output resource properties in a JSON format
		// the tft struct can be used to pull output variables of the TF module being invoked by this test and use the op object (a gjson struct)
		// to parse through the JSON results and assert the values of the resource against the constants defined above

		projectID := cloudBuildPrivatePoolT.GetStringOutput("project_id")

		// Worker Pool
		workerPoolAddress := gcloud.Run(t, fmt.Sprintf("compute addresses describe worker-pool-range --global"))
		assert.Equal("10.37.0.0", workerPoolAddress.Get("address").String(), fmt.Sprintf("Worker pool address range is 10.37.0.0"))
		assert.Equal("INTERNAL", workerPoolAddress.Get("addressType").String(), fmt.Sprintf("Worker pool address range is type INTERNAL"))
		assert.Equal("VPC_PEERING", workerPoolAddress.Get("purpose").String(), fmt.Sprintf("Worker pool address range is for VPC_PEERING"))

		privatePoolVPC := gcloud.Run(t, fmt.Sprintf("compute networks describe gke-private-pool-example-vpc"))

		privatePool := gcloud.Run(t, fmt.Sprintf("builds worker-pools describe cloudbuild-private-worker-pool --region=us-central1"))
		assert.Equal("RUNNING", privatePool.Get("state").String(), fmt.Sprintf("Worker pool is RUNNING"))

		// VPN Tunnels
		vpnTunnels := [6]string{"cloudbuild-to-gke-private-vpc-dev-remote-0", "cloudbuild-to-gke-private-vpc-dev-remote-1", "cloudbuild-to-gke-private-vpc-prod-remote-0", "cloudbuild-to-gke-private-vpc-prod-remote-1", "cloudbuild-to-gke-private-vpc-qa-remote-0", "cloudbuild-to-gke-private-vpc-qa-remote-1"}
		for _, tunnel := range vpnTunnels {
			tunnelStatus := gcloud.Run(t, fmt.Sprintf("compute vpn-tunnels describe %s --region=us-central1", tunnel))
			assert.Equal("Tunnel is up and running.", tunnelStatus.Get("detailedStatus").String(), fmt.Sprintf("%s is created and running", tunnel))
		}

	})
	// call the test function to execute the integration test
	cloudBuildPrivatePoolT.Test()
}
<commit_msg>update test<commit_after>/**
 * Copyright 2021 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// define test package name
package cloudbuild_private_pool

import (
	"fmt"
	"testing"

	// import the blueprints test framework modules for testing and assertions
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
	// "github.com/gruntwork-io/terratest/modules/terraform"
	"github.com/stretchr/testify/assert"
)

// name the function as Test*
func TestCloudBuildPrivatePoolExample(t *testing.T) {
	// define constants for all required assertions in the test case

	// initialize Terraform test from the Blueprints test framework
	cloudBuildPrivatePoolT := tft.NewTFBlueprintTest(t)

	// define and write a custom verifier for this test case call the default verify for confirming no additional changes
	cloudBuildPrivatePoolT.DefineVerify(func(assert *assert.Assertions) {
		// perform default verification ensuring Terraform reports no additional changes on an applied blueprint
		cloudBuildPrivatePoolT.DefaultVerify(assert)

		// invoke the gcloud module in the Blueprints test framework to run a gcloud command that will output resource properties in a JSON format
		// the tft struct can be used to pull output variables of the TF module being invoked by this test and use the op object (a gjson struct)
		// to parse through the JSON results and assert the values of the resource against the constants defined above

		projectID := cloudBuildPrivatePoolT.GetStringOutput("project_id")

		// Worker Pool
		workerPoolAddress := gcloud.Run(t, fmt.Sprintf("compute addresses describe worker-pool-range --global --project %s", projectID))
		assert.Equal("10.37.0.0", workerPoolAddress.Get("address").String(), "Worker pool address range is 10.37.0.0")
		assert.Equal("INTERNAL", workerPoolAddress.Get("addressType").String(), "Worker pool address range is type INTERNAL")
		assert.Equal("VPC_PEERING", workerPoolAddress.Get("purpose").String(), "Worker pool address range is for VPC_PEERING")

		privatePoolVPC := gcloud.Run(t, fmt.Sprintf("compute networks describe gke-private-pool-example-vpc --project %s", projectID))
		assert.contains(privatePoolVPC.Get("peerings.network").String(), "servicenetworking")

		privatePool := gcloud.Run(t, fmt.Sprintf("builds worker-pools describe cloudbuild-private-worker-pool --region=us-central1 --project %s", projectID))
		assert.Equal("RUNNING", privatePool.Get("state").String(), "Worker pool is RUNNING")

		// VPN Tunnels
		vpnTunnels := [6]string{"cloudbuild-to-gke-private-vpc-dev-remote-0", "cloudbuild-to-gke-private-vpc-dev-remote-1", "cloudbuild-to-gke-private-vpc-prod-remote-0", "cloudbuild-to-gke-private-vpc-prod-remote-1", "cloudbuild-to-gke-private-vpc-qa-remote-0", "cloudbuild-to-gke-private-vpc-qa-remote-1"}
		for _, tunnel := range vpnTunnels {
			tunnelStatus := gcloud.Run(t, fmt.Sprintf("compute vpn-tunnels describe %s --region=us-central1 --project %s", tunnel, projectID))
			assert.Equal("Tunnel is up and running.", tunnelStatus.Get("detailedStatus").String(), fmt.Sprintf("%s is created and running", tunnel))
		}

	})
	// call the test function to execute the integration test
	cloudBuildPrivatePoolT.Test()
}
<|endoftext|>


<commit_before>package viewer

import (
	"github.com/jayjanssen/myq-tools2/loader"
)

// A StateViewer represents the output of data from a State into a (usually) constrained width with a header and one or more lines of output per State
type StateViewer interface {
	// Single line help for the view
	GetShortHelp() string

	// Detailed multi-line help for the view
	GetHelp() []string

	// A list of sources that this view requires
	GetSources() []loader.Source

	// Header for this view, unclear if state is needed
	GetHeader(state *loader.State) []string

	// Data for this view based on the state
	GetData(state *loader.State) []string

	// Live views output the current timestamp whereas Runtime views output the delta in Uptime since the first state
	SetLive()
	SetRuntime()
}
<commit_msg>a StateViewer doesn't need to implement setting timecols<commit_after>package viewer

import (
	"github.com/jayjanssen/myq-tools2/loader"
)

// A StateViewer represents the output of data from a State into a (usually) constrained width with a header and one or more lines of output per State
type StateViewer interface {
	// Single line help for the view
	GetShortHelp() string

	// Detailed multi-line help for the view
	GetHelp() []string

	// A list of sources that this view requires
	GetSources() []loader.Source

	// Header for this view, unclear if state is needed
	GetHeader(state *loader.State) []string

	// Data for this view based on the state
	GetData(state *loader.State) []string
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/freneticmonkey/migrate/go/config"
	"github.com/freneticmonkey/migrate/go/git"
	"github.com/freneticmonkey/migrate/go/id"
	"github.com/freneticmonkey/migrate/go/migration"
	"github.com/freneticmonkey/migrate/go/mysql"
	"github.com/freneticmonkey/migrate/go/table"
	"github.com/freneticmonkey/migrate/go/util"
	"github.com/freneticmonkey/migrate/go/yaml"
	"github.com/urfave/cli"
)

// GetCreateCommand Create a new migration for the target project at the version indicated by hash
func GetCreateCommand() (setup cli.Command) {
	setup = cli.Command{
		Name:  "create",
		Usage: "This subcommand is used to create a migration and register it with the management database.",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "project",
				Value: "",
				Usage: "The target project",
			},
			cli.StringFlag{
				Name:  "version",
				Value: "",
				Usage: "The target git version",
			},
			cli.BoolFlag{
				Name:  "rollback",
				Usage: "Force a rollback (backward) migration to be created",
			},
		},
		Action: func(ctx *cli.Context) error {
			var project string
			var version string

			// Parse global flags
			parseGlobalFlags(ctx)

			// Setup the management database and configuration settings
			conf, err := configureManagement()

			if err != nil {
				return cli.NewExitError("Configuration Load failed.", 1)
			}

			// Override the project settings with the command line flags
			if ctx.IsSet("project") {
				project = ctx.String("project")
			}

			if ctx.IsSet("version") {
				version = ctx.String("version")
			}

			return create(project, version, conf)

		},
	}
	return setup
}

func create(project, version string, conf config.Config) *cli.ExitError {
	var problems int
	var ts string
	var info string
	var err error
	rollback := false

	// Override the project settings with the command line flags
	if project != "" {
		conf.Project.Name = project
	}

	if version != "" {
		conf.Project.Schema.Version = version
	}

	// if the version hasn't been defined
	if len(conf.Project.Name) == 0 {
		return cli.NewExitError("Creation failed.  Unable to generate a migration as no project was defined", 1)
	}

	if len(conf.Project.Schema.Version) == 0 {
		return cli.NewExitError("Creation failed.  Unable to generate a migration as no version was defined to migrate to", 1)
	}
	git.Clone(conf.Project)

	// Read the YAML files cloned from the repo
	err = yaml.ReadTables(conf.Options.WorkingPath)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read YAML Tables", 1)
	}
	problems, err = id.ValidateSchema(yaml.Schema, "YAML Schema")
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. YAML Validation Errors Detected", problems)
	}

	// Read the MySQL tables from the target database
	err = mysql.ReadTables()
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read MySQL Tables", 1)
	}
	problems, err = id.ValidateSchema(mysql.Schema, "Target Database Schema")
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Target Database Validation Errors Detected", problems)
	}

	forwardDiff, err := table.DiffTables(yaml.Schema, mysql.Schema)
	if util.ErrorCheckf(err, "Diff Failed while generating forward migration") {
		return cli.NewExitError("Create failed. Unable to generate a forward migration", 1)
	}
	forwardOps := mysql.GenerateAlters(forwardDiff)

	backwardDiff, err := table.DiffTables(mysql.Schema, yaml.Schema)
	if util.ErrorCheckf(err, "Diff Failed while generating backward migration") {
		return cli.NewExitError("Create failed. Unable to generate a backward migration", 1)
	}
	backwardOps := mysql.GenerateAlters(backwardDiff)

	ts, err = git.GetVersionTime(conf.Project.Name, conf.Project.Schema.Version)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to obtain Version Timestamp from Git checkout", 1)
	}
	info, err = git.GetVersionDetails(conf.Project.Name, conf.Project.Schema.Version)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to obtain Version Details from Git checkout", 1)
	}

	m, err := migration.New(migration.Param{
		Project:     conf.Project.Name,
		Version:     conf.Project.Schema.Version,
		Timestamp:   ts,
		Description: info,
		Forwards:    forwardOps,
		Backwards:   backwardOps,
		Rollback:    rollback,
	})
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to create new Migration in the management database", 1)
	}

	success := fmt.Sprintf("Created Migration successfully with ID: [%d]", m.MID)

	return cli.NewExitError(success, 0)
}
<commit_msg>Added missing rollback cli flag support<commit_after>package cmd

import (
	"fmt"

	"github.com/freneticmonkey/migrate/go/config"
	"github.com/freneticmonkey/migrate/go/git"
	"github.com/freneticmonkey/migrate/go/id"
	"github.com/freneticmonkey/migrate/go/migration"
	"github.com/freneticmonkey/migrate/go/mysql"
	"github.com/freneticmonkey/migrate/go/table"
	"github.com/freneticmonkey/migrate/go/util"
	"github.com/freneticmonkey/migrate/go/yaml"
	"github.com/urfave/cli"
)

// GetCreateCommand Create a new migration for the target project at the version indicated by hash
func GetCreateCommand() (setup cli.Command) {
	setup = cli.Command{
		Name:  "create",
		Usage: "This subcommand is used to create a migration and register it with the management database.",
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "project",
				Value: "",
				Usage: "The target project",
			},
			cli.StringFlag{
				Name:  "version",
				Value: "",
				Usage: "The target git version",
			},
			cli.BoolFlag{
				Name:  "rollback",
				Usage: "Force a rollback (backward) migration to be created",
			},
		},
		Action: func(ctx *cli.Context) error {
			var project string
			var version string
			var rollback bool

			// Parse global flags
			parseGlobalFlags(ctx)

			// Setup the management database and configuration settings
			conf, err := configureManagement()

			if err != nil {
				return cli.NewExitError("Configuration Load failed.", 1)
			}

			// Override the project settings with the command line flags
			if ctx.IsSet("project") {
				project = ctx.String("project")
			}

			if ctx.IsSet("version") {
				version = ctx.String("version")
			}

			if ctx.IsSet("rollback") {
				rollback = ctx.Bool("rollback")
			}

			return create(project, version, rollback, conf)

		},
	}
	return setup
}

func create(project, version string, rollback bool, conf config.Config) *cli.ExitError {
	var problems int
	var ts string
	var info string
	var err error

	// Override the project settings with the command line flags
	if project != "" {
		conf.Project.Name = project
	}

	if version != "" {
		conf.Project.Schema.Version = version
	}

	// if the version hasn't been defined
	if len(conf.Project.Name) == 0 {
		return cli.NewExitError("Creation failed.  Unable to generate a migration as no project was defined", 1)
	}

	if len(conf.Project.Schema.Version) == 0 {
		return cli.NewExitError("Creation failed.  Unable to generate a migration as no version was defined to migrate to", 1)
	}
	git.Clone(conf.Project)

	// Read the YAML files cloned from the repo
	err = yaml.ReadTables(conf.Options.WorkingPath)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read YAML Tables", 1)
	}
	problems, err = id.ValidateSchema(yaml.Schema, "YAML Schema")
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. YAML Validation Errors Detected", problems)
	}

	// Read the MySQL tables from the target database
	err = mysql.ReadTables()
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Unable to read MySQL Tables", 1)
	}
	problems, err = id.ValidateSchema(mysql.Schema, "Target Database Schema")
	if util.ErrorCheck(err) {
		return cli.NewExitError("Creation failed. Target Database Validation Errors Detected", problems)
	}

	forwardDiff, err := table.DiffTables(yaml.Schema, mysql.Schema)
	if util.ErrorCheckf(err, "Diff Failed while generating forward migration") {
		return cli.NewExitError("Create failed. Unable to generate a forward migration", 1)
	}
	forwardOps := mysql.GenerateAlters(forwardDiff)

	backwardDiff, err := table.DiffTables(mysql.Schema, yaml.Schema)
	if util.ErrorCheckf(err, "Diff Failed while generating backward migration") {
		return cli.NewExitError("Create failed. Unable to generate a backward migration", 1)
	}
	backwardOps := mysql.GenerateAlters(backwardDiff)

	ts, err = git.GetVersionTime(conf.Project.Name, conf.Project.Schema.Version)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to obtain Version Timestamp from Git checkout", 1)
	}
	info, err = git.GetVersionDetails(conf.Project.Name, conf.Project.Schema.Version)
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to obtain Version Details from Git checkout", 1)
	}

	m, err := migration.New(migration.Param{
		Project:     conf.Project.Name,
		Version:     conf.Project.Schema.Version,
		Timestamp:   ts,
		Description: info,
		Forwards:    forwardOps,
		Backwards:   backwardOps,
		Rollback:    rollback,
	})
	if util.ErrorCheck(err) {
		return cli.NewExitError("Create failed. Unable to create new Migration in the management database", 1)
	}

	success := fmt.Sprintf("Created Migration successfully with ID: [%d]", m.MID)

	return cli.NewExitError(success, 0)
}
<|endoftext|>


<commit_before>package gokismet

import (
	"encoding/json"
	"io/ioutil"
	"net/url"
	"os"
	"testing"
)

const (
	errMissingConfig = 100 + iota
	errInvalidConfig
	errIncompleteConfig
	errInvalidSite
)

type settings struct {
	// Akismet API Key
	APIKey string
	// Website to test for spam
	Site string
	// URL of an article on the provided website
	// We'll generate one if it isn't supplied
	Article string
	// IP Address to send in Akismet requests
	IP string
	// User Agent to send in Akismet requests
	UserAgent string
	// Whether or not to output Requests/Responses
	// to stdout
	Debug bool
}

var api *API
var comment *Comment

var config settings

// Test entry point
func TestMain(m *testing.M) {
	// Read the settings file
	b, err := ioutil.ReadFile("testconfig.json")
	if err != nil {
		os.Exit(errMissingConfig)
	}
	// Use the contents to populate the settings struct
	err = json.Unmarshal(b, &config)
	if err != nil {
		os.Exit(errInvalidConfig)
	}
	// Check that the settings have been populated correctly
	if config.APIKey == "" || config.Site == "" || config.IP == "" || config.UserAgent == "" {
		os.Exit(errIncompleteConfig)
	}
	// If there's no article URL specified in the config, make one up
	if config.Article == "" {
		u, err := url.Parse(config.Site)
		if err != nil {
			os.Exit(errInvalidSite)
		}
		u.Path = "blog/"
		u.RawQuery = "p=123"

		config.Article = u.String()
	}

	// Create the API object
	api = NewTestAPI()
	// Set up the debugger
	if config.Debug {
		api.SetDebugWriter(os.Stdout)
	} else {
		// Even if we're not debugging, still set a dummy Writer object.
		// That way we're still testing the logging code path. We're just
		// not outputting anything.
		api.SetDebugWriter(ioutil.Discard)
	}

	// Run the tests
	os.Exit(m.Run())
}

func defaultParams() url.Values {
	return url.Values{
		_Site:      {config.Site},
		_UserIP:    {config.IP},
		_UserAgent: {config.UserAgent},
		_Referer:   {"http://www.google.com"},
		_Page:      {config.Article},
		_Author:    {"gokismet tester"},
		_Type:      {"comment"},
		_Email:     {"hello@example.com"},
		_URL:       {"http://www.example.com"},
		_Content:   {"This is an example comment that does not contain anything spammy. In the absence of other settings Akismet should return a negative (non-spam) response for this comment. Cheers..."},
	}
}

func checkSpamStatus(t *testing.T, testName string, expectedStatus SpamStatus, status SpamStatus, err error) {
	if err != nil {
		t.Errorf("%s fail: %s", testName, err.Error())
		// No need to check the return value if the call failed
		return
	}
	if status != expectedStatus {
		t.Errorf("%s fail: Status %s, Expected %s", testName, status, expectedStatus)
	}
}

func TestAPIKeyNotVerified(t *testing.T) {
	params := url.Values{}

	// CheckComment should fail if the API key isn't verified
	_, err := api.CheckComment(&params)
	if err != errKeyNotVerified {
		t.Errorf("APIKeyNotVerified returned unexpected %s", err.Error())
	}

	// SubmitSpam should fail if the API key isn't verified
	err = api.SubmitSpam(&params)
	if err != errKeyNotVerified {
		t.Errorf("APIKeyNotVerified returned unexpected %s", err.Error())
	}

	// SubmitHam should fail if the API key isn't verified
	err = api.SubmitHam(&params)
	if err != errKeyNotVerified {
		t.Errorf("APIKeyNotVerified returned unexpected %s", err.Error())
	}
}

// Test key verification
func TestAPIVerifyKey(t *testing.T) {
	err := api.VerifyKey(config.APIKey, config.Site)
	if err != nil {
		t.Errorf("APIVerifyKey %s fail: %s", config.APIKey, err.Error())
	}
}

// To simulate a positive (spam) result, make a comment-check API call
// with the comment_author set to viagra-test-123 and all other required
// fields populated with typical values.
func TestAPICheckSpam(t *testing.T) {

	// Set up params for a positive result
	params := url.Values{
		_Site:      {config.Site},
		_UserIP:    {config.IP},
		_UserAgent: {config.UserAgent},
		_Author:    {"viagra-test-123"},
	}

	// And test them
	status, err := api.CheckComment(&params)
	checkSpamStatus(t, "APICheckSpam", StatusProbableSpam, status, err)

	// Add the discard flag to simulate blatant or "pervasive" spam
	params.Set("test_discard", "true")

	// And test again
	status, err = api.CheckComment(&params)
	checkSpamStatus(t, "APICheckSpam", StatusDefiniteSpam, status, err)
}

// To simulate a negative (not spam) result, make a comment-check API call
// with the user_role set to administrator and all other required fields
// populated with typical values.
func TestAPICheckHam(t *testing.T) {

	// Set up params for a negative result
	params := url.Values{
		_Site:       {config.Site},
		_UserIP:     {config.IP},
		_UserAgent:  {config.UserAgent},
		"user_role": {"administrator"},
	}

	// And test them
	status, err := api.CheckComment(&params)
	checkSpamStatus(t, "APICheckHam", StatusNotSpam, status, err)
}

// According to the Akismet docs, blog, user_ip and user_agent are required
// parameters for comment-check. In practice only the first two are actually
// required. API calls without a user agent will succeed.
//
// This is more a test of the Akismet API than a test of the gokismet code.
// It's really only here to flag any changes in the API behaviour.
func TestAPICheckRequired(t *testing.T) {

	var err error
	var params url.Values

	// Missing: blog, ip
	// Should throw an error
	params = defaultParams()
	params.Del(_Site)
	params.Del(_UserIP)

	_, err = api.CheckComment(&params)
	if err == nil {
		t.Errorf("APICheckRequired fail: call succeeded without blog or user ip set")
	}

	// Missing: blog
	// Should throw an error
	params = defaultParams()
	params.Del(_Site)

	_, err = api.CheckComment(&params)
	if err == nil {
		t.Errorf("APICheckRequired fail: call succeeded without blog set")
	}

	// Missing: ip
	// Should throw an error
	params = defaultParams()
	params.Del(_UserIP)

	_, err = api.CheckComment(&params)
	if err == nil {
		t.Errorf("APICheckRequired fail: call succeeded without user ip set")
	}

	// Missing: user agent
	// Should NOT thrown an error
	params = defaultParams()
	params.Del(_UserAgent)

	_, err = api.CheckComment(&params)
	if err != nil {
		t.Errorf("APICheckRequired fail: call failed without user agent set")
	}
}

func TestAPISubmitSpam(t *testing.T) {
	params := defaultParams()
	err := api.SubmitSpam(&params)
	if err != nil {
		t.Errorf("APISubmitSpam failed: %s", err.Error())
	}

	// TODO: Come up with a failing test for api.SubmitSpam. It seems
	// to work even with no parameters.
	err = api.SubmitSpam(&url.Values{})
	if err != nil {
		t.Errorf("APISubmitSpam (no params) failed: %s", err.Error())
	}
}

func TestAPISubmitHam(t *testing.T) {
	params := defaultParams()
	err := api.SubmitHam(&params)
	if err != nil {
		t.Errorf("APISubmitHam failed: %s", err.Error())
	}

	// TODO: Come up with a failing test for api.SubmitHam. It seems
	// to work even with no parameters.
	err = api.SubmitHam(&url.Values{})
	if err != nil {
		t.Errorf("APISubmitHam (no params) failed: %s", err.Error())
	}
}

func TestCommentNew(t *testing.T) {
	var err error
	// Note: If we use := here, Go will create a local Comment
	// object instead of initialising the global.
	comment, err = NewTestComment(config.APIKey, config.Site)
	if err != nil {
		// We can't continue without a Comment object
		t.Fatalf("CommentNew failed: %s", err.Error())
	}
	// This call is needed to make the code compile. But it highlights
	// the fact that we're not debugging the comment object (unlike the
	// api object). By debugging one object and not the other we test
	// both the logging and non-logging code paths.
	api.SetDebugWriter(nil)
}

func TestCommentCheck(t *testing.T) {
	if comment == nil {
		// We can't continue without a Comment object
		t.Fatal("CommentCheck fail: Comment object is nil.")
	}

	// Set up a non-spam comment
	comment.SetUserIP(config.IP)
	comment.SetUserAgent(config.UserAgent)
	comment.SetPage(config.Article)
	comment.SetAuthor("gokismet test")
	comment.SetEmail("hello@example.com")
	comment.SetContent("This is an example comment that does not contain anything spammy. In the absence of other settings Akismet should return a negative (non-spam) response for this comment. Cheers...")

	// And test it
	status, err := comment.Check()
	checkSpamStatus(t, "CommentCheck", StatusNotSpam, status, err)

	// Make the comment spammy
	comment.SetAuthor("viagra-test-123")

	// And test again
	status, err = comment.Check()
	checkSpamStatus(t, "CommentCheck", StatusProbableSpam, status, err)
}

func TestCommentReport(t *testing.T) {
	if comment == nil {
		// We can't continue without a Comment object
		t.Fatal("CommentReport fail: Comment object is nil.")
	}

	if err := comment.ReportSpam(); err != nil {
		t.Errorf("CommentReport Spam fail: %s", err.Error())
	}

	if err := comment.ReportNotSpam(); err != nil {
		t.Errorf("CommentReport NotSpam failed: %s", err.Error())
	}
}
<commit_msg>Replaced another T.Fatalf() with T.Errorf().<commit_after>package gokismet

import (
	"encoding/json"
	"io/ioutil"
	"net/url"
	"os"
	"testing"
)

const (
	errMissingConfig = 100 + iota
	errInvalidConfig
	errIncompleteConfig
	errInvalidSite
)

type settings struct {
	// Akismet API Key
	APIKey string
	// Website to test for spam
	Site string
	// URL of an article on the provided website
	// We'll generate one if it isn't supplied
	Article string
	// IP Address to send in Akismet requests
	IP string
	// User Agent to send in Akismet requests
	UserAgent string
	// Whether or not to output Requests/Responses
	// to stdout
	Debug bool
}

var api *API
var comment *Comment

var config settings

// Test entry point
func TestMain(m *testing.M) {
	// Read the settings file
	b, err := ioutil.ReadFile("testconfig.json")
	if err != nil {
		os.Exit(errMissingConfig)
	}
	// Use the contents to populate the settings struct
	err = json.Unmarshal(b, &config)
	if err != nil {
		os.Exit(errInvalidConfig)
	}
	// Check that the settings have been populated correctly
	if config.APIKey == "" || config.Site == "" || config.IP == "" || config.UserAgent == "" {
		os.Exit(errIncompleteConfig)
	}
	// If there's no article URL specified in the config, make one up
	if config.Article == "" {
		u, err := url.Parse(config.Site)
		if err != nil {
			os.Exit(errInvalidSite)
		}
		u.Path = "blog/"
		u.RawQuery = "p=123"

		config.Article = u.String()
	}

	// Create the API object
	api = NewTestAPI()
	// Set up the debugger
	if config.Debug {
		api.SetDebugWriter(os.Stdout)
	} else {
		// Even if we're not debugging, still set a dummy Writer object.
		// That way we're still testing the logging code path. We're just
		// not outputting anything.
		api.SetDebugWriter(ioutil.Discard)
	}

	// Run the tests
	os.Exit(m.Run())
}

func defaultParams() url.Values {
	return url.Values{
		_Site:      {config.Site},
		_UserIP:    {config.IP},
		_UserAgent: {config.UserAgent},
		_Referer:   {"http://www.google.com"},
		_Page:      {config.Article},
		_Author:    {"gokismet tester"},
		_Type:      {"comment"},
		_Email:     {"hello@example.com"},
		_URL:       {"http://www.example.com"},
		_Content:   {"This is an example comment that does not contain anything spammy. In the absence of other settings Akismet should return a negative (non-spam) response for this comment. Cheers..."},
	}
}

func checkSpamStatus(t *testing.T, testName string, expectedStatus SpamStatus, status SpamStatus, err error) {
	if err != nil {
		t.Errorf("%s fail: %s", testName, err.Error())
		// No need to check the return value if the call failed
		return
	}
	if status != expectedStatus {
		t.Errorf("%s fail: Status %s, Expected %s", testName, status, expectedStatus)
	}
}

func TestAPIKeyNotVerified(t *testing.T) {
	params := url.Values{}

	// CheckComment should fail if the API key isn't verified
	_, err := api.CheckComment(&params)
	if err != errKeyNotVerified {
		t.Errorf("APIKeyNotVerified returned unexpected %s", err.Error())
	}

	// SubmitSpam should fail if the API key isn't verified
	err = api.SubmitSpam(&params)
	if err != errKeyNotVerified {
		t.Errorf("APIKeyNotVerified returned unexpected %s", err.Error())
	}

	// SubmitHam should fail if the API key isn't verified
	err = api.SubmitHam(&params)
	if err != errKeyNotVerified {
		t.Errorf("APIKeyNotVerified returned unexpected %s", err.Error())
	}
}

// Test key verification
func TestAPIVerifyKey(t *testing.T) {
	err := api.VerifyKey(config.APIKey, config.Site)
	if err != nil {
		t.Errorf("APIVerifyKey %s fail: %s", config.APIKey, err.Error())
	}
}

// To simulate a positive (spam) result, make a comment-check API call
// with the comment_author set to viagra-test-123 and all other required
// fields populated with typical values.
func TestAPICheckSpam(t *testing.T) {

	// Set up params for a positive result
	params := url.Values{
		_Site:      {config.Site},
		_UserIP:    {config.IP},
		_UserAgent: {config.UserAgent},
		_Author:    {"viagra-test-123"},
	}

	// And test them
	status, err := api.CheckComment(&params)
	checkSpamStatus(t, "APICheckSpam", StatusProbableSpam, status, err)

	// Add the discard flag to simulate blatant or "pervasive" spam
	params.Set("test_discard", "true")

	// And test again
	status, err = api.CheckComment(&params)
	checkSpamStatus(t, "APICheckSpam", StatusDefiniteSpam, status, err)
}

// To simulate a negative (not spam) result, make a comment-check API call
// with the user_role set to administrator and all other required fields
// populated with typical values.
func TestAPICheckHam(t *testing.T) {

	// Set up params for a negative result
	params := url.Values{
		_Site:       {config.Site},
		_UserIP:     {config.IP},
		_UserAgent:  {config.UserAgent},
		"user_role": {"administrator"},
	}

	// And test them
	status, err := api.CheckComment(&params)
	checkSpamStatus(t, "APICheckHam", StatusNotSpam, status, err)
}

// According to the Akismet docs, blog, user_ip and user_agent are required
// parameters for comment-check. In practice only the first two are actually
// required. API calls without a user agent will succeed.
//
// This is more a test of the Akismet API than a test of the gokismet code.
// It's really only here to flag any changes in the API behaviour.
func TestAPICheckRequired(t *testing.T) {

	var err error
	var params url.Values

	// Missing: blog, ip
	// Should throw an error
	params = defaultParams()
	params.Del(_Site)
	params.Del(_UserIP)

	_, err = api.CheckComment(&params)
	if err == nil {
		t.Errorf("APICheckRequired fail: call succeeded without blog or user ip set")
	}

	// Missing: blog
	// Should throw an error
	params = defaultParams()
	params.Del(_Site)

	_, err = api.CheckComment(&params)
	if err == nil {
		t.Errorf("APICheckRequired fail: call succeeded without blog set")
	}

	// Missing: ip
	// Should throw an error
	params = defaultParams()
	params.Del(_UserIP)

	_, err = api.CheckComment(&params)
	if err == nil {
		t.Errorf("APICheckRequired fail: call succeeded without user ip set")
	}

	// Missing: user agent
	// Should NOT thrown an error
	params = defaultParams()
	params.Del(_UserAgent)

	_, err = api.CheckComment(&params)
	if err != nil {
		t.Errorf("APICheckRequired fail: call failed without user agent set")
	}
}

func TestAPISubmitSpam(t *testing.T) {
	params := defaultParams()
	err := api.SubmitSpam(&params)
	if err != nil {
		t.Errorf("APISubmitSpam failed: %s", err.Error())
	}

	// TODO: Come up with a failing test for api.SubmitSpam. It seems
	// to work even with no parameters.
	err = api.SubmitSpam(&url.Values{})
	if err != nil {
		t.Errorf("APISubmitSpam (no params) failed: %s", err.Error())
	}
}

func TestAPISubmitHam(t *testing.T) {
	params := defaultParams()
	err := api.SubmitHam(&params)
	if err != nil {
		t.Errorf("APISubmitHam failed: %s", err.Error())
	}

	// TODO: Come up with a failing test for api.SubmitHam. It seems
	// to work even with no parameters.
	err = api.SubmitHam(&url.Values{})
	if err != nil {
		t.Errorf("APISubmitHam (no params) failed: %s", err.Error())
	}
}

func TestCommentNew(t *testing.T) {
	var err error
	// Note: If we use := here, Go will create a local Comment
	// object instead of initialising the global.
	comment, err = NewTestComment(config.APIKey, config.Site)
	if err != nil {
		t.Errorf("CommentNew failed: %s", err.Error())
	}
}

func TestCommentCheck(t *testing.T) {
	if comment == nil {
		// We can't continue without a Comment object
		t.Fatal("CommentCheck fail: Comment object is nil.")
	}

	// Set up a non-spam comment
	comment.SetUserIP(config.IP)
	comment.SetUserAgent(config.UserAgent)
	comment.SetPage(config.Article)
	comment.SetAuthor("gokismet test")
	comment.SetEmail("hello@example.com")
	comment.SetContent("This is an example comment that does not contain anything spammy. In the absence of other settings Akismet should return a negative (non-spam) response for this comment. Cheers...")

	// And test it
	status, err := comment.Check()
	checkSpamStatus(t, "CommentCheck", StatusNotSpam, status, err)

	// Make the comment spammy
	comment.SetAuthor("viagra-test-123")

	// And test again
	status, err = comment.Check()
	checkSpamStatus(t, "CommentCheck", StatusProbableSpam, status, err)
}

func TestCommentReport(t *testing.T) {
	if comment == nil {
		// We can't continue without a Comment object
		t.Fatal("CommentReport fail: Comment object is nil.")
	}

	if err := comment.ReportSpam(); err != nil {
		t.Errorf("CommentReport Spam fail: %s", err.Error())
	}

	if err := comment.ReportNotSpam(); err != nil {
		t.Errorf("CommentReport NotSpam failed: %s", err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Rocky Bernstein.
// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"text/template"
)

/************** FIXME: put in a separate file ****************/
import (
	"encoding/json"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
)

type fmtResponse struct {
	Body  string
	Error string
}

var tmpdir string

// Handles a Go-source format request. This routine must be called via
// POST.
func FmtHandler(w http.ResponseWriter, req *http.Request) {
	resp := new(fmtResponse)
	if req.Method != "POST" {
		http.Error(w, "Forbidden, need POST", http.StatusForbidden)
		return
	}
	body, err := gofmt(req.FormValue("body"))
	if err != nil {
		resp.Error = err.Error()
	} else {
		resp.Body = body
	}
	json.NewEncoder(w).Encode(resp)
}

func gofmt(body string) (string, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "prog.go", body, parser.ParseComments)
	if err != nil {
		return "", err
	}
	ast.SortImports(fset, f)
	var buf bytes.Buffer
	config := &printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}
	err = config.Fprint(&buf, fset, f)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}
/*******************/

// CompileHandler is an HTTP handler that reads Go source code from the request,
// runs the program (returning any errors),
// and sends the program's output as the HTTP response.
func CompileHandler(w http.ResponseWriter, req *http.Request) {
	out, err := compile(req)
	if err != nil {
		error_(w, out, err)
		return
	}

	// write the output of x as the http response
	if *htmlOutput {
		w.Write(out)
	} else {
		output.Execute(w, out)
	}
}

/*******************/

const DefaultSaveName = "save"
const GoPathSuffix    = ".go"

// Handles a Go-source save request.
//
// We have to do this in Go since  HTML5 Doesn't grok file paths.
//
// In order to be able to distinguish relative versus absolute paths,
// we add an X in the URL. For example /save/X/tmp/save.go versus
// /save/Xtmp/save.go. The latter refers to ./tmp/save.go.
//
// This routine must be called via POST.
func SaveHandler(w http.ResponseWriter, req *http.Request) {
	filename := DefaultSaveName
	// +3 for enclosing "/X", e.g. "/save/X" not "save"
	SaveLen := len(DefaultSaveName) + 3
	if len(req.URL.Path) > SaveLen {
		filename = req.URL.Path[SaveLen:];
		if filename[0] != '/' {
			filename = filepath.Join(tmpdir, filename);
		}
	}
	if req.Method != "POST" {
		http.Error(w, "Forbidden, need POST", http.StatusForbidden)
		return
	}

	/** fmt.Printf("Req %s\n", req) **/
	body := new(bytes.Buffer)
	_, err := body.ReadFrom(req.Body)
	if err != nil {
		http.Error(w, "Server Error in SaveHandler reading Body text",
			http.StatusInternalServerError)
		return
	}
	req.Body.Close()

	if filename[len(filename)-len(GoPathSuffix):] != GoPathSuffix {
		filename += GoPathSuffix
	}
	err = ioutil.WriteFile(filename, body.Bytes(), 0600)
	if err != nil {
		fmt.Printf("Save Error: %s\n", err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Printf("File: %s saved\n", filename)
	w.Write([]byte(filename))
}

/*******************/

func ShareHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Printf("Redirecting\n");
	http.Redirect(w, req, "http://play.golang.org/share", http.StatusFound)
}

/*******************/

var (
	httpListen = flag.String("http", "127.0.0.1:3999",
		"host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
	resourceDir = "../static"
	resourceDirP = &resourceDir
	// resourceDir = flag.String("resource-root", "../static",
	// 	"Location of CSS and JavaScript resources")
)

var (
	// a source of numbers, for naming temporary files
	uniq = make(chan int)
)

func main() {
	flag.Parse()

	// source of unique numbers
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()

	// find real temporary directory (for rewriting filename in output)
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
	http.HandleFunc("/", edit)
	http.HandleFunc("/compile", CompileHandler)
	http.HandleFunc("/fmt",     FmtHandler)
	http.HandleFunc("/save",    SaveHandler)
	http.HandleFunc("/share",   ShareHandler)
	http.HandleFunc("/save/",   SaveHandler)

	http.Handle("/static/", http.StripPrefix("/static/",
		http.FileServer(http.Dir("../static"))))
	fmt.Printf("Runnning Go Play. Attempting to listening on %s\n",
		*httpListen)
	log.Fatal(http.ListenAndServe(*httpListen, nil))
}

var editTemplate = template.Must(template.ParseFiles("goplay.html"))

type Snippet struct {
	Body []byte
}

type editData struct {
	Snippet *Snippet
	ResourceDir string
}

// edit is an HTTP handler that renders the goplay interface.
// If a filename is supplied in the path component of the URI,
// its contents will be put in the interface's text area.
// Otherwise, the default "hello, world" program is displayed.
func edit(w http.ResponseWriter, req *http.Request) {
	data, err := ioutil.ReadFile(req.URL.Path[1:])
	if err != nil {
		data = []byte(hello)
	}

	snip := &Snippet{Body: data}
	editTemplate.Execute(w, &editData{snip, *resourceDirP})
}

var (
	commentRe = regexp.MustCompile(`(?m)^#.*\n`)
)

func compile(req *http.Request) (out []byte, err error) {
	// x is the base name for .go, .6, executable files
	x := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := x + ".go"
	bin := x
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// rewrite filename in error output
	defer func() {
		if err != nil {
			// drop messages from the go tool like '# _/compile0'
			out = commentRe.ReplaceAll(out, nil)
		}
		out = bytes.Replace(out, []byte(src+":"), []byte("main.go:"), -1)
	}()

	// write body to x.go
	body := new(bytes.Buffer)
	if _, err = body.ReadFrom(req.Body); err != nil {
		return
	}
	defer os.Remove(src)
	if err = ioutil.WriteFile(src, body.Bytes(), 0666); err != nil {
		return
	}

	// build x.go, creating x
	dir, file := filepath.Split(src)
	out, err = run(dir, "go", "build", "-o", bin, file)
	defer os.Remove(bin)
	if err != nil {
		return
	}

	// run x
	return run("", bin)
}

// error writes compile, link, or runtime errors to the HTTP connection.
// The JavaScript interface uses the 404 status code to identify the error.
func error_(w http.ResponseWriter, out []byte, err error) {
	w.WriteHeader(404)
	if out != nil {
		output.Execute(w, out)
	} else {
		output.Execute(w, err.Error())
	}
}

// run executes the specified command and returns its output and an error.
func run(dir string, args ...string) ([]byte, error) {
	var buf bytes.Buffer
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = &buf
	cmd.Stderr = cmd.Stdout
	err := cmd.Run()
	return buf.Bytes(), err
}

// HTML output template snippet
const outputText = `<pre>{{printf "%s" . |html}}</pre>`
var output = template.Must(template.New("output").Parse(outputText))

// Default program to start out with.
const hello = `package main

import "fmt"

func main() {
	fmt.Println("hello, world")
}
`
<commit_msg>Start using godoc doc conventions better.<commit_after>// Copyright 2013 Rocky Bernstein.
// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package main creates a HTTP server for a single HTML pages that
// controls editing, writing and running go programs.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"text/template"
)

/************** FIXME: put in a separate file ****************/
import (
	"encoding/json"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/token"
)

type fmtResponse struct {
	Body  string
	Error string
}

var tmpdir string

// FmtHandler handles a Go-source format request. The go source code
// is given in req and the formated output is passed back in w.  This
// routine must be called via an HTTP POST request.
func FmtHandler(w http.ResponseWriter, req *http.Request) {
	resp := new(fmtResponse)
	if req.Method != "POST" {
		http.Error(w, "Forbidden, need POST", http.StatusForbidden)
		return
	}
	body, err := gofmt(req.FormValue("body"))
	if err != nil {
		resp.Error = err.Error()
	} else {
		resp.Body = body
	}
	json.NewEncoder(w).Encode(resp)
}

func gofmt(body string) (string, error) {
	fset := token.NewFileSet()
	f, err := parser.ParseFile(fset, "prog.go", body, parser.ParseComments)
	if err != nil {
		return "", err
	}
	ast.SortImports(fset, f)
	var buf bytes.Buffer
	config := &printer.Config{Mode: printer.UseSpaces | printer.TabIndent, Tabwidth: 8}
	err = config.Fprint(&buf, fset, f)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}
/*******************/

// CompileHandler is an HTTP handler that reads Go source code from req,
// runs the program (returning any errors),
// and sends the program's output as the HTTP response to w.
func CompileHandler(w http.ResponseWriter, req *http.Request) {
	out, err := compile(req)
	if err != nil {
		error_(w, out, err)
		return
	}

	// write the output of x as the http response
	if *htmlOutput {
		w.Write(out)
	} else {
		output.Execute(w, out)
	}
}

/*******************/

const DefaultSaveName = "save"
const GoPathSuffix    = ".go"

// SaveHandler writes a Go program to file.
//
// We have to do this in Go since HTML5 Doesn't grok file paths.
//
// In order to be able to distinguish relative versus absolute paths,
// we add an X in the URL. For example /save/X/tmp/save.go versus
// /save/Xtmp/save.go. The latter refers to ./tmp/save.go.
//
// This routine must be called via POST.
func SaveHandler(w http.ResponseWriter, req *http.Request) {
	filename := DefaultSaveName
	// +3 for enclosing "/X", e.g. "/save/X" not "save"
	SaveLen := len(DefaultSaveName) + 3
	if len(req.URL.Path) > SaveLen {
		filename = req.URL.Path[SaveLen:];
		if filename[0] != '/' {
			filename = filepath.Join(tmpdir, filename);
		}
	}
	if req.Method != "POST" {
		http.Error(w, "Forbidden, need POST", http.StatusForbidden)
		return
	}

	/** fmt.Printf("Req %s\n", req) **/
	body := new(bytes.Buffer)
	_, err := body.ReadFrom(req.Body)
	if err != nil {
		http.Error(w, "Server Error in SaveHandler reading Body text",
			http.StatusInternalServerError)
		return
	}
	req.Body.Close()

	if filename[len(filename)-len(GoPathSuffix):] != GoPathSuffix {
		filename += GoPathSuffix
	}
	err = ioutil.WriteFile(filename, body.Bytes(), 0600)
	if err != nil {
		fmt.Printf("Save Error: %s\n", err.Error())
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	fmt.Printf("File: %s saved\n", filename)
	w.Write([]byte(filename))
}

/*******************/

func ShareHandler(w http.ResponseWriter, req *http.Request) {
	fmt.Printf("Redirecting\n");
	http.Redirect(w, req, "http://play.golang.org/share", http.StatusFound)
}

/*******************/

var (
	httpListen = flag.String("http", "127.0.0.1:3999",
		"host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
	resourceDir = "../static"
	resourceDirP = &resourceDir
	// resourceDir = flag.String("resource-root", "../static",
	// 	"Location of CSS and JavaScript resources")
)

var (
	// a source of numbers, for naming temporary files
	uniq = make(chan int)
)

func main() {
	flag.Parse()

	// source of unique numbers
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()

	// find real temporary directory (for rewriting filename in output)
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
	http.HandleFunc("/", edit)
	http.HandleFunc("/compile", CompileHandler)
	http.HandleFunc("/fmt",     FmtHandler)
	http.HandleFunc("/save",    SaveHandler)
	http.HandleFunc("/share",   ShareHandler)
	http.HandleFunc("/save/",   SaveHandler)

	http.Handle("/static/", http.StripPrefix("/static/",
		http.FileServer(http.Dir("../static"))))
	fmt.Printf("Runnning Go Play. Attempting to listening on %s\n",
		*httpListen)
	log.Fatal(http.ListenAndServe(*httpListen, nil))
}

var editTemplate = template.Must(template.ParseFiles("goplay.html"))

type Snippet struct {
	Body []byte
}

type editData struct {
	Snippet *Snippet
	ResourceDir string
}

// edit is an HTTP handler that renders the goplay interface.
// If a filename is supplied in the path component of the URI,
// its contents will be put in the interface's text area.
// Otherwise, the default "hello, world" program is displayed.
func edit(w http.ResponseWriter, req *http.Request) {
	data, err := ioutil.ReadFile(req.URL.Path[1:])
	if err != nil {
		data = []byte(hello)
	}

	snip := &Snippet{Body: data}
	editTemplate.Execute(w, &editData{snip, *resourceDirP})
}

var (
	commentRe = regexp.MustCompile(`(?m)^#.*\n`)
)

func compile(req *http.Request) (out []byte, err error) {
	// x is the base name for .go, .6, executable files
	x := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := x + ".go"
	bin := x
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// rewrite filename in error output
	defer func() {
		if err != nil {
			// drop messages from the go tool like '# _/compile0'
			out = commentRe.ReplaceAll(out, nil)
		}
		out = bytes.Replace(out, []byte(src+":"), []byte("main.go:"), -1)
	}()

	// write body to x.go
	body := new(bytes.Buffer)
	if _, err = body.ReadFrom(req.Body); err != nil {
		return
	}
	defer os.Remove(src)
	if err = ioutil.WriteFile(src, body.Bytes(), 0666); err != nil {
		return
	}

	// build x.go, creating x
	dir, file := filepath.Split(src)
	out, err = run(dir, "go", "build", "-o", bin, file)
	defer os.Remove(bin)
	if err != nil {
		return
	}

	// run x
	return run("", bin)
}

// error writes compile, link, or runtime errors to the HTTP connection.
// The JavaScript interface uses the 404 status code to identify the error.
func error_(w http.ResponseWriter, out []byte, err error) {
	w.WriteHeader(404)
	if out != nil {
		output.Execute(w, out)
	} else {
		output.Execute(w, err.Error())
	}
}

// run executes the specified command and returns its output and an error.
func run(dir string, args ...string) ([]byte, error) {
	var buf bytes.Buffer
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = &buf
	cmd.Stderr = cmd.Stdout
	err := cmd.Run()
	return buf.Bytes(), err
}

// HTML output template snippet
const outputText = `<pre>{{printf "%s" . |html}}</pre>`
var output = template.Must(template.New("output").Parse(outputText))

// Default program to start out with.
const hello = `package main

import "fmt"

func main() {
	fmt.Println("hello, world")
}
`
<|endoftext|>


<commit_before>package bheap

import "testing"

func compare(x, y interface{}) bool {
	return !CompareInt(x, y)
}

func TestPush(t *testing.T) {
	h := New(compare)
	for i := 0; i < 1<<20; i++ {
		h.Push(i)
	}
	for i := 0; i < 1<<20; i++ {
		if v, ok := h.Pop(); !ok || v.(int) != i {
			t.Fail()
		}
	}
}
<commit_msg>Make test coverage 100%<commit_after>package bheap

import (
	"math/rand"
	"sort"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestPush(t *testing.T) {
	h := New(CompareInt)
	assert.Equal(t, 0, h.Len())
	assert.True(t, h.IsEmpty())

	rand.Seed(time.Now().UnixNano())
	var s []int
	for i := 0; i < 1<<20; i++ {
		n := rand.Int()
		s = append(s, n)
		h.Push(n)
	}
	assert.False(t, h.IsEmpty())
	assert.Equal(t, 1<<20, h.Len())

	sort.Ints(s)
	for i := 1<<20 - 1; i >= 0; i-- {
		v, ok := h.Top()
		assert.True(t, ok)
		assert.Equal(t, s[i], v.(int))

		v, ok = h.Pop()
		assert.True(t, ok)
		assert.Equal(t, s[i], v.(int))
	}
	v, ok := h.Top()
	assert.False(t, ok)
	assert.Nil(t, v)

	v, ok = h.Pop()
	assert.False(t, ok)
	assert.Nil(t, v)

	h.Clean()
	assert.True(t, h.IsEmpty())

	h2 := New(CompareInt)
	for i := 0; i < 1<<10; i++ {
		h2.Push(i)
	}
	h2 = h2.Merge(nil)
	h2 = h2.Merge(h)
	assert.Equal(t, 1<<10, h2.Len())

	ss := []string{
		"abc",
		"ab",
		"hello",
	}
	h3 := New(CompareString)
	for _, s := range ss {
		h3.Push(s)
	}
}
<|endoftext|>


<commit_before>package gospell

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// GoSpell is main struct
type GoSpell struct {
	WordChars string              // from AFF file
	ireplacer *strings.Replacer   // input conversion
	Dict      map[string]struct{} // likely will contain some value later
}

// Input conversion does any character substitution before checking
func (s *GoSpell) InputConversion(raw []byte) string {
	sraw := string(raw)
	if s.ireplacer == nil {
		return sraw
	}
	return s.ireplacer.Replace(sraw)
}

// Spell checks to see if a given word is in the internal dictionaries
// TODO: add multiple dictionaries
func (s *GoSpell) Spell(word string) bool {
	//log.Printf("Checking %s", word)
	_, ok := s.Dict[word]
	return ok
}

// NewGoSpellReader creates a speller from io.Readers for aff and dic
// Hunspell files
func NewGoSpellReader(aff, dic io.Reader) (*GoSpell, error) {
	affix, err := NewAFF(aff)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(dic)
	// get first line
	if !scanner.Scan() {
		return nil, scanner.Err()
	}
	line := scanner.Text()
	i, err := strconv.ParseInt(line, 10, 64)
	if err != nil {
		return nil, err
	}
	gs := GoSpell{
		WordChars: affix.WordChars,
		Dict:      make(map[string]struct{}, i*5),
	}

	words := []string{}
	for scanner.Scan() {
		line := scanner.Text()
		words, err = affix.Expand(line, words)
		if err != nil {
			// Need to support Compound rules
			//return nil, fmt.Errorf("Unable to process %q: %s", line, err)
			continue
		}

		// this is about 100ms faster, than the full case iteration
		// below
		if false {
			for _, word := range words {
				gs.Dict[word] = struct{}{}
			}
		}

		if true {
			style := CaseStyle(words[0])
			for _, word := range words {
				switch style {
				case AllLower:
					gs.Dict[word] = struct{}{}
					gs.Dict[strings.Title(word)] = struct{}{}
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				case AllUpper:
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				case Title:
					gs.Dict[word] = struct{}{}
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				case Mixed:
					gs.Dict[word] = struct{}{}
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				default:
					gs.Dict[word] = struct{}{}
				}
			}
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	if len(affix.IconvReplacements) > 0 {
		gs.ireplacer = strings.NewReplacer(affix.IconvReplacements...)
	}
	//	log.Printf("Internal dictionary has %d entries", len(gs.Dict))
	return &gs, nil
}

// NewGoSpell from aff and dic Hunspell filenames
func NewGoSpell(affFile, dicFile string) (*GoSpell, error) {
	aff, err := os.Open(affFile)
	if err != nil {
		return nil, fmt.Errorf("Unable to open aff: %s", err)
	}
	defer aff.Close()
	dic, err := os.Open(dicFile)
	if err != nil {
		return nil, fmt.Errorf("Unable to open dic: %s", err)
	}
	defer dic.Close()
	h, err := NewGoSpellReader(aff, dic)
	return h, err
}
<commit_msg>fix doco<commit_after>package gospell

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
)

// GoSpell is main struct
type GoSpell struct {
	WordChars string              // from AFF file
	ireplacer *strings.Replacer   // input conversion
	Dict      map[string]struct{} // likely will contain some value later
}

// InputConversion does any character substitution before checking
//  This is based on the ICONV stanza
func (s *GoSpell) InputConversion(raw []byte) string {
	sraw := string(raw)
	if s.ireplacer == nil {
		return sraw
	}
	return s.ireplacer.Replace(sraw)
}

// Spell checks to see if a given word is in the internal dictionaries
// TODO: add multiple dictionaries
func (s *GoSpell) Spell(word string) bool {
	//log.Printf("Checking %s", word)
	_, ok := s.Dict[word]
	return ok
}

// NewGoSpellReader creates a speller from io.Readers for aff and dic
// Hunspell files
func NewGoSpellReader(aff, dic io.Reader) (*GoSpell, error) {
	affix, err := NewAFF(aff)
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(dic)
	// get first line
	if !scanner.Scan() {
		return nil, scanner.Err()
	}
	line := scanner.Text()
	i, err := strconv.ParseInt(line, 10, 64)
	if err != nil {
		return nil, err
	}
	gs := GoSpell{
		WordChars: affix.WordChars,
		Dict:      make(map[string]struct{}, i*5),
	}

	words := []string{}
	for scanner.Scan() {
		line := scanner.Text()
		words, err = affix.Expand(line, words)
		if err != nil {
			// Need to support Compound rules
			//return nil, fmt.Errorf("Unable to process %q: %s", line, err)
			continue
		}

		// this is about 100ms faster, than the full case iteration
		// below
		if false {
			for _, word := range words {
				gs.Dict[word] = struct{}{}
			}
		}

		if true {
			style := CaseStyle(words[0])
			for _, word := range words {
				switch style {
				case AllLower:
					gs.Dict[word] = struct{}{}
					gs.Dict[strings.Title(word)] = struct{}{}
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				case AllUpper:
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				case Title:
					gs.Dict[word] = struct{}{}
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				case Mixed:
					gs.Dict[word] = struct{}{}
					gs.Dict[strings.ToUpper(word)] = struct{}{}
				default:
					gs.Dict[word] = struct{}{}
				}
			}
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	if len(affix.IconvReplacements) > 0 {
		gs.ireplacer = strings.NewReplacer(affix.IconvReplacements...)
	}
	//	log.Printf("Internal dictionary has %d entries", len(gs.Dict))
	return &gs, nil
}

// NewGoSpell from aff and dic Hunspell filenames
func NewGoSpell(affFile, dicFile string) (*GoSpell, error) {
	aff, err := os.Open(affFile)
	if err != nil {
		return nil, fmt.Errorf("Unable to open aff: %s", err)
	}
	defer aff.Close()
	dic, err := os.Open(dicFile)
	if err != nil {
		return nil, fmt.Errorf("Unable to open dic: %s", err)
	}
	defer dic.Close()
	h, err := NewGoSpellReader(aff, dic)
	return h, err
}
<|endoftext|>