<commit_before>// Package torrent provides a torrent client implementation.
package torrent

import (
	"encoding/hex"
	"errors"
	"io"
	"math/rand"
	"net"
	"net/url"
	"time"

	"github.com/cenkalti/rain/internal/announcer"
	"github.com/cenkalti/rain/internal/bitfield"
	"github.com/cenkalti/rain/internal/clientversion"
	"github.com/cenkalti/rain/internal/downloader/acceptor"
	"github.com/cenkalti/rain/internal/downloader/addrlist"
	"github.com/cenkalti/rain/internal/downloader/handshaker/incominghandshaker"
	"github.com/cenkalti/rain/internal/downloader/handshaker/outgoinghandshaker"
	"github.com/cenkalti/rain/internal/downloader/infodownloader"
	"github.com/cenkalti/rain/internal/downloader/peer"
	"github.com/cenkalti/rain/internal/downloader/piece"
	"github.com/cenkalti/rain/internal/downloader/piecedownloader"
	"github.com/cenkalti/rain/internal/downloader/piecewriter"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/magnet"
	"github.com/cenkalti/rain/internal/metainfo"
	"github.com/cenkalti/rain/internal/mse"
	ip "github.com/cenkalti/rain/internal/peerconn"
	"github.com/cenkalti/rain/internal/semaphore"
	"github.com/cenkalti/rain/internal/torrentdata"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/cenkalti/rain/internal/tracker/httptracker"
	"github.com/cenkalti/rain/internal/tracker/udptracker"
	"github.com/cenkalti/rain/resume"
	"github.com/cenkalti/rain/storage"
	"github.com/cenkalti/rain/storage/filestorage"
)

const (
	parallelInfoDownloads  = 4
	parallelPieceDownloads = 4
	parallelPieceWrites    = 4 // TODO remove this
	maxPeerDial            = 40
	maxPeerAccept          = 40
)

var (
	// http://www.bittorrent.org/beps/bep_0020.html
	peerIDPrefix = []byte("-RN" + clientversion.Version + "-")
)

// Torrent connects to peers and downloads files from swarm.
type Torrent struct {
	spec *downloadSpec

	// Identifies the torrent being downloaded.
	infoHash [20]byte

	// Unique peer ID is generated per downloader.
	peerID [20]byte

	// TCP Port to listen for peer connections.
	port int

	// List of addresses to announce this torrent.
	trackers []string

	// Storage implementation to save the files in torrent.
	storage storage.Storage

	// Optional DB implementation to save resume state of the torrent.
	resume resume.DB

	// Contains info about files in torrent. This can be nil at start for magnet downloads.
	info *metainfo.Info

	// Bitfield for pieces we have. It is created after we got info.
	bitfield *bitfield.Bitfield

	// Data provides IO access to pieces in torrent.
	data *torrentdata.Data

	// Boolean state to to tell if all pieces are downloaded.
	completed bool

	// Contains state about the pieces in torrent.
	pieces []piece.Piece

	// Contains pieces in sorted order for piece selection function.
	sortedPieces []*piece.Piece

	// Peers are sent to this channel when they are disconnected.
	peerDisconnectedC chan *peer.Peer

	// All messages coming from peers are sent to this channel.
	messages chan peer.Message

	// We keep connected peers in this map after they complete handshake phase.
	connectedPeers map[*ip.Peer]*peer.Peer

	// Active piece downloads are kept in this map.
	pieceDownloads map[*ip.Peer]*piecedownloader.PieceDownloader

	// Active metadata downloads are kept in this map.
	infoDownloads map[*ip.Peer]*infodownloader.InfoDownloader

	// Downloader run loop sends a message to this channel for writing a piece to disk.
	writeRequestC chan piecewriter.Request

	// When a piece is written to the disk, a message is sent to this channel.
	writeResponseC chan piecewriter.Response

	// A peer is optimistically unchoked regardless of their download rate.
	optimisticUnchokedPeer *peer.Peer

	// This channel is closed once all pieces are downloaded and verified.
	completeC chan struct{}

	// If any unrecoverable error occurs, it will be sent to this channel and download will be stopped.
	errC chan error

	// When Stop() is called, it will close this channel to signal run() function to stop.
	closeC chan struct{}

	// This channel will be closed after run loop exists.
	closedC chan struct{}

	// These are the channels for sending a message to run() loop.
	statsCommandC       chan statsRequest       // Stats()
	startCommandC       chan struct{}           // Start()
	stopCommandC        chan struct{}           // Stop()
	notifyErrorCommandC chan notifyErrorCommand // NotifyError()

	// Trackers send announce responses to this channel.
	addrsFromTrackers chan []*net.TCPAddr

	// Keeps a list of peer addresses to connect.
	addrList *addrlist.AddrList

	// New raw connections created by OutgoingHandshaker are sent to here.
	newInConnC chan net.Conn

	// Keep a set of peer IDs to block duplicate connections.
	peerIDs map[[20]byte]struct{}

	// Listens for incoming peer connections.
	acceptor *acceptor.Acceptor

	// Special hash of info hash for encypted connection handshake.
	sKeyHash [20]byte

	// Responsible for writing downloaded pieces to disk.
	pieceWriters []*piecewriter.PieceWriter

	// Tracker implementations for giving to announcers.
	trackersInstances []tracker.Tracker

	// Announces the status of torrent to trackers to get peer addresses.
	announcers []*announcer.Announcer

	// List of peers in handshake state.
	incomingHandshakers map[string]*incominghandshaker.IncomingHandshaker
	outgoingHandshakers map[string]*outgoinghandshaker.OutgoingHandshaker

	// Handshake results are sent to these channels by handshakers.
	incomingHandshakerResultC chan incominghandshaker.Result
	outgoingHandshakerResultC chan outgoinghandshaker.Result

	// We keep connected and handshake completed peers here.
	incomingPeers []*peer.Peer
	outgoingPeers []*peer.Peer

	// When metadata of the torrent downloaded completely, a message is sent to this channel.
	infoDownloaderResultC chan infodownloader.Result

	// When a piece is downloaded completely a message is sent to this channel.
	pieceDownloaderResultC chan piecedownloader.Result

	// True after downloader is started with Start() method, false after Stop() is called.
	running bool

	// Announcers send a request to this channel to get information about the torrent.
	announcerRequests chan *announcer.Request

	// A timer that ticks periodically to keep a certain number of peers unchoked.
	unchokeTimer  *time.Ticker
	unchokeTimerC <-chan time.Time

	// A timer that ticks periodically to keep a random peer unchoked regardless of its upload rate.
	optimisticUnchokeTimer  *time.Ticker
	optimisticUnchokeTimerC <-chan time.Time

	// To limit the max number of peers to connect to.
	dialLimit *semaphore.Semaphore

	// To limit the max number of parallel piece downloads.
	pieceDownloaders *semaphore.Semaphore

	// To limit the max number of parallel metadata downloads.
	infoDownloaders *semaphore.Semaphore

	log logger.Logger
}

// New returns a new torrent by reading a torrent metainfo file.
func New(r io.Reader, port int, sto storage.Storage) (*Torrent, error) {
	m, err := metainfo.New(r)
	if err != nil {
		return nil, err
	}
	spec := &downloadSpec{
		InfoHash: m.Info.Hash,
		Trackers: m.GetTrackers(),
		Name:     m.Info.Name,
		Port:     port,
		Storage:  sto,
		Info:     m.Info,
	}
	return newTorrent(spec)
}

// NewMagnet returns a new torrent by parsing a magnet link.
func NewMagnet(magnetLink string, port int, sto storage.Storage) (*Torrent, error) {
	m, err := magnet.New(magnetLink)
	if err != nil {
		return nil, err
	}
	spec := &downloadSpec{
		InfoHash: m.InfoHash,
		Trackers: m.Trackers,
		Name:     m.Name,
		Port:     port,
		Storage:  sto,
	}
	return newTorrent(spec)
}

// NewResume returns a new torrent by loading all info from a resume.DB.
func NewResume(res resume.DB) (*Torrent, error) {
	spec, err := res.Read()
	if err != nil {
		return nil, err
	}
	if spec == nil {
		return nil, errors.New("no resume info")
	}
	return loadResumeSpec(spec, res)
}

// Name of the torrent.
// For magnet downloads name can change after metadata is downloaded but this method still returns the initial name.
func (t *Torrent) Name() string {
	return t.spec.Name
}

// InfoHash string encoded in hex.
// InfoHash is a unique value that identifies the files in torrent.
func (t *Torrent) InfoHash() string {
	return hex.EncodeToString(t.spec.InfoHash[:])
}

// SetResume adds resume capability to the torrent.
// It must be called before Start() is called.
func (t *Torrent) SetResume(res resume.DB) error {
	spec, err := res.Read()
	if err != nil {
		return err
	}
	if spec == nil {
		return t.writeResume(res)
	}
	t2, err := loadResumeSpec(spec, res)
	if err != nil {
		return err
	}
	if t.InfoHash() != t2.InfoHash() {
		t2.Close()
		return errors.New("invalid resume file (info hashes does not match)")
	}
	t.Close()
	*t = *t2
	return nil
}

func loadResumeSpec(spec *resume.Spec, res resume.DB) (*Torrent, error) {
	var err error
	dspec := &downloadSpec{
		Port:     spec.Port,
		Trackers: spec.Trackers,
		Name:     spec.Name,
		Resume:   res,
	}
	copy(dspec.InfoHash[:], spec.InfoHash)
	if len(spec.Info) > 0 {
		dspec.Info, err = metainfo.NewInfo(spec.Info)
		if err != nil {
			return nil, err
		}
		if len(spec.Bitfield) > 0 {
			dspec.Bitfield = bitfield.New(dspec.Info.NumPieces)
			copy(dspec.Bitfield.Bytes(), spec.Bitfield)
		}
	}
	switch spec.StorageType {
	case filestorage.StorageType:
		dspec.Storage = &filestorage.FileStorage{}
	default:
		return nil, errors.New("unknown storage type: " + spec.StorageType)
	}
	err = dspec.Storage.Load(spec.StorageArgs)
	if err != nil {
		return nil, err
	}
	return newTorrent(dspec)
}

func (t *Torrent) writeResume(res resume.DB) error {
	rspec := &resume.Spec{
		InfoHash:    t.spec.InfoHash[:],
		Port:        t.spec.Port,
		Name:        t.spec.Name,
		Trackers:    t.spec.Trackers,
		StorageType: t.spec.Storage.Type(),
		StorageArgs: t.spec.Storage.Args(),
	}
	if t.spec.Info != nil {
		rspec.Info = t.spec.Info.Bytes
	}
	if t.spec.Bitfield != nil {
		rspec.Bitfield = t.spec.Bitfield.Bytes()
	}
	return res.Write(rspec)
}

func newTorrent(spec *downloadSpec) (*Torrent, error) {
	logName := spec.Name
	if len(logName) > 8 {
		logName = logName[:8]
	}
	d := &Torrent{
		spec:     spec,
		infoHash: spec.InfoHash,
		trackers: spec.Trackers,
		port:     spec.Port,
		storage:  spec.Storage,
		resume:   spec.Resume,
		info:     spec.Info,
		bitfield: spec.Bitfield,
		log:      logger.New("download " + logName),

		peerDisconnectedC:         make(chan *peer.Peer),
		messages:                  make(chan peer.Message),
		connectedPeers:            make(map[*ip.Peer]*peer.Peer),
		pieceDownloads:            make(map[*ip.Peer]*piecedownloader.PieceDownloader),
		infoDownloads:             make(map[*ip.Peer]*infodownloader.InfoDownloader),
		writeRequestC:             make(chan piecewriter.Request),
		writeResponseC:            make(chan piecewriter.Response),
		completeC:                 make(chan struct{}),
		closeC:                    make(chan struct{}),
		closedC:                   make(chan struct{}),
		statsCommandC:             make(chan statsRequest),
		notifyErrorCommandC:       make(chan notifyErrorCommand),
		addrsFromTrackers:         make(chan []*net.TCPAddr),
		addrList:                  addrlist.New(),
		peerIDs:                   make(map[[20]byte]struct{}),
		newInConnC:                make(chan net.Conn),
		sKeyHash:                  mse.HashSKey(spec.InfoHash[:]),
		infoDownloaderResultC:     make(chan infodownloader.Result),
		pieceDownloaderResultC:    make(chan piecedownloader.Result),
		incomingHandshakers:       make(map[string]*incominghandshaker.IncomingHandshaker),
		outgoingHandshakers:       make(map[string]*outgoinghandshaker.OutgoingHandshaker),
		incomingHandshakerResultC: make(chan incominghandshaker.Result),
		outgoingHandshakerResultC: make(chan outgoinghandshaker.Result),
		startCommandC:             make(chan struct{}),
		stopCommandC:              make(chan struct{}),
		announcerRequests:         make(chan *announcer.Request),
		dialLimit:                 semaphore.New(maxPeerDial),
		pieceDownloaders:          semaphore.New(parallelPieceDownloads),
		infoDownloaders:           semaphore.New(parallelPieceDownloads),
	}
	copy(d.peerID[:], peerIDPrefix)
	_, err := rand.Read(d.peerID[len(peerIDPrefix):]) // nolint: gosec
	if err != nil {
		return nil, err
	}
	d.trackersInstances, err = parseTrackers(d.trackers, d.log)
	if err != nil {
		return nil, err
	}
	go d.run()
	return d, nil
}

func parseTrackers(trackers []string, log logger.Logger) ([]tracker.Tracker, error) {
	var ret []tracker.Tracker
	for _, s := range trackers {
		u, err := url.Parse(s)
		if err != nil {
			log.Warningln("cannot parse tracker url:", err)
			continue
		}
		switch u.Scheme {
		case "http", "https":
			ret = append(ret, httptracker.New(u))
		case "udp":
			ret = append(ret, udptracker.New(u))
		default:
			log.Warningln("unsupported tracker scheme: %s", u.Scheme)
		}
	}
	if len(ret) == 0 {
		return nil, errors.New("no tracker found")
	}
	return ret, nil
}
<commit_msg>rename import<commit_after>// Package torrent provides a torrent client implementation.
package torrent

import (
	"encoding/hex"
	"errors"
	"io"
	"math/rand"
	"net"
	"net/url"
	"time"

	"github.com/cenkalti/rain/internal/announcer"
	"github.com/cenkalti/rain/internal/bitfield"
	"github.com/cenkalti/rain/internal/clientversion"
	"github.com/cenkalti/rain/internal/downloader/acceptor"
	"github.com/cenkalti/rain/internal/downloader/addrlist"
	"github.com/cenkalti/rain/internal/downloader/handshaker/incominghandshaker"
	"github.com/cenkalti/rain/internal/downloader/handshaker/outgoinghandshaker"
	"github.com/cenkalti/rain/internal/downloader/infodownloader"
	"github.com/cenkalti/rain/internal/downloader/peer"
	"github.com/cenkalti/rain/internal/downloader/piece"
	"github.com/cenkalti/rain/internal/downloader/piecedownloader"
	"github.com/cenkalti/rain/internal/downloader/piecewriter"
	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/internal/magnet"
	"github.com/cenkalti/rain/internal/metainfo"
	"github.com/cenkalti/rain/internal/mse"
	"github.com/cenkalti/rain/internal/peerconn"
	"github.com/cenkalti/rain/internal/semaphore"
	"github.com/cenkalti/rain/internal/torrentdata"
	"github.com/cenkalti/rain/internal/tracker"
	"github.com/cenkalti/rain/internal/tracker/httptracker"
	"github.com/cenkalti/rain/internal/tracker/udptracker"
	"github.com/cenkalti/rain/resume"
	"github.com/cenkalti/rain/storage"
	"github.com/cenkalti/rain/storage/filestorage"
)

const (
	parallelInfoDownloads  = 4
	parallelPieceDownloads = 4
	parallelPieceWrites    = 4 // TODO remove this
	maxPeerDial            = 40
	maxPeerAccept          = 40
)

var (
	// http://www.bittorrent.org/beps/bep_0020.html
	peerIDPrefix = []byte("-RN" + clientversion.Version + "-")
)

// Torrent connects to peers and downloads files from swarm.
type Torrent struct {
	spec *downloadSpec

	// Identifies the torrent being downloaded.
	infoHash [20]byte

	// Unique peer ID is generated per downloader.
	peerID [20]byte

	// TCP Port to listen for peer connections.
	port int

	// List of addresses to announce this torrent.
	trackers []string

	// Storage implementation to save the files in torrent.
	storage storage.Storage

	// Optional DB implementation to save resume state of the torrent.
	resume resume.DB

	// Contains info about files in torrent. This can be nil at start for magnet downloads.
	info *metainfo.Info

	// Bitfield for pieces we have. It is created after we got info.
	bitfield *bitfield.Bitfield

	// Data provides IO access to pieces in torrent.
	data *torrentdata.Data

	// Boolean state to to tell if all pieces are downloaded.
	completed bool

	// Contains state about the pieces in torrent.
	pieces []piece.Piece

	// Contains pieces in sorted order for piece selection function.
	sortedPieces []*piece.Piece

	// Peers are sent to this channel when they are disconnected.
	peerDisconnectedC chan *peer.Peer

	// All messages coming from peers are sent to this channel.
	messages chan peer.Message

	// We keep connected peers in this map after they complete handshake phase.
	connectedPeers map[*peerconn.Peer]*peer.Peer

	// Active piece downloads are kept in this map.
	pieceDownloads map[*peerconn.Peer]*piecedownloader.PieceDownloader

	// Active metadata downloads are kept in this map.
	infoDownloads map[*peerconn.Peer]*infodownloader.InfoDownloader

	// Downloader run loop sends a message to this channel for writing a piece to disk.
	writeRequestC chan piecewriter.Request

	// When a piece is written to the disk, a message is sent to this channel.
	writeResponseC chan piecewriter.Response

	// A peer is optimistically unchoked regardless of their download rate.
	optimisticUnchokedPeer *peer.Peer

	// This channel is closed once all pieces are downloaded and verified.
	completeC chan struct{}

	// If any unrecoverable error occurs, it will be sent to this channel and download will be stopped.
	errC chan error

	// When Stop() is called, it will close this channel to signal run() function to stop.
	closeC chan struct{}

	// This channel will be closed after run loop exists.
	closedC chan struct{}

	// These are the channels for sending a message to run() loop.
	statsCommandC       chan statsRequest       // Stats()
	startCommandC       chan struct{}           // Start()
	stopCommandC        chan struct{}           // Stop()
	notifyErrorCommandC chan notifyErrorCommand // NotifyError()

	// Trackers send announce responses to this channel.
	addrsFromTrackers chan []*net.TCPAddr

	// Keeps a list of peer addresses to connect.
	addrList *addrlist.AddrList

	// New raw connections created by OutgoingHandshaker are sent to here.
	newInConnC chan net.Conn

	// Keep a set of peer IDs to block duplicate connections.
	peerIDs map[[20]byte]struct{}

	// Listens for incoming peer connections.
	acceptor *acceptor.Acceptor

	// Special hash of info hash for encypted connection handshake.
	sKeyHash [20]byte

	// Responsible for writing downloaded pieces to disk.
	pieceWriters []*piecewriter.PieceWriter

	// Tracker implementations for giving to announcers.
	trackersInstances []tracker.Tracker

	// Announces the status of torrent to trackers to get peer addresses.
	announcers []*announcer.Announcer

	// List of peers in handshake state.
	incomingHandshakers map[string]*incominghandshaker.IncomingHandshaker
	outgoingHandshakers map[string]*outgoinghandshaker.OutgoingHandshaker

	// Handshake results are sent to these channels by handshakers.
	incomingHandshakerResultC chan incominghandshaker.Result
	outgoingHandshakerResultC chan outgoinghandshaker.Result

	// We keep connected and handshake completed peers here.
	incomingPeers []*peer.Peer
	outgoingPeers []*peer.Peer

	// When metadata of the torrent downloaded completely, a message is sent to this channel.
	infoDownloaderResultC chan infodownloader.Result

	// When a piece is downloaded completely a message is sent to this channel.
	pieceDownloaderResultC chan piecedownloader.Result

	// True after downloader is started with Start() method, false after Stop() is called.
	running bool

	// Announcers send a request to this channel to get information about the torrent.
	announcerRequests chan *announcer.Request

	// A timer that ticks periodically to keep a certain number of peers unchoked.
	unchokeTimer  *time.Ticker
	unchokeTimerC <-chan time.Time

	// A timer that ticks periodically to keep a random peer unchoked regardless of its upload rate.
	optimisticUnchokeTimer  *time.Ticker
	optimisticUnchokeTimerC <-chan time.Time

	// To limit the max number of peers to connect to.
	dialLimit *semaphore.Semaphore

	// To limit the max number of parallel piece downloads.
	pieceDownloaders *semaphore.Semaphore

	// To limit the max number of parallel metadata downloads.
	infoDownloaders *semaphore.Semaphore

	log logger.Logger
}

// New returns a new torrent by reading a torrent metainfo file.
func New(r io.Reader, port int, sto storage.Storage) (*Torrent, error) {
	m, err := metainfo.New(r)
	if err != nil {
		return nil, err
	}
	spec := &downloadSpec{
		InfoHash: m.Info.Hash,
		Trackers: m.GetTrackers(),
		Name:     m.Info.Name,
		Port:     port,
		Storage:  sto,
		Info:     m.Info,
	}
	return newTorrent(spec)
}

// NewMagnet returns a new torrent by parsing a magnet link.
func NewMagnet(magnetLink string, port int, sto storage.Storage) (*Torrent, error) {
	m, err := magnet.New(magnetLink)
	if err != nil {
		return nil, err
	}
	spec := &downloadSpec{
		InfoHash: m.InfoHash,
		Trackers: m.Trackers,
		Name:     m.Name,
		Port:     port,
		Storage:  sto,
	}
	return newTorrent(spec)
}

// NewResume returns a new torrent by loading all info from a resume.DB.
func NewResume(res resume.DB) (*Torrent, error) {
	spec, err := res.Read()
	if err != nil {
		return nil, err
	}
	if spec == nil {
		return nil, errors.New("no resume info")
	}
	return loadResumeSpec(spec, res)
}

// Name of the torrent.
// For magnet downloads name can change after metadata is downloaded but this method still returns the initial name.
func (t *Torrent) Name() string {
	return t.spec.Name
}

// InfoHash string encoded in hex.
// InfoHash is a unique value that identifies the files in torrent.
func (t *Torrent) InfoHash() string {
	return hex.EncodeToString(t.spec.InfoHash[:])
}

// SetResume adds resume capability to the torrent.
// It must be called before Start() is called.
func (t *Torrent) SetResume(res resume.DB) error {
	spec, err := res.Read()
	if err != nil {
		return err
	}
	if spec == nil {
		return t.writeResume(res)
	}
	t2, err := loadResumeSpec(spec, res)
	if err != nil {
		return err
	}
	if t.InfoHash() != t2.InfoHash() {
		t2.Close()
		return errors.New("invalid resume file (info hashes does not match)")
	}
	t.Close()
	*t = *t2
	return nil
}

func loadResumeSpec(spec *resume.Spec, res resume.DB) (*Torrent, error) {
	var err error
	dspec := &downloadSpec{
		Port:     spec.Port,
		Trackers: spec.Trackers,
		Name:     spec.Name,
		Resume:   res,
	}
	copy(dspec.InfoHash[:], spec.InfoHash)
	if len(spec.Info) > 0 {
		dspec.Info, err = metainfo.NewInfo(spec.Info)
		if err != nil {
			return nil, err
		}
		if len(spec.Bitfield) > 0 {
			dspec.Bitfield = bitfield.New(dspec.Info.NumPieces)
			copy(dspec.Bitfield.Bytes(), spec.Bitfield)
		}
	}
	switch spec.StorageType {
	case filestorage.StorageType:
		dspec.Storage = &filestorage.FileStorage{}
	default:
		return nil, errors.New("unknown storage type: " + spec.StorageType)
	}
	err = dspec.Storage.Load(spec.StorageArgs)
	if err != nil {
		return nil, err
	}
	return newTorrent(dspec)
}

func (t *Torrent) writeResume(res resume.DB) error {
	rspec := &resume.Spec{
		InfoHash:    t.spec.InfoHash[:],
		Port:        t.spec.Port,
		Name:        t.spec.Name,
		Trackers:    t.spec.Trackers,
		StorageType: t.spec.Storage.Type(),
		StorageArgs: t.spec.Storage.Args(),
	}
	if t.spec.Info != nil {
		rspec.Info = t.spec.Info.Bytes
	}
	if t.spec.Bitfield != nil {
		rspec.Bitfield = t.spec.Bitfield.Bytes()
	}
	return res.Write(rspec)
}

func newTorrent(spec *downloadSpec) (*Torrent, error) {
	logName := spec.Name
	if len(logName) > 8 {
		logName = logName[:8]
	}
	d := &Torrent{
		spec:     spec,
		infoHash: spec.InfoHash,
		trackers: spec.Trackers,
		port:     spec.Port,
		storage:  spec.Storage,
		resume:   spec.Resume,
		info:     spec.Info,
		bitfield: spec.Bitfield,
		log:      logger.New("download " + logName),

		peerDisconnectedC:         make(chan *peer.Peer),
		messages:                  make(chan peer.Message),
		connectedPeers:            make(map[*peerconn.Peer]*peer.Peer),
		pieceDownloads:            make(map[*peerconn.Peer]*piecedownloader.PieceDownloader),
		infoDownloads:             make(map[*peerconn.Peer]*infodownloader.InfoDownloader),
		writeRequestC:             make(chan piecewriter.Request),
		writeResponseC:            make(chan piecewriter.Response),
		completeC:                 make(chan struct{}),
		closeC:                    make(chan struct{}),
		closedC:                   make(chan struct{}),
		statsCommandC:             make(chan statsRequest),
		notifyErrorCommandC:       make(chan notifyErrorCommand),
		addrsFromTrackers:         make(chan []*net.TCPAddr),
		addrList:                  addrlist.New(),
		peerIDs:                   make(map[[20]byte]struct{}),
		newInConnC:                make(chan net.Conn),
		sKeyHash:                  mse.HashSKey(spec.InfoHash[:]),
		infoDownloaderResultC:     make(chan infodownloader.Result),
		pieceDownloaderResultC:    make(chan piecedownloader.Result),
		incomingHandshakers:       make(map[string]*incominghandshaker.IncomingHandshaker),
		outgoingHandshakers:       make(map[string]*outgoinghandshaker.OutgoingHandshaker),
		incomingHandshakerResultC: make(chan incominghandshaker.Result),
		outgoingHandshakerResultC: make(chan outgoinghandshaker.Result),
		startCommandC:             make(chan struct{}),
		stopCommandC:              make(chan struct{}),
		announcerRequests:         make(chan *announcer.Request),
		dialLimit:                 semaphore.New(maxPeerDial),
		pieceDownloaders:          semaphore.New(parallelPieceDownloads),
		infoDownloaders:           semaphore.New(parallelPieceDownloads),
	}
	copy(d.peerID[:], peerIDPrefix)
	_, err := rand.Read(d.peerID[len(peerIDPrefix):]) // nolint: gosec
	if err != nil {
		return nil, err
	}
	d.trackersInstances, err = parseTrackers(d.trackers, d.log)
	if err != nil {
		return nil, err
	}
	go d.run()
	return d, nil
}

func parseTrackers(trackers []string, log logger.Logger) ([]tracker.Tracker, error) {
	var ret []tracker.Tracker
	for _, s := range trackers {
		u, err := url.Parse(s)
		if err != nil {
			log.Warningln("cannot parse tracker url:", err)
			continue
		}
		switch u.Scheme {
		case "http", "https":
			ret = append(ret, httptracker.New(u))
		case "udp":
			ret = append(ret, udptracker.New(u))
		default:
			log.Warningln("unsupported tracker scheme: %s", u.Scheme)
		}
	}
	if len(ret) == 0 {
		return nil, errors.New("no tracker found")
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package towerfall

import (
	"fmt"
	"sort"
	"time"
)

type CompleteSnapshot map[string]*Snapshot

type Snapshot struct {
	Person *Person         `json:"person"`
	Total  *PlayerSnapshot `json:"total"`
	Rank   int             `json:"rank"`
	// The Tournaments map contains the same data structure, but
	// separated per tournament
	Tournaments map[string]*PlayerSnapshot `json:"tournaments"`
}

// A PlayerSnapshot is a sum of all total stats about a player
type PlayerSnapshot struct {
	Shots    int           `json:"shots"`
	Sweeps   int           `json:"sweeps"`
	Kills    int           `json:"kills"`
	Self     int           `json:"self"`
	Matches  int           `json:"matches"`
	Rounds   int           `json:"rounds"`
	Score    int           `json:"score"`
	Playtime time.Duration `json:"playtime"`
	Wins     int           `json:"wins"`
}

// NewSnapshot returns a full snapshot
func NewSnapshot(s *Server) CompleteSnapshot {
	ss := make(map[string]*Snapshot)
	s.DB.LoadPeople()
	for _, p := range s.DB.People {
		ss[p.ID] = &Snapshot{
			Person:      p,
			Tournaments: make(map[string]*PlayerSnapshot),
		}
	}

	// Calculate the per-tournament data points
	for _, t := range s.DB.Tournaments {
		tid := t.ID
		for _, m := range t.Matches {
			for _, p := range m.Players {
				pid := p.Person.ID
				if _, ok := ss[pid]; !ok {
					fmt.Println("Snapshot not set for player", p.Person)
					ss[p.Person.ID] = &Snapshot{
						Person:      p.Person,
						Tournaments: make(map[string]*PlayerSnapshot),
					}
				}
				if _, ok := ss[pid].Tournaments[tid]; !ok {
					ss[pid].Tournaments[tid] = &PlayerSnapshot{}
				}

				ss[pid].Tournaments[tid].Matches++
				ss[pid].Tournaments[tid].Rounds += len(m.Rounds)
				ss[pid].Tournaments[tid].Shots += p.Shots
				ss[pid].Tournaments[tid].Sweeps += p.Sweeps
				ss[pid].Tournaments[tid].Kills += p.Kills
				ss[pid].Tournaments[tid].Self += p.Self
				ss[pid].Tournaments[tid].Score += p.Score()
				ss[pid].Tournaments[tid].Playtime += m.Duration()
			}

		}

		// Only do the winner calculations if someone actually won
		if !t.Ended.IsZero() && len(t.Winners) > 0 {
			winner := t.Winners[0].Person.ID
			ss[winner].Tournaments[tid].Wins++
		}
	}

	// Summarize the per-tournament data points into the totals
	for pid := range ss {
		if ss[pid].Total == nil {
			ss[pid].Total = &PlayerSnapshot{}
		}

		for tid := range ss[pid].Tournaments {
			t := ss[pid].Tournaments[tid]

			ss[pid].Total.Matches += t.Matches
			ss[pid].Total.Rounds += t.Rounds
			ss[pid].Total.Shots += t.Shots
			ss[pid].Total.Sweeps += t.Sweeps
			ss[pid].Total.Kills += t.Kills
			ss[pid].Total.Self += t.Self
			ss[pid].Total.Score += t.Score
			ss[pid].Total.Playtime += t.Playtime
		}
	}

	// Add the Rank attribute to all the players
	ranked := make([]*Snapshot, len(ss))
	x := 0
	for _, p := range ss {
		ranked[x] = p
		x++
	}
	for x, p := range SortByRank(ranked) {
		ss[p.Person.ID].Rank = x + 1 // The +1 fixes zero-index.
	}

	return CompleteSnapshot(ss)
}

// ByRank is a sort.Interface that sorts players by tournament wins
// and then by total score.
type ByRank []*Snapshot

func (b ByRank) Len() int {
	return len(b)
}

func (b ByRank) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

func (b ByRank) Less(i, j int) bool {
	// If a person is not actively playing, they are definitely less.
	if b[i].Person.Disabled != b[j].Person.Disabled {
		return b[j].Person.Disabled
	}

	if b[i].Total.Wins == b[j].Total.Wins {
		return b[i].Total.Score > b[j].Total.Score
	}

	return b[i].Total.Wins > b[j].Total.Wins
}

func SortByRank(ps []*Snapshot) []*Snapshot {
	sort.Sort(ByRank(ps))
	return ps
}
<commit_msg>If no person is set, skip stat collection<commit_after>package towerfall

import (
	"fmt"
	"sort"
	"time"
)

type CompleteSnapshot map[string]*Snapshot

type Snapshot struct {
	Person *Person         `json:"person"`
	Total  *PlayerSnapshot `json:"total"`
	Rank   int             `json:"rank"`
	// The Tournaments map contains the same data structure, but
	// separated per tournament
	Tournaments map[string]*PlayerSnapshot `json:"tournaments"`
}

// A PlayerSnapshot is a sum of all total stats about a player
type PlayerSnapshot struct {
	Shots    int           `json:"shots"`
	Sweeps   int           `json:"sweeps"`
	Kills    int           `json:"kills"`
	Self     int           `json:"self"`
	Matches  int           `json:"matches"`
	Rounds   int           `json:"rounds"`
	Score    int           `json:"score"`
	Playtime time.Duration `json:"playtime"`
	Wins     int           `json:"wins"`
}

// NewSnapshot returns a full snapshot
func NewSnapshot(s *Server) CompleteSnapshot {
	ss := make(map[string]*Snapshot)
	s.DB.LoadPeople()
	for _, p := range s.DB.People {
		ss[p.ID] = &Snapshot{
			Person:      p,
			Tournaments: make(map[string]*PlayerSnapshot),
		}
	}

	// Calculate the per-tournament data points
	for _, t := range s.DB.Tournaments {
		tid := t.ID
		for _, m := range t.Matches {
			for _, p := range m.Players {
				if p.Person == nil {
					continue
				}

				if _, ok := ss[pid]; !ok {
					fmt.Println("Snapshot not set for player", p.Person)
					ss[p.Person.ID] = &Snapshot{
						Person:      p.Person,
						Tournaments: make(map[string]*PlayerSnapshot),
					}
				}
				if _, ok := ss[pid].Tournaments[tid]; !ok {
					ss[pid].Tournaments[tid] = &PlayerSnapshot{}
				}

				ss[pid].Tournaments[tid].Matches++
				ss[pid].Tournaments[tid].Rounds += len(m.Rounds)
				ss[pid].Tournaments[tid].Shots += p.Shots
				ss[pid].Tournaments[tid].Sweeps += p.Sweeps
				ss[pid].Tournaments[tid].Kills += p.Kills
				ss[pid].Tournaments[tid].Self += p.Self
				ss[pid].Tournaments[tid].Score += p.Score()
				ss[pid].Tournaments[tid].Playtime += m.Duration()
			}

		}

		// Only do the winner calculations if someone actually won
		if !t.Ended.IsZero() && len(t.Winners) > 0 {
			winner := t.Winners[0].Person.ID
			ss[winner].Tournaments[tid].Wins++
		}
	}

	// Summarize the per-tournament data points into the totals
	for pid := range ss {
		if ss[pid].Total == nil {
			ss[pid].Total = &PlayerSnapshot{}
		}

		for tid := range ss[pid].Tournaments {
			t := ss[pid].Tournaments[tid]

			ss[pid].Total.Matches += t.Matches
			ss[pid].Total.Rounds += t.Rounds
			ss[pid].Total.Shots += t.Shots
			ss[pid].Total.Sweeps += t.Sweeps
			ss[pid].Total.Kills += t.Kills
			ss[pid].Total.Self += t.Self
			ss[pid].Total.Score += t.Score
			ss[pid].Total.Playtime += t.Playtime
		}
	}

	// Add the Rank attribute to all the players
	ranked := make([]*Snapshot, len(ss))
	x := 0
	for _, p := range ss {
		ranked[x] = p
		x++
	}
	for x, p := range SortByRank(ranked) {
		ss[p.Person.ID].Rank = x + 1 // The +1 fixes zero-index.
	}

	return CompleteSnapshot(ss)
}

// ByRank is a sort.Interface that sorts players by tournament wins
// and then by total score.
type ByRank []*Snapshot

func (b ByRank) Len() int {
	return len(b)
}

func (b ByRank) Swap(i, j int) {
	b[i], b[j] = b[j], b[i]
}

func (b ByRank) Less(i, j int) bool {
	// If a person is not actively playing, they are definitely less.
	if b[i].Person.Disabled != b[j].Person.Disabled {
		return b[j].Person.Disabled
	}

	if b[i].Total.Wins == b[j].Total.Wins {
		return b[i].Total.Score > b[j].Total.Score
	}

	return b[i].Total.Wins > b[j].Total.Wins
}

func SortByRank(ps []*Snapshot) []*Snapshot {
	sort.Sort(ByRank(ps))
	return ps
}
<|endoftext|>


<commit_before>package tracker

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
)

type TrackResult struct {
	TargetURL         string
	TargetText        string
	TargetStatusCode  int
	ResultStatusCode  int
	ResultTextMatched bool
}

func Perform(targetURL, targetText string, targetStatusCode int) (TrackResult, error) {
	trackResult := TrackResult{targetURL, targetText, targetStatusCode, 0, false}

	resp, err := http.Get(targetURL)
	if err != nil {
		return trackResult, err
	}

	trackResult.ResultStatusCode = resp.StatusCode

	if trackResult.TargetStatusCode == 0 {
		// TODO Log that status comparison was skipped because it was not set
	} else if trackResult.ResultStatusCode != targetStatusCode {
		return trackResult, fmt.Errorf("StatusCodeMatchError: Looked for (%d), but found (%d)", trackResult.TargetStatusCode, trackResult.ResultStatusCode)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return trackResult, err
	}

	match, err := regexp.MatchString(targetText, string(body))
	if err != nil {
		return trackResult, err
	}

	trackResult.ResultTextMatched = match

	if !trackResult.ResultTextMatched {
		return trackResult, fmt.Errorf("TextMatchError: Looked for (%s)", trackResult.TargetText)
	}

	return trackResult, nil
}
<commit_msg>Added timestamps to TrackResult struct<commit_after>package tracker

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"time"
)

type TrackResult struct {
	TargetURL         string
	TargetText        string
	TargetStatusCode  int
	ResultStatusCode  int
	ResultTextMatched bool
	Timestamp         time.Time
}

func Perform(targetURL, targetText string, targetStatusCode int) (TrackResult, error) {
	trackResult := TrackResult{targetURL, targetText, targetStatusCode, 0, false, time.Now()}

	resp, err := http.Get(targetURL)
	if err != nil {
		return trackResult, err
	}

	trackResult.ResultStatusCode = resp.StatusCode

	if trackResult.TargetStatusCode == 0 {
		// TODO Log that status comparison was skipped because it was not set
	} else if trackResult.ResultStatusCode != targetStatusCode {
		return trackResult, fmt.Errorf("StatusCodeMatchError: Looked for (%d), but found (%d)", trackResult.TargetStatusCode, trackResult.ResultStatusCode)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return trackResult, err
	}

	match, err := regexp.MatchString(targetText, string(body))
	if err != nil {
		return trackResult, err
	}

	trackResult.ResultTextMatched = match

	if !trackResult.ResultTextMatched {
		return trackResult, fmt.Errorf("TextMatchError: Looked for (%s)", trackResult.TargetText)
	}

	return trackResult, nil
}
<|endoftext|>


<commit_before>package tracker

import (
	"context"

	"../entities"
)

//NewClient returns new instance of tracker client
func NewClient() *Client {
	return &Client{}
}

//Client to tracker
type Client struct{}

//Project return project by id or err if not foind project
func (c *Client) Project(ctx context.Context, tr entities.Tracker, pid entities.ProjectID) (*entities.Project, error) {
	panic("not implemented")
}

//Projects return project list and total amount of projects
func (c *Client) Projects(ctx context.Context, tr entities.Tracker, p entities.Pagination) ([]entities.Project, int64, error) {
	panic("not implemented")
}

//ProjectIssues return issues assigned to user and total amount
func (c *Client) ProjectIssues(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, p entities.Pagination) ([]entities.Issue, int64, error) {
	panic("not implemented")
}

//UserInfo returns user info from tracker t
func (c *Client) UserInfo(ctx context.Context, tr entities.Tracker) (*entities.User, error) {
	panic("not implemented")
}

//Issue query issue by ID from tracker t or err if not found
func (c *Client) Issue(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, iid entities.IssueID) (*entities.Issue, error) {
	panic("not implemented")
}

//IssueByURL query issue by URL ar err if not found
func (c *Client) IssueByURL(ctx context.Context, tr entities.Tracker, url entities.IssueURL) (*entities.Issue, error) {
	panic("not implemented")
}

//CreateIssue for projectID and assign it to user
func (c *Client) CreateIssue(ctx context.Context, tr entities.Tracker, issue entities.NewIssue, pid entities.ProjectID) (*entities.Issue, error) {
	panic("not implemented")
}

//UpdateIssueProgress updates issue progress by issue id
func (c *Client) UpdateIssueProgress(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, is entities.IssueID, pr entities.Progress) error {
	panic("not implemented")
}

//TotalReports receive date as UNIX timestamp (seconds) and return total reported time at this day in seconds
func (c *Client) TotalReports(ctx context.Context, t entities.Tracker, date int64) (int64, error) {
	panic("not implemented")
}

//CreateReport for user
func (c *Client) CreateReport(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, rep entities.Report) error {
	panic("not implemented")
}
<commit_msg>Add logger<commit_after>package tracker

import (
	"context"

	"github.com/powerman/narada-go/narada"

	"../entities"
)

var log = narada.NewLog("tracker client: ")

//NewClient returns new instance of tracker client
func NewClient() *Client {
	return &Client{}
}

//Client to tracker
type Client struct{}

//Project return project by id or err if not foind project
func (c *Client) Project(ctx context.Context, tr entities.Tracker, pid entities.ProjectID) (*entities.Project, error) {
	panic("not implemented")
}

//Projects return project list and total amount of projects
func (c *Client) Projects(ctx context.Context, tr entities.Tracker, p entities.Pagination) ([]entities.Project, int64, error) {
	panic("not implemented")
}

//ProjectIssues return issues assigned to user and total amount
func (c *Client) ProjectIssues(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, p entities.Pagination) ([]entities.Issue, int64, error) {
	panic("not implemented")
}

//UserInfo returns user info from tracker t
func (c *Client) UserInfo(ctx context.Context, tr entities.Tracker) (*entities.User, error) {
	panic("not implemented")
}

//Issue query issue by ID from tracker t or err if not found
func (c *Client) Issue(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, iid entities.IssueID) (*entities.Issue, error) {
	panic("not implemented")
}

//IssueByURL query issue by URL ar err if not found
func (c *Client) IssueByURL(ctx context.Context, tr entities.Tracker, url entities.IssueURL) (*entities.Issue, error) {
	panic("not implemented")
}

//CreateIssue for projectID and assign it to user
func (c *Client) CreateIssue(ctx context.Context, tr entities.Tracker, issue entities.NewIssue, pid entities.ProjectID) (*entities.Issue, error) {
	panic("not implemented")
}

//UpdateIssueProgress updates issue progress by issue id
func (c *Client) UpdateIssueProgress(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, is entities.IssueID, pr entities.Progress) error {
	panic("not implemented")
}

//TotalReports receive date as UNIX timestamp (seconds) and return total reported time at this day in seconds
func (c *Client) TotalReports(ctx context.Context, t entities.Tracker, date int64) (int64, error) {
	panic("not implemented")
}

//CreateReport for user
func (c *Client) CreateReport(ctx context.Context, tr entities.Tracker, pid entities.ProjectID, rep entities.Report) error {
	panic("not implemented")
}
<|endoftext|>


<commit_before>package transfer

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"strings"

	"github.com/github/git-lfs/localstorage"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

type customAdapterWorkerContext struct {
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
}

type customAdapterInitRequest struct {
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}
type customAdapterInitResponse struct {
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterUploadRequest struct {
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path"`
	Action *api.LinkRelation `json:"action"`
}
type customAdapterUploadResponse struct {
	Oid   string           `json:"oid"`
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterDownloadRequest struct {
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Action *api.LinkRelation `json:"action"`
}
type customAdapterTransferResponse struct { // common between upload/download
	Oid   string           `json:"oid"`
	Path  string           `json:"path,omitempty"` // always blank for upload
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTerminateRequest struct {
	Complete bool `json:"complete"`
}
type customAdapterProgressResponse struct {
	Oid            string `json:"oid"`
	BytesSoFar     int64  `json:"bytesSoFar"`
	BytesSinceLast int    `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{cmd, outp, bufio.NewReader(outp), inp}

	// send initiate message
	op := "upload"
	if a.direction == Download {
		op = "download"
	}
	initReq := &customAdapterInitRequest{op, a.concurrent, a.originalConcurrency}
	var initResp customAdapterInitResponse
	err = a.exchangeMessage(ctx, initReq, &initResp)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}

	tracerx.Printf("xfer: %q for worker %d started OK", a.name, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// readResponse reads one of 1..N possible responses and populates the first one which
// was unmarshalled correctly. This allows us to listen for one of possibly many responses
// Returns the index of the item which was populated
func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext, possResps []interface{}) (int, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return 0, err
	}
	for i, resp := range possResps {
		if json.Unmarshal([]byte(line), resp) == nil {
			return i, nil
		}
	}
	return 0, fmt.Errorf("Response %q did not match any of possible responses %v", string(line), possResps)

}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req, resp interface{}) error {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return err
	}
	// Read response if needed
	if resp != nil {
		_, err = a.readResponse(ctx, []interface{}{resp})
		return err
	}
	return nil
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	termReq := &customAdapterTerminateRequest{true}
	err := a.exchangeMessage(ctx, termReq, nil)
	if err != nil {
		return err
	}
	ctx.stdin.Close()
	ctx.stdout.Close()
	return ctx.cmd.Wait()
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q, aborting: %v", a.name, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	var req interface{}
	if a.direction == Download {
		rel, ok := t.Object.Rel("download")
		if !ok {
			return errors.New("Object not found on the server.")
		}
		req = &customAdapterDownloadRequest{t.Object.Oid, t.Object.Size, rel}
	} else {
		rel, ok := t.Object.Rel("upload")
		if !ok {
			return errors.New("Object not found on the server.")
		}
		req = &customAdapterUploadRequest{t.Object.Oid, t.Object.Size, localstorage.Objects().ObjectPath(t.Object.Oid), rel}
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	possResps := []interface{}{&customAdapterProgressResponse{}, &customAdapterTransferResponse{}}
	var complete bool
	for !complete {
		respIdx, err := a.readResponse(customCtx, possResps)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch respIdx {
		case 0:
			// Progress
			prog := possResps[0].(customAdapterProgressResponse)
			if prog.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", prog.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, prog.BytesSoFar, prog.BytesSinceLast)
			}
			wasAuthOk = prog.BytesSoFar > 0
		case 1:
			// Download/Upload complete
			comp := possResps[0].(customAdapterTransferResponse)
			if comp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", comp.Oid, t.Object.Oid)
			}
			if comp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, comp.Error.Error())
			}
			wasAuthOk = true
			complete = true
		}
		// Call auth on first progress or success
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	// Send verify if successful upload
	if a.direction == Upload {
		return api.VerifyUpload(t.Object)
	}
	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		if match := pathRegex.FindStringSubmatch(k); match != nil {
			name := match[1]
			path := v
			var args string
			var concurrent bool
			var direction string
			// retrieve other values
			args, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
			concurrent = config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
			direction, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
			if len(direction) == 0 {
				direction = "both"
			} else {
				direction = strings.ToLower(direction)
			}

			// Separate closure for each since we need to capture vars above
			newfunc := func(name string, dir Direction) TransferAdapter {
				return newCustomAdapter(name, dir, path, args, concurrent)
			}

			if direction == "download" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Download, newfunc)
			}
			if direction == "upload" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Upload, newfunc)
			}

		}
	}

}

func init() {
	ConfigureCustomAdapters()
}
<commit_msg>Simplify request structs a little & reuse with omitempty instead<commit_after>package transfer

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os/exec"
	"regexp"
	"strings"

	"github.com/github/git-lfs/localstorage"

	"github.com/github/git-lfs/api"
	"github.com/github/git-lfs/subprocess"
	"github.com/rubyist/tracerx"

	"github.com/github/git-lfs/config"
)

// Adapter for custom transfer via external process
type customAdapter struct {
	*adapterBase
	path                string
	args                string
	concurrent          bool
	originalConcurrency int
}

type customAdapterWorkerContext struct {
	cmd         *exec.Cmd
	stdout      io.ReadCloser
	bufferedOut *bufio.Reader
	stdin       io.WriteCloser
}

type customAdapterInitRequest struct {
	Operation           string `json:"operation"`
	Concurrent          bool   `json:"concurrent"`
	ConcurrentTransfers int    `json:"concurrenttransfers"`
}
type customAdapterInitResponse struct {
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTransferRequest struct { // common between upload/download
	Oid    string            `json:"oid"`
	Size   int64             `json:"size"`
	Path   string            `json:"path,omitempty"`
	Action *api.LinkRelation `json:"action"`
}
type customAdapterTransferResponse struct { // common between upload/download
	Oid   string           `json:"oid"`
	Path  string           `json:"path,omitempty"` // always blank for upload
	Error *api.ObjectError `json:"error,omitempty"`
}
type customAdapterTerminateRequest struct {
	Complete bool `json:"complete"`
}
type customAdapterProgressResponse struct {
	Oid            string `json:"oid"`
	BytesSoFar     int64  `json:"bytesSoFar"`
	BytesSinceLast int    `json:"bytesSinceLast"`
}

func (a *customAdapter) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	// If config says not to launch multiple processes, downgrade incoming value
	useConcurrency := maxConcurrency
	if !a.concurrent {
		useConcurrency = 1
	}
	a.originalConcurrency = maxConcurrency

	tracerx.Printf("xfer: Custom transfer adapter %q using concurrency %d", a.name, useConcurrency)

	// Use common workers impl, but downgrade workers to number of processes
	return a.adapterBase.Begin(useConcurrency, cb, completion)
}

func (a *customAdapter) ClearTempStorage() error {
	// no action requred
	return nil
}

func (a *customAdapter) WorkerStarting(workerNum int) (interface{}, error) {

	// Start a process per worker
	// If concurrent = false we have already dialled back workers to 1
	tracerx.Printf("xfer: starting up custom transfer process %q for worker %d", a.name, workerNum)
	cmd := subprocess.ExecCommand(a.path, a.args)
	outp, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdout for custom transfer command %q remote: %v", a.path, err)
	}
	inp, err := cmd.StdinPipe()
	if err != nil {
		return nil, fmt.Errorf("Failed to get stdin for custom transfer command %q remote: %v", a.path, err)
	}
	err = cmd.Start()
	if err != nil {
		return nil, fmt.Errorf("Failed to start custom transfer command %q remote: %v", a.path, err)
	}
	// Set up buffered reader/writer since we operate on lines
	ctx := &customAdapterWorkerContext{cmd, outp, bufio.NewReader(outp), inp}

	// send initiate message
	initReq := &customAdapterInitRequest{a.getOperationName(), a.concurrent, a.originalConcurrency}
	var initResp customAdapterInitResponse
	err = a.exchangeMessage(ctx, initReq, &initResp)
	if err != nil {
		a.abortWorkerProcess(ctx)
		return nil, err
	}

	tracerx.Printf("xfer: %q for worker %d started OK", a.name, workerNum)

	// Save this process context and use in future callbacks
	return ctx, nil
}

func (a *customAdapter) getOperationName() string {
	if a.direction == Download {
		return "download"
	}
	return "upload"
}

// sendMessage sends a JSON message to the custom adapter process
func (a *customAdapter) sendMessage(ctx *customAdapterWorkerContext, req interface{}) error {
	b, err := json.Marshal(req)
	if err != nil {
		return err
	}
	// Line oriented JSON
	b = append(b, '\n')
	_, err = ctx.stdin.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// readResponse reads one of 1..N possible responses and populates the first one which
// was unmarshalled correctly. This allows us to listen for one of possibly many responses
// Returns the index of the item which was populated
func (a *customAdapter) readResponse(ctx *customAdapterWorkerContext, possResps []interface{}) (int, error) {
	line, err := ctx.bufferedOut.ReadString('\n')
	if err != nil {
		return 0, err
	}
	for i, resp := range possResps {
		if json.Unmarshal([]byte(line), resp) == nil {
			return i, nil
		}
	}
	return 0, fmt.Errorf("Response %q did not match any of possible responses %v", string(line), possResps)

}

// exchangeMessage sends a message to a process and reads a response if resp != nil
// Only fatal errors to communicate return an error, errors may be embedded in reply
func (a *customAdapter) exchangeMessage(ctx *customAdapterWorkerContext, req, resp interface{}) error {

	err := a.sendMessage(ctx, req)
	if err != nil {
		return err
	}
	// Read response if needed
	if resp != nil {
		_, err = a.readResponse(ctx, []interface{}{resp})
		return err
	}
	return nil
}

// shutdownWorkerProcess terminates gracefully a custom adapter process
// returns an error if it couldn't shut down gracefully (caller may abortWorkerProcess)
func (a *customAdapter) shutdownWorkerProcess(ctx *customAdapterWorkerContext) error {
	termReq := &customAdapterTerminateRequest{true}
	err := a.exchangeMessage(ctx, termReq, nil)
	if err != nil {
		return err
	}
	ctx.stdin.Close()
	ctx.stdout.Close()
	return ctx.cmd.Wait()
}

// abortWorkerProcess terminates & aborts untidily, most probably breakdown of comms or internal error
func (a *customAdapter) abortWorkerProcess(ctx *customAdapterWorkerContext) {
	ctx.stdin.Close()
	ctx.stdout.Close()
	ctx.cmd.Process.Kill()
}
func (a *customAdapter) WorkerEnding(workerNum int, ctx interface{}) {
	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		tracerx.Printf("Context object for custom transfer %q was of the wrong type", a.name)
		return
	}

	err := a.shutdownWorkerProcess(customCtx)
	if err != nil {
		tracerx.Printf("xfer: error finishing up custom transfer process %q, aborting: %v", a.name, err)
		a.abortWorkerProcess(customCtx)
	}
}

func (a *customAdapter) DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error {
	if ctx == nil {
		return fmt.Errorf("Custom transfer %q was not properly initialized, see previous errors", a.name)
	}

	customCtx, ok := ctx.(*customAdapterWorkerContext)
	if !ok {
		return fmt.Errorf("Context object for custom transfer %q was of the wrong type", a.name)
	}
	var authCalled bool

	rel, ok := t.Object.Rel(a.getOperationName())
	if !ok {
		return errors.New("Object not found on the server.")
	}
	req := &customAdapterTransferRequest{t.Object.Oid, t.Object.Size, "", rel}
	if a.direction == Upload {
		req.Path = localstorage.Objects().ObjectPath(t.Object.Oid)
	}
	err := a.sendMessage(customCtx, req)
	if err != nil {
		return err
	}

	// 1..N replies (including progress & one of download / upload)
	possResps := []interface{}{&customAdapterProgressResponse{}, &customAdapterTransferResponse{}}
	var complete bool
	for !complete {
		respIdx, err := a.readResponse(customCtx, possResps)
		if err != nil {
			return err
		}
		var wasAuthOk bool
		switch respIdx {
		case 0:
			// Progress
			prog := possResps[0].(customAdapterProgressResponse)
			if prog.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", prog.Oid, t.Object.Oid)
			}
			if cb != nil {
				cb(t.Name, t.Object.Size, prog.BytesSoFar, prog.BytesSinceLast)
			}
			wasAuthOk = prog.BytesSoFar > 0
		case 1:
			// Download/Upload complete
			comp := possResps[0].(customAdapterTransferResponse)
			if comp.Oid != t.Object.Oid {
				return fmt.Errorf("Unexpected oid %q in response, expecting %q", comp.Oid, t.Object.Oid)
			}
			if comp.Error != nil {
				return fmt.Errorf("Error transferring %q: %v", t.Object.Oid, comp.Error.Error())
			}
			wasAuthOk = true
			complete = true
		}
		// Call auth on first progress or success
		if wasAuthOk && authOkFunc != nil && !authCalled {
			authOkFunc()
			authCalled = true
		}
	}

	// Send verify if successful upload
	if a.direction == Upload {
		return api.VerifyUpload(t.Object)
	}
	return nil
}

func newCustomAdapter(name string, dir Direction, path, args string, concurrent bool) *customAdapter {
	c := &customAdapter{newAdapterBase(name, dir, nil), path, args, concurrent, 3}
	// self implements impl
	c.transferImpl = c
	return c
}

// Initialise custom adapters based on current config
func ConfigureCustomAdapters() {
	pathRegex := regexp.MustCompile(`lfs.customtransfer.([^.]+).path`)
	for k, v := range config.Config.AllGitConfig() {
		if match := pathRegex.FindStringSubmatch(k); match != nil {
			name := match[1]
			path := v
			var args string
			var concurrent bool
			var direction string
			// retrieve other values
			args, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.args", name))
			concurrent = config.Config.GitConfigBool(fmt.Sprintf("lfs.customtransfer.%s.concurrent", name), true)
			direction, _ = config.Config.GitConfig(fmt.Sprintf("lfs.customtransfer.%s.direction", name))
			if len(direction) == 0 {
				direction = "both"
			} else {
				direction = strings.ToLower(direction)
			}

			// Separate closure for each since we need to capture vars above
			newfunc := func(name string, dir Direction) TransferAdapter {
				return newCustomAdapter(name, dir, path, args, concurrent)
			}

			if direction == "download" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Download, newfunc)
			}
			if direction == "upload" || direction == "both" {
				RegisterNewTransferAdapterFunc(name, Upload, newfunc)
			}

		}
	}

}

func init() {
	ConfigureCustomAdapters()
}
<|endoftext|>


<commit_before>package twitter

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/gin-gonic/gin"
	"github.com/kurrik/oauth1a"
	"github.com/kurrik/twittergo"
	"net/http"
	"net/url"
	"os"
	"time"
)

type Twitter struct{}

var (
	TWITTER_COSUMER_KEY     = os.Getenv("TWITTER_CONSUMER_KEY")
	TWITTER_CONSUMER_SECRET = os.Getenv("TWITTER_CONSUMER_SECRET")
	REDISTOGO, _ = url.Parse(os.Getenv("REDISTOGO"))
)

func (tw Twitter) LoadCredentials() (client *twittergo.Client, err error) {
	config := &oauth1a.ClientConfig{
		ConsumerKey:    TWITTER_COSUMER_KEY,
		ConsumerSecret: TWITTER_CONSUMER_SECRET,
	}
	user := oauth1a.NewAuthorizedConfig("", "")
	client = twittergo.NewClient(config, user)
	return
}

func (tw Twitter) Memoize(resp map[string]string, tweetId uint64, key string, value string) map[string]string {
	conn, _ := redis.Dial("tcp", REDISTOGO.Host)
	cache, _ := redis.String(conn.Do("HGET", fmt.Sprintf("tweet:%v", tweetId), key))
    if cache == "" {
		conn.Do("HSET", fmt.Sprintf("tweet:%v", tweetId), key, value)
		cache = value
	}
	resp[key] = cache
	return resp
}

func (tw Twitter) TweetsFromResults(c *gin.Context, results *twittergo.SearchResults, t chan map[string]string, done chan bool) {
	conn, _ := redis.Dial("tcp", REDISTOGO.Host)
	fmt.Printf("start TweetsFromResults \n")

	for _, tweet := range results.Statuses() {

		user := tweet.User()
		entities := tweet["entities"].(map[string]interface{})
		urls := entities["urls"].([]interface{})

		if len(urls) > 0 {
			url := urls[0].(map[string]interface{})

			resp := make(map[string]string)

			reply, _ := redis.Values(conn.Do("KEYS", fmt.Sprintf("tweet:%v", tweet.Id())))
			fmt.Printf("values: %v", reply)
			if len(reply) == 0 {
				conn.Do("RPUSH", "tweets", tweet.Id())
			}

			resp = tw.Memoize(resp, tweet.Id(), "tweet", fmt.Sprintf("%v", tweet.Text()))
			resp = tw.Memoize(resp, tweet.Id(), "image", fmt.Sprintf("%v", url["expanded_url"]))
			resp = tw.Memoize(resp, tweet.Id(), "user", fmt.Sprintf("%v (@%v) ", user.Name(), user.ScreenName()))
			resp = tw.Memoize(resp, tweet.Id(), "created", fmt.Sprintf("%v", tweet.CreatedAt().Format(time.RFC1123)))
			t <- resp
		}
	}
	fmt.Printf("end TweetsFromResults \n")
}

func (tw Twitter) RetrieveMaxId() (string, bool){
	conn, _ := redis.Dial("tcp", REDISTOGO.Host)
	l, _ := redis.Int(conn.Do("LLEN", "tweets"))
	reply, _ := redis.String( conn.Do("LINDEX", "tweets", l-1) )
	if reply == "" {
		return reply, false
	} else {
		return reply, true
	}
}

func (tw Twitter) ConstructParams() url.Values {
	query := url.Values{}
	query.Set("q", "#beautifulsky")
	query.Set("result_type", "mixed")
	query.Set("count", "100")

	if maxId, present := tw.RetrieveMaxId(); present {
		fmt.Printf("maxId: %v", maxId)
		query.Set("max_id", maxId)
	}
	return query
}

func (tw Twitter) SearchTweets( k chan *twittergo.SearchResults, r chan *twittergo.APIResponse) {
	var (
		err     error
		results *twittergo.SearchResults
		req     *http.Request
		resp    *twittergo.APIResponse
		client *twittergo.Client
	)
	client, _ = tw.LoadCredentials()

	query := tw.ConstructParams()

	url := fmt.Sprintf("/1.1/search/tweets.json?%v", query.Encode())
	req, err = http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Printf("Could not parse request: %v\n", err)
	}

	resp, err = client.SendRequest(req)
	if err != nil {
		fmt.Printf("Could not send request: %v\n", err)
	}

	results = &twittergo.SearchResults{}
	err = resp.Parse(results)
	if err != nil {
		fmt.Printf("Problem parsing response: %v\n", err)
	}

	k <- results
	close(k)
	r <- resp
}

func (tw Twitter) TweetsFromCache(t chan map[string]string, cacheDone chan bool) {
    conn, _ := redis.Dial("tcp", REDISTOGO.Host)

	l, _ := redis.Int(conn.Do("LLEN", "tweets"))
	for i:=0; i< l; i++ {
		tweetId, _ := redis.String(conn.Do("LINDEX", "tweets", i))
		var resp map[string]string = make(map[string]string)
		for _, key := range []string{"tweet","image","user","created"} {
			reply,_ := redis.String( conn.Do("HGET", "tweet:"+tweetId, key) )
			resp[key] = reply
		}
		t <- resp
	}
	cacheDone <- true
}

func (tw Twitter) PrintRateLimit(r chan *twittergo.APIResponse) {
	fmt.Printf("start PrintRateLimit \n")
	resp := <-r
	if resp.HasRateLimit() {
		fmt.Printf("Rate limit:           %v\n", resp.RateLimit())
		fmt.Printf("Rate limit remaining: %v\n", resp.RateLimitRemaining())
		fmt.Printf("Rate limit reset:     %v\n", resp.RateLimitReset())
	} else {
		fmt.Printf("Could not parse rate limit from response.\n")
	}
	fmt.Printf("end PrintRateLimit \n")
}
<commit_msg>fix heroku url<commit_after>package twitter

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"github.com/gin-gonic/gin"
	"github.com/kurrik/oauth1a"
	"github.com/kurrik/twittergo"
	"net/http"
	"net/url"
	"os"
	"time"
)

type Twitter struct{}

var (
	TWITTER_COSUMER_KEY     = os.Getenv("TWITTER_CONSUMER_KEY")
	TWITTER_CONSUMER_SECRET = os.Getenv("TWITTER_CONSUMER_SECRET")
	REDISTOGO, _ = url.Parse(os.Getenv("REDISTOGO_URL"))
)

func (tw Twitter) LoadCredentials() (client *twittergo.Client, err error) {
	config := &oauth1a.ClientConfig{
		ConsumerKey:    TWITTER_COSUMER_KEY,
		ConsumerSecret: TWITTER_CONSUMER_SECRET,
	}
	user := oauth1a.NewAuthorizedConfig("", "")
	client = twittergo.NewClient(config, user)
	return
}

func (tw Twitter) Memoize(resp map[string]string, tweetId uint64, key string, value string) map[string]string {
	conn, _ := redis.Dial("tcp", REDISTOGO.Host)
	cache, _ := redis.String(conn.Do("HGET", fmt.Sprintf("tweet:%v", tweetId), key))
    if cache == "" {
		conn.Do("HSET", fmt.Sprintf("tweet:%v", tweetId), key, value)
		cache = value
	}
	resp[key] = cache
	return resp
}

func (tw Twitter) TweetsFromResults(c *gin.Context, results *twittergo.SearchResults, t chan map[string]string, done chan bool) {
	conn, _ := redis.Dial("tcp", REDISTOGO.Host)
	fmt.Printf("start TweetsFromResults \n")

	for _, tweet := range results.Statuses() {

		user := tweet.User()
		entities := tweet["entities"].(map[string]interface{})
		urls := entities["urls"].([]interface{})

		if len(urls) > 0 {
			url := urls[0].(map[string]interface{})

			resp := make(map[string]string)

			reply, _ := redis.Values(conn.Do("KEYS", fmt.Sprintf("tweet:%v", tweet.Id())))
			fmt.Printf("values: %v", reply)
			if len(reply) == 0 {
				conn.Do("RPUSH", "tweets", tweet.Id())
			}

			resp = tw.Memoize(resp, tweet.Id(), "tweet", fmt.Sprintf("%v", tweet.Text()))
			resp = tw.Memoize(resp, tweet.Id(), "image", fmt.Sprintf("%v", url["expanded_url"]))
			resp = tw.Memoize(resp, tweet.Id(), "user", fmt.Sprintf("%v (@%v) ", user.Name(), user.ScreenName()))
			resp = tw.Memoize(resp, tweet.Id(), "created", fmt.Sprintf("%v", tweet.CreatedAt().Format(time.RFC1123)))
			t <- resp
		}
	}
	fmt.Printf("end TweetsFromResults \n")
}

func (tw Twitter) RetrieveMaxId() (string, bool){
	conn, _ := redis.Dial("tcp", REDISTOGO.Host)
	l, _ := redis.Int(conn.Do("LLEN", "tweets"))
	reply, _ := redis.String( conn.Do("LINDEX", "tweets", l-1) )
	if reply == "" {
		return reply, false
	} else {
		return reply, true
	}
}

func (tw Twitter) ConstructParams() url.Values {
	query := url.Values{}
	query.Set("q", "#beautifulsky")
	query.Set("result_type", "mixed")
	query.Set("count", "100")

	if maxId, present := tw.RetrieveMaxId(); present {
		fmt.Printf("maxId: %v", maxId)
		query.Set("max_id", maxId)
	}
	return query
}

func (tw Twitter) SearchTweets( k chan *twittergo.SearchResults, r chan *twittergo.APIResponse) {
	var (
		err     error
		results *twittergo.SearchResults
		req     *http.Request
		resp    *twittergo.APIResponse
		client *twittergo.Client
	)
	client, _ = tw.LoadCredentials()

	query := tw.ConstructParams()

	url := fmt.Sprintf("/1.1/search/tweets.json?%v", query.Encode())
	req, err = http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Printf("Could not parse request: %v\n", err)
	}

	resp, err = client.SendRequest(req)
	if err != nil {
		fmt.Printf("Could not send request: %v\n", err)
	}

	results = &twittergo.SearchResults{}
	err = resp.Parse(results)
	if err != nil {
		fmt.Printf("Problem parsing response: %v\n", err)
	}

	k <- results
	close(k)
	r <- resp
}

func (tw Twitter) TweetsFromCache(t chan map[string]string, cacheDone chan bool) {
    conn, _ := redis.Dial("tcp", REDISTOGO.Host)

	l, _ := redis.Int(conn.Do("LLEN", "tweets"))
	for i:=0; i< l; i++ {
		tweetId, _ := redis.String(conn.Do("LINDEX", "tweets", i))
		var resp map[string]string = make(map[string]string)
		for _, key := range []string{"tweet","image","user","created"} {
			reply,_ := redis.String( conn.Do("HGET", "tweet:"+tweetId, key) )
			resp[key] = reply
		}
		t <- resp
	}
	cacheDone <- true
}

func (tw Twitter) PrintRateLimit(r chan *twittergo.APIResponse) {
	fmt.Printf("start PrintRateLimit \n")
	resp := <-r
	if resp.HasRateLimit() {
		fmt.Printf("Rate limit:           %v\n", resp.RateLimit())
		fmt.Printf("Rate limit remaining: %v\n", resp.RateLimitRemaining())
		fmt.Printf("Rate limit reset:     %v\n", resp.RateLimitReset())
	} else {
		fmt.Printf("Could not parse rate limit from response.\n")
	}
	fmt.Printf("end PrintRateLimit \n")
}
<|endoftext|>


<commit_before>package drouter

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	dockertypes "github.com/docker/engine-api/types"
	"github.com/vishvananda/netlink"
	"golang.org/x/net/context"
	"net"
	"strings"
)

type container struct {
	handle *netlink.Handle
}

func newContainerFromID(id string) (*container, error) {
	cjson, err := dockerClient.ContainerInspect(context.Background(), id)
	if err != nil {
		return nil, err
	}
	if !cjson.State.Running {
		return &container{}, nil
	}

	ch, err := netlinkHandleFromPid(cjson.State.Pid)
	if err != nil {
		return nil, err
	}

	return &container{handle: ch}, nil
}

//adds all known routes for provided container
func (c *container) addAllRoutes() error {
	//Loop through all static routes, ensure each one is installed in the container
	log.Info("Syncing static routes.")
	//add routes for all the static routes
	for _, sr := range staticRoutes {
		go c.addRoute(sr)
	}

	//Loop through all discovered networks, ensure each one is installed in the container
	//Unless it is covered by a static route already
	log.Info("Syncing discovered routes.")
	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
	if err != nil {
		log.Error("Failed to get my routes.")
		log.Error(err)
	}

Subnets:
	for _, r := range routes {
		if r.Gw != nil {
			continue
		}
		if localShortcut {
			if subnetEqualSubnet(r.Dst, networkID(p2p.network)) {
				continue
			}
		}
		for _, sr := range staticRoutes {
			if subnetContainsSubnet(sr, r.Dst) {
				log.Debugf("Skipping route %v covered by %v.", r.Dst, sr)
				break Subnets
			}
		}
		go c.addRoute(r.Dst)
	}

	return nil
}

func (c *container) addRoute(prefix *net.IPNet) {
	gateway, err := c.getPathIP()
	if err != nil {
		log.Error(err)
		return
	}

	if (prefix.IP.To4() == nil) != (gateway.To4() == nil) {
		// Dst is a different IP family
		return
	}

	route := &netlink.Route{
		Dst: prefix,
		Gw:  gateway,
	}

	log.Infof("Adding route to %v via %v.", prefix, gateway)
	err = c.handle.RouteAdd(route)
	if err != nil {
		if !strings.Contains(err.Error(), "file exists") {
			log.Error(err)
			return
		}
	}
}

func (c *container) delRoutesVia(sn *net.IPNet) {
	routes, err := c.getRoutes()
	if err != nil {
		log.Error("Failed to get container route table.")
		log.Error(err)
		return
	}

	//get all drouter ips
	ips, err := netlink.AddrList(nil, netlink.FAMILY_V4)
	if err != nil {
		log.Error("Failed to get drouter ip addresses.")
		log.Error(err)
		return
	}

	for _, r := range routes {
		if r.Dst == nil {
			continue
		}

		for _, ipaddr := range ips {
			if !r.Gw.Equal(ipaddr.IP) {
				continue
			}
			if !sn.Contains(r.Gw) {
				continue
			}
			err = c.handle.RouteDel(&r)
			if err != nil {
				log.Errorf("Failed to delete container route to %v via %v", r.Dst, r.Gw)
				continue
			}
		}
	}
}

func (c *container) delRoutes(prefix *net.IPNet) {
	//get all container routes
	routes, err := c.getRoutes()
	if err != nil {
		log.Error("Failed to get container route table.")
		log.Error(err)
		return
	}

	//get all drouter ips
	ips, err := netlink.AddrList(nil, netlink.FAMILY_V4)
	if err != nil {
		log.Error("Failed to get drouter ip addresses.")
		log.Error(err)
		return
	}

	for _, r := range routes {
		if r.Dst == nil {
			continue
		}
		if !subnetContainsSubnet(prefix, r.Dst) {
			continue
		}

		for _, ipaddr := range ips {
			if !r.Gw.Equal(ipaddr.IP) {
				continue
			}
			err := c.handle.RouteDel(&r)
			if err != nil {
				log.Errorf("Failed to delete container route to %v via %v", r.Dst, r.Gw)
				continue
			}
		}
	}
}

//sets the provided container's default route to the provided gateway
func (c *container) replaceGateway(gateway net.IP) error {
	log.Debugf("container.replaceGateway(%v)", gateway)

	var defr *netlink.Route
	//replace containers default gateway with drouter
	routes, err := c.getRoutes()
	if err != nil {
		return err
	}
	log.Debugf("container routes: %v", routes)
	for _, r := range routes {
		if r.Dst != nil {
			// Not the container gateway
			continue
		}

		defr = &r
	}

	//bail if the container gateway is already set to gateway
	if gateway.Equal(defr.Gw) {
		return nil
	}

	log.Debugf("Remove existing default route: %v", defr)
	err = c.handle.RouteDel(defr)
	if err != nil {
		return err
	}

	if gateway == nil || gateway.Equal(net.IP{}) {
		return nil
	}

	defr.Gw = gateway
	err = c.handle.RouteAdd(defr)
	if err != nil {
		return err
	}
	log.Debugf("Default route changed to: %v", defr)

	return nil
}

// called during a network connect event
func (c *container) connectEvent(drn *network) error {
	if !drn.isConnected() {
		drn.connect()
		//return now because the routeEvent will trigger routes to be installed in this container
		return nil
	}

	//let's push our routes into this new container
	gateway, err := c.getPathIP()
	if err != nil {
		log.Error("Failed to get container path IP.")
		return err
	}

	if localGateway {
		go c.replaceGateway(gateway)
	} else {
		go c.addAllRoutes()
	}
	return nil
}

// called during a network disconnect event
func (c *container) disconnectEvent(drn *network) error {

	for _, ic := range drn.IPAM.Config {
		subnet, err := netlink.ParseIPNet(ic.Subnet)
		if err != nil {
			log.Errorf("Failed to parse ipam config subnet: %v", ic.Subnet)
			log.Error(err)
			continue
		}
		c.delRoutesVia(subnet)
	}

	if _, err := c.getPathIP(); err == nil {
		c.addAllRoutes()
	}

	if aggressive {
		return nil
	}

	//if not aggressive mode, then we disconnect from the network if this is the last connected container

	//loop through all the containers
	dockerContainers, err := dockerClient.ContainerList(context.Background(), dockertypes.ContainerListOptions{})
	if err != nil {
		return err
	}
	for _, dc := range dockerContainers {
		if dc.HostConfig.NetworkMode == "host" {
			continue
		}
		if dc.ID == selfContainerID {
			continue
		}

		for n, _ := range dc.NetworkSettings.Networks {
			if n == drn.Name {
				// This netowork is still in use
				return nil
			}
		}
	}

	networkDisconnectWG.Add(len(drn.IPAM.Config))
	go drn.disconnect()

	return nil
}

//returns a drouter IP that is on some same network as provided container
func (c *container) getPathIP() (net.IP, error) {
	if c.handle == nil {
		return nil, fmt.Errorf("No namespace handle for container, is it running?")
	}

	addrs, err := c.handle.AddrList(nil, netlink.FAMILY_V4)
	if err != nil {
		log.Error("Failed to list container addresses.")
		return nil, err
	}

	for _, addr := range addrs {
		if addr.Label == "lo" {
			continue
		}

		log.Debugf("Getting my route to container IP: %v", addr.IP)
		srcRoutes, err := netlink.RouteGet(addr.IP)
		if err != nil {
			log.Error(err)
			continue
		}
		for _, srcRoute := range srcRoutes {
			if srcRoute.Gw == nil {
				return srcRoute.Src, nil
			}
		}
	}

	return nil, fmt.Errorf("No direct connection to container.")
}

func (c *container) getRoutes() ([]netlink.Route, error) {
	if c.handle != nil {
		return c.handle.RouteList(nil, netlink.FAMILY_ALL)
	}

	return make([]netlink.Route, 0), nil
}
<commit_msg>fix add route delete if exists and different<commit_after>package drouter

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	dockertypes "github.com/docker/engine-api/types"
	"github.com/vishvananda/netlink"
	"golang.org/x/net/context"
	"net"
	"strings"
)

type container struct {
	handle *netlink.Handle
}

func newContainerFromID(id string) (*container, error) {
	cjson, err := dockerClient.ContainerInspect(context.Background(), id)
	if err != nil {
		return nil, err
	}
	if !cjson.State.Running {
		return &container{}, nil
	}

	ch, err := netlinkHandleFromPid(cjson.State.Pid)
	if err != nil {
		return nil, err
	}

	return &container{handle: ch}, nil
}

//adds all known routes for provided container
func (c *container) addAllRoutes() error {
	//Loop through all static routes, ensure each one is installed in the container
	log.Info("Syncing static routes.")
	//add routes for all the static routes
	for _, sr := range staticRoutes {
		go c.addRoute(sr)
	}

	//Loop through all discovered networks, ensure each one is installed in the container
	//Unless it is covered by a static route already
	log.Info("Syncing discovered routes.")
	routes, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
	if err != nil {
		log.Error("Failed to get my routes.")
		log.Error(err)
	}

Subnets:
	for _, r := range routes {
		if r.Gw != nil {
			continue
		}
		if localShortcut {
			if subnetEqualSubnet(r.Dst, networkID(p2p.network)) {
				continue
			}
		}
		for _, sr := range staticRoutes {
			if subnetContainsSubnet(sr, r.Dst) {
				log.Debugf("Skipping route %v covered by %v.", r.Dst, sr)
				break Subnets
			}
		}
		go c.addRoute(r.Dst)
	}

	return nil
}

func (c *container) addRoute(prefix *net.IPNet) {
	gateway, err := c.getPathIP()
	if err != nil {
		log.Error(err)
		return
	}

	if (prefix.IP.To4() == nil) != (gateway.To4() == nil) {
		// Dst is a different IP family
		return
	}

	route := &netlink.Route{
		Dst: prefix,
		Gw:  gateway,
	}

	log.Infof("Adding route to %v via %v.", prefix, gateway)
	err = c.handle.RouteAdd(route)
	if err != nil {
		if !strings.Contains(err.Error(), "file exists") {
			log.Error(err)
			return
		}
		routes, err2 := c.handle.RouteGet(prefix.IP)
		if err2 != nil {
			log.Errorf("Failed to get container routes to: %v.", prefix.IP)
			log.Error(err)
			return
		}
		for _, r := range routes {
			if r.Gw.Equal(gateway) {
				return
			}
			err := c.handle.RouteDel(&netlink.Route{Dst: prefix, Gw: r.Gw})
			if err != nil {
				log.Errorf("Failed to delete route to %v via %v.", prefix, r.Gw)
				log.Error(err)
				return
			}
		}
		c.addRoute(prefix)
	}
}

func (c *container) delRoutesVia(sn *net.IPNet) {
	routes, err := c.getRoutes()
	if err != nil {
		log.Error("Failed to get container route table.")
		log.Error(err)
		return
	}

	//get all drouter ips
	ips, err := netlink.AddrList(nil, netlink.FAMILY_V4)
	if err != nil {
		log.Error("Failed to get drouter ip addresses.")
		log.Error(err)
		return
	}

	for _, r := range routes {
		if r.Dst == nil {
			continue
		}

		for _, ipaddr := range ips {
			if !r.Gw.Equal(ipaddr.IP) {
				continue
			}
			if !sn.Contains(r.Gw) {
				continue
			}
			err = c.handle.RouteDel(&r)
			if err != nil {
				log.Errorf("Failed to delete container route to %v via %v", r.Dst, r.Gw)
				continue
			}
		}
	}
}

func (c *container) delRoutes(prefix *net.IPNet) {
	//get all container routes
	routes, err := c.getRoutes()
	if err != nil {
		log.Error("Failed to get container route table.")
		log.Error(err)
		return
	}

	//get all drouter ips
	ips, err := netlink.AddrList(nil, netlink.FAMILY_V4)
	if err != nil {
		log.Error("Failed to get drouter ip addresses.")
		log.Error(err)
		return
	}

	for _, r := range routes {
		if r.Dst == nil {
			continue
		}
		if !subnetContainsSubnet(prefix, r.Dst) {
			continue
		}

		for _, ipaddr := range ips {
			if !r.Gw.Equal(ipaddr.IP) {
				continue
			}
			err := c.handle.RouteDel(&r)
			if err != nil {
				log.Errorf("Failed to delete container route to %v via %v", r.Dst, r.Gw)
				continue
			}
		}
	}
}

//sets the provided container's default route to the provided gateway
func (c *container) replaceGateway(gateway net.IP) error {
	log.Debugf("container.replaceGateway(%v)", gateway)

	var defr *netlink.Route
	//replace containers default gateway with drouter
	routes, err := c.getRoutes()
	if err != nil {
		return err
	}
	log.Debugf("container routes: %v", routes)
	for _, r := range routes {
		if r.Dst != nil {
			// Not the container gateway
			continue
		}

		defr = &r
	}

	//bail if the container gateway is already set to gateway
	if gateway.Equal(defr.Gw) {
		return nil
	}

	log.Debugf("Remove existing default route: %v", defr)
	err = c.handle.RouteDel(defr)
	if err != nil {
		return err
	}

	if gateway == nil || gateway.Equal(net.IP{}) {
		return nil
	}

	defr.Gw = gateway
	err = c.handle.RouteAdd(defr)
	if err != nil {
		return err
	}
	log.Debugf("Default route changed to: %v", defr)

	return nil
}

// called during a network connect event
func (c *container) connectEvent(drn *network) error {
	if !drn.isConnected() {
		drn.connect()
		//return now because the routeEvent will trigger routes to be installed in this container
		return nil
	}

	//let's push our routes into this new container
	gateway, err := c.getPathIP()
	if err != nil {
		log.Error("Failed to get container path IP.")
		return err
	}

	if localGateway {
		go c.replaceGateway(gateway)
	} else {
		go c.addAllRoutes()
	}
	return nil
}

// called during a network disconnect event
func (c *container) disconnectEvent(drn *network) error {

	for _, ic := range drn.IPAM.Config {
		subnet, err := netlink.ParseIPNet(ic.Subnet)
		if err != nil {
			log.Errorf("Failed to parse ipam config subnet: %v", ic.Subnet)
			log.Error(err)
			continue
		}
		c.delRoutesVia(subnet)
	}

	if _, err := c.getPathIP(); err == nil {
		c.addAllRoutes()
	}

	if aggressive {
		return nil
	}

	//if not aggressive mode, then we disconnect from the network if this is the last connected container

	//loop through all the containers
	dockerContainers, err := dockerClient.ContainerList(context.Background(), dockertypes.ContainerListOptions{})
	if err != nil {
		return err
	}
	for _, dc := range dockerContainers {
		if dc.HostConfig.NetworkMode == "host" {
			continue
		}
		if dc.ID == selfContainerID {
			continue
		}

		for n, _ := range dc.NetworkSettings.Networks {
			if n == drn.Name {
				// This netowork is still in use
				return nil
			}
		}
	}

	networkDisconnectWG.Add(len(drn.IPAM.Config))
	go drn.disconnect()

	return nil
}

//returns a drouter IP that is on some same network as provided container
func (c *container) getPathIP() (net.IP, error) {
	if c.handle == nil {
		return nil, fmt.Errorf("No namespace handle for container, is it running?")
	}

	addrs, err := c.handle.AddrList(nil, netlink.FAMILY_V4)
	if err != nil {
		log.Error("Failed to list container addresses.")
		return nil, err
	}

	for _, addr := range addrs {
		if addr.Label == "lo" {
			continue
		}

		log.Debugf("Getting my route to container IP: %v", addr.IP)
		srcRoutes, err := netlink.RouteGet(addr.IP)
		if err != nil {
			log.Error(err)
			continue
		}
		for _, srcRoute := range srcRoutes {
			if srcRoute.Gw == nil {
				return srcRoute.Src, nil
			}
		}
	}

	return nil, fmt.Errorf("No direct connection to container.")
}

func (c *container) getRoutes() ([]netlink.Route, error) {
	if c.handle != nil {
		return c.handle.RouteList(nil, netlink.FAMILY_ALL)
	}

	return make([]netlink.Route, 0), nil
}
<|endoftext|>


<commit_before>package main

import (
	"../../dynago"

	"code.google.com/p/gcfg"
	"github.com/gobs/cmd"
	"github.com/gobs/pretty"

	"fmt"
	"log"
	"os"
	"path"
	"strings"
)

const (
	CONFIG_FILE  = ".dynagorc"
	HISTORY_FILE = ".dynago_history"
)

// the configuration should look like the following
// (with multiple profiles and a selected one)
//
// [dynago]
// profile=xxx
//
// [profile "xxx"]
// region=us-west-1
// accessKey=XXXXXXXX
// secretKey=YYYYYYYY

type Config struct {
	Dynago struct {
		// define default profile
		Profile string
	}

	// list of named profiles
	Profile map[string]*struct {
		Region    string
		AccessKey string
		SecretKey string
	}
}

func ReadConfig(configFile string, config *Config) {
	// configFile in current directory or full path
	if _, err := os.Stat(configFile); err != nil {
		if strings.Contains(configFile, "/") {
			return
		}

		// configFile in home directory
		configFile = path.Join(os.Getenv("HOME"), configFile)
		if _, err := os.Stat(configFile); err != nil {
			return
		}
	}

	err := gcfg.ReadFileInto(config, configFile)
	if err != nil {
		log.Fatal(err)
	}
}

func CompletionFunction(text string, line string, start, stop int) []string {
	return nil
}

func main() {
	var config Config
	ReadConfig(CONFIG_FILE, &config)

	selected := config.Dynago.Profile

	if len(os.Args) > 1 {
		// there is at least one parameter:
		// override the selected profile
		selected = os.Args[1]
	}

	profile := config.Profile[selected]
	if profile == nil {
		log.Fatal("no profile selected")
	}

	db := dynago.NewDBClient()

	if len(profile.Region) > 0 {
		db.WithRegion(profile.Region)
	}

	if len(profile.AccessKey) > 0 {
		db.WithCredentials(profile.AccessKey, profile.SecretKey)
	}

	commander := &cmd.Cmd{HistoryFile: HISTORY_FILE, Complete: CompletionFunction, EnableShell: true}
	commander.Prompt = "dynagosh> "
	commander.Init()

	commander.Add(cmd.Command{"config",
		`
		config : display current configuration
		`,
		func(string) (stop bool) {
			pretty.PrettyPrint(config)
			return
		}})

	commander.Add(cmd.Command{"list",
		`
                list : display list of available tables
                `,
		func(string) (stop bool) {
			tables, err := db.ListTables()

			if err != nil {
				fmt.Println(err)
				return
			}

			fmt.Println("Available tables")

			for _, tableName := range tables {
				fmt.Println("  ", tableName)
			}

			return
		}})

	commander.Add(cmd.Command{"describe",
		`
                describe {table} : display table configuration
                `,
		func(line string) (stop bool) {
			tableName := line
			table, err := db.DescribeTable(tableName)
			if err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
			}

			return
		}})

	commander.Commands["ls"] = commander.Commands["list"]

	commander.CmdLoop()
}
<commit_msg>Adding completion on table names<commit_after>package main

import (
	"../../dynago"

	"code.google.com/p/gcfg"
	"github.com/gobs/cmd"
	"github.com/gobs/pretty"

	"fmt"
	"log"
	"os"
	"path"
	"strings"
)

const (
	CONFIG_FILE  = ".dynagorc"
	HISTORY_FILE = ".dynago_history"
)

// the configuration should look like the following
// (with multiple profiles and a selected one)
//
// [dynago]
// profile=xxx
//
// [profile "xxx"]
// region=us-west-1
// accessKey=XXXXXXXX
// secretKey=YYYYYYYY

type Config struct {
	Dynago struct {
		// define default profile
		Profile string
	}

	// list of named profiles
	Profile map[string]*struct {
		Region    string
		AccessKey string
		SecretKey string
	}
}

func ReadConfig(configFile string, config *Config) {
	// configFile in current directory or full path
	if _, err := os.Stat(configFile); err != nil {
		if strings.Contains(configFile, "/") {
			return
		}

		// configFile in home directory
		configFile = path.Join(os.Getenv("HOME"), configFile)
		if _, err := os.Stat(configFile); err != nil {
			return
		}
	}

	err := gcfg.ReadFileInto(config, configFile)
	if err != nil {
		log.Fatal(err)
	}
}

var (
	table_list []string
)

func CompletionFunction(text string, line string, start, stop int) []string {
	if len(table_list) > 0 {
	        matches := make([]string, 0, len(table_list))

		for _, w := range table_list {
			if strings.HasPrefix(w, text) {
				matches = append(matches, w)
			}
		}

		return matches
	}

	return nil
}

func main() {
	var config Config
	ReadConfig(CONFIG_FILE, &config)

	selected := config.Dynago.Profile

	if len(os.Args) > 1 {
		// there is at least one parameter:
		// override the selected profile
		selected = os.Args[1]
	}

	profile := config.Profile[selected]
	if profile == nil {
		log.Fatal("no profile selected")
	}

	db := dynago.NewDBClient()

	if len(profile.Region) > 0 {
		db.WithRegion(profile.Region)
	}

	if len(profile.AccessKey) > 0 {
		db.WithCredentials(profile.AccessKey, profile.SecretKey)
	}

	commander := &cmd.Cmd{HistoryFile: HISTORY_FILE, Complete: CompletionFunction, EnableShell: true}
	commander.Prompt = "dynagosh> "
	commander.Init()

	commander.Add(cmd.Command{"config",
		`
		config : display current configuration
		`,
		func(string) (stop bool) {
			pretty.PrettyPrint(config)
			return
		}})

	commander.Add(cmd.Command{"list",
		`
                list : display list of available tables
                `,
		func(string) (stop bool) {
			tables, err := db.ListTables()

			if err != nil {
				fmt.Println(err)
				return
			}

			fmt.Println("Available tables")

			for _, tableName := range tables {
				fmt.Println("  ", tableName)
			}

			if len(tables) > 0 {
				table_list = tables
			}

			return
		}})

	commander.Add(cmd.Command{"describe",
		`
                describe {table} : display table configuration
                `,
		func(line string) (stop bool) {
			tableName := line
			table, err := db.DescribeTable(tableName)
			if err != nil {
				fmt.Println(err)
			} else {
				pretty.PrettyPrint(table)
			}

			return
		}})

	commander.Commands["ls"] = commander.Commands["list"]

	commander.CmdLoop()
}
<|endoftext|>