<commit_before>package eth

import (
	"container/list"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethdb-go"
	"github.com/ethereum/ethutil-go"
	"github.com/ethereum/ethwire-go"
	"log"
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"
)

func eachPeer(peers *list.List, callback func(*Peer, *list.Element)) {
	// Loop thru the peers and close them (if we had them)
	for e := peers.Front(); e != nil; e = e.Next() {
		if peer, ok := e.Value.(*Peer); ok {
			callback(peer, e)
		}
	}
}

const (
	processReapingTimeout = 60 // TODO increase
)

type Ethereum struct {
	// Channel for shutting down the ethereum
	shutdownChan chan bool
	// DB interface
	//db *ethdb.LDBDatabase
	db *ethdb.MemDatabase
	// Block manager for processing new blocks and managing the block chain
	BlockManager *ethchain.BlockManager
	// The transaction pool. Transaction can be pushed on this pool
	// for later including in the blocks
	TxPool *ethchain.TxPool
	// Peers (NYI)
	peers *list.List
	// Nonce
	Nonce uint64

	Addr net.Addr

	nat NAT

	peerMut sync.Mutex

	// Capabilities for outgoing peers
	serverCaps Caps
}

func New(caps Caps) (*Ethereum, error) {
	//db, err := ethdb.NewLDBDatabase()
	db, err := ethdb.NewMemDatabase()
	if err != nil {
		return nil, err
	}

	ethutil.Config.Db = db

	nat, err := Discover()
	if err != nil {
		log.Printf("Can't discover upnp: %v", err)
	}
	log.Println(nat)

	nonce, _ := ethutil.RandomUint64()
	ethereum := &Ethereum{
		shutdownChan: make(chan bool),
		db:           db,
		peers:        list.New(),
		Nonce:        nonce,
		serverCaps:   caps,
		nat:          nat,
	}
	ethereum.TxPool = ethchain.NewTxPool()
	ethereum.TxPool.Speaker = ethereum
	ethereum.BlockManager = ethchain.NewBlockManager(ethereum)

	ethereum.TxPool.BlockManager = ethereum.BlockManager
	ethereum.BlockManager.TransactionPool = ethereum.TxPool

	return ethereum, nil
}

func (s *Ethereum) AddPeer(conn net.Conn) {
	peer := NewPeer(conn, s, true)

	if peer != nil {
		s.peers.PushBack(peer)
		peer.Start(false)
	}
}

func (s *Ethereum) ProcessPeerList(addrs []string) {
	for _, addr := range addrs {
		// TODO Probably requires some sanity checks
		s.ConnectToPeer(addr)
	}
}

func (s *Ethereum) ConnectToPeer(addr string) error {
	var alreadyConnected bool

	eachPeer(s.peers, func(p *Peer, v *list.Element) {
		if p.conn == nil {
			return
		}
		phost, _, _ := net.SplitHostPort(p.conn.RemoteAddr().String())
		ahost, _, _ := net.SplitHostPort(addr)

		if phost == ahost {
			alreadyConnected = true
			return
		}
	})

	if alreadyConnected {
		return nil
	}

	peer := NewOutboundPeer(addr, s, s.serverCaps)

	s.peers.PushBack(peer)

	return nil
}

func (s *Ethereum) OutboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	outboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if !p.inbound && p.conn != nil {
			outboundPeers[length] = p
			length++
		}
	})

	return outboundPeers[:length]
}

func (s *Ethereum) InboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if p.inbound {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) InOutPeers() []*Peer {
	// Reap the dead peers first
	s.reapPeers()

	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		// Only return peers with an actual ip
		if len(p.host) > 0 {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) Broadcast(msgType ethwire.MsgType, data []interface{}) {
	msg := ethwire.NewMessage(msgType, data)
	s.BroadcastMsg(msg)
}

func (s *Ethereum) BroadcastMsg(msg *ethwire.Msg) {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.QueueMessage(msg)
	})
}

func (s *Ethereum) Peers() *list.List {
	return s.peers
}

func (s *Ethereum) reapPeers() {
	s.peerMut.Lock()
	defer s.peerMut.Unlock()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if atomic.LoadInt32(&p.disconnect) == 1 || (p.inbound && (time.Now().Unix()-p.lastPong) > int64(5*time.Minute)) {
			s.peers.Remove(e)
		}
	})
}

func (s *Ethereum) ReapDeadPeerHandler() {
	reapTimer := time.NewTicker(processReapingTimeout * time.Second)

	for {
		select {
		case <-reapTimer.C:
			s.reapPeers()
		}
	}
}

// FIXME
func (s *Ethereum) upnpUpdateThread() {
	// Go off immediately to prevent code duplication, thereafter we renew
	// lease every 15 minutes.
	timer := time.NewTimer(0 * time.Second)
	lport, _ := strconv.ParseInt("30303", 10, 16)
	first := true
out:
	for {
		select {
		case <-timer.C:
			listenPort, err := s.nat.AddPortMapping("TCP", int(lport), int(lport), "eth listen port", 20*60)
			if err != nil {
				log.Printf("can't add UPnP port mapping: %v\n", err)
			}
			if first && err == nil {
				externalip, err := s.nat.GetExternalAddress()
				if err != nil {
					log.Printf("UPnP can't get external address: %v\n", err)
					continue out
				}
				// externalip, listenport
				log.Println("Successfully bound via UPnP to", externalip, listenPort)
				first = false
			}
			timer.Reset(time.Minute * 15)
		case <-s.shutdownChan:
			break out
		}
	}

	timer.Stop()

	if err := s.nat.DeletePortMapping("tcp", int(lport), int(lport)); err != nil {
		log.Printf("unable to remove UPnP port mapping: %v\n", err)
	} else {
		log.Printf("succesfully disestablished UPnP port mapping\n")
	}
}

// Start the ethereum
func (s *Ethereum) Start() {
	// Bind to addr and port
	ln, err := net.Listen("tcp", ":30303")
	if err != nil {
		log.Println("Connection listening disabled. Acting as client")
	} else {
		s.Addr = ln.Addr()
		// Starting accepting connections
		log.Println("Ready and accepting connections")
		// Start the peer handler
		go s.peerHandler(ln)
	}

	go s.upnpUpdateThread()

	// Start the reaping processes
	go s.ReapDeadPeerHandler()

	// Start the tx pool
	s.TxPool.Start()
}

func (s *Ethereum) peerHandler(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println(err)

			continue
		}

		go s.AddPeer(conn)
	}
}

func (s *Ethereum) Stop() {
	// Close the database
	defer s.db.Close()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.Stop()
	})

	s.shutdownChan <- true

	s.TxPool.Stop()
}

// This function will wait for a shutdown and resumes main thread execution
func (s *Ethereum) WaitForShutdown() {
	<-s.shutdownChan
}
<commit_msg>removed upnp<commit_after>package eth

import (
	"container/list"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethdb-go"
	"github.com/ethereum/ethutil-go"
	"github.com/ethereum/ethwire-go"
	"log"
	"net"
	"sync"
	"sync/atomic"
	"time"
)

func eachPeer(peers *list.List, callback func(*Peer, *list.Element)) {
	// Loop thru the peers and close them (if we had them)
	for e := peers.Front(); e != nil; e = e.Next() {
		if peer, ok := e.Value.(*Peer); ok {
			callback(peer, e)
		}
	}
}

const (
	processReapingTimeout = 60 // TODO increase
)

type Ethereum struct {
	// Channel for shutting down the ethereum
	shutdownChan chan bool
	// DB interface
	//db *ethdb.LDBDatabase
	db *ethdb.MemDatabase
	// Block manager for processing new blocks and managing the block chain
	BlockManager *ethchain.BlockManager
	// The transaction pool. Transaction can be pushed on this pool
	// for later including in the blocks
	TxPool *ethchain.TxPool
	// Peers (NYI)
	peers *list.List
	// Nonce
	Nonce uint64

	Addr net.Addr

	peerMut sync.Mutex

	// Capabilities for outgoing peers
	serverCaps Caps
}

func New(caps Caps) (*Ethereum, error) {
	//db, err := ethdb.NewLDBDatabase()
	db, err := ethdb.NewMemDatabase()
	if err != nil {
		return nil, err
	}

	ethutil.Config.Db = db

	nonce, _ := ethutil.RandomUint64()
	ethereum := &Ethereum{
		shutdownChan: make(chan bool),
		db:           db,
		peers:        list.New(),
		Nonce:        nonce,
		serverCaps:   caps,
	}
	ethereum.TxPool = ethchain.NewTxPool()
	ethereum.TxPool.Speaker = ethereum
	ethereum.BlockManager = ethchain.NewBlockManager(ethereum)

	ethereum.TxPool.BlockManager = ethereum.BlockManager
	ethereum.BlockManager.TransactionPool = ethereum.TxPool

	return ethereum, nil
}

func (s *Ethereum) AddPeer(conn net.Conn) {
	peer := NewPeer(conn, s, true)

	if peer != nil {
		s.peers.PushBack(peer)
		peer.Start(false)
	}
}

func (s *Ethereum) ProcessPeerList(addrs []string) {
	for _, addr := range addrs {
		// TODO Probably requires some sanity checks
		s.ConnectToPeer(addr)
	}
}

func (s *Ethereum) ConnectToPeer(addr string) error {
	var alreadyConnected bool

	eachPeer(s.peers, func(p *Peer, v *list.Element) {
		if p.conn == nil {
			return
		}
		phost, _, _ := net.SplitHostPort(p.conn.RemoteAddr().String())
		ahost, _, _ := net.SplitHostPort(addr)

		if phost == ahost {
			alreadyConnected = true
			return
		}
	})

	if alreadyConnected {
		return nil
	}

	peer := NewOutboundPeer(addr, s, s.serverCaps)

	s.peers.PushBack(peer)

	return nil
}

func (s *Ethereum) OutboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	outboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if !p.inbound && p.conn != nil {
			outboundPeers[length] = p
			length++
		}
	})

	return outboundPeers[:length]
}

func (s *Ethereum) InboundPeers() []*Peer {
	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if p.inbound {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) InOutPeers() []*Peer {
	// Reap the dead peers first
	s.reapPeers()

	// Create a new peer slice with at least the length of the total peers
	inboundPeers := make([]*Peer, s.peers.Len())
	length := 0
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		// Only return peers with an actual ip
		if len(p.host) > 0 {
			inboundPeers[length] = p
			length++
		}
	})

	return inboundPeers[:length]
}

func (s *Ethereum) Broadcast(msgType ethwire.MsgType, data []interface{}) {
	msg := ethwire.NewMessage(msgType, data)
	s.BroadcastMsg(msg)
}

func (s *Ethereum) BroadcastMsg(msg *ethwire.Msg) {
	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.QueueMessage(msg)
	})
}

func (s *Ethereum) Peers() *list.List {
	return s.peers
}

func (s *Ethereum) reapPeers() {
	s.peerMut.Lock()
	defer s.peerMut.Unlock()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		if atomic.LoadInt32(&p.disconnect) == 1 || (p.inbound && (time.Now().Unix()-p.lastPong) > int64(5*time.Minute)) {
			s.peers.Remove(e)
		}
	})
}

func (s *Ethereum) ReapDeadPeerHandler() {
	reapTimer := time.NewTicker(processReapingTimeout * time.Second)

	for {
		select {
		case <-reapTimer.C:
			s.reapPeers()
		}
	}
}

// Start the ethereum
func (s *Ethereum) Start() {
	// Bind to addr and port
	ln, err := net.Listen("tcp", ":30303")
	if err != nil {
		log.Println("Connection listening disabled. Acting as client")
	} else {
		s.Addr = ln.Addr()
		// Starting accepting connections
		log.Println("Ready and accepting connections")
		// Start the peer handler
		go s.peerHandler(ln)
	}

	// Start the reaping processes
	go s.ReapDeadPeerHandler()

	// Start the tx pool
	s.TxPool.Start()
}

func (s *Ethereum) peerHandler(listener net.Listener) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println(err)

			continue
		}

		go s.AddPeer(conn)
	}
}

func (s *Ethereum) Stop() {
	// Close the database
	defer s.db.Close()

	eachPeer(s.peers, func(p *Peer, e *list.Element) {
		p.Stop()
	})

	s.shutdownChan <- true

	s.TxPool.Stop()
}

// This function will wait for a shutdown and resumes main thread execution
func (s *Ethereum) WaitForShutdown() {
	<-s.shutdownChan
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethutil-go"
	"github.com/obscuren/secp256k1-go"
	"log"
	"os"
	"os/signal"
	"runtime"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)

			s.Stop()
		}
	}()
}

func CreateKeyPair(force bool) {
	data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
	if len(data) == 0 || force {
		log.Println("Generating new address and keypair")

		pub, prv := secp256k1.GenerateKeyPair()

		log.Printf("Your new address is %x\n", ethutil.Sha3Bin(pub)[12:])

		ethutil.Config.Db.Put([]byte("KeyRing"), ethutil.Encode([]interface{}{prv, ethutil.Sha3Bin(pub)[12:]}))
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	Init()

	ethchain.InitFees()
	ethutil.ReadConfig()

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}

	if GenAddr {
		fmt.Println("This action overwrites your old private key. Are you sure? (y/n)")

		var r string
		fmt.Scanln(&r)
		for ; ; fmt.Scanln(&r) {
			if r == "n" || r == "y" {
				break
			} else {
				fmt.Println("Yes or no?", r)
			}
		}

		if r == "y" {
			CreateKeyPair(true)
		}
		os.Exit(0)
	} else {
		CreateKeyPair(false)
	}

	if ShowGenesis {
		fmt.Println(ethereum.BlockManager.BlockChain().Genesis())
		os.Exit(0)
	}

	log.Printf("Starting Ethereum v%s\n", ethutil.Config.Ver)

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}

	RegisterInterupts(ethereum)

	ethereum.Start()

	if StartMining {
		log.Printf("Dev Test Mining started...\n")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {
			pow := &ethchain.EasyPow{}
			data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
			keyRing := ethutil.NewValueFromBytes(data)
			addr := keyRing.Get(1).Bytes()

			for {
				txs := ethereum.TxPool.Flush()
				// Create a new block which we're going to mine
				block := ethereum.BlockManager.BlockChain().NewBlock(addr, txs)
				// Apply all transactions to the block
				ethereum.BlockManager.ApplyTransactions(block, block.Transactions())

				ethereum.BlockManager.AccumelateRewards(block, block)

				// Search the nonce
				block.Nonce = pow.Search(block)
				err := ethereum.BlockManager.ProcessBlock(block)
				if err != nil {
					log.Println(err)
				} else {
					log.Println("\n+++++++ MINED BLK +++++++\n", ethereum.BlockManager.BlockChain().CurrentBlock)
				}
			}
		}()
	}

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<commit_msg>Added pub key to keyring<commit_after>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/ethchain-go"
	"github.com/ethereum/ethutil-go"
	"github.com/obscuren/secp256k1-go"
	"log"
	"os"
	"os/signal"
	"runtime"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)

			s.Stop()
		}
	}()
}

func CreateKeyPair(force bool) {
	data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
	if len(data) == 0 || force {
		log.Println("Generating new address and keypair")

		pub, prv := secp256k1.GenerateKeyPair()
		addr := ethutil.Sha3Bin(pub)[12:]

		log.Printf("Your new address is %x\n", addr)
		log.Printf("Your new pubkey is %x (%d)\n", pub, len(pub))

		keyRing := ethutil.NewValue([]interface{}{prv, addr, pub})
		ethutil.Config.Db.Put([]byte("KeyRing"), keyRing.Encode())
	}
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	Init()

	ethchain.InitFees()
	ethutil.ReadConfig()

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}

	if GenAddr {
		fmt.Println("This action overwrites your old private key. Are you sure? (y/n)")

		var r string
		fmt.Scanln(&r)
		for ; ; fmt.Scanln(&r) {
			if r == "n" || r == "y" {
				break
			} else {
				fmt.Println("Yes or no?", r)
			}
		}

		if r == "y" {
			CreateKeyPair(true)
		}
		os.Exit(0)
	} else {
		CreateKeyPair(false)
	}

	if ShowGenesis {
		fmt.Println(ethereum.BlockManager.BlockChain().Genesis())
		os.Exit(0)
	}

	log.Printf("Starting Ethereum v%s\n", ethutil.Config.Ver)

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}

	RegisterInterupts(ethereum)

	ethereum.Start()

	if StartMining {
		log.Printf("Miner started\n")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {
			pow := &ethchain.EasyPow{}
			data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
			keyRing := ethutil.NewValueFromBytes(data)
			addr := keyRing.Get(1).Bytes()

			for {
				txs := ethereum.TxPool.Flush()
				// Create a new block which we're going to mine
				block := ethereum.BlockManager.BlockChain().NewBlock(addr, txs)
				// Apply all transactions to the block
				ethereum.BlockManager.ApplyTransactions(block, block.Transactions())

				ethereum.BlockManager.AccumelateRewards(block, block)

				// Search the nonce
				block.Nonce = pow.Search(block)
				err := ethereum.BlockManager.ProcessBlock(block)
				if err != nil {
					log.Println(err)
				} else {
					log.Println("\n+++++++ MINED BLK +++++++\n", ethereum.BlockManager.BlockChain().CurrentBlock)
				}
			}
		}()
	}

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<|endoftext|>


<commit_before>package interfaces

import (
	"fmt"
	"strconv"
	"time"

	"github.com/marcusolsson/goddd/application"
	"github.com/marcusolsson/goddd/domain/cargo"
	"github.com/marcusolsson/goddd/domain/location"
	"github.com/marcusolsson/goddd/domain/voyage"
)

type cargoDTO struct {
	TrackingID           string     `json:"trackingID"`
	StatusText           string     `json:"statusText"`
	Origin               string     `json:"origin"`
	Destination          string     `json:"destination"`
	ETA                  string     `json:"eta"`
	NextExpectedActivity string     `json:"nextExpectedActivity"`
	Misrouted            bool       `json:"misrouted"`
	Routed               bool       `json:"routed"`
	ArrivalDeadline      string     `json:"arrivalDeadline"`
	Events               []eventDTO `json:"events"`
	Legs                 []legDTO   `json:"legs"`
}

type locationDTO struct {
	UNLocode string `json:"locode"`
	Name     string `json:"name"`
}

type RouteCandidateDTO struct {
	Legs []legDTO `json:"legs"`
}

type legDTO struct {
	VoyageNumber string    `json:"voyageNumber"`
	From         string    `json:"from"`
	To           string    `json:"to"`
	LoadTime     time.Time `json:"loadTime"`
	UnloadTime   time.Time `json:"unloadTime"`
}

type eventDTO struct {
	Description string `json:"description"`
	Expected    bool   `json:"expected"`
}

type BookingServiceFacade interface {
	BookNewCargo(origin, destination string, arrivalDeadline string) (string, error)
	LoadCargoForRouting(trackingID string) (cargoDTO, error)
	AssignCargoToRoute(trackingID string, candidate RouteCandidateDTO) error
	ChangeDestination(trackingID string, destinationUNLocode string) error
	RequestRoutesForCargo(trackingID string) []RouteCandidateDTO
	ListShippingLocations() []locationDTO
	ListAllCargos() []cargoDTO
}

type bookingServiceFacade struct {
	cargoRepository         cargo.Repository
	locationRepository      location.Repository
	handlingEventRepository cargo.HandlingEventRepository
	bookingService          application.BookingService
}

func (f *bookingServiceFacade) BookNewCargo(origin, destination string, arrivalDeadline string) (string, error) {
	millis, _ := strconv.ParseInt(fmt.Sprintf("%s", arrivalDeadline), 10, 64)
	trackingID, err := f.bookingService.BookNewCargo(location.UNLocode(origin), location.UNLocode(destination), time.Unix(millis/1000, 0))

	return string(trackingID), err
}

func (f *bookingServiceFacade) LoadCargoForRouting(trackingID string) (cargoDTO, error) {
	c, err := f.cargoRepository.Find(cargo.TrackingID(trackingID))

	if err != nil {
		return cargoDTO{}, err
	}

	return assemble(c, f.handlingEventRepository), nil
}

func (f *bookingServiceFacade) AssignCargoToRoute(trackingID string, candidate RouteCandidateDTO) error {
	var legs []cargo.Leg
	for _, l := range candidate.Legs {
		legs = append(legs, cargo.Leg{
			VoyageNumber:   voyage.Number(l.VoyageNumber),
			LoadLocation:   location.UNLocode(l.From),
			UnloadLocation: location.UNLocode(l.To),
		})
	}

	return f.bookingService.AssignCargoToRoute(cargo.Itinerary{Legs: legs}, cargo.TrackingID(trackingID))
}

func (f *bookingServiceFacade) ChangeDestination(trackingID string, destinationUNLocode string) error {
	return f.bookingService.ChangeDestination(cargo.TrackingID(trackingID), location.UNLocode(destinationUNLocode))
}

func (f *bookingServiceFacade) RequestRoutesForCargo(trackingID string) []RouteCandidateDTO {
	itineraries := f.bookingService.RequestPossibleRoutesForCargo(cargo.TrackingID(trackingID))

	var candidates []RouteCandidateDTO
	for _, itin := range itineraries {
		var legs []legDTO
		for _, leg := range itin.Legs {
			legs = append(legs, legDTO{
				VoyageNumber: string(leg.VoyageNumber),
				From:         string(leg.LoadLocation),
				To:           string(leg.UnloadLocation),
				LoadTime:     leg.LoadTime,
				UnloadTime:   leg.UnloadTime,
			})
		}
		candidates = append(candidates, RouteCandidateDTO{Legs: legs})
	}

	return candidates
}

func (f *bookingServiceFacade) ListShippingLocations() []locationDTO {
	locations := f.locationRepository.FindAll()

	dtos := make([]locationDTO, len(locations))
	for i, loc := range locations {
		dtos[i] = locationDTO{
			UNLocode: string(loc.UNLocode),
			Name:     loc.Name,
		}
	}

	return dtos
}

func (f *bookingServiceFacade) ListAllCargos() []cargoDTO {
	cargos := f.cargoRepository.FindAll()
	dtos := make([]cargoDTO, len(cargos))

	for i, c := range cargos {
		dtos[i] = assemble(c, f.handlingEventRepository)
	}

	return dtos
}

func NewBookingServiceFacade(cargoRepository cargo.Repository, locationRepository location.Repository, handlingEventRepository cargo.HandlingEventRepository, bookingService application.BookingService) BookingServiceFacade {
	return &bookingServiceFacade{cargoRepository, locationRepository, handlingEventRepository, bookingService}
}

func assemble(c cargo.Cargo, her cargo.HandlingEventRepository) cargoDTO {
	return cargoDTO{
		TrackingID:           string(c.TrackingID),
		Origin:               string(c.Origin),
		Destination:          string(c.RouteSpecification.Destination),
		ETA:                  c.Delivery.ETA.Format(time.RFC3339),
		NextExpectedActivity: "",
		Misrouted:            c.Delivery.RoutingStatus == cargo.Misrouted,
		Routed:               !c.Itinerary.IsEmpty(),
		ArrivalDeadline:      c.RouteSpecification.ArrivalDeadline.Format(time.RFC3339),
		StatusText:           assembleStatusText(c),
		Legs:                 assembleLegs(c),
		Events:               assembleEvents(c, her),
	}
}

func assembleStatusText(c cargo.Cargo) string {
	switch c.Delivery.TransportStatus {
	case cargo.NotReceived:
		return "Not received"
	case cargo.InPort:
		return fmt.Sprintf("In port %s", c.Delivery.LastKnownLocation)
	case cargo.OnboardCarrier:
		return fmt.Sprintf("Onboard voyage %s", c.Delivery.CurrentVoyage)
	case cargo.Claimed:
		return "Claimed"
	default:
		return "Unknown"
	}
}

func assembleLegs(c cargo.Cargo) []legDTO {
	var legs []legDTO
	for _, l := range c.Itinerary.Legs {
		legs = append(legs, legDTO{
			VoyageNumber: string(l.VoyageNumber),
			From:         string(l.LoadLocation),
			To:           string(l.UnloadLocation),
		})
	}
	return legs
}

func assembleEvents(c cargo.Cargo, r cargo.HandlingEventRepository) []eventDTO {
	h := r.QueryHandlingHistory(c.TrackingID)
	events := make([]eventDTO, len(h.HandlingEvents))
	for i, e := range h.HandlingEvents {
		var description string

		switch e.Activity.Type {
		case cargo.NotHandled:
			description = "Cargo has not yet been received."
		case cargo.Receive:
			description = fmt.Sprintf("Received in %s, at %s", e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Load:
			description = fmt.Sprintf("Loaded onto voyage %s in %s, at %s.", e.Activity.VoyageNumber, e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Unload:
			description = fmt.Sprintf("Unloaded off voyage %s in %s, at %s.", e.Activity.VoyageNumber, e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Claim:
			description = fmt.Sprintf("Claimed in %s, at %s.", e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Customs:
			description = fmt.Sprintf("Cleared customs in %s, at %s.", e.Activity.Location, time.Now().Format(time.RFC3339))
		default:
			description = "[Unknown status]"
		}

		events[i] = eventDTO{Description: description, Expected: c.Itinerary.IsExpected(e)}
	}

	return events
}
<commit_msg>Fix field name.<commit_after>package interfaces

import (
	"fmt"
	"strconv"
	"time"

	"github.com/marcusolsson/goddd/application"
	"github.com/marcusolsson/goddd/domain/cargo"
	"github.com/marcusolsson/goddd/domain/location"
	"github.com/marcusolsson/goddd/domain/voyage"
)

type cargoDTO struct {
	TrackingID           string     `json:"trackingId"`
	StatusText           string     `json:"statusText"`
	Origin               string     `json:"origin"`
	Destination          string     `json:"destination"`
	ETA                  string     `json:"eta"`
	NextExpectedActivity string     `json:"nextExpectedActivity"`
	Misrouted            bool       `json:"misrouted"`
	Routed               bool       `json:"routed"`
	ArrivalDeadline      string     `json:"arrivalDeadline"`
	Events               []eventDTO `json:"events"`
	Legs                 []legDTO   `json:"legs"`
}

type locationDTO struct {
	UNLocode string `json:"locode"`
	Name     string `json:"name"`
}

type RouteCandidateDTO struct {
	Legs []legDTO `json:"legs"`
}

type legDTO struct {
	VoyageNumber string    `json:"voyageNumber"`
	From         string    `json:"from"`
	To           string    `json:"to"`
	LoadTime     time.Time `json:"loadTime"`
	UnloadTime   time.Time `json:"unloadTime"`
}

type eventDTO struct {
	Description string `json:"description"`
	Expected    bool   `json:"expected"`
}

type BookingServiceFacade interface {
	BookNewCargo(origin, destination string, arrivalDeadline string) (string, error)
	LoadCargoForRouting(trackingID string) (cargoDTO, error)
	AssignCargoToRoute(trackingID string, candidate RouteCandidateDTO) error
	ChangeDestination(trackingID string, destinationUNLocode string) error
	RequestRoutesForCargo(trackingID string) []RouteCandidateDTO
	ListShippingLocations() []locationDTO
	ListAllCargos() []cargoDTO
}

type bookingServiceFacade struct {
	cargoRepository         cargo.Repository
	locationRepository      location.Repository
	handlingEventRepository cargo.HandlingEventRepository
	bookingService          application.BookingService
}

func (f *bookingServiceFacade) BookNewCargo(origin, destination string, arrivalDeadline string) (string, error) {
	millis, _ := strconv.ParseInt(fmt.Sprintf("%s", arrivalDeadline), 10, 64)
	trackingID, err := f.bookingService.BookNewCargo(location.UNLocode(origin), location.UNLocode(destination), time.Unix(millis/1000, 0))

	return string(trackingID), err
}

func (f *bookingServiceFacade) LoadCargoForRouting(trackingID string) (cargoDTO, error) {
	c, err := f.cargoRepository.Find(cargo.TrackingID(trackingID))

	if err != nil {
		return cargoDTO{}, err
	}

	return assemble(c, f.handlingEventRepository), nil
}

func (f *bookingServiceFacade) AssignCargoToRoute(trackingID string, candidate RouteCandidateDTO) error {
	var legs []cargo.Leg
	for _, l := range candidate.Legs {
		legs = append(legs, cargo.Leg{
			VoyageNumber:   voyage.Number(l.VoyageNumber),
			LoadLocation:   location.UNLocode(l.From),
			UnloadLocation: location.UNLocode(l.To),
		})
	}

	return f.bookingService.AssignCargoToRoute(cargo.Itinerary{Legs: legs}, cargo.TrackingID(trackingID))
}

func (f *bookingServiceFacade) ChangeDestination(trackingID string, destinationUNLocode string) error {
	return f.bookingService.ChangeDestination(cargo.TrackingID(trackingID), location.UNLocode(destinationUNLocode))
}

func (f *bookingServiceFacade) RequestRoutesForCargo(trackingID string) []RouteCandidateDTO {
	itineraries := f.bookingService.RequestPossibleRoutesForCargo(cargo.TrackingID(trackingID))

	var candidates []RouteCandidateDTO
	for _, itin := range itineraries {
		var legs []legDTO
		for _, leg := range itin.Legs {
			legs = append(legs, legDTO{
				VoyageNumber: string(leg.VoyageNumber),
				From:         string(leg.LoadLocation),
				To:           string(leg.UnloadLocation),
				LoadTime:     leg.LoadTime,
				UnloadTime:   leg.UnloadTime,
			})
		}
		candidates = append(candidates, RouteCandidateDTO{Legs: legs})
	}

	return candidates
}

func (f *bookingServiceFacade) ListShippingLocations() []locationDTO {
	locations := f.locationRepository.FindAll()

	dtos := make([]locationDTO, len(locations))
	for i, loc := range locations {
		dtos[i] = locationDTO{
			UNLocode: string(loc.UNLocode),
			Name:     loc.Name,
		}
	}

	return dtos
}

func (f *bookingServiceFacade) ListAllCargos() []cargoDTO {
	cargos := f.cargoRepository.FindAll()
	dtos := make([]cargoDTO, len(cargos))

	for i, c := range cargos {
		dtos[i] = assemble(c, f.handlingEventRepository)
	}

	return dtos
}

func NewBookingServiceFacade(cargoRepository cargo.Repository, locationRepository location.Repository, handlingEventRepository cargo.HandlingEventRepository, bookingService application.BookingService) BookingServiceFacade {
	return &bookingServiceFacade{cargoRepository, locationRepository, handlingEventRepository, bookingService}
}

func assemble(c cargo.Cargo, her cargo.HandlingEventRepository) cargoDTO {
	return cargoDTO{
		TrackingID:           string(c.TrackingID),
		Origin:               string(c.Origin),
		Destination:          string(c.RouteSpecification.Destination),
		ETA:                  c.Delivery.ETA.Format(time.RFC3339),
		NextExpectedActivity: "",
		Misrouted:            c.Delivery.RoutingStatus == cargo.Misrouted,
		Routed:               !c.Itinerary.IsEmpty(),
		ArrivalDeadline:      c.RouteSpecification.ArrivalDeadline.Format(time.RFC3339),
		StatusText:           assembleStatusText(c),
		Legs:                 assembleLegs(c),
		Events:               assembleEvents(c, her),
	}
}

func assembleStatusText(c cargo.Cargo) string {
	switch c.Delivery.TransportStatus {
	case cargo.NotReceived:
		return "Not received"
	case cargo.InPort:
		return fmt.Sprintf("In port %s", c.Delivery.LastKnownLocation)
	case cargo.OnboardCarrier:
		return fmt.Sprintf("Onboard voyage %s", c.Delivery.CurrentVoyage)
	case cargo.Claimed:
		return "Claimed"
	default:
		return "Unknown"
	}
}

func assembleLegs(c cargo.Cargo) []legDTO {
	var legs []legDTO
	for _, l := range c.Itinerary.Legs {
		legs = append(legs, legDTO{
			VoyageNumber: string(l.VoyageNumber),
			From:         string(l.LoadLocation),
			To:           string(l.UnloadLocation),
		})
	}
	return legs
}

func assembleEvents(c cargo.Cargo, r cargo.HandlingEventRepository) []eventDTO {
	h := r.QueryHandlingHistory(c.TrackingID)
	events := make([]eventDTO, len(h.HandlingEvents))
	for i, e := range h.HandlingEvents {
		var description string

		switch e.Activity.Type {
		case cargo.NotHandled:
			description = "Cargo has not yet been received."
		case cargo.Receive:
			description = fmt.Sprintf("Received in %s, at %s", e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Load:
			description = fmt.Sprintf("Loaded onto voyage %s in %s, at %s.", e.Activity.VoyageNumber, e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Unload:
			description = fmt.Sprintf("Unloaded off voyage %s in %s, at %s.", e.Activity.VoyageNumber, e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Claim:
			description = fmt.Sprintf("Claimed in %s, at %s.", e.Activity.Location, time.Now().Format(time.RFC3339))
		case cargo.Customs:
			description = fmt.Sprintf("Cleared customs in %s, at %s.", e.Activity.Location, time.Now().Format(time.RFC3339))
		default:
			description = "[Unknown status]"
		}

		events[i] = eventDTO{Description: description, Expected: c.Itinerary.IsExpected(e)}
	}

	return events
}
<|endoftext|>


<commit_before>package sexp

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"unicode"
)

type panicReader struct {
	sr *strings.Reader
}

func (pr panicReader) readRune() (c rune) {
	c, _, err := pr.sr.ReadRune()
	if err != nil {
		panic(err)
	}
	return
}

func Parse(data []byte) (exp interface{}, rest []byte, err error) {
	defer func() {
		if x := recover(); x != nil {
			if err, _ = x.(error); err == nil {
				panic(x)
			}
		}
	}()

	exp, rest = ParsePanic(data)
	return
}

func ParsePanic(data []byte) (list []interface{}, rest []byte) {
	sr := strings.NewReader(string(data))

	inComment := false

	for {
		c, _, err := sr.ReadRune()
		if err != nil {
			if err == io.EOF {
				return
			}
			panic(err)
		}

		if inComment {
			if c == '\n' {
				inComment = false
			}
		} else {
			if c == ';' {
				inComment = true
			} else if !unicode.IsSpace(c) {
				if err := sr.UnreadRune(); err != nil {
					panic(err)
				}
				break
			}
		}
	}

	pr := panicReader{sr}

	for {
		exp, ok, _ := parse(pr)
		if ok {
			list = exp.([]interface{})

			var err error
			rest, err = ioutil.ReadAll(sr)
			if err != nil {
				panic(err)
			}

			return
		}
	}
}

func parse(pr panicReader) (exp interface{}, ok, end bool) {
	var c rune

	for {
		c = pr.readRune()
		if !unicode.IsSpace(c) {
			break
		}
	}

	switch {
	case c == ';':
		skipComment(pr)

	case c == '(':
		exp = parseList(pr)
		ok = true

	case c == ')':
		end = true

	case c == '"':
		exp, end = parseString(pr)
		ok = true

	default:
		exp, end = parseToken(pr, c)
		ok = true
	}

	return
}

func parseList(pr panicReader) interface{} {
	var list []interface{}

	for {
		item, ok, end := parse(pr)
		if ok {
			list = append(list, item)
		}
		if end {
			break
		}
	}

	return list
}

func parseString(pr panicReader) (s string, end bool) {
	var buf []rune

	for {
		c := pr.readRune()

		if c == '"' {
			break
		}

		if c == '\\' {
			c = pr.readRune()

			switch c {
			case '"', '\\':

			case 'n':
				c = '\n'

			case 'r':
				c = '\r'

			case '0':
				c = 0

			default:
				panic(fmt.Errorf("'\\%c' in string literal not handled", c))
			}
		}

		buf = append(buf, c)
	}

	c := pr.readRune()
	switch {
	case c == ')':
		end = true

	case unicode.IsSpace(c):

	default:
		panic(errors.New("trailing data after string literal"))
	}

	s = string(buf)
	return
}

func parseToken(pr panicReader, c rune) (s string, end bool) {
	buf := []rune{c}

	for {
		c := pr.readRune()

		if c == ')' {
			end = true
			break
		}

		if unicode.IsSpace(c) {
			break
		}

		buf = append(buf, c)
	}

	s = string(buf)
	return
}

func skipComment(pr panicReader) {
	for {
		c := pr.readRune()
		if c == '\n' {
			break
		}
	}
}
<commit_msg>sexp refactoring<commit_after>package sexp

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"unicode"
)

type panicReader struct {
	sr *strings.Reader
}

func (pr panicReader) readRune() (c rune) {
	c, _, err := pr.sr.ReadRune()
	if err != nil {
		panic(err)
	}
	return
}

func (pr panicReader) unreadRune() {
	if err := pr.sr.UnreadRune(); err != nil {
		panic(err)
	}
}

func Parse(data []byte) (exp interface{}, rest []byte, err error) {
	defer func() {
		if x := recover(); x != nil {
			if err, _ = x.(error); err == nil {
				panic(x)
			}
		}
	}()

	exp, rest = ParsePanic(data)
	return
}

func ParsePanic(data []byte) (list []interface{}, rest []byte) {
	sr := strings.NewReader(string(data))
	pr := panicReader{sr}

	inComment := false

	for {
		c, _, err := sr.ReadRune()
		if err != nil {
			if err == io.EOF {
				return
			}
			panic(err)
		}

		if inComment {
			if c == '\n' {
				inComment = false
			}
		} else {
			if c == ';' {
				inComment = true
			} else if !unicode.IsSpace(c) {
				pr.unreadRune()
				break
			}
		}
	}

	for {
		exp, ok, _ := parse(pr)
		if ok {
			list = exp.([]interface{})

			var err error
			rest, err = ioutil.ReadAll(sr)
			if err != nil {
				panic(err)
			}

			return
		}
	}
}

func parse(pr panicReader) (x interface{}, ok, end bool) {
	for {
		if !unicode.IsSpace(pr.readRune()) {
			pr.unreadRune()
			break
		}
	}

	switch pr.readRune() {
	case ';':
		skipComment(pr)

	case '(':
		x = parseList(pr)
		ok = true

	case ')':
		end = true

	case '"':
		x = parseString(pr)
		ok = true

	default:
		pr.unreadRune()
		x = parseToken(pr)
		ok = true
	}

	return
}

func parseList(pr panicReader) interface{} {
	var list []interface{}

	for {
		item, ok, end := parse(pr)
		if ok {
			list = append(list, item)
		}
		if end {
			break
		}
	}

	return list
}

func parseString(pr panicReader) string {
	var buf []rune

	for {
		c := pr.readRune()

		if c == ';' {
			skipComment(pr)
			continue
		}

		if c == '"' {
			break
		}

		if c == '\\' {
			c = pr.readRune()

			switch c {
			case '"', '\\':

			case 'n':
				c = '\n'

			case 'r':
				c = '\r'

			case '0':
				c = 0

			default:
				panic(fmt.Errorf("'\\%c' in string literal not handled", c))
			}
		}

		buf = append(buf, c)
	}

	c := pr.readRune()

	switch {
	case c == ')':
		pr.unreadRune()

	case unicode.IsSpace(c):

	default:
		panic(errors.New("trailing data after string literal"))
	}

	return string(buf)
}

func parseToken(pr panicReader) string {
	var buf []rune

	for {
		c := pr.readRune()

		if c == ')' {
			pr.unreadRune()
			break
		}

		if c == '"' {
			panic(errors.New("invalid character inside token: '\"'"))
		}

		if unicode.IsSpace(c) {
			break
		}

		buf = append(buf, c)
	}

	return string(buf)
}

func skipComment(pr panicReader) {
	for pr.readRune() != '\n' {
	}

	pr.unreadRune()
}
<|endoftext|>


<commit_before>// Package hadapt provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package hadapt

import (
	"fmt"
	"math"
)

const (
	bufferInitSize   = 200
	bufferGrowFactor = 2
)

// Grid is the interface that an sparse grid should satisfy in order to be used
// in the algorithm.
type Grid interface {
	Dimensionality() uint16
	ComputeNodes(levels []uint8, orders []uint32) []float64
	ComputeChildren(levels []uint8, orders []uint32) ([]uint8, []uint32)
}

// Basis is the interface that a functional basis should satisfy in order to be
// used in the algorithm.
type Basis interface {
	Evaluate(point []float64, levels []uint8, orders []uint32) float64
}

// Self represents a particular instantiation of the algorithm.
type Self struct {
	grid         Grid
	basis        Basis
	minLevel     uint8
	maxLevel     uint8
	absTolerance float64
	relTolerance float64
}

// New creates an instance of the algorithm for the given sparse grid and
// functional basis.
func New(grid Grid, basis Basis) *Self {
	return &Self{
		grid:         grid,
		basis:        basis,
		minLevel:     1,
		maxLevel:     9,
		absTolerance: 1e-4,
		relTolerance: 1e-2,
	}
}

// Surrogate is the result of Construct, which represents an interpolant for a
// function.
type Surrogate struct {
	level     uint8
	inCount   uint16
	nodeCount uint32

	levels    []uint8
	orders    []uint32
	surpluses []float64
}

func (s *Surrogate) initialize(inCount uint16) {
	s.inCount = inCount
	s.nodeCount = bufferInitSize

	s.levels = make([]uint8, bufferInitSize*inCount)
	s.orders = make([]uint32, bufferInitSize*inCount)
	s.surpluses = make([]float64, bufferInitSize)
}

func (s *Surrogate) finalize(level uint8, nodeCount uint32) {
	s.level = level
	s.nodeCount = nodeCount

	s.levels = s.levels[0 : nodeCount*uint32(s.inCount)]
	s.orders = s.orders[0 : nodeCount*uint32(s.inCount)]
	s.surpluses = s.surpluses[0:nodeCount]
}

func (s *Surrogate) resize(nodeCount uint32) {
	if nodeCount <= s.nodeCount {
		return
	}

	if count := bufferGrowFactor * s.nodeCount; count > nodeCount {
		nodeCount = count
	}

	levels := make([]uint8, nodeCount*uint32(s.inCount))
	orders := make([]uint32, nodeCount*uint32(s.inCount))
	surpluses := make([]float64, nodeCount)

	copy(levels, s.levels[0:s.nodeCount*uint32(s.inCount)])
	copy(orders, s.orders[0:s.nodeCount*uint32(s.inCount)])
	copy(surpluses, s.surpluses[0:s.nodeCount])

	s.nodeCount = nodeCount
	s.levels = levels
	s.orders = orders
	s.surpluses = surpluses
}

// String returns a string containing human-friendly information about the
// surrogate.
func (s *Surrogate) String() string {
	return fmt.Sprintf("Surrogate{ inputs: %d, levels: %d, nodes: %d }",
		s.inCount, s.level, s.nodeCount)
}

// Construct takes a function and yields a surrogate/interpolant for it, which
// can be further fed to Evaluate for the actual interpolation.
func (self *Self) Construct(target func([]float64) []float64) *Surrogate {
	inc := uint32(self.grid.Dimensionality())

	surrogate := new(Surrogate)
	surrogate.initialize(uint16(inc))

	level := uint8(0)
	nodeCount := uint32(0)

	minValue := math.Inf(1)
	maxValue := math.Inf(-1)

	newc := uint32(1)
	oldc := uint32(0)

	levels := make([]uint8, newc*inc)
	orders := make([]uint32, newc*inc)

	for {
		surrogate.resize(oldc + newc)

		copy(surrogate.levels[oldc*inc:], levels)
		copy(surrogate.orders[oldc*inc:], orders)

		nodes := self.grid.ComputeNodes(levels, orders)
		values := target(nodes)

		for i := uint32(0); i < newc; i++ {
			surrogate.surpluses[oldc+i] = values[i] -
				evaluate(self.basis, inc, oldc,
					nodes[i*inc:(i+1)*inc],
					surrogate.levels[0:oldc*inc],
					surrogate.orders[0:oldc*inc],
					surrogate.surpluses[0:oldc])
		}

		nodeCount += newc

		if level >= self.maxLevel {
			break
		}

		for i := range values {
			if values[i] < minValue {
				minValue = values[i]
			}
			if values[i] > maxValue {
				maxValue = values[i]
			}
		}

		if level >= self.minLevel {
			k := 0

			for i := uint32(0); i < newc; i++ {
				absError := math.Abs(surrogate.surpluses[oldc+i])
				relError := absError / (maxValue - minValue)

				if absError <= self.absTolerance &&
					relError <= self.relTolerance {

					continue
				}

				for j := uint32(0); j < inc; j++ {
					levels[k] = levels[i*inc+j]
					orders[k] = orders[i*inc+j]
					k++
				}
			}

			levels = levels[0:k]
			orders = orders[0:k]
		}

		levels, orders = self.grid.ComputeChildren(levels, orders)

		oldc += newc
		newc = uint32(len(levels)) / inc

		if newc == 0 {
			break
		}

		level++
	}

	surrogate.finalize(level, nodeCount)
	return surrogate
}

// Evaluate takes a surrogate produced by Construct and evaluates it at the
// given points.
func (self *Self) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	inc := uint32(self.grid.Dimensionality())
	pointCount := uint32(len(points)) / inc

	values := make([]float64, pointCount)

	for i := uint32(0); i < pointCount; i++ {
		values[i] = evaluate(self.basis, inc, surrogate.nodeCount, points[i*inc:(i+1)*inc],
			surrogate.levels, surrogate.orders, surrogate.surpluses)
	}

	return values
}

func evaluate(basis Basis, inc uint32, sc uint32, point []float64,
	levels []uint8, orders []uint32, surpluses []float64) (value float64) {

	for i := uint32(0); i < sc; i++ {
		value += surpluses[i] * basis.Evaluate(point,
			levels[i*inc:(i+1)*inc], orders[i*inc:(i+1)*inc])
	}

	return value
}
<commit_msg>Minor renaming<commit_after>// Package hadapt provides an algorithm for adaptive hierarchical interpolation
// with local refinements.
package hadapt

import (
	"fmt"
	"math"
)

const (
	bufferInitSize   = 200
	bufferGrowFactor = 2
)

// Grid is the interface that an sparse grid should satisfy in order to be used
// in the algorithm.
type Grid interface {
	Dimensionality() uint16
	ComputeNodes(levels []uint8, orders []uint32) []float64
	ComputeChildren(levels []uint8, orders []uint32) ([]uint8, []uint32)
}

// Basis is the interface that a functional basis should satisfy in order to be
// used in the algorithm.
type Basis interface {
	Evaluate(point []float64, levels []uint8, orders []uint32) float64
}

// Self represents a particular instantiation of the algorithm.
type Self struct {
	grid         Grid
	basis        Basis
	minLevel     uint8
	maxLevel     uint8
	absTolerance float64
	relTolerance float64
}

// New creates an instance of the algorithm for the given sparse grid and
// functional basis.
func New(grid Grid, basis Basis) *Self {
	return &Self{
		grid:         grid,
		basis:        basis,
		minLevel:     1,
		maxLevel:     9,
		absTolerance: 1e-4,
		relTolerance: 1e-2,
	}
}

// Surrogate is the result of Construct, which represents an interpolant for a
// function.
type Surrogate struct {
	level     uint8
	inCount   uint16
	nodeCount uint32

	levels    []uint8
	orders    []uint32
	surpluses []float64
}

func (s *Surrogate) initialize(inCount uint16) {
	s.inCount = inCount
	s.nodeCount = bufferInitSize

	s.levels = make([]uint8, bufferInitSize*inCount)
	s.orders = make([]uint32, bufferInitSize*inCount)
	s.surpluses = make([]float64, bufferInitSize)
}

func (s *Surrogate) finalize(level uint8, nodeCount uint32) {
	s.level = level
	s.nodeCount = nodeCount

	s.levels = s.levels[0 : nodeCount*uint32(s.inCount)]
	s.orders = s.orders[0 : nodeCount*uint32(s.inCount)]
	s.surpluses = s.surpluses[0:nodeCount]
}

func (s *Surrogate) resize(nodeCount uint32) {
	if nodeCount <= s.nodeCount {
		return
	}

	if count := bufferGrowFactor * s.nodeCount; count > nodeCount {
		nodeCount = count
	}

	levels := make([]uint8, nodeCount*uint32(s.inCount))
	orders := make([]uint32, nodeCount*uint32(s.inCount))
	surpluses := make([]float64, nodeCount)

	copy(levels, s.levels[0:s.nodeCount*uint32(s.inCount)])
	copy(orders, s.orders[0:s.nodeCount*uint32(s.inCount)])
	copy(surpluses, s.surpluses[0:s.nodeCount])

	s.nodeCount = nodeCount
	s.levels = levels
	s.orders = orders
	s.surpluses = surpluses
}

// String returns a string containing human-friendly information about the
// surrogate.
func (s *Surrogate) String() string {
	return fmt.Sprintf("Surrogate{ inputs: %d, levels: %d, nodes: %d }",
		s.inCount, s.level, s.nodeCount)
}

// Construct takes a function and yields a surrogate/interpolant for it, which
// can be further fed to Evaluate for the actual interpolation.
func (self *Self) Construct(target func([]float64) []float64) *Surrogate {
	inc := uint32(self.grid.Dimensionality())

	surrogate := new(Surrogate)
	surrogate.initialize(uint16(inc))

	level := uint8(0)
	nodeCount := uint32(0)

	minValue := math.Inf(1)
	maxValue := math.Inf(-1)

	newc := uint32(1)
	oldc := uint32(0)

	levels := make([]uint8, newc*inc)
	orders := make([]uint32, newc*inc)

	for {
		surrogate.resize(oldc + newc)

		copy(surrogate.levels[oldc*inc:], levels)
		copy(surrogate.orders[oldc*inc:], orders)

		nodes := self.grid.ComputeNodes(levels, orders)
		values := target(nodes)

		for i := uint32(0); i < newc; i++ {
			surrogate.surpluses[oldc+i] = values[i] -
				evaluate(self.basis, inc, oldc,
					nodes[i*inc:(i+1)*inc],
					surrogate.levels[0:oldc*inc],
					surrogate.orders[0:oldc*inc],
					surrogate.surpluses[0:oldc])
		}

		nodeCount += newc

		if level >= self.maxLevel {
			break
		}

		for i := range values {
			if values[i] < minValue {
				minValue = values[i]
			}
			if values[i] > maxValue {
				maxValue = values[i]
			}
		}

		if level >= self.minLevel {
			k := 0

			for i := uint32(0); i < newc; i++ {
				absError := math.Abs(surrogate.surpluses[oldc+i])
				relError := absError / (maxValue - minValue)

				if absError <= self.absTolerance &&
					relError <= self.relTolerance {

					continue
				}

				for j := uint32(0); j < inc; j++ {
					levels[k] = levels[i*inc+j]
					orders[k] = orders[i*inc+j]
					k++
				}
			}

			levels = levels[0:k]
			orders = orders[0:k]
		}

		levels, orders = self.grid.ComputeChildren(levels, orders)

		oldc += newc
		newc = uint32(len(levels)) / inc

		if newc == 0 {
			break
		}

		level++
	}

	surrogate.finalize(level, nodeCount)
	return surrogate
}

// Evaluate takes a surrogate produced by Construct and evaluates it at the
// given points.
func (self *Self) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	inc := uint32(self.grid.Dimensionality())
	pointCount := uint32(len(points)) / inc

	values := make([]float64, pointCount)

	for i := uint32(0); i < pointCount; i++ {
		values[i] = evaluate(self.basis, inc, surrogate.nodeCount, points[i*inc:(i+1)*inc],
			surrogate.levels, surrogate.orders, surrogate.surpluses)
	}

	return values
}

func evaluate(basis Basis, inc uint32, surplusCount uint32, point []float64,
	levels []uint8, orders []uint32, surpluses []float64) (value float64) {

	for i := uint32(0); i < surplusCount; i++ {
		value += surpluses[i] * basis.Evaluate(point,
			levels[i*inc:(i+1)*inc], orders[i*inc:(i+1)*inc])
	}

	return value
}
<|endoftext|>


<commit_before>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sort"
	"strings"
)

// Repo is an F-Droid repository holding apps and apks
type Repo struct {
	Apps []App `xml:"application"`
}

// App is an Android application
type App struct {
	ID      string `xml:"id"`
	Name    string `xml:"name"`
	Summary string `xml:"summary"`
	Desc    string `xml:"desc"`
	License string `xml:"license"`
	Categs  string `xml:"categories"`
	CVName  string `xml:"marketversion"`
	CVCode  uint   `xml:"marketvercode"`
	Website string `xml:"web"`
	Source  string `xml:"source"`
	Tracker string `xml:"tracker"`
	Apks    []Apk  `xml:"package"`
	CurApk  *Apk
}

// Apk is an Android package
type Apk struct {
	VName   string `xml:"version"`
	VCode   uint   `xml:"versioncode"`
	Size    int    `xml:"size"`
	MinSdk  int    `xml:"sdkver"`
}

func (app *App) prepareData() {
	for _, apk := range app.Apks {
		app.CurApk = &apk
		if app.CVCode >= apk.VCode {
			break
		}
	}
}

func (app *App) writeShort(w io.Writer) {
	fmt.Fprintf(w, "%s %s %s\n", app.ID, app.Name, app.CurApk.VName)
	fmt.Fprintf(w, "    %s\n", app.Summary)
}

func (app *App) writeDetailed(w io.Writer) {
	p := func(title string, format string, args ...interface{}) {
		if format == "" {
			fmt.Fprintln(w, title)
		} else {
			fmt.Fprintf(w, "%s %s\n", title, fmt.Sprintf(format, args...))
		}
	}
	p("Name             :", "%s", app.Name)
	p("Summary          :", "%s", app.Summary)
	p("Current Version  :", "%s (%d)", app.CurApk.VName, app.CurApk.VCode)
	p("Upstream Version :", "%s (%d)", app.CVName, app.CVCode)
	p("License          :", "%s", app.License)
	if app.Categs != "" {
		p("Categories       :", "%s", app.Categs)
	}
	if app.Website != "" {
		p("Website          :", "%s", app.Website)
	}
	if app.Source != "" {
		p("Source           :", "%s", app.Source)
	}
	if app.Tracker != "" {
		p("Tracker          :", "%s", app.Tracker)
	}
	// p("Description     :", "%s", app.Desc) // TODO: parse html, 80 column wrapping
	fmt.Println()
	p("Available Versions :", "")
	for _, apk := range app.Apks {
		fmt.Println()
		p("    Name   :", "%s (%d)", apk.VName, apk.VCode)
		p("    Size   :", "%d", apk.Size)
		p("    MinSdk :", "%d", apk.MinSdk)
	}
}

const indexName = "index.jar"

func updateIndex() {
	url := fmt.Sprintf("%s/%s", *repoURL, indexName)
	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("Failed to fetch '%s': %s", url, err)
	}
	defer resp.Body.Close()
	out, err := os.Create(indexName)
	if err != nil {
		log.Fatalf("Failed to create file '%s': %s", indexName, err)
	}
	defer out.Close()
	if _, err := io.Copy(out, resp.Body); err != nil {
		log.Fatal(err)
	}
}

func loadApps() map[string]App {
	r, err := zip.OpenReader(indexName)
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()
	buf := new(bytes.Buffer)

	for _, f := range r.File {
		if f.Name != "index.xml" {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		if _, err = io.Copy(buf, rc); err != nil {
			log.Fatal(err)
		}
		rc.Close()
		break
	}

	var repo Repo
	if err := xml.Unmarshal(buf.Bytes(), &repo); err != nil {
		log.Fatalf("Could not read xml: %s", err)
	}
	apps := make(map[string]App)

	for i := range repo.Apps {
		app := repo.Apps[i]
		app.prepareData()
		apps[app.ID] = app
	}
	return apps
}

func appMatches(fields []string, terms []string) bool {
	for _, field := range fields {
		for _, term := range terms {
			if !strings.Contains(field, term) {
				goto next
			}
		}
		return true
	next:
	}
	return false
}

func filterAppsSearch(apps *map[string]App, terms []string) {
	for _, term := range terms {
		term = strings.ToLower(term)
	}
	for appID, app := range *apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			// strings.ToLower(app.Desc), // TODO remove html
		}
		if !appMatches(fields, terms) {
			delete(*apps, appID)
		}
	}
}

type appList []App

func (al appList) Len() int           { return len(al) }
func (al appList) Swap(i, j int)      { al[i], al[j] = al[j], al[i] }
func (al appList) Less(i, j int) bool { return al[i].ID < al[j].ID }

func sortedApps(apps map[string]App) []App {
	list := make(appList, 0, len(apps))
	for appID := range apps {
		list = append(list, apps[appID])
	}
	sort.Sort(list)
	return list
}

var repoURL = flag.String("r", "https://f-droid.org/repo", "repository address")

func init() {
	flag.Usage = func() {
		p := func(args ...interface{}) {
			fmt.Fprintln(os.Stderr, args...)
		}
		p("Usage: fdroidcl [-h] <command> [<args>]")
		p()
		p("Available commands:")
		p("   update           Update the index")
		p("   list             List all available apps")
		p("   search <term...> Search available apps")
		p("   show <appid..>   Show detailed info of an app")
	}
}

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(2)
	}

	cmd := flag.Args()[0]
	args := flag.Args()[1:]

	switch cmd {
	case "update":
		updateIndex()
	case "list":
		apps := loadApps()
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "search":
		apps := loadApps()
		filterAppsSearch(&apps, args)
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "show":
		apps := loadApps()
		for _, appID := range args {
			app, e := apps[appID]
			if !e {
				log.Fatalf("Could not find app with ID '%s'", appID)
			}
			app.writeDetailed(os.Stdout)
		}
	default:
		fmt.Fprintf(os.Stderr, "Unrecognised command '%s'\n\n", cmd)
		flag.Usage()
		os.Exit(2)
	}
}
<commit_msg>Separate app ids and names<commit_after>/* Copyright (c) 2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"archive/zip"
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"sort"
	"strings"
)

// Repo is an F-Droid repository holding apps and apks
type Repo struct {
	Apps []App `xml:"application"`
}

// App is an Android application
type App struct {
	ID      string `xml:"id"`
	Name    string `xml:"name"`
	Summary string `xml:"summary"`
	Desc    string `xml:"desc"`
	License string `xml:"license"`
	Categs  string `xml:"categories"`
	CVName  string `xml:"marketversion"`
	CVCode  uint   `xml:"marketvercode"`
	Website string `xml:"web"`
	Source  string `xml:"source"`
	Tracker string `xml:"tracker"`
	Apks    []Apk  `xml:"package"`
	CurApk  *Apk
}

// Apk is an Android package
type Apk struct {
	VName   string `xml:"version"`
	VCode   uint   `xml:"versioncode"`
	Size    int    `xml:"size"`
	MinSdk  int    `xml:"sdkver"`
}

func (app *App) prepareData() {
	for _, apk := range app.Apks {
		app.CurApk = &apk
		if app.CVCode >= apk.VCode {
			break
		}
	}
}

func (app *App) writeShort(w io.Writer) {
	fmt.Fprintf(w, "%s | %s %s\n", app.ID, app.Name, app.CurApk.VName)
	fmt.Fprintf(w, "    %s\n", app.Summary)
}

func (app *App) writeDetailed(w io.Writer) {
	p := func(title string, format string, args ...interface{}) {
		if format == "" {
			fmt.Fprintln(w, title)
		} else {
			fmt.Fprintf(w, "%s %s\n", title, fmt.Sprintf(format, args...))
		}
	}
	p("Name             :", "%s", app.Name)
	p("Summary          :", "%s", app.Summary)
	p("Current Version  :", "%s (%d)", app.CurApk.VName, app.CurApk.VCode)
	p("Upstream Version :", "%s (%d)", app.CVName, app.CVCode)
	p("License          :", "%s", app.License)
	if app.Categs != "" {
		p("Categories       :", "%s", app.Categs)
	}
	if app.Website != "" {
		p("Website          :", "%s", app.Website)
	}
	if app.Source != "" {
		p("Source           :", "%s", app.Source)
	}
	if app.Tracker != "" {
		p("Tracker          :", "%s", app.Tracker)
	}
	// p("Description     :", "%s", app.Desc) // TODO: parse html, 80 column wrapping
	fmt.Println()
	p("Available Versions :", "")
	for _, apk := range app.Apks {
		fmt.Println()
		p("    Name   :", "%s (%d)", apk.VName, apk.VCode)
		p("    Size   :", "%d", apk.Size)
		p("    MinSdk :", "%d", apk.MinSdk)
	}
}

const indexName = "index.jar"

func updateIndex() {
	url := fmt.Sprintf("%s/%s", *repoURL, indexName)
	resp, err := http.Get(url)
	if err != nil {
		log.Fatalf("Failed to fetch '%s': %s", url, err)
	}
	defer resp.Body.Close()
	out, err := os.Create(indexName)
	if err != nil {
		log.Fatalf("Failed to create file '%s': %s", indexName, err)
	}
	defer out.Close()
	if _, err := io.Copy(out, resp.Body); err != nil {
		log.Fatal(err)
	}
}

func loadApps() map[string]App {
	r, err := zip.OpenReader(indexName)
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()
	buf := new(bytes.Buffer)

	for _, f := range r.File {
		if f.Name != "index.xml" {
			continue
		}
		rc, err := f.Open()
		if err != nil {
			log.Fatal(err)
		}
		if _, err = io.Copy(buf, rc); err != nil {
			log.Fatal(err)
		}
		rc.Close()
		break
	}

	var repo Repo
	if err := xml.Unmarshal(buf.Bytes(), &repo); err != nil {
		log.Fatalf("Could not read xml: %s", err)
	}
	apps := make(map[string]App)

	for i := range repo.Apps {
		app := repo.Apps[i]
		app.prepareData()
		apps[app.ID] = app
	}
	return apps
}

func appMatches(fields []string, terms []string) bool {
	for _, field := range fields {
		for _, term := range terms {
			if !strings.Contains(field, term) {
				goto next
			}
		}
		return true
	next:
	}
	return false
}

func filterAppsSearch(apps *map[string]App, terms []string) {
	for _, term := range terms {
		term = strings.ToLower(term)
	}
	for appID, app := range *apps {
		fields := []string{
			strings.ToLower(app.ID),
			strings.ToLower(app.Name),
			strings.ToLower(app.Summary),
			// strings.ToLower(app.Desc), // TODO remove html
		}
		if !appMatches(fields, terms) {
			delete(*apps, appID)
		}
	}
}

type appList []App

func (al appList) Len() int           { return len(al) }
func (al appList) Swap(i, j int)      { al[i], al[j] = al[j], al[i] }
func (al appList) Less(i, j int) bool { return al[i].ID < al[j].ID }

func sortedApps(apps map[string]App) []App {
	list := make(appList, 0, len(apps))
	for appID := range apps {
		list = append(list, apps[appID])
	}
	sort.Sort(list)
	return list
}

var repoURL = flag.String("r", "https://f-droid.org/repo", "repository address")

func init() {
	flag.Usage = func() {
		p := func(args ...interface{}) {
			fmt.Fprintln(os.Stderr, args...)
		}
		p("Usage: fdroidcl [-h] <command> [<args>]")
		p()
		p("Available commands:")
		p("   update           Update the index")
		p("   list             List all available apps")
		p("   search <term...> Search available apps")
		p("   show <appid..>   Show detailed info of an app")
	}
}

func main() {
	flag.Parse()
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(2)
	}

	cmd := flag.Args()[0]
	args := flag.Args()[1:]

	switch cmd {
	case "update":
		updateIndex()
	case "list":
		apps := loadApps()
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "search":
		apps := loadApps()
		filterAppsSearch(&apps, args)
		for _, app := range sortedApps(apps) {
			app.writeShort(os.Stdout)
		}
	case "show":
		apps := loadApps()
		for _, appID := range args {
			app, e := apps[appID]
			if !e {
				log.Fatalf("Could not find app with ID '%s'", appID)
			}
			app.writeDetailed(os.Stdout)
		}
	default:
		fmt.Fprintf(os.Stderr, "Unrecognised command '%s'\n\n", cmd)
		flag.Usage()
		os.Exit(2)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"strconv"

	"github.com/couchbase/gomemcached/client"
	log "github.com/couchbaselabs/clog"
	"github.com/couchbaselabs/go-couchbase"
)

type TAPFeed struct {
	name       string
	url        string
	poolName   string
	bucketName string
	bucketUUID string
	pf         StreamPartitionFunc
	streams    map[string]Stream
	closeCh    chan bool
	doneCh     chan bool
	doneErr    error
	doneMsg    string
}

func NewTAPFeed(name, url, poolName, bucketName, bucketUUID string,
	pf StreamPartitionFunc, streams map[string]Stream) (*TAPFeed, error) {
	return &TAPFeed{
		name:       name,
		url:        url,
		poolName:   poolName,
		bucketName: bucketName,
		bucketUUID: bucketUUID,
		pf:         pf,
		streams:    streams,
		closeCh:    make(chan bool),
		doneCh:     make(chan bool),
		doneErr:    nil,
		doneMsg:    "",
	}, nil
}

func (t *TAPFeed) Name() string {
	return t.name
}

func (t *TAPFeed) Start() error {
	log.Printf("TAPFeed.Start, name: %s", t.Name())

	go ExponentialBackoffLoop(t.Name(),
		func() int {
			progress, err := t.feed()
			if err != nil {
				log.Printf("TAPFeed name: %s, progress: %d, err: %v",
					t.Name(), progress, err)
			}
			return progress
		},
		FEED_SLEEP_INIT_MS,  // Milliseconds.
		FEED_BACKOFF_FACTOR, // Backoff.
		FEED_SLEEP_MAX_MS)

	return nil
}

func (t *TAPFeed) feed() (int, error) {
	select {
	case <-t.closeCh:
		t.doneErr = nil
		t.doneMsg = "closeCh closed"
		close(t.doneCh)
		return -1, nil
	default:
	}

	bucket, err := couchbase.GetBucket(t.url, t.poolName, t.bucketName)
	if err != nil {
		return 0, err
	}
	defer bucket.Close()

	if t.bucketUUID != "" && t.bucketUUID != bucket.UUID {
		bucket.Close()
		return -1, fmt.Errorf("error: mismatched bucket uuid,"+
			"bucketName: %s, bucketUUID: %s, bucket.UUID: %s",
			t.bucketName, t.bucketUUID, bucket.UUID)
	}

	args := memcached.TapArguments{}

	vbuckets, err := ParsePartitionsToVBucketIds(t.streams)
	if err != nil {
		return -1, err
	}
	if len(vbuckets) > 0 {
		args.VBuckets = vbuckets
	}

	feed, err := bucket.StartTapFeed(&args)
	if err != nil {
		return 0, err
	}
	defer feed.Close()

	// TODO: maybe TAPFeed should do a rollback to zero if it finds it
	// needs to do a full backfill.
	// TODO: this TAPFeed implementation currently only works against
	// a couchbase cluster that has just a single node.

	log.Printf("TapFeed: running, url: %s,"+
		" poolName: %s, bucketName: %s, vbuckets: %#v",
		t.url, t.poolName, t.bucketName, vbuckets)

	for {
		select {
		case <-t.closeCh:
			t.doneErr = nil
			t.doneMsg = "closeCh closed"
			close(t.doneCh)
			return -1, nil

		case req, alive := <-feed.C:
			if !alive {
				break
			}

			log.Printf("TapFeed: received from url: %s,"+
				" poolName: %s, bucketName: %s, req: %#v",
				t.url, t.poolName, t.bucketName, req)

			partition := fmt.Sprintf("%d", req.VBucket)
			stream, err := t.pf(req.Key, partition, t.streams)
			if err != nil {
				return 1, fmt.Errorf("error: TAPFeed:"+
					" partition func error from url: %s,"+
					" poolName: %s, bucketName: %s, req: %#v, streams: %#v, err: %v",
					t.url, t.poolName, t.bucketName, req, t.streams, err)
			}

			if req.Opcode == memcached.TapMutation {
				stream <- &StreamRequest{
					Op:  STREAM_OP_UPDATE,
					Key: req.Key,
					Val: req.Value,
				}
			} else if req.Opcode == memcached.TapDeletion {
				stream <- &StreamRequest{
					Op:  STREAM_OP_DELETE,
					Key: req.Key,
				}
			} else {
			}
		}
	}

	return 1, nil
}

func (t *TAPFeed) Close() error {
	close(t.closeCh)
	<-t.doneCh
	return t.doneErr
}

func (t *TAPFeed) Streams() map[string]Stream {
	return t.streams
}

func ParsePartitionsToVBucketIds(streams map[string]Stream) ([]uint16, error) {
	vbuckets := make([]uint16, 0, len(streams))
	for partition, _ := range streams {
		if partition != "" {
			vbId, err := strconv.Atoi(partition)
			if err != nil {
				return nil, fmt.Errorf("error: could not parse partition: %s, err: %v",
					partition, err)
			}
			vbuckets = append(vbuckets, uint16(vbId))
		}
	}
	return vbuckets, nil
}
<commit_msg>log TapOpcode as string<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"fmt"
	"strconv"

	"github.com/couchbase/gomemcached/client"
	log "github.com/couchbaselabs/clog"
	"github.com/couchbaselabs/go-couchbase"
)

type TAPFeed struct {
	name       string
	url        string
	poolName   string
	bucketName string
	bucketUUID string
	pf         StreamPartitionFunc
	streams    map[string]Stream
	closeCh    chan bool
	doneCh     chan bool
	doneErr    error
	doneMsg    string
}

func NewTAPFeed(name, url, poolName, bucketName, bucketUUID string,
	pf StreamPartitionFunc, streams map[string]Stream) (*TAPFeed, error) {
	return &TAPFeed{
		name:       name,
		url:        url,
		poolName:   poolName,
		bucketName: bucketName,
		bucketUUID: bucketUUID,
		pf:         pf,
		streams:    streams,
		closeCh:    make(chan bool),
		doneCh:     make(chan bool),
		doneErr:    nil,
		doneMsg:    "",
	}, nil
}

func (t *TAPFeed) Name() string {
	return t.name
}

func (t *TAPFeed) Start() error {
	log.Printf("TAPFeed.Start, name: %s", t.Name())

	go ExponentialBackoffLoop(t.Name(),
		func() int {
			progress, err := t.feed()
			if err != nil {
				log.Printf("TAPFeed name: %s, progress: %d, err: %v",
					t.Name(), progress, err)
			}
			return progress
		},
		FEED_SLEEP_INIT_MS,  // Milliseconds.
		FEED_BACKOFF_FACTOR, // Backoff.
		FEED_SLEEP_MAX_MS)

	return nil
}

func (t *TAPFeed) feed() (int, error) {
	select {
	case <-t.closeCh:
		t.doneErr = nil
		t.doneMsg = "closeCh closed"
		close(t.doneCh)
		return -1, nil
	default:
	}

	bucket, err := couchbase.GetBucket(t.url, t.poolName, t.bucketName)
	if err != nil {
		return 0, err
	}
	defer bucket.Close()

	if t.bucketUUID != "" && t.bucketUUID != bucket.UUID {
		bucket.Close()
		return -1, fmt.Errorf("error: mismatched bucket uuid,"+
			"bucketName: %s, bucketUUID: %s, bucket.UUID: %s",
			t.bucketName, t.bucketUUID, bucket.UUID)
	}

	args := memcached.TapArguments{}

	vbuckets, err := ParsePartitionsToVBucketIds(t.streams)
	if err != nil {
		return -1, err
	}
	if len(vbuckets) > 0 {
		args.VBuckets = vbuckets
	}

	feed, err := bucket.StartTapFeed(&args)
	if err != nil {
		return 0, err
	}
	defer feed.Close()

	// TODO: maybe TAPFeed should do a rollback to zero if it finds it
	// needs to do a full backfill.
	// TODO: this TAPFeed implementation currently only works against
	// a couchbase cluster that has just a single node.

	log.Printf("TapFeed: running, url: %s,"+
		" poolName: %s, bucketName: %s, vbuckets: %#v",
		t.url, t.poolName, t.bucketName, vbuckets)

	for {
		select {
		case <-t.closeCh:
			t.doneErr = nil
			t.doneMsg = "closeCh closed"
			close(t.doneCh)
			return -1, nil

		case req, alive := <-feed.C:
			if !alive {
				break
			}

			log.Printf("TapFeed: received from url: %s,"+
				" poolName: %s, bucketName: %s, opcode: %s, req: %#v",
				t.url, t.poolName, t.bucketName, req.Opcode, req)

			partition := fmt.Sprintf("%d", req.VBucket)
			stream, err := t.pf(req.Key, partition, t.streams)
			if err != nil {
				return 1, fmt.Errorf("error: TAPFeed:"+
					" partition func error from url: %s,"+
					" poolName: %s, bucketName: %s, req: %#v, streams: %#v, err: %v",
					t.url, t.poolName, t.bucketName, req, t.streams, err)
			}

			if req.Opcode == memcached.TapMutation {
				stream <- &StreamRequest{
					Op:  STREAM_OP_UPDATE,
					Key: req.Key,
					Val: req.Value,
				}
			} else if req.Opcode == memcached.TapDeletion {
				stream <- &StreamRequest{
					Op:  STREAM_OP_DELETE,
					Key: req.Key,
				}
			} else {
			}
		}
	}

	return 1, nil
}

func (t *TAPFeed) Close() error {
	close(t.closeCh)
	<-t.doneCh
	return t.doneErr
}

func (t *TAPFeed) Streams() map[string]Stream {
	return t.streams
}

func ParsePartitionsToVBucketIds(streams map[string]Stream) ([]uint16, error) {
	vbuckets := make([]uint16, 0, len(streams))
	for partition, _ := range streams {
		if partition != "" {
			vbId, err := strconv.Atoi(partition)
			if err != nil {
				return nil, fmt.Errorf("error: could not parse partition: %s, err: %v",
					partition, err)
			}
			vbuckets = append(vbuckets, uint16(vbId))
		}
	}
	return vbuckets, nil
}
<|endoftext|>


<commit_before>// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm)
// go-aah/essentials source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package ess

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Excludes is handly filepath match manipulation
type Excludes []string

// Validate helps to evalute the pattern are valid
// `Match` method is from error and focus on match
func (e *Excludes) Validate() error {
	for _, pattern := range *e {
		if _, err := filepath.Match(pattern, "abc/b/c"); err != nil {
			return fmt.Errorf("unable to evalute pattern: %v", pattern)
		}
	}
	return nil
}

// Match evalutes given file with available patterns returns true if matches
// otherwise false. `Match` internally uses the `filepath.Match`
//
// Note: `Match` ignore pattern errors, use `Validate` method to ensure
// you have correct exclude patterns
func (e *Excludes) Match(file string) bool {
	for _, pattern := range *e {
		if match, _ := filepath.Match(pattern, file); match {
			return match
		}
	}
	return false
}

// IsFileExists return true is file or directory is exists, otherwise returns
// false. It also take cares of symlink path as well
func IsFileExists(filename string) bool {
	_, err := os.Lstat(filename)
	return err == nil
}

// IsDirEmpty returns true if the given directory is empty also returns true if
// directory not exists. Otherwise returns false
func IsDirEmpty(path string) bool {
	if !IsFileExists(path) {
		// directory not exists
		return true
	}
	dir, _ := os.Open(path)
	defer CloseQuietly(dir)
	results, _ := dir.Readdir(1)
	return len(results) == 0
}

// IsDir returns true if the given `path` is directory otherwise returns false.
// Also returns false if path is not exists
func IsDir(path string) bool {
	info, err := os.Lstat(path)
	return err == nil && info.IsDir()
}

// ApplyFileMode applies the given file mode to the target{file|directory}
func ApplyFileMode(target string, mode os.FileMode) error {
	err := os.Chmod(target, mode)
	if err != nil {
		return fmt.Errorf("unable to apply mode: %v, to given file or directory: %v", mode, target)
	}
	return nil
}

// LineCnt counts no. of lines on file
func LineCnt(fileName string) int {
	f, err := os.Open(fileName)
	if err != nil {
		return 0
	}
	defer CloseQuietly(f)

	return LineCntr(f)
}

// LineCntr counts no. of lines for given reader
func LineCntr(r io.Reader) int {
	buf := make([]byte, 8196)
	count := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return count
		}

		count += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return count
}

// Walk method extends filepath.Walk to also follows symlinks.
// Always returns the path of the file or directory also path
// is inline to name of symlink
func Walk(srcDir string, walkFn filepath.WalkFunc) error {
	return doWalk(srcDir, srcDir, walkFn)
}

func doWalk(fname string, linkName string, walkFn filepath.WalkFunc) error {
	fsWalkFn := func(path string, info os.FileInfo, err error) error {
		var name string
		name, err = filepath.Rel(fname, path)
		if err != nil {
			return err
		}

		path = filepath.Join(linkName, name)

		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
			var symlinkPath string
			symlinkPath, err = filepath.EvalSymlinks(path)
			if err != nil {
				return err
			}

			// https://github.com/golang/go/blob/master/src/path/filepath/path.go#L392
			info, err = os.Lstat(symlinkPath)
			if err != nil {
				return walkFn(path, info, err)
			}

			if info.IsDir() {
				return doWalk(symlinkPath, path, walkFn)
			}
		}

		return walkFn(path, info, err)
	}

	return filepath.Walk(fname, fsWalkFn)
}

// CopyFile copies the given source file into destination
func CopyFile(dest, src string) (int64, error) {
	if !IsFileExists(src) {
		return 0, fmt.Errorf("source file does not exists: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return 0, fmt.Errorf("destination file already exists: %v", dest)
	}

	destFile, err := os.Create(dest)
	if err != nil {
		return 0, fmt.Errorf("unable to create dest file: %v", dest)
	}
	defer CloseQuietly(destFile)

	srcFile, err := os.Open(src)
	if err != nil {
		return 0, fmt.Errorf("unable to open source file: %v", src)
	}
	defer CloseQuietly(srcFile)

	copiedBytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return 0, fmt.Errorf("unable to copy file from %v to %v (%v)",
			src, dest, err)
	}

	return copiedBytes, nil
}

// CopyDir copies entire directory, sub directories and files into destination
// and it excludes give file matches
func CopyDir(dest, src string, excludes Excludes) error {
	if !IsFileExists(src) {
		return fmt.Errorf("source dir does not exists: %v", src)
	}

	src = filepath.Clean(src)
	srcInfo, _ := os.Lstat(src)
	if !srcInfo.IsDir() {
		return fmt.Errorf("source is not directory: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return fmt.Errorf("destination dir already exists: %v", dest)
	}

	if err := excludes.Validate(); err != nil {
		return err
	}

	return Walk(src, func(srcPath string, info os.FileInfo, err error) error {
		if excludes.Match(filepath.Base(srcPath)) {
			if info.IsDir() {
				// excluding directory
				return filepath.SkipDir
			}
			// excluding file
			return nil
		}

		relativeSrcPath := strings.TrimLeft(srcPath[len(src):], string(filepath.Separator))
		destPath := filepath.Join(dest, relativeSrcPath)

		if info.IsDir() {
			// directory permissions is not preserved from source
			return MkDirAll(destPath, 0755)
		}

		// copy source into destination
		if _, err = CopyFile(destPath, srcPath); err != nil {
			return err
		}

		// Apply source permision into target as well
		// so file permissions are preserved
		return ApplyFileMode(destPath, info.Mode())
	})
}

// DeleteFiles method deletes give files or directories. ensure your supplying
// appropriate paths.
func DeleteFiles(files ...string) (errs []error) {
	for _, f := range files {
		if !IsFileExists(f) {
			errs = append(errs, fmt.Errorf("path does not exists: %s", f))
			continue
		}

		var err error
		if IsDir(f) {
			err = os.RemoveAll(f)
		} else {
			err = os.Remove(f)
		}

		if err != nil {
			errs = append(errs, err)
		}
	}

	return
}

// DirsPath method returns all directories absolute path from given base path recursively.
func DirsPath(basePath string, recursive bool) (pdirs []string, err error) {
	return DirsPathExcludes(basePath, recursive, Excludes{})
}

// DirsPathExcludes method returns all directories absolute path from given base path recursively
// excluding the excludes list.
func DirsPathExcludes(basePath string, recursive bool, excludes Excludes) (pdirs []string, err error) {
	if err = excludes.Validate(); err != nil {
		return
	}

	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if excludes.Match(filepath.Base(srcPath)) {
				if info.IsDir() {
					// excluding directory
					return filepath.SkipDir
				}
				// excluding file
				return nil
			}

			if info.IsDir() {
				pdirs = append(pdirs, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if v.IsDir() && !excludes.Match(v.Name()) {
			pdirs = append(pdirs, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// FilesPath method returns all files absolute path from given base path recursively.
func FilesPath(basePath string, recursive bool) (files []string, err error) {
	return FilesPathExcludes(basePath, recursive, Excludes{})
}

// FilesPathExcludes method returns all files absolute path from given base path recursively
// excluding the excludes list.
func FilesPathExcludes(basePath string, recursive bool, excludes Excludes) (files []string, err error) {
	if err = excludes.Validate(); err != nil {
		return
	}

	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if excludes.Match(filepath.Base(srcPath)) {
				if info.IsDir() {
					// excluding directory
					return filepath.SkipDir
				}
				// excluding file
				return nil
			}

			if !info.IsDir() {
				files = append(files, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if !v.IsDir() && !excludes.Match(v.Name()) {
			files = append(files, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// StripExt method returns name of the file without extension.
//    E.g.: index.html => index
func StripExt(name string) string {
	if IsStrEmpty(name) {
		return name
	}

	idx := strings.LastIndexByte(name, '.')
	if idx > 0 {
		return name[:idx]
	}

	return name
}
<commit_msg>improvement and test update<commit_after>// Copyright (c) Jeevanandam M. (https://github.com/jeevatkm)
// go-aah/essentials source code and usage is governed by a MIT style
// license that can be found in the LICENSE file.

package ess

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Excludes is handly filepath match manipulation
type Excludes []string

// Validate helps to evalute the pattern are valid
// `Match` method is from error and focus on match
func (e *Excludes) Validate() error {
	for _, pattern := range *e {
		if _, err := filepath.Match(pattern, "abc/b/c"); err != nil {
			return fmt.Errorf("unable to evalute pattern: %v", pattern)
		}
	}
	return nil
}

// Match evalutes given file with available patterns returns true if matches
// otherwise false. `Match` internally uses the `filepath.Match`
//
// Note: `Match` ignore pattern errors, use `Validate` method to ensure
// you have correct exclude patterns
func (e *Excludes) Match(file string) bool {
	for _, pattern := range *e {
		if match, _ := filepath.Match(pattern, file); match {
			return match
		}
	}
	return false
}

// IsFileExists return true is file or directory is exists, otherwise returns
// false. It also take cares of symlink path as well
func IsFileExists(filename string) bool {
	_, err := os.Lstat(filename)
	return err == nil
}

// IsDirEmpty returns true if the given directory is empty also returns true if
// directory not exists. Otherwise returns false
func IsDirEmpty(path string) bool {
	if !IsFileExists(path) {
		// directory not exists
		return true
	}
	dir, _ := os.Open(path)
	defer CloseQuietly(dir)
	results, _ := dir.Readdir(1)
	return len(results) == 0
}

// IsDir returns true if the given `path` is directory otherwise returns false.
// Also returns false if path is not exists
func IsDir(path string) bool {
	info, err := os.Lstat(path)
	return err == nil && info.IsDir()
}

// ApplyFileMode applies the given file mode to the target{file|directory}
func ApplyFileMode(target string, mode os.FileMode) error {
	err := os.Chmod(target, mode)
	if err != nil {
		return fmt.Errorf("unable to apply mode: %v, to given file or directory: %v", mode, target)
	}
	return nil
}

// LineCnt counts no. of lines on file
func LineCnt(fileName string) int {
	f, err := os.Open(fileName)
	if err != nil {
		return 0
	}
	defer CloseQuietly(f)

	return LineCntr(f)
}

// LineCntr counts no. of lines for given reader
func LineCntr(r io.Reader) int {
	buf := make([]byte, 8196)
	count := 0
	lineSep := []byte{'\n'}

	for {
		c, err := r.Read(buf)
		if err != nil && err != io.EOF {
			return count
		}

		count += bytes.Count(buf[:c], lineSep)

		if err == io.EOF {
			break
		}
	}

	return count
}

// Walk method extends filepath.Walk to also follows symlinks.
// Always returns the path of the file or directory also path
// is inline to name of symlink
func Walk(srcDir string, walkFn filepath.WalkFunc) error {
	return doWalk(srcDir, srcDir, walkFn)
}

func doWalk(fname string, linkName string, walkFn filepath.WalkFunc) error {
	fsWalkFn := func(path string, info os.FileInfo, err error) error {
		var name string
		name, err = filepath.Rel(fname, path)
		if err != nil {
			return err
		}

		path = filepath.Join(linkName, name)

		if err == nil && info.Mode()&os.ModeSymlink == os.ModeSymlink {
			var symlinkPath string
			symlinkPath, err = filepath.EvalSymlinks(path)
			if err != nil {
				return err
			}

			// https://github.com/golang/go/blob/master/src/path/filepath/path.go#L392
			info, err = os.Lstat(symlinkPath)
			if err != nil {
				return walkFn(path, info, err)
			}

			if info.IsDir() {
				return doWalk(symlinkPath, path, walkFn)
			}
		}

		return walkFn(path, info, err)
	}

	return filepath.Walk(fname, fsWalkFn)
}

// CopyFile copies the given source file into destination
func CopyFile(dest, src string) (int64, error) {
	if !IsFileExists(src) {
		return 0, fmt.Errorf("source file does not exists: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return 0, fmt.Errorf("destination file already exists: %v", dest)
	}

	destFile, err := os.Create(dest)
	if err != nil {
		return 0, fmt.Errorf("unable to create dest file: %v", dest)
	}
	defer CloseQuietly(destFile)

	srcFile, err := os.Open(src)
	if err != nil {
		return 0, fmt.Errorf("unable to open source file: %v", src)
	}
	defer CloseQuietly(srcFile)

	copiedBytes, err := io.Copy(destFile, srcFile)
	if err != nil {
		return 0, fmt.Errorf("unable to copy file from %v to %v (%v)",
			src, dest, err)
	}

	return copiedBytes, nil
}

// CopyDir copies entire directory, sub directories and files into destination
// and it excludes give file matches
func CopyDir(dest, src string, excludes Excludes) error {
	if !IsFileExists(src) {
		return fmt.Errorf("source dir does not exists: %v", src)
	}

	src = filepath.Clean(src)
	srcInfo, _ := os.Lstat(src)
	if !srcInfo.IsDir() {
		return fmt.Errorf("source is not directory: %v", src)
	}

	baseName := filepath.Base(src)
	if !strings.HasSuffix(dest, baseName) {
		dest = filepath.Join(dest, baseName)
	}

	if IsFileExists(dest) {
		return fmt.Errorf("destination dir already exists: %v", dest)
	}

	if err := excludes.Validate(); err != nil {
		return err
	}

	return Walk(src, func(srcPath string, info os.FileInfo, err error) error {
		if excludes.Match(filepath.Base(srcPath)) {
			if info.IsDir() {
				// excluding directory
				return filepath.SkipDir
			}
			// excluding file
			return nil
		}

		relativeSrcPath := strings.TrimLeft(srcPath[len(src):], string(filepath.Separator))
		destPath := filepath.Join(dest, relativeSrcPath)

		if info.IsDir() {
			// directory permissions is not preserved from source
			return MkDirAll(destPath, 0755)
		}

		// copy source into destination
		if _, err = CopyFile(destPath, srcPath); err != nil {
			return err
		}

		// Apply source permision into target as well
		// so file permissions are preserved
		return ApplyFileMode(destPath, info.Mode())
	})
}

// DeleteFiles method deletes give files or directories. ensure your supplying
// appropriate paths.
func DeleteFiles(files ...string) (errs []error) {
	for _, f := range files {
		if !IsFileExists(f) {
			errs = append(errs, fmt.Errorf("path does not exists: %s", f))
			continue
		}

		var err error
		if IsDir(f) {
			err = os.RemoveAll(f)
		} else {
			err = os.Remove(f)
		}

		if err != nil {
			errs = append(errs, err)
		}
	}

	return
}

// DirsPath method returns all directories absolute path from given base path recursively.
func DirsPath(basePath string, recursive bool) (pdirs []string, err error) {
	return DirsPathExcludes(basePath, recursive, Excludes{})
}

// DirsPathExcludes method returns all directories absolute path from given base path recursively
// excluding the excludes list.
func DirsPathExcludes(basePath string, recursive bool, excludes Excludes) (pdirs []string, err error) {
	if err = excludes.Validate(); err != nil {
		return
	}

	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if info.IsDir() && excludes.Match(filepath.Base(srcPath)) {
				// excluding directory
				return filepath.SkipDir
			}

			if info.IsDir() {
				pdirs = append(pdirs, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if v.IsDir() && !excludes.Match(v.Name()) {
			pdirs = append(pdirs, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// FilesPath method returns all files absolute path from given base path recursively.
func FilesPath(basePath string, recursive bool) (files []string, err error) {
	return FilesPathExcludes(basePath, recursive, Excludes{})
}

// FilesPathExcludes method returns all files absolute path from given base path recursively
// excluding the excludes list.
func FilesPathExcludes(basePath string, recursive bool, excludes Excludes) (files []string, err error) {
	if err = excludes.Validate(); err != nil {
		return
	}

	if recursive {
		err = Walk(basePath, func(srcPath string, info os.FileInfo, err error) error {
			if !info.IsDir() && excludes.Match(filepath.Base(srcPath)) {
				// excluding file
				return nil
			}

			if !info.IsDir() {
				files = append(files, srcPath)
			}
			return nil
		})
		return
	}

	var list []os.FileInfo
	list, err = ioutil.ReadDir(basePath)
	if err != nil {
		return
	}

	for _, v := range list {
		if !v.IsDir() && !excludes.Match(v.Name()) {
			files = append(files, filepath.Join(basePath, v.Name()))
		}
	}

	return
}

// StripExt method returns name of the file without extension.
//    E.g.: index.html => index
func StripExt(name string) string {
	if IsStrEmpty(name) {
		return name
	}

	idx := strings.LastIndexByte(name, '.')
	if idx > 0 {
		return name[:idx]
	}

	return name
}
<|endoftext|>


<commit_before>// Package fireauth provides an access token generator for the Firebase v3.0.0+ API.
package fireauth

import (
	"encoding/json"
	"errors"
	"strconv"
	"time"

	"github.com/knq/jwt"
)

const (
	// Audience is the required value for the audience ("aud") field for
	// Firebase.
	Audience = "https://identitytoolkit.googleapis.com/google.identity.identitytoolkit.v1.IdentityToolkit"

	// DefaultExpirationDuration is the default expiration duration.
	DefaultExpirationDuration time.Duration = 2 * time.Hour
)

// Claims is the JWT payload claims for a Firebase access token.
type Claims struct {
	// Issuer ("iss") identifies the principal that issued the JWT.
	Issuer string `json:"iss,omitempty"`

	// Subject ("sub") identifies the principal that is the subject of the JWT.
	Subject string `json:"sub,omitempty"`

	// Audience ("aud") identifies the recipients that the JWT is intended for.
	Audience string `json:"aud,omitempty"`

	// IssuedAt ("iat") identifies the time at which the JWT was issued.
	IssuedAt json.Number `json:"iat,omitempty"`

	// NotBefore ("nbf") identifies the time before which the JWT MUST NOT be
	// accepted for processing.
	NotBefore json.Number `json:"nbf,omitempty"`

	// Expiration ("exp") identifies the expiration time on or after which the
	// JWT MUST NOT be accepted for processing.
	Expiration json.Number `json:"exp,omitempty"`

	// UserID identifies the authenticated user to Firebase.
	//
	// This is copied from the parent, but can be overridden with the UserID
	// TokenOption.
	UserID string `json:"uid,omitempty"`

	// Claims is the additional authenticated user data for Firebase.
	//
	// This can be set with the AuthData TokenOption, and is then available in
	// the Firebase `auth` and `request.auth` variables.
	Claims interface{} `json:"claims,omitempty"`
}

// fireauth wraps a token signer with the required data.
type fireauth struct {
	signer jwt.Signer

	projectID           string
	serviceAccountEmail string

	enableExpiration bool
	enableIssuedAt   bool
	enableNotBefore  bool

	expirationDuration time.Duration
}

// New creates a new fireauth using the provided options.
func New(opts ...Option) (*fireauth, error) {
	var err error

	// default options
	f := &fireauth{
		enableExpiration: true,
		enableIssuedAt:   true,
		enableNotBefore:  true,

		expirationDuration: DefaultExpirationDuration,
	}

	// apply options
	for _, o := range opts {
		err = o(f)
		if err != nil {
			return nil, err
		}
	}

	// check that signer is defined
	if f.signer == nil {
		return nil, errors.New("no private key was provided")
	}

	// check issuer and subject
	if f.serviceAccountEmail == "" {
		return nil, errors.New("no service account email was provided")
	}

	return f, nil
}

// Token generates a token with the provided token options.
func (f *fireauth) Token(opts ...TokenOption) ([]byte, error) {
	tok := &Claims{
		Issuer:   f.serviceAccountEmail,
		Subject:  f.serviceAccountEmail,
		Audience: Audience,
	}

	now := time.Now()
	n := json.Number(strconv.FormatInt(now.Unix(), 10))

	// set expiration
	if f.enableExpiration {
		tok.Expiration = json.Number(strconv.FormatInt(now.Add(f.expirationDuration).Unix(), 10))
	}

	// set issued at
	if f.enableIssuedAt {
		tok.IssuedAt = n
	}

	//  set not before
	if f.enableNotBefore {
		tok.NotBefore = n
	}

	// apply options
	for _, o := range opts {
		o(tok)
	}

	// encode
	return f.signer.Encode(tok)
}

// TokenString generates a token with the provided token options.
func (f *fireauth) TokenString(opts ...TokenOption) (string, error) {
	buf, err := f.Token(opts...)
	if err != nil {
		return "", err
	}

	return string(buf), nil
}
<commit_msg>Minor change<commit_after>// Package fireauth provides a simple access token generator for the Firebase v3.0.0+ API.
package fireauth

import (
	"encoding/json"
	"errors"
	"strconv"
	"time"

	"github.com/knq/jwt"
)

const (
	// Audience is the required value for the audience ("aud") field for
	// Firebase.
	Audience = "https://identitytoolkit.googleapis.com/google.identity.identitytoolkit.v1.IdentityToolkit"

	// DefaultExpirationDuration is the default expiration duration.
	DefaultExpirationDuration time.Duration = 2 * time.Hour
)

// Claims is the JWT payload claims for a Firebase access token.
type Claims struct {
	// Issuer ("iss") identifies the principal that issued the JWT.
	Issuer string `json:"iss,omitempty"`

	// Subject ("sub") identifies the principal that is the subject of the JWT.
	Subject string `json:"sub,omitempty"`

	// Audience ("aud") identifies the recipients that the JWT is intended for.
	Audience string `json:"aud,omitempty"`

	// IssuedAt ("iat") identifies the time at which the JWT was issued.
	IssuedAt json.Number `json:"iat,omitempty"`

	// NotBefore ("nbf") identifies the time before which the JWT MUST NOT be
	// accepted for processing.
	NotBefore json.Number `json:"nbf,omitempty"`

	// Expiration ("exp") identifies the expiration time on or after which the
	// JWT MUST NOT be accepted for processing.
	Expiration json.Number `json:"exp,omitempty"`

	// UserID identifies the authenticated user to Firebase.
	//
	// This is copied from the parent, but can be overridden with the UserID
	// TokenOption.
	UserID string `json:"uid,omitempty"`

	// Claims is the additional authenticated user data for Firebase.
	//
	// This can be set with the AuthData TokenOption, and is then available in
	// the Firebase `auth` and `request.auth` variables.
	Claims interface{} `json:"claims,omitempty"`
}

// fireauth wraps a token signer with the required data.
type fireauth struct {
	signer jwt.Signer

	projectID           string
	serviceAccountEmail string

	enableExpiration bool
	enableIssuedAt   bool
	enableNotBefore  bool

	expirationDuration time.Duration
}

// New creates a new fireauth using the provided options.
func New(opts ...Option) (*fireauth, error) {
	var err error

	// default options
	f := &fireauth{
		enableExpiration: true,
		enableIssuedAt:   true,
		enableNotBefore:  true,

		expirationDuration: DefaultExpirationDuration,
	}

	// apply options
	for _, o := range opts {
		err = o(f)
		if err != nil {
			return nil, err
		}
	}

	// check that signer is defined
	if f.signer == nil {
		return nil, errors.New("no private key was provided")
	}

	// check issuer and subject
	if f.serviceAccountEmail == "" {
		return nil, errors.New("no service account email was provided")
	}

	return f, nil
}

// Token generates a token with the provided token options.
func (f *fireauth) Token(opts ...TokenOption) ([]byte, error) {
	tok := &Claims{
		Issuer:   f.serviceAccountEmail,
		Subject:  f.serviceAccountEmail,
		Audience: Audience,
	}

	now := time.Now()
	n := json.Number(strconv.FormatInt(now.Unix(), 10))

	// set expiration
	if f.enableExpiration {
		tok.Expiration = json.Number(strconv.FormatInt(now.Add(f.expirationDuration).Unix(), 10))
	}

	// set issued at
	if f.enableIssuedAt {
		tok.IssuedAt = n
	}

	//  set not before
	if f.enableNotBefore {
		tok.NotBefore = n
	}

	// apply options
	for _, o := range opts {
		o(tok)
	}

	// encode
	return f.signer.Encode(tok)
}

// TokenString generates a token with the provided token options.
func (f *fireauth) TokenString(opts ...TokenOption) (string, error) {
	buf, err := f.Token(opts...)
	if err != nil {
		return "", err
	}

	return string(buf), nil
}
<|endoftext|>