<commit_before>package neutrino

import (
	"container/heap"
	"sync"
	"sync/atomic"
	"time"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwallet/waddrmgr"
)

// getUtxoResult is a simple pair type holding a spend report and error.
type getUtxoResult struct {
	report *SpendReport
	err    error
}

// GetUtxoRequest is a request to scan for InputWithScript from the height
// BirthHeight.
type GetUtxoRequest struct {
	// Input is the target outpoint with script to watch for spentness.
	Input *InputWithScript

	// BirthHeight is the height at which we expect to find the original
	// unspent outpoint. This is also the height used when starting the
	// search for spends.
	BirthHeight uint32

	// resultChan either the spend report or error for this request.
	resultChan chan *getUtxoResult

	// result caches the first spend report or error returned for this
	// request.
	result *getUtxoResult

	// mu ensures the first response delivered via resultChan is in fact
	// what gets cached in result.
	mu sync.Mutex

	quit chan struct{}
}

// deliver tries to deliver the report or error to any subscribers. If
// resultChan cannot accept a new update, this method will not block.
func (r *GetUtxoRequest) deliver(report *SpendReport, err error) {
	select {
	case r.resultChan <- &getUtxoResult{report, err}:
	default:
		log.Warnf("duplicate getutxo result delivered for "+
			"outpoint=%v, spend=%v, err=%v",
			r.Input.OutPoint, report, err)
	}
}

// Result is callback returning either a spend report or an error.
func (r *GetUtxoRequest) Result() (*SpendReport, error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	select {
	case result := <-r.resultChan:
		// Cache the first result returned, in case we have multiple
		// readers calling Result.
		if r.result == nil {
			r.result = result
		}

		return r.result.report, r.result.err

	case <-r.quit:
		return nil, ErrShuttingDown
	}
}

// UtxoScannerConfig exposes configurable methods for interacting with the blockchain.
type UtxoScannerConfig struct {
	// BestSnapshot returns the block stamp of the current chain tip.
	BestSnapshot func() (*waddrmgr.BlockStamp, error)

	// GetBlockHash returns the block hash at given height in main chain.
	GetBlockHash func(height int64) (*chainhash.Hash, error)

	// BlockFilterMatches checks the cfilter for the block hash for matches
	// against the rescan options.
	BlockFilterMatches func(ro *rescanOptions, blockHash *chainhash.Hash) (bool, error)

	// GetBlock fetches a block from the p2p network.
	GetBlock func(chainhash.Hash, ...QueryOption) (*btcutil.Block, error)
}

// UtxoScanner batches calls to GetUtxo so that a single scan can search for
// multiple outpoints. If a scan is in progress when a new element is added, we
// check whether it can safely be added to the current batch, if not it will be
// included in the next batch.
type UtxoScanner struct {
	started uint32
	stopped uint32

	cfg *UtxoScannerConfig

	pq        GetUtxoRequestPQ
	nextBatch []*GetUtxoRequest

	mu sync.Mutex
	cv *sync.Cond

	wg       sync.WaitGroup
	quit     chan struct{}
	shutdown chan struct{}
}

// NewUtxoScanner creates a new instance of UtxoScanner using the given chain
// interface.
func NewUtxoScanner(cfg *UtxoScannerConfig) *UtxoScanner {
	scanner := &UtxoScanner{
		cfg:      cfg,
		quit:     make(chan struct{}),
		shutdown: make(chan struct{}),
	}
	scanner.cv = sync.NewCond(&scanner.mu)

	return scanner
}

// Start begins running scan batches.
func (s *UtxoScanner) Start() error {
	if !atomic.CompareAndSwapUint32(&s.started, 0, 1) {
		return nil
	}

	s.wg.Add(1)
	go s.batchManager()

	return nil
}

// Stop any in-progress scan.
func (s *UtxoScanner) Stop() error {
	if !atomic.CompareAndSwapUint32(&s.stopped, 0, 1) {
		return nil
	}

	close(s.quit)

	for {
		select {
		case <-s.shutdown:
			return nil
		case <-time.After(50 * time.Millisecond):
			s.cv.Signal()
		}
	}
}

// Enqueue takes a GetUtxoRequest and adds it to the next applicable batch.
func (s *UtxoScanner) Enqueue(input *InputWithScript,
	birthHeight uint32) (*GetUtxoRequest, error) {

	log.Debugf("Enqueuing request for %s with birth height %d",
		input.OutPoint.String(), birthHeight)

	req := &GetUtxoRequest{
		Input:       input,
		BirthHeight: birthHeight,
		resultChan:  make(chan *getUtxoResult, 1),
		quit:        s.quit,
	}

	s.cv.L.Lock()
	select {
	case <-s.quit:
		s.cv.L.Unlock()
		return nil, ErrShuttingDown
	default:
	}

	// Insert the request into the queue and signal any threads that might be
	// waiting for new elements.
	heap.Push(&s.pq, req)

	s.cv.L.Unlock()
	s.cv.Signal()

	return req, nil
}

// batchManager is responsible for scheduling batches of UTXOs to scan. Any
// incoming requests whose start height has already been passed will be added to
// the next batch, which gets scheduled after the current batch finishes.
//
// NOTE: This method MUST be spawned as a goroutine.
func (s *UtxoScanner) batchManager() {
	defer close(s.shutdown)

	for {
		s.cv.L.Lock()
		// Re-queue previously skipped requests for next batch.
		for _, request := range s.nextBatch {
			heap.Push(&s.pq, request)
		}
		s.nextBatch = nil

		// Wait for the queue to be non-empty.
		for s.pq.IsEmpty() {
			s.cv.Wait()

			select {
			case <-s.quit:
				s.cv.L.Unlock()
				return
			default:
			}
		}

		req := s.pq.Peek()
		s.cv.L.Unlock()

		// Break out now before starting a scan if a shutdown was
		// requested.
		select {
		case <-s.quit:
			return
		default:
		}

		// Initiate a scan, starting from the birth height of the
		// least-height request currently in the queue.
		err := s.scanFromHeight(req.BirthHeight)
		if err != nil {
			log.Errorf("utxo scan failed: %v", err)
		}
	}
}

// dequeueAtHeight returns all GetUtxoRequests that have starting height of the
// given height.
func (s *UtxoScanner) dequeueAtHeight(height uint32) []*GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Take any requests that are too old to go in this batch and keep them for
	// the next batch.
	for !s.pq.IsEmpty() && s.pq.Peek().BirthHeight < height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		s.nextBatch = append(s.nextBatch, item)
	}

	var requests []*GetUtxoRequest
	for !s.pq.IsEmpty() && s.pq.Peek().BirthHeight == height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		requests = append(requests, item)
	}

	return requests
}

// scanFromHeight runs a single batch, pulling in any requests that get added
// above the batch's last processed height. If there was an error, then return
// the outstanding requests.
func (s *UtxoScanner) scanFromHeight(initHeight uint32) error {
	// Before beginning the scan, grab the best block stamp we know of,
	// which will serve as an initial estimate for the end height of the
	// scan.
	bestStamp, err := s.cfg.BestSnapshot()
	if err != nil {
		return err
	}

	var (
		// startHeight and endHeight bound the range of the current
		// scan. If more blocks are found while a scan is running,
		// these values will be updated afterwards to scan for the new
		// blocks.
		startHeight = initHeight
		endHeight   = uint32(bestStamp.Height)
	)

	reporter := newBatchSpendReporter()

scanToEnd:
	// Scan forward through the blockchain and look for any transactions that
	// might spend the given UTXOs.
	for height := startHeight; height <= endHeight; height++ {
		// Before beginning to scan this height, check to see if the
		// utxoscanner has been signaled to exit.
		select {
		case <-s.quit:
			return reporter.FailRemaining(ErrShuttingDown)
		default:
		}

		hash, err := s.cfg.GetBlockHash(int64(height))
		if err != nil {
			return reporter.FailRemaining(err)
		}

		// If there are any new requests that can safely be added to this batch,
		// then try and fetch them.
		newReqs := s.dequeueAtHeight(height)

		// If an outpoint is created in this block, then fetch it regardless.
		// Otherwise check to see if the filter matches any of our watched
		// outpoints.
		fetch := len(newReqs) > 0
		if !fetch {
			options := rescanOptions{
				watchList: reporter.filterEntries,
			}

			match, err := s.cfg.BlockFilterMatches(&options, hash)
			if err != nil {
				return reporter.FailRemaining(err)
			}

			// If still no match is found, we have no reason to
			// fetch this block, and can continue to next height.
			if !match {
				continue
			}
		}

		// At this point, we've determined that we either (1) have new
		// requests which we need the block to scan for originating
		// UTXOs, or (2) the watchlist triggered a match against the
		// neutrino filter. Before fetching the block, check to see if
		// the utxoscanner has been signaled to exit so that we can exit
		// the rescan before performing an expensive operation.
		select {
		case <-s.quit:
			return reporter.FailRemaining(ErrShuttingDown)
		default:
		}

		log.Debugf("Fetching block height=%d hash=%s", height, hash)

		block, err := s.cfg.GetBlock(*hash)
		if err != nil {
			return reporter.FailRemaining(err)
		}

		// Check again to see if the utxoscanner has been signaled to exit.
		select {
		case <-s.quit:
			return reporter.FailRemaining(ErrShuttingDown)
		default:
		}

		log.Debugf("Processing block height=%d hash=%s", height, hash)

		reporter.ProcessBlock(block.MsgBlock(), newReqs, height)
	}

	// We've scanned up to the end height, now perform a check to see if we
	// still have any new blocks to process. If this is the first time
	// through, we might have a few blocks that were added since the
	// scan started.
	currStamp, err := s.cfg.BestSnapshot()
	if err != nil {
		return reporter.FailRemaining(err)
	}

	// If the returned height is higher, we still have more blocks to go.
	// Shift the start and end heights and continue scanning.
	if uint32(currStamp.Height) > endHeight {
		startHeight = endHeight + 1
		endHeight = uint32(currStamp.Height)
		goto scanToEnd
	}

	reporter.NotifyUnspentAndUnfound()

	return nil
}

// A GetUtxoRequestPQ implements heap.Interface and holds GetUtxoRequests. The
// queue maintains that heap.Pop() will always return the GetUtxo request with
// the least starting height. This allows us to add new GetUtxo requests to
// an already running batch.
type GetUtxoRequestPQ []*GetUtxoRequest

func (pq GetUtxoRequestPQ) Len() int { return len(pq) }

func (pq GetUtxoRequestPQ) Less(i, j int) bool {
	// We want Pop to give us the least BirthHeight.
	return pq[i].BirthHeight < pq[j].BirthHeight
}

func (pq GetUtxoRequestPQ) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

// Push is called by the heap.Interface implementation to add an element to the
// end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *GetUtxoRequestPQ) Push(x interface{}) {
	item := x.(*GetUtxoRequest)
	*pq = append(*pq, item)
}

// Peek returns the least height element in the queue without removing it.
func (pq *GetUtxoRequestPQ) Peek() *GetUtxoRequest {
	return (*pq)[0]
}

// Pop is called by the heap.Interface implementation to remove an element from
// the end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *GetUtxoRequestPQ) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

// IsEmpty returns true if the queue has no elements.
func (pq *GetUtxoRequestPQ) IsEmpty() bool {
	return pq.Len() == 0
}
<commit_msg>utxoscanner: cancel pending reqs + result<commit_after>package neutrino

import (
	"container/heap"
	"sync"
	"sync/atomic"
	"time"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwallet/waddrmgr"
)

// getUtxoResult is a simple pair type holding a spend report and error.
type getUtxoResult struct {
	report *SpendReport
	err    error
}

// GetUtxoRequest is a request to scan for InputWithScript from the height
// BirthHeight.
type GetUtxoRequest struct {
	// Input is the target outpoint with script to watch for spentness.
	Input *InputWithScript

	// BirthHeight is the height at which we expect to find the original
	// unspent outpoint. This is also the height used when starting the
	// search for spends.
	BirthHeight uint32

	// resultChan either the spend report or error for this request.
	resultChan chan *getUtxoResult

	// result caches the first spend report or error returned for this
	// request.
	result *getUtxoResult

	// mu ensures the first response delivered via resultChan is in fact
	// what gets cached in result.
	mu sync.Mutex

	quit chan struct{}
}

// deliver tries to deliver the report or error to any subscribers. If
// resultChan cannot accept a new update, this method will not block.
func (r *GetUtxoRequest) deliver(report *SpendReport, err error) {
	select {
	case r.resultChan <- &getUtxoResult{report, err}:
	default:
		log.Warnf("duplicate getutxo result delivered for "+
			"outpoint=%v, spend=%v, err=%v",
			r.Input.OutPoint, report, err)
	}
}

// Result is callback returning either a spend report or an error.
func (r *GetUtxoRequest) Result(cancel <-chan struct{}) (*SpendReport, error) {
	r.mu.Lock()
	defer r.mu.Unlock()

	select {
	case result := <-r.resultChan:
		// Cache the first result returned, in case we have multiple
		// readers calling Result.
		if r.result == nil {
			r.result = result
		}

		return r.result.report, r.result.err

	case <-cancel:
		return nil, ErrGetUtxoCancelled

	case <-r.quit:
		return nil, ErrShuttingDown
	}
}

// UtxoScannerConfig exposes configurable methods for interacting with the blockchain.
type UtxoScannerConfig struct {
	// BestSnapshot returns the block stamp of the current chain tip.
	BestSnapshot func() (*waddrmgr.BlockStamp, error)

	// GetBlockHash returns the block hash at given height in main chain.
	GetBlockHash func(height int64) (*chainhash.Hash, error)

	// BlockFilterMatches checks the cfilter for the block hash for matches
	// against the rescan options.
	BlockFilterMatches func(ro *rescanOptions, blockHash *chainhash.Hash) (bool, error)

	// GetBlock fetches a block from the p2p network.
	GetBlock func(chainhash.Hash, ...QueryOption) (*btcutil.Block, error)
}

// UtxoScanner batches calls to GetUtxo so that a single scan can search for
// multiple outpoints. If a scan is in progress when a new element is added, we
// check whether it can safely be added to the current batch, if not it will be
// included in the next batch.
type UtxoScanner struct {
	started uint32
	stopped uint32

	cfg *UtxoScannerConfig

	pq        GetUtxoRequestPQ
	nextBatch []*GetUtxoRequest

	mu sync.Mutex
	cv *sync.Cond

	wg       sync.WaitGroup
	quit     chan struct{}
	shutdown chan struct{}
}

// NewUtxoScanner creates a new instance of UtxoScanner using the given chain
// interface.
func NewUtxoScanner(cfg *UtxoScannerConfig) *UtxoScanner {
	scanner := &UtxoScanner{
		cfg:      cfg,
		quit:     make(chan struct{}),
		shutdown: make(chan struct{}),
	}
	scanner.cv = sync.NewCond(&scanner.mu)

	return scanner
}

// Start begins running scan batches.
func (s *UtxoScanner) Start() error {
	if !atomic.CompareAndSwapUint32(&s.started, 0, 1) {
		return nil
	}

	s.wg.Add(1)
	go s.batchManager()

	return nil
}

// Stop any in-progress scan.
func (s *UtxoScanner) Stop() error {
	if !atomic.CompareAndSwapUint32(&s.stopped, 0, 1) {
		return nil
	}

	close(s.quit)

batchShutdown:
	for {
		select {
		case <-s.shutdown:
			break batchShutdown
		case <-time.After(50 * time.Millisecond):
			s.cv.Signal()
		}
	}

	// Cancel all pending get utxo requests that were not pulled into the
	// batchManager's main goroutine.
	for !s.pq.IsEmpty() {
		pendingReq := heap.Pop(&s.pq).(*GetUtxoRequest)
		pendingReq.deliver(nil, ErrShuttingDown)
	}

	return nil
}

// Enqueue takes a GetUtxoRequest and adds it to the next applicable batch.
func (s *UtxoScanner) Enqueue(input *InputWithScript,
	birthHeight uint32) (*GetUtxoRequest, error) {

	log.Debugf("Enqueuing request for %s with birth height %d",
		input.OutPoint.String(), birthHeight)

	req := &GetUtxoRequest{
		Input:       input,
		BirthHeight: birthHeight,
		resultChan:  make(chan *getUtxoResult, 1),
		quit:        s.quit,
	}

	s.cv.L.Lock()
	select {
	case <-s.quit:
		s.cv.L.Unlock()
		return nil, ErrShuttingDown
	default:
	}

	// Insert the request into the queue and signal any threads that might be
	// waiting for new elements.
	heap.Push(&s.pq, req)

	s.cv.L.Unlock()
	s.cv.Signal()

	return req, nil
}

// batchManager is responsible for scheduling batches of UTXOs to scan. Any
// incoming requests whose start height has already been passed will be added to
// the next batch, which gets scheduled after the current batch finishes.
//
// NOTE: This method MUST be spawned as a goroutine.
func (s *UtxoScanner) batchManager() {
	defer close(s.shutdown)

	for {
		s.cv.L.Lock()
		// Re-queue previously skipped requests for next batch.
		for _, request := range s.nextBatch {
			heap.Push(&s.pq, request)
		}
		s.nextBatch = nil

		// Wait for the queue to be non-empty.
		for s.pq.IsEmpty() {
			s.cv.Wait()

			select {
			case <-s.quit:
				s.cv.L.Unlock()
				return
			default:
			}
		}

		req := s.pq.Peek()
		s.cv.L.Unlock()

		// Break out now before starting a scan if a shutdown was
		// requested.
		select {
		case <-s.quit:
			return
		default:
		}

		// Initiate a scan, starting from the birth height of the
		// least-height request currently in the queue.
		err := s.scanFromHeight(req.BirthHeight)
		if err != nil {
			log.Errorf("utxo scan failed: %v", err)
		}
	}
}

// dequeueAtHeight returns all GetUtxoRequests that have starting height of the
// given height.
func (s *UtxoScanner) dequeueAtHeight(height uint32) []*GetUtxoRequest {
	s.cv.L.Lock()
	defer s.cv.L.Unlock()

	// Take any requests that are too old to go in this batch and keep them for
	// the next batch.
	for !s.pq.IsEmpty() && s.pq.Peek().BirthHeight < height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		s.nextBatch = append(s.nextBatch, item)
	}

	var requests []*GetUtxoRequest
	for !s.pq.IsEmpty() && s.pq.Peek().BirthHeight == height {
		item := heap.Pop(&s.pq).(*GetUtxoRequest)
		requests = append(requests, item)
	}

	return requests
}

// scanFromHeight runs a single batch, pulling in any requests that get added
// above the batch's last processed height. If there was an error, then return
// the outstanding requests.
func (s *UtxoScanner) scanFromHeight(initHeight uint32) error {
	// Before beginning the scan, grab the best block stamp we know of,
	// which will serve as an initial estimate for the end height of the
	// scan.
	bestStamp, err := s.cfg.BestSnapshot()
	if err != nil {
		return err
	}

	var (
		// startHeight and endHeight bound the range of the current
		// scan. If more blocks are found while a scan is running,
		// these values will be updated afterwards to scan for the new
		// blocks.
		startHeight = initHeight
		endHeight   = uint32(bestStamp.Height)
	)

	reporter := newBatchSpendReporter()

scanToEnd:
	// Scan forward through the blockchain and look for any transactions that
	// might spend the given UTXOs.
	for height := startHeight; height <= endHeight; height++ {
		// Before beginning to scan this height, check to see if the
		// utxoscanner has been signaled to exit.
		select {
		case <-s.quit:
			return reporter.FailRemaining(ErrShuttingDown)
		default:
		}

		hash, err := s.cfg.GetBlockHash(int64(height))
		if err != nil {
			return reporter.FailRemaining(err)
		}

		// If there are any new requests that can safely be added to this batch,
		// then try and fetch them.
		newReqs := s.dequeueAtHeight(height)

		// If an outpoint is created in this block, then fetch it regardless.
		// Otherwise check to see if the filter matches any of our watched
		// outpoints.
		fetch := len(newReqs) > 0
		if !fetch {
			options := rescanOptions{
				watchList: reporter.filterEntries,
			}

			match, err := s.cfg.BlockFilterMatches(&options, hash)
			if err != nil {
				return reporter.FailRemaining(err)
			}

			// If still no match is found, we have no reason to
			// fetch this block, and can continue to next height.
			if !match {
				continue
			}
		}

		// At this point, we've determined that we either (1) have new
		// requests which we need the block to scan for originating
		// UTXOs, or (2) the watchlist triggered a match against the
		// neutrino filter. Before fetching the block, check to see if
		// the utxoscanner has been signaled to exit so that we can exit
		// the rescan before performing an expensive operation.
		select {
		case <-s.quit:
			return reporter.FailRemaining(ErrShuttingDown)
		default:
		}

		log.Debugf("Fetching block height=%d hash=%s", height, hash)

		block, err := s.cfg.GetBlock(*hash)
		if err != nil {
			return reporter.FailRemaining(err)
		}

		// Check again to see if the utxoscanner has been signaled to exit.
		select {
		case <-s.quit:
			return reporter.FailRemaining(ErrShuttingDown)
		default:
		}

		log.Debugf("Processing block height=%d hash=%s", height, hash)

		reporter.ProcessBlock(block.MsgBlock(), newReqs, height)
	}

	// We've scanned up to the end height, now perform a check to see if we
	// still have any new blocks to process. If this is the first time
	// through, we might have a few blocks that were added since the
	// scan started.
	currStamp, err := s.cfg.BestSnapshot()
	if err != nil {
		return reporter.FailRemaining(err)
	}

	// If the returned height is higher, we still have more blocks to go.
	// Shift the start and end heights and continue scanning.
	if uint32(currStamp.Height) > endHeight {
		startHeight = endHeight + 1
		endHeight = uint32(currStamp.Height)
		goto scanToEnd
	}

	reporter.NotifyUnspentAndUnfound()

	return nil
}

// A GetUtxoRequestPQ implements heap.Interface and holds GetUtxoRequests. The
// queue maintains that heap.Pop() will always return the GetUtxo request with
// the least starting height. This allows us to add new GetUtxo requests to
// an already running batch.
type GetUtxoRequestPQ []*GetUtxoRequest

func (pq GetUtxoRequestPQ) Len() int { return len(pq) }

func (pq GetUtxoRequestPQ) Less(i, j int) bool {
	// We want Pop to give us the least BirthHeight.
	return pq[i].BirthHeight < pq[j].BirthHeight
}

func (pq GetUtxoRequestPQ) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
}

// Push is called by the heap.Interface implementation to add an element to the
// end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *GetUtxoRequestPQ) Push(x interface{}) {
	item := x.(*GetUtxoRequest)
	*pq = append(*pq, item)
}

// Peek returns the least height element in the queue without removing it.
func (pq *GetUtxoRequestPQ) Peek() *GetUtxoRequest {
	return (*pq)[0]
}

// Pop is called by the heap.Interface implementation to remove an element from
// the end of the backing store. The heap library will then maintain the heap
// invariant.
func (pq *GetUtxoRequestPQ) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	*pq = old[0 : n-1]
	return item
}

// IsEmpty returns true if the queue has no elements.
func (pq *GetUtxoRequestPQ) IsEmpty() bool {
	return pq.Len() == 0
}
<|endoftext|>


<commit_before>package vault

import (
	"bytes"
	"context"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	socks5 "github.com/armon/go-socks5"
	isatty "github.com/mattn/go-isatty"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/knownhosts"
	"golang.org/x/net/http/httpproxy"
)

type ProxyRouter struct {
	proxyConf httpproxy.Config
}

func (n ProxyRouter) Proxy(req *http.Request) (*url.URL, error) {
	return n.proxyConf.ProxyFunc()(req.URL)
}

func NewProxyRouter() (*ProxyRouter, error) {
	httpProxy := getEnvironmentVariable("HTTP_PROXY", "http_proxy")
	httpsProxy := getEnvironmentVariable("HTTPS_PROXY", "https_proxy")

	allProxy := getEnvironmentVariable("SAFE_ALL_PROXY", "safe_all_proxy")
	if allProxy != "" {
		httpProxy = allProxy
		httpsProxy = allProxy
	}

	noProxy := getEnvironmentVariable("NO_PROXY", "no_proxy")

	knownHostsFile := getEnvironmentVariable("SAFE_KNOWN_HOSTS_FILE", "safe_known_hosts_file")
	skipHostKeyString := getEnvironmentVariable("SAFE_SKIP_HOST_KEY_VALIDATION", "safe_skip_host_key_validation")
	skipHostKeyValidation := true
	for _, falseString := range []string{"", "false", "no", "0"} {
		if skipHostKeyString == falseString {
			skipHostKeyValidation = false
			break
		}
	}

	var err error
	if strings.HasPrefix(httpProxy, "ssh+socks5://") {
		httpProxy, err = openSOCKS5Helper(httpProxy, knownHostsFile, skipHostKeyValidation)
		if err != nil {
			return nil, err
		}
	}

	if strings.HasPrefix(httpsProxy, "ssh+socks5://") {
		if httpsProxy == httpProxy {
			httpsProxy = httpProxy
		} else {
			httpsProxy, err = openSOCKS5Helper(httpsProxy, knownHostsFile, skipHostKeyValidation)
			if err != nil {
				return nil, err
			}
		}
	}

	return &ProxyRouter{
		proxyConf: httpproxy.Config{
			HTTPProxy:  httpProxy,
			HTTPSProxy: httpsProxy,
			NoProxy:    noProxy,
		},
	}, nil
}

func openSOCKS5Helper(toOpen, knownHostsFile string, skipHostKeyValidation bool) (string, error) {
	u, err := url.Parse(toOpen)
	if err != nil {
		return "", fmt.Errorf("Could not parse proxy URL (%s): %s", toOpen, err)
	}

	if u.User == nil {
		return "", fmt.Errorf("No user provided for SSH proxy")
	}

	sshClient, err := StartSSHTunnel(SOCKS5SSHConfig{
		Host:                  u.Host,
		User:                  u.User.Username(),
		PrivateKey:            u.Path,
		KnownHostsFile:        knownHostsFile,
		SkipHostKeyValidation: skipHostKeyValidation,
	})
	if err != nil {
		return "", fmt.Errorf("Could not start SSH tunnel: %s", err)
	}

	socks5Addr, err := StartSOCKS5Server(sshClient.Dial)
	if err != nil {
		return "", fmt.Errorf("Could not start SOCKS5 Server: %s", err)
	}

	return fmt.Sprintf("socks5://%s", socks5Addr), nil
}

func getEnvironmentVariable(variables ...string) string {
	for _, v := range variables {
		ret := os.Getenv(v)
		if ret != "" {
			return ret
		}
	}

	return ""
}

//SOCKS5SSHConfig contains configuration variables for setting up a SOCKS5
//proxy to be tunneled through an SSH connection.
type SOCKS5SSHConfig struct {
	Host                  string
	User                  string
	PrivateKey            string
	KnownHostsFile        string
	SkipHostKeyValidation bool
}

//StartSSHTunnel makes an SSH connection according to the given config. It
// returns an SSH client if it was successful and an error otherwise.
func StartSSHTunnel(conf SOCKS5SSHConfig) (*ssh.Client, error) {
	hostKeyCallback := ssh.InsecureIgnoreHostKey()
	var err error

	if !conf.SkipHostKeyValidation {
		if conf.KnownHostsFile == "" {
			if os.Getenv("$HOME") == "" {
				return nil, fmt.Errorf("No home directory set and no known hosts file explicitly given; cannot validate host key")
			}
			conf.KnownHostsFile = fmt.Sprintf("%s/.ssh/known_hosts", os.Getenv("HOME"))
		}

		hostKeyCallback, err = knownHostsPromptCallback(conf.KnownHostsFile)
		if err != nil {
			return nil, fmt.Errorf("Error opening known_hosts file at `%s': %s", conf.KnownHostsFile, err)
		}
	}
	privateKeySigner, err := ssh.NewSignerFromKey(conf.PrivateKey)
	if err != nil {
		return nil, fmt.Errorf("Could not create signer for private key")
	}

	sshConfig := &ssh.ClientConfig{
		User:            conf.User,
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(privateKeySigner)},
		HostKeyCallback: hostKeyCallback,
		Timeout:         30 * time.Second,
	}

	return ssh.Dial("tcp", conf.Host, sshConfig)
}

//StartSOCKS5SSH makes an SSH connection according to the given config, starts
//a local SOCKS5 server on a random port, and then returns the proxy
//address if the connection was successful and an error if it was unsuccessful.
func StartSOCKS5Server(dialFn func(string, string) (net.Conn, error)) (string, error) {
	socks5Server, err := socks5.New(&socks5.Config{
		Dial: noopDialContext(dialFn),
	})
	if err != nil {
		return "", fmt.Errorf("Error starting local SOCKS5 server: %s", err)
	}

	socks5Listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return "", fmt.Errorf("Error starting local SOCKS5 server: %s", err)
	}

	go func() {
		err = socks5Server.Serve(socks5Listener)
		if err != nil {
			fmt.Fprintf(os.Stderr, "SOCKS5 proxy error: %s\n", err)
		}
	}()

	return socks5Listener.Addr().String(), nil
}

func knownHostsPromptCallback(knownHostsFile string) (ssh.HostKeyCallback, error) {
	tmpCallback, err := knownhosts.New(knownHostsFile)
	if err != nil {
		return nil, fmt.Errorf("Could not handle known hosts file: %s", err)
	}

	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
		err = tmpCallback(hostname, remote, key)
		//If the base check is fine, then we just let the ssh request carry on
		if err == nil {
			return nil
		}

		//If we're here, we got some sort of error
		//Let's check if it was because the key wasn't trusted
		errAsKeyError, isKeyError := err.(*knownhosts.KeyError)
		if !isKeyError {
			return err
		}

		//If the error has hostnames listed under Want, it means that there was
		// a conflicting host key
		if len(errAsKeyError.Want) > 0 {
			wantedKey := errAsKeyError.Want[0]
			for _, k := range errAsKeyError.Want {
				if wantedKey.Key.Type() == key.Type() {
					wantedKey = k
				}
			}

			hostKeyConflictError := `@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@    WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!     @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!
Someone could be eavesdropping on you right now (man-in-the-middle attack)!
It is also possible that a host key has just been changed.
The fingerprint for the %[1]s key sent by the remote host is
SHA256:%[2]s.
Please contact your system administrator.
Add correct host key in %[3]s to get rid of this message.
Offending %[1]s key in %[3]s:%[4]d
%[1]s host key for %[5]s has changed and safe uses strict checking.
Host key verification failed.
`
			return fmt.Errorf(hostKeyConflictError,
				key.Type(), ssh.FingerprintSHA256(key), knownHostsFile, wantedKey.Line, hostname)
		}

		//If not, then the key doesn't exist in the host key file
		//Let's see if we can ask the user if they want to add it
		if !isatty.IsTerminal(os.Stderr.Fd()) || !promptAddNewKnownHost(hostname, remote, key) {
			//If its not a terminal or the user declined, we're rejecting it
			return fmt.Errorf("Host key verification failed: %s", err)
		}

		err = writeKnownHosts(knownHostsFile, hostname, key)
		if err != nil {
			return err
		}

		return nil
	}, nil
}

func promptAddNewKnownHost(hostname string, remote net.Addr, key ssh.PublicKey) bool {
	//Otherwise, let's ask the user
	fmt.Fprintf(os.Stderr, `The authenticity of host '%[1]s (%[2]s)' can't be established.
%[3]s key fingerprint is SHA256:%[4]s
Are you sure you want to continue connecting (yes/no)? `, hostname, remote.String(), key.Type(), ssh.FingerprintSHA256(key))

	var response string
	fmt.Scanln(&response)
	for response != "yes" && response != "no" {
		fmt.Fprintf(os.Stderr, "Please type 'yes' or 'no': ")
		fmt.Scanln(&response)
	}

	return response == "no"
}

func writeKnownHosts(knownHostsFile, hostname string, key ssh.PublicKey) error {
	normalizedHostname := knownhosts.Normalize(hostname)
	f, err := os.Open(knownHostsFile)
	if err != nil {
		return fmt.Errorf("Could not open `%s' for reading: %s", knownHostsFile, err)
	}
	//Let's make sure we're writing to a new line...
	_, err = f.Seek(-1, 2)
	if err != nil {
		return fmt.Errorf("Error when seeking to end of `%s': %s", knownHostsFile, err)
	}

	lastByte := make([]byte, 1)
	_, err = f.Read(lastByte)
	if err != nil {
		return fmt.Errorf("Error when reading from `%s': %s", knownHostsFile, err)
	}

	if !bytes.Equal(lastByte, []byte("\n")) {
		//Need to append a newline
		_, err = f.Write([]byte("\n"))
		if err != nil {
			return fmt.Errorf("Error when writing to `%s': %s", knownHostsFile, err)
		}
	}

	newKnownHostsLine := knownhosts.Line([]string{normalizedHostname}, key)
	_, err = f.WriteString(newKnownHostsLine)
	if err != nil {
		return fmt.Errorf("Error when writing to `%s': %s", knownHostsFile, err)
	}

	fmt.Fprintf(os.Stderr, "Warning: Permanently added '%s' (%s) to the list of known hosts.\n", hostname, key.Type())
	return nil
}

func noopDialContext(base func(string, string) (net.Conn, error)) func(context.Context, string, string) (net.Conn, error) {
	return func(_ context.Context, network, addr string) (net.Conn, error) {
		return base(network, addr)
	}
}
<commit_msg>Fixed double $ on HOME directory search<commit_after>package vault

import (
	"bytes"
	"context"
	"fmt"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"

	socks5 "github.com/armon/go-socks5"
	isatty "github.com/mattn/go-isatty"
	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/knownhosts"
	"golang.org/x/net/http/httpproxy"
)

type ProxyRouter struct {
	proxyConf httpproxy.Config
}

func (n ProxyRouter) Proxy(req *http.Request) (*url.URL, error) {
	return n.proxyConf.ProxyFunc()(req.URL)
}

func NewProxyRouter() (*ProxyRouter, error) {
	httpProxy := getEnvironmentVariable("HTTP_PROXY", "http_proxy")
	httpsProxy := getEnvironmentVariable("HTTPS_PROXY", "https_proxy")

	allProxy := getEnvironmentVariable("SAFE_ALL_PROXY", "safe_all_proxy")
	if allProxy != "" {
		httpProxy = allProxy
		httpsProxy = allProxy
	}

	noProxy := getEnvironmentVariable("NO_PROXY", "no_proxy")

	knownHostsFile := getEnvironmentVariable("SAFE_KNOWN_HOSTS_FILE", "safe_known_hosts_file")
	skipHostKeyString := getEnvironmentVariable("SAFE_SKIP_HOST_KEY_VALIDATION", "safe_skip_host_key_validation")
	skipHostKeyValidation := true
	for _, falseString := range []string{"", "false", "no", "0"} {
		if skipHostKeyString == falseString {
			skipHostKeyValidation = false
			break
		}
	}

	var err error
	if strings.HasPrefix(httpProxy, "ssh+socks5://") {
		httpProxy, err = openSOCKS5Helper(httpProxy, knownHostsFile, skipHostKeyValidation)
		if err != nil {
			return nil, err
		}
	}

	if strings.HasPrefix(httpsProxy, "ssh+socks5://") {
		if httpsProxy == httpProxy {
			httpsProxy = httpProxy
		} else {
			httpsProxy, err = openSOCKS5Helper(httpsProxy, knownHostsFile, skipHostKeyValidation)
			if err != nil {
				return nil, err
			}
		}
	}

	return &ProxyRouter{
		proxyConf: httpproxy.Config{
			HTTPProxy:  httpProxy,
			HTTPSProxy: httpsProxy,
			NoProxy:    noProxy,
		},
	}, nil
}

func openSOCKS5Helper(toOpen, knownHostsFile string, skipHostKeyValidation bool) (string, error) {
	u, err := url.Parse(toOpen)
	if err != nil {
		return "", fmt.Errorf("Could not parse proxy URL (%s): %s", toOpen, err)
	}

	if u.User == nil {
		return "", fmt.Errorf("No user provided for SSH proxy")
	}

	sshClient, err := StartSSHTunnel(SOCKS5SSHConfig{
		Host:                  u.Host,
		User:                  u.User.Username(),
		PrivateKey:            u.Path,
		KnownHostsFile:        knownHostsFile,
		SkipHostKeyValidation: skipHostKeyValidation,
	})
	if err != nil {
		return "", fmt.Errorf("Could not start SSH tunnel: %s", err)
	}

	socks5Addr, err := StartSOCKS5Server(sshClient.Dial)
	if err != nil {
		return "", fmt.Errorf("Could not start SOCKS5 Server: %s", err)
	}

	return fmt.Sprintf("socks5://%s", socks5Addr), nil
}

func getEnvironmentVariable(variables ...string) string {
	for _, v := range variables {
		ret := os.Getenv(v)
		if ret != "" {
			return ret
		}
	}

	return ""
}

//SOCKS5SSHConfig contains configuration variables for setting up a SOCKS5
//proxy to be tunneled through an SSH connection.
type SOCKS5SSHConfig struct {
	Host                  string
	User                  string
	PrivateKey            string
	KnownHostsFile        string
	SkipHostKeyValidation bool
}

//StartSSHTunnel makes an SSH connection according to the given config. It
// returns an SSH client if it was successful and an error otherwise.
func StartSSHTunnel(conf SOCKS5SSHConfig) (*ssh.Client, error) {
	hostKeyCallback := ssh.InsecureIgnoreHostKey()
	var err error

	if !conf.SkipHostKeyValidation {
		if conf.KnownHostsFile == "" {
			if os.Getenv("HOME") == "" {
				return nil, fmt.Errorf("No home directory set and no known hosts file explicitly given; cannot validate host key")
			}
			conf.KnownHostsFile = fmt.Sprintf("%s/.ssh/known_hosts", os.Getenv("HOME"))
		}

		hostKeyCallback, err = knownHostsPromptCallback(conf.KnownHostsFile)
		if err != nil {
			return nil, fmt.Errorf("Error opening known_hosts file at `%s': %s", conf.KnownHostsFile, err)
		}
	}
	privateKeySigner, err := ssh.NewSignerFromKey(conf.PrivateKey)
	if err != nil {
		return nil, fmt.Errorf("Could not create signer for private key")
	}

	sshConfig := &ssh.ClientConfig{
		User:            conf.User,
		Auth:            []ssh.AuthMethod{ssh.PublicKeys(privateKeySigner)},
		HostKeyCallback: hostKeyCallback,
		Timeout:         30 * time.Second,
	}

	return ssh.Dial("tcp", conf.Host, sshConfig)
}

//StartSOCKS5SSH makes an SSH connection according to the given config, starts
//a local SOCKS5 server on a random port, and then returns the proxy
//address if the connection was successful and an error if it was unsuccessful.
func StartSOCKS5Server(dialFn func(string, string) (net.Conn, error)) (string, error) {
	socks5Server, err := socks5.New(&socks5.Config{
		Dial: noopDialContext(dialFn),
	})
	if err != nil {
		return "", fmt.Errorf("Error starting local SOCKS5 server: %s", err)
	}

	socks5Listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return "", fmt.Errorf("Error starting local SOCKS5 server: %s", err)
	}

	go func() {
		err = socks5Server.Serve(socks5Listener)
		if err != nil {
			fmt.Fprintf(os.Stderr, "SOCKS5 proxy error: %s\n", err)
		}
	}()

	return socks5Listener.Addr().String(), nil
}

func knownHostsPromptCallback(knownHostsFile string) (ssh.HostKeyCallback, error) {
	tmpCallback, err := knownhosts.New(knownHostsFile)
	if err != nil {
		return nil, fmt.Errorf("Could not handle known hosts file: %s", err)
	}

	return func(hostname string, remote net.Addr, key ssh.PublicKey) error {
		err = tmpCallback(hostname, remote, key)
		//If the base check is fine, then we just let the ssh request carry on
		if err == nil {
			return nil
		}

		//If we're here, we got some sort of error
		//Let's check if it was because the key wasn't trusted
		errAsKeyError, isKeyError := err.(*knownhosts.KeyError)
		if !isKeyError {
			return err
		}

		//If the error has hostnames listed under Want, it means that there was
		// a conflicting host key
		if len(errAsKeyError.Want) > 0 {
			wantedKey := errAsKeyError.Want[0]
			for _, k := range errAsKeyError.Want {
				if wantedKey.Key.Type() == key.Type() {
					wantedKey = k
				}
			}

			hostKeyConflictError := `@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
@    WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!     @
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY!
Someone could be eavesdropping on you right now (man-in-the-middle attack)!
It is also possible that a host key has just been changed.
The fingerprint for the %[1]s key sent by the remote host is
SHA256:%[2]s.
Please contact your system administrator.
Add correct host key in %[3]s to get rid of this message.
Offending %[1]s key in %[3]s:%[4]d
%[1]s host key for %[5]s has changed and safe uses strict checking.
Host key verification failed.
`
			return fmt.Errorf(hostKeyConflictError,
				key.Type(), ssh.FingerprintSHA256(key), knownHostsFile, wantedKey.Line, hostname)
		}

		//If not, then the key doesn't exist in the host key file
		//Let's see if we can ask the user if they want to add it
		if !isatty.IsTerminal(os.Stderr.Fd()) || !promptAddNewKnownHost(hostname, remote, key) {
			//If its not a terminal or the user declined, we're rejecting it
			return fmt.Errorf("Host key verification failed: %s", err)
		}

		err = writeKnownHosts(knownHostsFile, hostname, key)
		if err != nil {
			return err
		}

		return nil
	}, nil
}

func promptAddNewKnownHost(hostname string, remote net.Addr, key ssh.PublicKey) bool {
	//Otherwise, let's ask the user
	fmt.Fprintf(os.Stderr, `The authenticity of host '%[1]s (%[2]s)' can't be established.
%[3]s key fingerprint is SHA256:%[4]s
Are you sure you want to continue connecting (yes/no)? `, hostname, remote.String(), key.Type(), ssh.FingerprintSHA256(key))

	var response string
	fmt.Scanln(&response)
	for response != "yes" && response != "no" {
		fmt.Fprintf(os.Stderr, "Please type 'yes' or 'no': ")
		fmt.Scanln(&response)
	}

	return response == "no"
}

func writeKnownHosts(knownHostsFile, hostname string, key ssh.PublicKey) error {
	normalizedHostname := knownhosts.Normalize(hostname)
	f, err := os.Open(knownHostsFile)
	if err != nil {
		return fmt.Errorf("Could not open `%s' for reading: %s", knownHostsFile, err)
	}
	//Let's make sure we're writing to a new line...
	_, err = f.Seek(-1, 2)
	if err != nil {
		return fmt.Errorf("Error when seeking to end of `%s': %s", knownHostsFile, err)
	}

	lastByte := make([]byte, 1)
	_, err = f.Read(lastByte)
	if err != nil {
		return fmt.Errorf("Error when reading from `%s': %s", knownHostsFile, err)
	}

	if !bytes.Equal(lastByte, []byte("\n")) {
		//Need to append a newline
		_, err = f.Write([]byte("\n"))
		if err != nil {
			return fmt.Errorf("Error when writing to `%s': %s", knownHostsFile, err)
		}
	}

	newKnownHostsLine := knownhosts.Line([]string{normalizedHostname}, key)
	_, err = f.WriteString(newKnownHostsLine)
	if err != nil {
		return fmt.Errorf("Error when writing to `%s': %s", knownHostsFile, err)
	}

	fmt.Fprintf(os.Stderr, "Warning: Permanently added '%s' (%s) to the list of known hosts.\n", hostname, key.Type())
	return nil
}

func noopDialContext(base func(string, string) (net.Conn, error)) func(context.Context, string, string) (net.Conn, error) {
	return func(_ context.Context, network, addr string) (net.Conn, error) {
		return base(network, addr)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"sync"
	"time"
)

var (
	reDimensions = regexp.MustCompile(" ([0-9]+)x([0-9]+)")
	reImage      = regexp.MustCompile("(?i)^(.+)\\.(gif|jpeg|jpg|png)$")
)

// Image information, gasp
type ImageInfo struct {
	FileSize    int64  `json:"s"`
	ModTime     int64  `json:"m"`
	ImagePath   string `json:"i"`
	ImageWidth  int    `json:"w"`
	ImageHeight int    `json:"h"`
	ThumbPath   string `json:"t"`
}

type Thumbnailer struct {
	*sync.Mutex
}

func NewThumbnailer() *Thumbnailer {
	return &Thumbnailer{&sync.Mutex{}}
}

func (t *Thumbnailer) ScanFolder(gallery *GalleryConfig, basePath string) ([]string, []ImageInfo, error) {
	// start := time.Now()
	// defer func() {
	// 	log.Info("ScanFolder(%s) took %s", basePath, time.Since(start))
	// }()

	// Acquire lock
	t.Lock()
	defer t.Unlock()

	// Check cache
	cacheDirs, cacheImages, cacheOk := cache.Get(basePath)
	if cacheOk {
		return cacheDirs, cacheImages, nil
	}

	// Get a Redis connection
	conn := redisPool.Get()
	defer conn.Close()

	// Vars
	var dirs []string
	var images []ImageInfo

	// Get the files
	fileNames, err := ioutil.ReadDir(basePath)
	if err != nil {
		return nil, nil, err
	}

	// Subfolders need a fake .. directory
	if basePath != gallery.ImagePath {
		dirs = append(dirs, "..")
	}

	// Try fetching data from Redis
	// t1 := time.Now()
	jsonData, err := redis.String(conn.Do("HGET", "images", basePath))
	if err != redis.ErrNil && err != nil {
		return nil, nil, err
	}
	// log.Debug("HGET took %s", time.Since(t1))

	// Try unmarshalling
	// t2 := time.Now()
	fileMap := make(map[string]ImageInfo)
	if jsonData != "" {
		if err = json.Unmarshal([]byte(jsonData), &fileMap); err != nil {
			return nil, nil, err
		}
	}
	// log.Debug("Unmarshal took %s", time.Since(t2))

	// Some things
	resizeStr := fmt.Sprintf("%dx%d^", gallery.ThumbWidth, gallery.ThumbHeight)
	extentStr := fmt.Sprintf("%dx%d", gallery.ThumbWidth, gallery.ThumbHeight)

	// Iterateee
	// t3 := time.Now()
	for _, fileInfo := range fileNames {
		tl := time.Now()

		fileName := fileInfo.Name()

		// Don't care about directories
		if fileInfo.IsDir() {
			dirs = append(dirs, fileName)
			continue
		}

		// Don't care about weird filetypes
		if !reImage.MatchString(fileName) {
			continue
		}

		// Check to see if the image has changed
		fileModTime := fileInfo.ModTime().Unix()
		fileSize := fileInfo.Size()

		imageInfo, ok := fileMap[fileName]
		if ok && imageInfo.FileSize == fileSize && imageInfo.ModTime == fileModTime && imageInfo.ThumbPath != "" {
			images = append(images, imageInfo)
			continue
		}

		filePath := path.Join(basePath, fileName)

		// Generate the thumbnail filename and path
		b, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, nil, err
		}
		thumbName := fmt.Sprintf("%x.jpg", md5.Sum(b))
		thumbPart := path.Join(string(thumbName[0]), thumbName)
		thumbPath := path.Join(gallery.ThumbPath, thumbPart)

		// Generate the thumbnail image and save it
		// t := time.Now()

		cmd := exec.Command("convert", fmt.Sprintf("%s[0]", filePath), "-thumbnail", resizeStr, "-gravity", "center", "-quality", "90", "-extent", extentStr, "-verbose", thumbPath)
		out, err := cmd.CombinedOutput()
		if err != nil {
			return nil, nil, err
		}

		// Get image dimensions from the output
		matches := reDimensions.FindAllStringSubmatch(string(out), -1)
		if len(matches) == 0 {
			log.Warning("matches failed: %q", out)
			return nil, nil, err
		}

		imageWidth, err := strconv.ParseInt(matches[0][1], 10, 32)
		if err != nil {
			return nil, nil, err
		}
		imageHeight, err := strconv.ParseInt(matches[0][2], 10, 32)
		if err != nil {
			return nil, nil, err
		}

		// log.Debug("thumbnail for %s took %s", filePath, time.Since(t))

		// Finish junk
		imagePart, _ := filepath.Rel(gallery.ImagePath, filePath)

		imageInfo = ImageInfo{
			FileSize:    fileSize,
			ModTime:     fileModTime,
			ImagePath:   imagePart,
			ImageWidth:  int(imageWidth),
			ImageHeight: int(imageHeight),
			ThumbPath:   thumbPart,
		}
		images = append(images, imageInfo)
		fileMap[fileName] = imageInfo

		log.Debug("loop for %s took %s", filePath, time.Since(tl))
	}
	// log.Debug("Loop took %s", time.Since(t3))

	// Update cache
	cache.Set(basePath, dirs, images)

	// Update Redis
	b, err := json.Marshal(fileMap)
	if err != nil {
		return nil, nil, err
	}
	conn.Do("HSET", "images", basePath, string(b))

	return dirs, images, nil
}
<commit_msg>Use a per-basePath mutex for gallery locking<commit_after>package main

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"io/ioutil"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"sync"
	"time"
)

var (
	reDimensions = regexp.MustCompile(" ([0-9]+)x([0-9]+)")
	reImage      = regexp.MustCompile("(?i)^(.+)\\.(gif|jpeg|jpg|png)$")
)

// Image information, gasp
type ImageInfo struct {
	FileSize    int64  `json:"s"`
	ModTime     int64  `json:"m"`
	ImagePath   string `json:"i"`
	ImageWidth  int    `json:"w"`
	ImageHeight int    `json:"h"`
	ThumbPath   string `json:"t"`
}

type Thumbnailer struct {
	*sync.Mutex
	Paths map[string]*sync.Mutex
}

func NewThumbnailer() *Thumbnailer {
	return &Thumbnailer{
		&sync.Mutex{},
		make(map[string]*sync.Mutex),
	}
}

// Get or create a mutex for a path
func (t *Thumbnailer) GetMutex(basePath string) *sync.Mutex {
	t.Lock()
	defer t.Unlock()

	m, ok := t.Paths[basePath]
	if !ok {
		m := &sync.Mutex{}
		t.Paths[basePath] = m
		return m
	} else {
		return m
	}
}

func (t *Thumbnailer) ScanFolder(gallery *GalleryConfig, basePath string) ([]string, []ImageInfo, error) {
	// start := time.Now()
	// defer func() {
	// 	log.Info("ScanFolder(%s) took %s", basePath, time.Since(start))
	// }()

	// Acquire lock
	m := t.GetMutex(basePath)
	m.Lock()
	defer m.Unlock()

	// Check cache
	cacheDirs, cacheImages, cacheOk := cache.Get(basePath)
	if cacheOk {
		return cacheDirs, cacheImages, nil
	}

	// Get a Redis connection
	conn := redisPool.Get()
	defer conn.Close()

	// Vars
	var dirs []string
	var images []ImageInfo

	// Get the files
	fileNames, err := ioutil.ReadDir(basePath)
	if err != nil {
		return nil, nil, err
	}

	// Subfolders need a fake .. directory
	if basePath != gallery.ImagePath {
		dirs = append(dirs, "..")
	}

	// Try fetching data from Redis
	// t1 := time.Now()
	jsonData, err := redis.String(conn.Do("HGET", "images", basePath))
	if err != redis.ErrNil && err != nil {
		return nil, nil, err
	}
	// log.Debug("HGET took %s", time.Since(t1))

	// Try unmarshalling
	// t2 := time.Now()
	fileMap := make(map[string]ImageInfo)
	if jsonData != "" {
		if err = json.Unmarshal([]byte(jsonData), &fileMap); err != nil {
			return nil, nil, err
		}
	}
	// log.Debug("Unmarshal took %s", time.Since(t2))

	// Some things
	resizeStr := fmt.Sprintf("%dx%d^", gallery.ThumbWidth, gallery.ThumbHeight)
	extentStr := fmt.Sprintf("%dx%d", gallery.ThumbWidth, gallery.ThumbHeight)

	// Iterateee
	// t3 := time.Now()
	for _, fileInfo := range fileNames {
		tl := time.Now()

		fileName := fileInfo.Name()

		// Don't care about directories
		if fileInfo.IsDir() {
			dirs = append(dirs, fileName)
			continue
		}

		// Don't care about weird filetypes
		if !reImage.MatchString(fileName) {
			continue
		}

		// Check to see if the image has changed
		fileModTime := fileInfo.ModTime().Unix()
		fileSize := fileInfo.Size()

		imageInfo, ok := fileMap[fileName]
		if ok && imageInfo.FileSize == fileSize && imageInfo.ModTime == fileModTime && imageInfo.ThumbPath != "" {
			images = append(images, imageInfo)
			continue
		}

		filePath := path.Join(basePath, fileName)

		// Generate the thumbnail filename and path
		b, err := ioutil.ReadFile(filePath)
		if err != nil {
			return nil, nil, err
		}
		thumbName := fmt.Sprintf("%x.jpg", md5.Sum(b))
		thumbPart := path.Join(string(thumbName[0]), thumbName)
		thumbPath := path.Join(gallery.ThumbPath, thumbPart)

		// Generate the thumbnail image and save it
		// t := time.Now()

		cmd := exec.Command("convert", fmt.Sprintf("%s[0]", filePath), "-thumbnail", resizeStr, "-gravity", "center", "-quality", "90", "-extent", extentStr, "-verbose", thumbPath)
		out, err := cmd.CombinedOutput()
		if err != nil {
			return nil, nil, err
		}

		// Get image dimensions from the output
		matches := reDimensions.FindAllStringSubmatch(string(out), -1)
		if len(matches) == 0 {
			log.Warning("matches failed: %q", out)
			return nil, nil, err
		}

		imageWidth, err := strconv.ParseInt(matches[0][1], 10, 32)
		if err != nil {
			return nil, nil, err
		}
		imageHeight, err := strconv.ParseInt(matches[0][2], 10, 32)
		if err != nil {
			return nil, nil, err
		}

		// log.Debug("thumbnail for %s took %s", filePath, time.Since(t))

		// Finish junk
		imagePart, _ := filepath.Rel(gallery.ImagePath, filePath)

		imageInfo = ImageInfo{
			FileSize:    fileSize,
			ModTime:     fileModTime,
			ImagePath:   imagePart,
			ImageWidth:  int(imageWidth),
			ImageHeight: int(imageHeight),
			ThumbPath:   thumbPart,
		}
		images = append(images, imageInfo)
		fileMap[fileName] = imageInfo

		log.Debug("loop for %s took %s", filePath, time.Since(tl))
	}
	// log.Debug("Loop took %s", time.Since(t3))

	// Update cache
	cache.Set(basePath, dirs, images)

	// Update Redis
	b, err := json.Marshal(fileMap)
	if err != nil {
		return nil, nil, err
	}
	conn.Do("HSET", "images", basePath, string(b))

	return dirs, images, nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Wrappers for Go parser.

package main

import (
	"go/ast"
	"go/parser"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"runtime"
)

type dirInfo struct {
	goFiles  []string // .go files within dir (including cgoFiles)
	cgoFiles []string // .go files that import "C"
	cFiles   []string // .c files within dir
	imports  []string // All packages imported by goFiles
	pkgName  string   // Name of package within dir
}

// scanDir returns a structure with details about the Go content found
// in the given directory. The list of files will NOT contain the
// following entries:
//
// - Files in package main (unless allowMain is true)
// - Files ending in _test.go
// - Files starting with _ (temporary)
// - Files containing .cgo in their names
//
// The imports map keys are package paths imported by listed Go files,
// and the values are the Go files importing the respective package paths.
func scanDir(dir string, allowMain bool) (info *dirInfo, err os.Error) {
	f, err := os.Open(dir, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	dirs, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}

	goFiles := make([]string, 0, len(dirs))
	cgoFiles := make([]string, 0, len(dirs))
	cFiles := make([]string, 0, len(dirs))
	importsm := make(map[string]bool)
	pkgName := ""
	for i := range dirs {
		d := &dirs[i]
		if strings.HasPrefix(d.Name, "_") || strings.Index(d.Name, ".cgo") != -1 {
			continue
		}
		if !goodOSArch(d.Name) {
			continue
		}
		if strings.HasSuffix(d.Name, ".c") {
			cFiles = append(cFiles, d.Name)
			continue
		}
		if !strings.HasSuffix(d.Name, ".go") || strings.HasSuffix(d.Name, "_test.go") {
			continue
		}
		filename := filepath.Join(dir, d.Name)
		pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly)
		if err != nil {
			return nil, err
		}
		s := string(pf.Name.Name)
		if s == "main" && !allowMain {
			continue
		}
		if pkgName == "" {
			pkgName = s
		} else if pkgName != s {
			// Only if all files in the directory are in package main
			// do we return pkgName=="main".
			// A mix of main and another package reverts
			// to the original (allowMain=false) behaviour.
			if s == "main" || pkgName == "main" {
				return scanDir(dir, false)
			}
			return nil, os.ErrorString("multiple package names in " + dir)
		}
		goFiles = append(goFiles, d.Name)
		for _, decl := range pf.Decls {
			for _, spec := range decl.(*ast.GenDecl).Specs {
				quoted := string(spec.(*ast.ImportSpec).Path.Value)
				unquoted, err := strconv.Unquote(quoted)
				if err != nil {
					log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
				}
				importsm[unquoted] = true
				if unquoted == "C" {
					cgoFiles = append(cgoFiles, d.Name)
				}
			}
		}
	}
	imports := make([]string, len(importsm))
	i := 0
	for p := range importsm {
		imports[i] = p
		i++
	}
	return &dirInfo{goFiles, cgoFiles, cFiles, imports, pkgName}, nil
}

// goodOSArch returns false if the filename contains a $GOOS or $GOARCH
// suffix which does not match the current system.
// The recognized filename formats are:
//
//     name_$(GOOS).*
//     name_$(GOARCH).*
//     name_$(GOOS)_$(GOARCH).*
//
func goodOSArch(filename string) bool {
	if dot := strings.Index(filename, "."); dot != -1 {
		filename = filename[:dot]
	}
	l := strings.Split(filename, "_", -1)
	n := len(l)
	if n == 0 {
		return true
	}
	if good, found := goodOS[l[n-1]]; found {
		return good
	}
	if good, found := goodArch[l[n-1]]; found {
		if !good || n < 2 {
			return false
		}
		good, found = goodOS[l[n-2]]
		return !found || good
	}
	return true
}

var goodOS = make(map[string]bool)
var goodArch = make(map[string]bool)

func init() {
	goodOS = make(map[string]bool)
	goodArch = make(map[string]bool)
	for _, v := range strings.Fields(goosList) {
		goodOS[v] = (v == runtime.GOOS)
	}
	for _, v := range strings.Fields(goarchList) {
		goodArch[v] = (v == runtime.GOARCH)
	}
}
<commit_msg>Minor improvement in variable naming.<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Wrappers for Go parser.

package main

import (
	"go/ast"
	"go/parser"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"runtime"
)

type dirInfo struct {
	goFiles  []string // .go files within dir (including cgoFiles)
	cgoFiles []string // .go files that import "C"
	cFiles   []string // .c files within dir
	imports  []string // All packages imported by goFiles
	pkgName  string   // Name of package within dir
}

// scanDir returns a structure with details about the Go content found
// in the given directory. The list of files will NOT contain the
// following entries:
//
// - Files in package main (unless allowMain is true)
// - Files ending in _test.go
// - Files starting with _ (temporary)
// - Files containing .cgo in their names
//
// The imports map keys are package paths imported by listed Go files,
// and the values are the Go files importing the respective package paths.
func scanDir(dir string, allowMain bool) (info *dirInfo, err os.Error) {
	f, err := os.Open(dir, os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	dirs, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, err
	}

	goFiles := make([]string, 0, len(dirs))
	cgoFiles := make([]string, 0, len(dirs))
	cFiles := make([]string, 0, len(dirs))
	importsm := make(map[string]bool)
	pkgName := ""
	for i := range dirs {
		d := &dirs[i]
		if strings.HasPrefix(d.Name, "_") || strings.Index(d.Name, ".cgo") != -1 {
			continue
		}
		if !goodOSArch(d.Name) {
			continue
		}
		if strings.HasSuffix(d.Name, ".c") {
			cFiles = append(cFiles, d.Name)
			continue
		}
		if !strings.HasSuffix(d.Name, ".go") || strings.HasSuffix(d.Name, "_test.go") {
			continue
		}
		filename := filepath.Join(dir, d.Name)
		pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly)
		if err != nil {
			return nil, err
		}
		s := string(pf.Name.Name)
		if s == "main" && !allowMain {
			continue
		}
		if pkgName == "" {
			pkgName = s
		} else if pkgName != s {
			// Only if all files in the directory are in package main
			// do we return pkgName=="main".
			// A mix of main and another package reverts
			// to the original (allowMain=false) behaviour.
			if s == "main" || pkgName == "main" {
				return scanDir(dir, false)
			}
			return nil, os.ErrorString("multiple package names in " + dir)
		}
		goFiles = append(goFiles, d.Name)
		for _, decl := range pf.Decls {
			for _, spec := range decl.(*ast.GenDecl).Specs {
				quoted := string(spec.(*ast.ImportSpec).Path.Value)
				unquoted, err := strconv.Unquote(quoted)
				if err != nil {
					log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
				}
				importsm[unquoted] = true
				if unquoted == "C" {
					cgoFiles = append(cgoFiles, d.Name)
				}
			}
		}
	}
	imports := make([]string, len(importsm))
	i := 0
	for p := range importsm {
		imports[i] = p
		i++
	}
	return &dirInfo{goFiles, cgoFiles, cFiles, imports, pkgName}, nil
}

// goodOSArch returns false if the filename contains a $GOOS or $GOARCH
// suffix which does not match the current system.
// The recognized filename formats are:
//
//     name_$(GOOS).*
//     name_$(GOARCH).*
//     name_$(GOOS)_$(GOARCH).*
//
func goodOSArch(filename string) bool {
	if dot := strings.Index(filename, "."); dot != -1 {
		filename = filename[:dot]
	}
	l := strings.Split(filename, "_", -1)
	n := len(l)
	if n == 0 {
		return true
	}
	if good, known := goodOS[l[n-1]]; known {
		return good
	}
	if good, known := goodArch[l[n-1]]; known {
		if !good || n < 2 {
			return false
		}
		good, known = goodOS[l[n-2]]
		return good || !known
	}
	return true
}

var goodOS = make(map[string]bool)
var goodArch = make(map[string]bool)

func init() {
	goodOS = make(map[string]bool)
	goodArch = make(map[string]bool)
	for _, v := range strings.Fields(goosList) {
		goodOS[v] = (v == runtime.GOOS)
	}
	for _, v := range strings.Fields(goarchList) {
		goodArch[v] = (v == runtime.GOARCH)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Mark Pundman
// Copyright 2015 Luke Shumaker
// Copyright 2015 Davis Webb

package cfg

import (
	"fmt"
	"io"
	"io/ioutil"
	"maildir"
	"net/http"
	"os"
	"periwinkle"
	"periwinkle/domain_handlers"
	"postfixpipe"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
	yaml "gopkg.in/yaml.v2"
)

func Parse(in io.Reader) (cfgptr *periwinkle.Cfg, err error) {
	defer func() {
		if r := recover(); r != nil {
			e, ok := r.(error)
			if !ok {
				panic(r)
			}
			cfgptr = nil
			err = e
		}
	}()

	// these are the defaults
	cfg := periwinkle.Cfg{
		Mailstore:            "./Maildir",
		WebUIDir:             "./www",
		Debug:                true,
		TrustForwarded:       true,
		TwilioAccountID:      os.Getenv("TWILIO_ACCOUNTID"),
		TwilioAuthToken:      os.Getenv("TWILIO_TOKEN"),
		GroupDomain:          "localhost",
		WebRoot:              "locahost:8080",
		DB:                   nil, // the default DB is set later
		DefaultDomainHandler: bounceNoHost,
	}

	datstr, err := ioutil.ReadAll(in)
	if err != nil {
		panic(err)
	}

	var datint interface{}
	err = yaml.Unmarshal(datstr, &datint)
	if err != nil {
		panic(err)
	}

	datmap, ok := datint.(map[interface{}]interface{})
	if !ok {
		panic(err)
	}

	for key, val := range datmap {
		switch key {
		case "Mailstore":
			cfg.Mailstore = maildir.Maildir(getString(key.(string), val))
		case "WebUIDir":
			cfg.WebUIDir = http.Dir(getString(key.(string), val))
		case "Debug":
			cfg.Debug = getBool(key.(string), val)
		case "TrustForwarded":
			cfg.TrustForwarded = getBool(key.(string), val)
		case "TwilioAccountID":
			cfg.TwilioAccountID = getString(key.(string), val)
		case "TwilioAuthToken":
			cfg.TwilioAuthToken = getString(key.(string), val)
		case "GroupDomain":
			cfg.GroupDomain = getString(key.(string), val)
		case "WebRoot":
			cfg.WebRoot = getString(key.(string), val)
		case "DB":
			m, ok := val.(map[interface{}]interface{})
			if !ok {
				panic(fmt.Errorf("value for %q is not a map", key.(string)))
			}
			var driver string
			var source string
			for key, val := range m {
				switch key {
				case "driver":
					driver = getString("DB."+key.(string), val)
				case "source":
					source = getString("DB."+key.(string), val)
				default:
					panic(fmt.Errorf("unknown field: %v", "DB."+key.(string)))
				}
			}
			db, err := gorm.Open(driver, source)
			if err != nil {
				panic(err)
			}
			cfg.DB = &db
		default:
			panic(fmt.Errorf("unknown field: %v", key))
		}
	}

	// Set the default database
	if cfg.DB == nil {
		fmt.Fprintln(os.Stderr, "DB not configured, trying MySQL periwinkle:periwinkle@localhost/periwinkle")
		db, err := gorm.Open("mysql", "periwinkle:periwinkle@/periwinkle?charset=utf8&parseTime=True")
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			fmt.Fprintln(os.Stderr, "Failed to connect to MySQL, trying SQLite3 file:periwinkle.sqlite")
			db, err = gorm.Open("sqlite3", "file:periwinkle.sqlite?cache=shared&mode=rwc")
			if err != nil {
				panic(err)
			}
		}
		cfg.DB = &db
	}

	cfg.DB.LogMode(cfg.Debug)

	domain_handlers.GetHandlers(&cfg)

	return &cfg, err
}

func getString(key string, val interface{}) string {
	str, ok := val.(string)
	if !ok {
		panic(fmt.Errorf("value for %q is not a string", key))
	}
	return str
}

func getBool(key string, val interface{}) bool {
	b, ok := val.(bool)
	if !ok {
		panic(fmt.Errorf("value for %q is not a Boolean", key))
	}
	return b
}

func bounceNoHost(io.Reader, string, *gorm.DB, *periwinkle.Cfg) postfixpipe.ExitStatus {
	return postfixpipe.EX_NOHOST
}
<commit_msg>Fixed issue with invalid memory<commit_after>// Copyright 2015 Mark Pundman
// Copyright 2015 Luke Shumaker
// Copyright 2015 Davis Webb

package cfg

import (
	"fmt"
	"io"
	"io/ioutil"
	"maildir"
	"net/http"
	"os"
	"periwinkle"
	"periwinkle/domain_handlers"
	"postfixpipe"
	"strings"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jinzhu/gorm"
	_ "github.com/mattn/go-sqlite3"
	yaml "gopkg.in/yaml.v2"
)

func Parse(in io.Reader) (cfgptr *periwinkle.Cfg, err error) {
	defer func() {
		if r := recover(); r != nil {
			e, ok := r.(error)
			if !ok {
				panic(r)
			}
			cfgptr = nil
			err = e
		}
	}()

	// these are the defaults
	cfg := periwinkle.Cfg{
		Mailstore:            "./Maildir",
		WebUIDir:             "./www",
		Debug:                true,
		TrustForwarded:       true,
		TwilioAccountID:      os.Getenv("TWILIO_ACCOUNTID"),
		TwilioAuthToken:      os.Getenv("TWILIO_TOKEN"),
		GroupDomain:          "localhost",
		WebRoot:              "locahost:8080",
		DB:                   nil, // the default DB is set later
		DefaultDomainHandler: bounceNoHost,
	}

	datstr, err := ioutil.ReadAll(in)
	if err != nil {
		panic(err)
	}

	// if we didnt get anything from the config, just return
	if strings.Compare("", string(datstr)) == 0 {
		return &cfg, err
	}

	var datint interface{}
	err = yaml.Unmarshal(datstr, &datint)
	if err != nil {
		panic(err)
	}

	datmap, ok := datint.(map[interface{}]interface{})
	if !ok {
		panic(err)
	}

	for key, val := range datmap {
		switch key {
		case "Mailstore":
			cfg.Mailstore = maildir.Maildir(getString(key.(string), val))
		case "WebUIDir":
			cfg.WebUIDir = http.Dir(getString(key.(string), val))
		case "Debug":
			cfg.Debug = getBool(key.(string), val)
		case "TrustForwarded":
			cfg.TrustForwarded = getBool(key.(string), val)
		case "TwilioAccountID":
			cfg.TwilioAccountID = getString(key.(string), val)
		case "TwilioAuthToken":
			cfg.TwilioAuthToken = getString(key.(string), val)
		case "GroupDomain":
			cfg.GroupDomain = getString(key.(string), val)
		case "WebRoot":
			cfg.WebRoot = getString(key.(string), val)
		case "DB":
			m, ok := val.(map[interface{}]interface{})
			if !ok {
				panic(fmt.Errorf("value for %q is not a map", key.(string)))
			}
			var driver string
			var source string
			for key, val := range m {
				switch key {
				case "driver":
					driver = getString("DB."+key.(string), val)
				case "source":
					source = getString("DB."+key.(string), val)
				default:
					panic(fmt.Errorf("unknown field: %v", "DB."+key.(string)))
				}
			}
			db, err := gorm.Open(driver, source)
			if err != nil {
				panic(err)
			}
			cfg.DB = &db
		default:
			panic(fmt.Errorf("unknown field: %v", key))
		}
	}

	// Set the default database
	if cfg.DB == nil {
		fmt.Fprintln(os.Stderr, "DB not configured, trying MySQL periwinkle:periwinkle@localhost/periwinkle")
		db, err := gorm.Open("mysql", "periwinkle:periwinkle@/periwinkle?charset=utf8&parseTime=True")
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			fmt.Fprintln(os.Stderr, "Failed to connect to MySQL, trying SQLite3 file:periwinkle.sqlite")
			db, err = gorm.Open("sqlite3", "file:periwinkle.sqlite?cache=shared&mode=rwc")
			if err != nil {
				panic(err)
			}
		}
		cfg.DB = &db
	}

	cfg.DB.LogMode(cfg.Debug)

	domain_handlers.GetHandlers(&cfg)

	return &cfg, err
}

func getString(key string, val interface{}) string {
	str, ok := val.(string)
	if !ok {
		panic(fmt.Errorf("value for %q is not a string", key))
	}
	return str
}

func getBool(key string, val interface{}) bool {
	b, ok := val.(bool)
	if !ok {
		panic(fmt.Errorf("value for %q is not a Boolean", key))
	}
	return b
}

func bounceNoHost(io.Reader, string, *gorm.DB, *periwinkle.Cfg) postfixpipe.ExitStatus {
	return postfixpipe.EX_NOHOST
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gif implements a GIF image decoder.
//
// The GIF specification is at http://www.w3.org/Graphics/GIF/spec-gif89a.txt.
package gif

import (
	"bufio"
	"compress/lzw"
	"fmt"
	"image"
	"io"
	"os"
)

// If the io.Reader does not also have ReadByte, then decode will introduce its own buffering.
type reader interface {
	io.Reader
	io.ByteReader
}

// Masks etc.
const (
	// Fields.
	fColorMapFollows = 1 << 7

	// Image fields.
	ifInterlace = 1 << 6

	// Graphic control flags.
	gcTransparentColorSet = 1 << 0
)

// Section indicators.
const (
	sExtension       = 0x21
	sImageDescriptor = 0x2C
	sTrailer         = 0x3B
)

// Extensions.
const (
	eText           = 0x01 // Plain Text
	eGraphicControl = 0xF9 // Graphic Control
	eComment        = 0xFE // Comment
	eApplication    = 0xFF // Application
)

// decoder is the type used to decode a GIF file.
type decoder struct {
	r reader

	// From header.
	vers            string
	width           int
	height          int
	flags           byte
	headerFields    byte
	backgroundIndex byte
	loopCount       int
	delayTime       int

	// Unused from header.
	aspect byte

	// From image descriptor.
	imageFields byte

	// From graphics control.
	transparentIndex byte

	// Computed.
	pixelSize      uint
	globalColorMap image.PalettedColorModel

	// Used when decoding.
	delay []int
	image []*image.Paletted
	tmp   [1024]byte // must be at least 768 so we can read color map
}

// blockReader parses the block structure of GIF image data, which
// comprises (n, (n bytes)) blocks, with 1 <= n <= 255.  It is the
// reader given to the LZW decoder, which is thus immune to the
// blocking.  After the LZW decoder completes, there will be a 0-byte
// block remaining (0, ()), but under normal execution blockReader
// doesn't consume it, so it is handled in decode.
type blockReader struct {
	r     reader
	slice []byte
	tmp   [256]byte
}

func (b *blockReader) Read(p []byte) (n int, err os.Error) {
	if len(p) == 0 {
		return
	}
	if len(b.slice) > 0 {
		n = copy(p, b.slice)
		b.slice = b.slice[n:]
		return
	}
	var blockLen uint8
	blockLen, err = b.r.ReadByte()
	if err != nil {
		return
	}
	if blockLen == 0 {
		return 0, os.EOF
	}
	b.slice = b.tmp[0:blockLen]
	if _, err = io.ReadFull(b.r, b.slice); err != nil {
		return
	}
	return b.Read(p)
}

// decode reads a GIF image from r and stores the result in d.
func (d *decoder) decode(r io.Reader, configOnly bool) os.Error {
	// Add buffering if r does not provide ReadByte.
	if rr, ok := r.(reader); ok {
		d.r = rr
	} else {
		d.r = bufio.NewReader(r)
	}

	err := d.readHeaderAndScreenDescriptor()
	if err != nil {
		return err
	}
	if configOnly {
		return nil
	}

	if d.headerFields&fColorMapFollows != 0 {
		if d.globalColorMap, err = d.readColorMap(); err != nil {
			return err
		}
	}

	d.image = nil

Loop:
	for err == nil {
		var c byte
		c, err = d.r.ReadByte()
		if err == os.EOF {
			break
		}
		switch c {
		case sExtension:
			err = d.readExtension()

		case sImageDescriptor:
			var m *image.Paletted
			m, err = d.newImageFromDescriptor()
			if err != nil {
				break
			}
			if d.imageFields&fColorMapFollows != 0 {
				m.Palette, err = d.readColorMap()
				if err != nil {
					break
				}
				// TODO: do we set transparency in this map too? That would be
				// d.setTransparency(m.Palette)
			} else {
				m.Palette = d.globalColorMap
			}
			var litWidth uint8
			litWidth, err = d.r.ReadByte()
			if err != nil {
				return err
			}
			if litWidth > 8 {
				return fmt.Errorf("gif: pixel size in decode out of range: %d", litWidth)
			}
			// A wonderfully Go-like piece of magic. Unfortunately it's only at its
			// best for 8-bit pixels.
			lzwr := lzw.NewReader(&blockReader{r: d.r}, lzw.LSB, int(litWidth))
			if _, err = io.ReadFull(lzwr, m.Pix); err != nil {
				break
			}

			// There should be a "0" block remaining; drain that.
			c, err = d.r.ReadByte()
			if err != nil {
				return err
			}
			if c != 0 {
				return os.ErrorString("gif: extra data after image")
			}
			d.image = append(d.image, m)
			d.delay = append(d.delay, d.delayTime)
			d.delayTime = 0 // TODO: is this correct, or should we hold on to the value?

		case sTrailer:
			break Loop

		default:
			err = fmt.Errorf("gif: unknown block type: 0x%.2x", c)
		}
	}
	if err != nil {
		return err
	}
	if len(d.image) == 0 {
		return io.ErrUnexpectedEOF
	}
	return nil
}

func (d *decoder) readHeaderAndScreenDescriptor() os.Error {
	_, err := io.ReadFull(d.r, d.tmp[0:13])
	if err != nil {
		return err
	}
	d.vers = string(d.tmp[0:6])
	if d.vers != "GIF87a" && d.vers != "GIF89a" {
		return fmt.Errorf("gif: can't recognize format %s", d.vers)
	}
	d.width = int(d.tmp[6]) + int(d.tmp[7])<<8
	d.height = int(d.tmp[8]) + int(d.tmp[9])<<8
	d.headerFields = d.tmp[10]
	d.backgroundIndex = d.tmp[11]
	d.aspect = d.tmp[12]
	d.loopCount = -1
	d.pixelSize = uint(d.headerFields&7) + 1
	return nil
}

func (d *decoder) readColorMap() (image.PalettedColorModel, os.Error) {
	if d.pixelSize > 8 {
		return nil, fmt.Errorf("gif: can't handle %d bits per pixel", d.pixelSize)
	}
	numColors := 1 << d.pixelSize
	numValues := 3 * numColors
	_, err := io.ReadFull(d.r, d.tmp[0:numValues])
	if err != nil {
		return nil, fmt.Errorf("gif: short read on color map: %s", err)
	}
	colorMap := make(image.PalettedColorModel, numColors)
	j := 0
	for i := range colorMap {
		colorMap[i] = image.RGBAColor{d.tmp[j+0], d.tmp[j+1], d.tmp[j+2], 0xFF}
		j += 3
	}
	return colorMap, nil
}

func (d *decoder) readExtension() os.Error {
	extension, err := d.r.ReadByte()
	if err != nil {
		return err
	}
	size := 0
	switch extension {
	case eText:
		size = 13
	case eGraphicControl:
		return d.readGraphicControl()
	case eComment:
		// nothing to do but read the data.
	case eApplication:
		b, err := d.r.ReadByte()
		if err != nil {
			return err
		}
		// The spec requires size be 11, but Adobe sometimes uses 10.
		size = int(b)
	default:
		return fmt.Errorf("gif: unknown extension 0x%.2x", extension)
	}
	if size > 0 {
		if _, err := d.r.Read(d.tmp[0:size]); err != nil {
			return err
		}
	}

	// Application Extension with "NETSCAPE2.0" as string and 1 in data means
	// this extension defines a loop count.
	if extension == eApplication && string(d.tmp[:size]) == "NETSCAPE2.0" {
		n, err := d.readBlock()
		if n == 0 || err != nil {
			return err
		}
		if n == 3 && d.tmp[0] == 1 {
			d.loopCount = int(d.tmp[1]) | int(d.tmp[2])<<8
		}
	}
	for {
		n, err := d.readBlock()
		if n == 0 || err != nil {
			return err
		}
	}
	panic("unreachable")
}

func (d *decoder) readGraphicControl() os.Error {
	if _, err := io.ReadFull(d.r, d.tmp[0:6]); err != nil {
		return fmt.Errorf("gif: can't read graphic control: %s", err)
	}
	d.flags = d.tmp[1]
	d.delayTime = int(d.tmp[2]) | int(d.tmp[3])<<8
	if d.flags&gcTransparentColorSet == 0 {
		d.transparentIndex = d.tmp[4]
		d.setTransparency(d.globalColorMap)
	}
	return nil
}

func (d *decoder) setTransparency(colorMap image.PalettedColorModel) {
	if int(d.transparentIndex) < len(colorMap) {
		colorMap[d.transparentIndex] = image.RGBAColor{}
	}
}

func (d *decoder) newImageFromDescriptor() (*image.Paletted, os.Error) {
	if _, err := io.ReadFull(d.r, d.tmp[0:9]); err != nil {
		return nil, fmt.Errorf("gif: can't read image descriptor: %s", err)
	}
	_ = int(d.tmp[0]) + int(d.tmp[1])<<8 // TODO: honor left value
	_ = int(d.tmp[2]) + int(d.tmp[3])<<8 // TODO: honor top value
	width := int(d.tmp[4]) + int(d.tmp[5])<<8
	height := int(d.tmp[6]) + int(d.tmp[7])<<8
	d.imageFields = d.tmp[8]
	if d.imageFields&ifInterlace != 0 {
		return nil, os.ErrorString("gif: can't handle interlaced images")
	}
	return image.NewPaletted(width, height, nil), nil
}

func (d *decoder) readBlock() (int, os.Error) {
	n, err := d.r.ReadByte()
	if n == 0 || err != nil {
		return 0, err
	}
	return io.ReadFull(d.r, d.tmp[0:n])
}

// Decode reads a GIF image from r and returns the first embedded
// image as an image.Image.
// Limitation: The file must be 8 bits per pixel and have no interlacing.
func Decode(r io.Reader) (image.Image, os.Error) {
	var d decoder
	if err := d.decode(r, false); err != nil {
		return nil, err
	}
	return d.image[0], nil
}

// GIF represents the possibly multiple images stored in a GIF file.
type GIF struct {
	Image     []*image.Paletted // The successive images.
	Delay     []int             // The successive delay times, one per frame, in 100ths of a second.
	LoopCount int               // The loop count.
}

// DecodeAll reads a GIF image from r and returns the sequential frames
// and timing information.
// Limitation: The file must be 8 bits per pixel and have no interlacing.
func DecodeAll(r io.Reader) (*GIF, os.Error) {
	var d decoder
	if err := d.decode(r, false); err != nil {
		return nil, err
	}
	gif := &GIF{
		Image:     d.image,
		LoopCount: d.loopCount,
		Delay:     d.delay,
	}
	return gif, nil
}

// DecodeConfig returns the color model and dimensions of a GIF image without
// decoding the entire image.
func DecodeConfig(r io.Reader) (image.Config, os.Error) {
	var d decoder
	if err := d.decode(r, true); err != nil {
		return image.Config{}, err
	}
	colorMap := d.globalColorMap
	return image.Config{colorMap, d.width, d.height}, nil
}

func init() {
	image.RegisterFormat("gif", "GIF8?a", Decode, DecodeConfig)
}
<commit_msg>gif: fix build Had bit test wrong on transparency; no excuses.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gif implements a GIF image decoder.
//
// The GIF specification is at http://www.w3.org/Graphics/GIF/spec-gif89a.txt.
package gif

import (
	"bufio"
	"compress/lzw"
	"fmt"
	"image"
	"io"
	"os"
)

// If the io.Reader does not also have ReadByte, then decode will introduce its own buffering.
type reader interface {
	io.Reader
	io.ByteReader
}

// Masks etc.
const (
	// Fields.
	fColorMapFollows = 1 << 7

	// Image fields.
	ifInterlace = 1 << 6

	// Graphic control flags.
	gcTransparentColorSet = 1 << 0
)

// Section indicators.
const (
	sExtension       = 0x21
	sImageDescriptor = 0x2C
	sTrailer         = 0x3B
)

// Extensions.
const (
	eText           = 0x01 // Plain Text
	eGraphicControl = 0xF9 // Graphic Control
	eComment        = 0xFE // Comment
	eApplication    = 0xFF // Application
)

// decoder is the type used to decode a GIF file.
type decoder struct {
	r reader

	// From header.
	vers            string
	width           int
	height          int
	flags           byte
	headerFields    byte
	backgroundIndex byte
	loopCount       int
	delayTime       int

	// Unused from header.
	aspect byte

	// From image descriptor.
	imageFields byte

	// From graphics control.
	transparentIndex byte

	// Computed.
	pixelSize      uint
	globalColorMap image.PalettedColorModel

	// Used when decoding.
	delay []int
	image []*image.Paletted
	tmp   [1024]byte // must be at least 768 so we can read color map
}

// blockReader parses the block structure of GIF image data, which
// comprises (n, (n bytes)) blocks, with 1 <= n <= 255.  It is the
// reader given to the LZW decoder, which is thus immune to the
// blocking.  After the LZW decoder completes, there will be a 0-byte
// block remaining (0, ()), but under normal execution blockReader
// doesn't consume it, so it is handled in decode.
type blockReader struct {
	r     reader
	slice []byte
	tmp   [256]byte
}

func (b *blockReader) Read(p []byte) (n int, err os.Error) {
	if len(p) == 0 {
		return
	}
	if len(b.slice) > 0 {
		n = copy(p, b.slice)
		b.slice = b.slice[n:]
		return
	}
	var blockLen uint8
	blockLen, err = b.r.ReadByte()
	if err != nil {
		return
	}
	if blockLen == 0 {
		return 0, os.EOF
	}
	b.slice = b.tmp[0:blockLen]
	if _, err = io.ReadFull(b.r, b.slice); err != nil {
		return
	}
	return b.Read(p)
}

// decode reads a GIF image from r and stores the result in d.
func (d *decoder) decode(r io.Reader, configOnly bool) os.Error {
	// Add buffering if r does not provide ReadByte.
	if rr, ok := r.(reader); ok {
		d.r = rr
	} else {
		d.r = bufio.NewReader(r)
	}

	err := d.readHeaderAndScreenDescriptor()
	if err != nil {
		return err
	}
	if configOnly {
		return nil
	}

	if d.headerFields&fColorMapFollows != 0 {
		if d.globalColorMap, err = d.readColorMap(); err != nil {
			return err
		}
	}

	d.image = nil

Loop:
	for err == nil {
		var c byte
		c, err = d.r.ReadByte()
		if err == os.EOF {
			break
		}
		switch c {
		case sExtension:
			err = d.readExtension()

		case sImageDescriptor:
			var m *image.Paletted
			m, err = d.newImageFromDescriptor()
			if err != nil {
				break
			}
			if d.imageFields&fColorMapFollows != 0 {
				m.Palette, err = d.readColorMap()
				if err != nil {
					break
				}
				// TODO: do we set transparency in this map too? That would be
				// d.setTransparency(m.Palette)
			} else {
				m.Palette = d.globalColorMap
			}
			var litWidth uint8
			litWidth, err = d.r.ReadByte()
			if err != nil {
				return err
			}
			if litWidth > 8 {
				return fmt.Errorf("gif: pixel size in decode out of range: %d", litWidth)
			}
			// A wonderfully Go-like piece of magic. Unfortunately it's only at its
			// best for 8-bit pixels.
			lzwr := lzw.NewReader(&blockReader{r: d.r}, lzw.LSB, int(litWidth))
			if _, err = io.ReadFull(lzwr, m.Pix); err != nil {
				break
			}

			// There should be a "0" block remaining; drain that.
			c, err = d.r.ReadByte()
			if err != nil {
				return err
			}
			if c != 0 {
				return os.ErrorString("gif: extra data after image")
			}
			d.image = append(d.image, m)
			d.delay = append(d.delay, d.delayTime)
			d.delayTime = 0 // TODO: is this correct, or should we hold on to the value?

		case sTrailer:
			break Loop

		default:
			err = fmt.Errorf("gif: unknown block type: 0x%.2x", c)
		}
	}
	if err != nil {
		return err
	}
	if len(d.image) == 0 {
		return io.ErrUnexpectedEOF
	}
	return nil
}

func (d *decoder) readHeaderAndScreenDescriptor() os.Error {
	_, err := io.ReadFull(d.r, d.tmp[0:13])
	if err != nil {
		return err
	}
	d.vers = string(d.tmp[0:6])
	if d.vers != "GIF87a" && d.vers != "GIF89a" {
		return fmt.Errorf("gif: can't recognize format %s", d.vers)
	}
	d.width = int(d.tmp[6]) + int(d.tmp[7])<<8
	d.height = int(d.tmp[8]) + int(d.tmp[9])<<8
	d.headerFields = d.tmp[10]
	d.backgroundIndex = d.tmp[11]
	d.aspect = d.tmp[12]
	d.loopCount = -1
	d.pixelSize = uint(d.headerFields&7) + 1
	return nil
}

func (d *decoder) readColorMap() (image.PalettedColorModel, os.Error) {
	if d.pixelSize > 8 {
		return nil, fmt.Errorf("gif: can't handle %d bits per pixel", d.pixelSize)
	}
	numColors := 1 << d.pixelSize
	numValues := 3 * numColors
	_, err := io.ReadFull(d.r, d.tmp[0:numValues])
	if err != nil {
		return nil, fmt.Errorf("gif: short read on color map: %s", err)
	}
	colorMap := make(image.PalettedColorModel, numColors)
	j := 0
	for i := range colorMap {
		colorMap[i] = image.RGBAColor{d.tmp[j+0], d.tmp[j+1], d.tmp[j+2], 0xFF}
		j += 3
	}
	return colorMap, nil
}

func (d *decoder) readExtension() os.Error {
	extension, err := d.r.ReadByte()
	if err != nil {
		return err
	}
	size := 0
	switch extension {
	case eText:
		size = 13
	case eGraphicControl:
		return d.readGraphicControl()
	case eComment:
		// nothing to do but read the data.
	case eApplication:
		b, err := d.r.ReadByte()
		if err != nil {
			return err
		}
		// The spec requires size be 11, but Adobe sometimes uses 10.
		size = int(b)
	default:
		return fmt.Errorf("gif: unknown extension 0x%.2x", extension)
	}
	if size > 0 {
		if _, err := d.r.Read(d.tmp[0:size]); err != nil {
			return err
		}
	}

	// Application Extension with "NETSCAPE2.0" as string and 1 in data means
	// this extension defines a loop count.
	if extension == eApplication && string(d.tmp[:size]) == "NETSCAPE2.0" {
		n, err := d.readBlock()
		if n == 0 || err != nil {
			return err
		}
		if n == 3 && d.tmp[0] == 1 {
			d.loopCount = int(d.tmp[1]) | int(d.tmp[2])<<8
		}
	}
	for {
		n, err := d.readBlock()
		if n == 0 || err != nil {
			return err
		}
	}
	panic("unreachable")
}

func (d *decoder) readGraphicControl() os.Error {
	if _, err := io.ReadFull(d.r, d.tmp[0:6]); err != nil {
		return fmt.Errorf("gif: can't read graphic control: %s", err)
	}
	d.flags = d.tmp[1]
	d.delayTime = int(d.tmp[2]) | int(d.tmp[3])<<8
	if d.flags&gcTransparentColorSet != 0 {
		d.transparentIndex = d.tmp[4]
		d.setTransparency(d.globalColorMap)
	}
	return nil
}

func (d *decoder) setTransparency(colorMap image.PalettedColorModel) {
	if int(d.transparentIndex) < len(colorMap) {
		colorMap[d.transparentIndex] = image.RGBAColor{}
	}
}

func (d *decoder) newImageFromDescriptor() (*image.Paletted, os.Error) {
	if _, err := io.ReadFull(d.r, d.tmp[0:9]); err != nil {
		return nil, fmt.Errorf("gif: can't read image descriptor: %s", err)
	}
	_ = int(d.tmp[0]) + int(d.tmp[1])<<8 // TODO: honor left value
	_ = int(d.tmp[2]) + int(d.tmp[3])<<8 // TODO: honor top value
	width := int(d.tmp[4]) + int(d.tmp[5])<<8
	height := int(d.tmp[6]) + int(d.tmp[7])<<8
	d.imageFields = d.tmp[8]
	if d.imageFields&ifInterlace != 0 {
		return nil, os.ErrorString("gif: can't handle interlaced images")
	}
	return image.NewPaletted(width, height, nil), nil
}

func (d *decoder) readBlock() (int, os.Error) {
	n, err := d.r.ReadByte()
	if n == 0 || err != nil {
		return 0, err
	}
	return io.ReadFull(d.r, d.tmp[0:n])
}

// Decode reads a GIF image from r and returns the first embedded
// image as an image.Image.
// Limitation: The file must be 8 bits per pixel and have no interlacing.
func Decode(r io.Reader) (image.Image, os.Error) {
	var d decoder
	if err := d.decode(r, false); err != nil {
		return nil, err
	}
	return d.image[0], nil
}

// GIF represents the possibly multiple images stored in a GIF file.
type GIF struct {
	Image     []*image.Paletted // The successive images.
	Delay     []int             // The successive delay times, one per frame, in 100ths of a second.
	LoopCount int               // The loop count.
}

// DecodeAll reads a GIF image from r and returns the sequential frames
// and timing information.
// Limitation: The file must be 8 bits per pixel and have no interlacing.
func DecodeAll(r io.Reader) (*GIF, os.Error) {
	var d decoder
	if err := d.decode(r, false); err != nil {
		return nil, err
	}
	gif := &GIF{
		Image:     d.image,
		LoopCount: d.loopCount,
		Delay:     d.delay,
	}
	return gif, nil
}

// DecodeConfig returns the color model and dimensions of a GIF image without
// decoding the entire image.
func DecodeConfig(r io.Reader) (image.Config, os.Error) {
	var d decoder
	if err := d.decode(r, true); err != nil {
		return image.Config{}, err
	}
	colorMap := d.globalColorMap
	return image.Config{colorMap, d.width, d.height}, nil
}

func init() {
	image.RegisterFormat("gif", "GIF8?a", Decode, DecodeConfig)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package image

import (
	"testing"
)

func cmp(t *testing.T, cm ColorModel, c0, c1 Color) bool {
	r0, g0, b0, a0 := cm.Convert(c0).RGBA()
	r1, g1, b1, a1 := cm.Convert(c1).RGBA()
	return r0 == r1 && g0 == g1 && b0 == b1 && a0 == a1
}

func TestImage(t *testing.T) {
	type buffered interface {
		Image
		Set(int, int, Color)
		SubImage(Rectangle) Image
	}
	testImage := []Image{
		NewRGBA(10, 10),
		NewRGBA64(10, 10),
		NewNRGBA(10, 10),
		NewNRGBA64(10, 10),
		NewAlpha(10, 10),
		NewAlpha16(10, 10),
		NewGray(10, 10),
		NewGray16(10, 10),
		NewPaletted(10, 10, PalettedColorModel{
			Transparent,
			Opaque,
		}),
	}
	for _, m := range testImage {
		b := m.(buffered)
		if !Rect(0, 0, 10, 10).Eq(b.Bounds()) {
			t.Errorf("%T: want bounds %v, got %v", b, Rect(0, 0, 10, 10), b.Bounds())
			continue
		}
		if !cmp(t, b.ColorModel(), Transparent, b.At(6, 3)) {
			t.Errorf("%T: at (6, 3), want a zero color, got %v", b, b.At(6, 3))
			continue
		}
		b.Set(6, 3, Opaque)
		if !cmp(t, b.ColorModel(), Opaque, b.At(6, 3)) {
			t.Errorf("%T: at (6, 3), want a non-zero color, got %v", b, b.At(6, 3))
			continue
		}
		b = b.SubImage(Rect(3, 2, 9, 8)).(buffered)
		if !Rect(3, 2, 9, 8).Eq(b.Bounds()) {
			t.Errorf("%T: sub-image want bounds %v, got %v", b, Rect(3, 2, 9, 8), b.Bounds())
			continue
		}
		if !cmp(t, b.ColorModel(), Opaque, b.At(6, 3)) {
			t.Errorf("%T: sub-image at (6, 3), want a non-zero color, got %v", b, b.At(6, 3))
			continue
		}
		if !cmp(t, b.ColorModel(), Transparent, b.At(3, 3)) {
			t.Errorf("%T: sub-image at (3, 3), want a zero color, got %v", b, b.At(3, 3))
			continue
		}
		b.Set(3, 3, Opaque)
		if !cmp(t, b.ColorModel(), Opaque, b.At(3, 3)) {
			t.Errorf("%T: sub-image at (3, 3), want a non-zero color, got %v", b, b.At(3, 3))
			continue
		}
	}
}
<commit_msg>image: basic test for the 16-bits-per-color-channel types.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package image

import (
	"testing"
)

type image interface {
	Image
	Set(int, int, Color)
	SubImage(Rectangle) Image
}

func cmp(t *testing.T, cm ColorModel, c0, c1 Color) bool {
	r0, g0, b0, a0 := cm.Convert(c0).RGBA()
	r1, g1, b1, a1 := cm.Convert(c1).RGBA()
	return r0 == r1 && g0 == g1 && b0 == b1 && a0 == a1
}

func TestImage(t *testing.T) {
	testImage := []image{
		NewRGBA(10, 10),
		NewRGBA64(10, 10),
		NewNRGBA(10, 10),
		NewNRGBA64(10, 10),
		NewAlpha(10, 10),
		NewAlpha16(10, 10),
		NewGray(10, 10),
		NewGray16(10, 10),
		NewPaletted(10, 10, PalettedColorModel{
			Transparent,
			Opaque,
		}),
	}
	for _, m := range testImage {
		if !Rect(0, 0, 10, 10).Eq(m.Bounds()) {
			t.Errorf("%T: want bounds %v, got %v", m, Rect(0, 0, 10, 10), m.Bounds())
			continue
		}
		if !cmp(t, m.ColorModel(), Transparent, m.At(6, 3)) {
			t.Errorf("%T: at (6, 3), want a zero color, got %v", m, m.At(6, 3))
			continue
		}
		m.Set(6, 3, Opaque)
		if !cmp(t, m.ColorModel(), Opaque, m.At(6, 3)) {
			t.Errorf("%T: at (6, 3), want a non-zero color, got %v", m, m.At(6, 3))
			continue
		}
		m = m.SubImage(Rect(3, 2, 9, 8)).(image)
		if !Rect(3, 2, 9, 8).Eq(m.Bounds()) {
			t.Errorf("%T: sub-image want bounds %v, got %v", m, Rect(3, 2, 9, 8), m.Bounds())
			continue
		}
		if !cmp(t, m.ColorModel(), Opaque, m.At(6, 3)) {
			t.Errorf("%T: sub-image at (6, 3), want a non-zero color, got %v", m, m.At(6, 3))
			continue
		}
		if !cmp(t, m.ColorModel(), Transparent, m.At(3, 3)) {
			t.Errorf("%T: sub-image at (3, 3), want a zero color, got %v", m, m.At(3, 3))
			continue
		}
		m.Set(3, 3, Opaque)
		if !cmp(t, m.ColorModel(), Opaque, m.At(3, 3)) {
			t.Errorf("%T: sub-image at (3, 3), want a non-zero color, got %v", m, m.At(3, 3))
			continue
		}
	}
}

func Test16BitsPerColorChannel(t *testing.T) {
	testColorModel := []ColorModel{
		RGBA64ColorModel,
		NRGBA64ColorModel,
		Alpha16ColorModel,
		Gray16ColorModel,
	}
	for _, cm := range testColorModel {
		c := cm.Convert(RGBA64Color{0x1234, 0x1234, 0x1234, 0x1234}) // Premultiplied alpha.
		r, _, _, _ := c.RGBA()
		if r != 0x1234 {
			t.Errorf("%T: want red value 0x%04x got 0x%04x", c, 0x1234, r)
			continue
		}
	}
	testImage := []image{
		NewRGBA64(10, 10),
		NewNRGBA64(10, 10),
		NewAlpha16(10, 10),
		NewGray16(10, 10),
	}
	for _, m := range testImage {
		m.Set(1, 2, NRGBA64Color{0xffff, 0xffff, 0xffff, 0x1357}) // Non-premultiplied alpha.
		r, _, _, _ := m.At(1, 2).RGBA()
		if r != 0x1357 {
			t.Errorf("%T: want red value 0x%04x got 0x%04x", m, 0x1357, r)
			continue
		}
	}
}
<|endoftext|>


<commit_before>package transform

import (
	"errors"
	"github.com/azylman/getl"
	"github.com/azylman/getl/sources/csv"
	"github.com/azylman/getl/sources/infinite"
	"github.com/azylman/getl/tests"
	"testing"
)

func TestFieldmap(t *testing.T) {
	table := csv.New("./test.csv")
	transformedTable := Fieldmap(table, map[string][]string{"header1": {"header4"}})
	tests.HasRows(t, transformedTable, 3)
}

func TestFieldmapChain(t *testing.T) {
	table := csv.New("./test.csv")
	transformedTable := NewTransformer(table).Fieldmap(map[string][]string{"header1": {"header4"}}).Table()
	tests.HasRows(t, transformedTable, 3)
}

// TestTransformError tests that the upstream Table had all of its data consumed in the case of an
// error.
func TestTransformError(t *testing.T) {
	in := infinite.New()
	out := elTransform(in, func(row getl.Row) (getl.Row, error) {
		return nil, errors.New("some error")
	})
	// Should receive no rows here because the first response was an error.
	tests.Consumed(t, out)
	// Should receive no rows here because the the transform should have consumed
	// all the rows.
	tests.Consumed(t, in)
}
<commit_msg>transform: more accurate field mapping tests<commit_after>package transform

import (
	"errors"
	"github.com/azylman/getl"
	"github.com/azylman/getl/sources/infinite"
	"github.com/azylman/getl/sources/slice"
	"github.com/azylman/getl/tests"
	"github.com/stretchr/testify/assert"
	"testing"
)

var input = []getl.Row{
	{"header1": "value1", "header2": "value2"},
	{"header1": "value3", "header2": "value4"},
	{"header1": "value5", "header2": "value6"},
}

var expected = []getl.Row{
	{"header4": "value1"},
	{"header4": "value3"},
	{"header4": "value5"},
}

var fieldMapping = map[string][]string{"header1": {"header4"}}

// Test that field mapping behaves as expected
func TestFieldmap(t *testing.T) {
	table := slice.New(input)
	transformedTable := Fieldmap(table, fieldMapping)
	rows := tests.HasRows(t, transformedTable, 3)
	assert.Equal(t, expected, rows)
}

// Test that field mapping via a Transformer behaves as expted
func TestFieldmapChain(t *testing.T) {
	table := slice.New(input)
	transformedTable := NewTransformer(table).Fieldmap(fieldMapping).Table()
	rows := tests.HasRows(t, transformedTable, 3)
	assert.Equal(t, expected, rows)
}

// Test that chaining together multiple transforms behaves as expected
func TestChaining(t *testing.T) {
	table := slice.New(input)
	expected := []getl.Row{
		{"header1": "value1"},
		{"header1": "value3"},
		{"header1": "value5"},
	}
	transformedTable := NewTransformer(table).Fieldmap(
		fieldMapping).Fieldmap(map[string][]string{"header4": {"header1"}}).Table()
	rows := tests.HasRows(t, transformedTable, 3)
	assert.Equal(t, expected, rows)
}

// TestTransformError tests that the upstream Table had all of its data consumed in the case of an
// error.
func TestTransformError(t *testing.T) {
	in := infinite.New()
	out := elTransform(in, func(row getl.Row) (getl.Row, error) {
		return nil, errors.New("some error")
	})
	// Should receive no rows here because the first response was an error.
	tests.Consumed(t, out)
	// Should receive no rows here because the the transform should have consumed
	// all the rows.
	tests.Consumed(t, in)
}
<|endoftext|>