<commit_before>package httpfetch

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"
)

const DefaultUserAgent = "Sequell httpfetch/1.0"

type Fetcher struct {
	HTTPClient                   *http.Client
	Quiet                        bool
	ConnectTimeout               time.Duration
	ReadTimeout                  time.Duration
	UserAgent                    string
	MaxConcurrentRequestsPerHost int
	Logger                       *log.Logger
	logWriter                    Logger

	// Queues for each host, monitored by the service goroutine.
	hostQueues       map[string]chan<- *FetchRequest
	hostWaitGroup    sync.WaitGroup
	enqueueWaitGroup sync.WaitGroup
}

// New returns a new Fetcher for parallel downloads. Fetcher
// methods are not threadsafe.
func New() *Fetcher {
	writer := CreateLogger()
	return &Fetcher{
		HTTPClient:                   DefaultHTTPClient,
		ConnectTimeout:               DefaultConnectTimeout,
		ReadTimeout:                  DefaultReadTimeout,
		UserAgent:                    DefaultUserAgent,
		MaxConcurrentRequestsPerHost: 5,
		logWriter:                    writer,
		Logger:                       log.New(writer, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile),
		hostQueues:                   map[string]chan<- *FetchRequest{},
	}
}

var DefaultConnectTimeout = 10 * time.Second
var DefaultReadTimeout = 20 * time.Second
var DefaultHTTPTransport = http.Transport{
	Dial: dialer(DefaultConnectTimeout, DefaultReadTimeout),
	ResponseHeaderTimeout: DefaultConnectTimeout,
}
var DefaultHTTPClient = &http.Client{
	Transport: &DefaultHTTPTransport,
}

func (h *Fetcher) SetLogWriter(writer io.Writer) {
	h.logWriter.SetWriter(writer)
}

type unbufferedWriter struct {
	file *os.File
}

func (uw unbufferedWriter) Write(b []byte) (n int, err error) {
	n, err = uw.file.Write(b)
	if err != nil {
		_ = uw.file.Sync()
	}
	return
}

func (h *Fetcher) SetLogFile(file *os.File) {
	h.SetLogWriter(unbufferedWriter{file: file})
}

func (h *Fetcher) Logf(format string, rest ...interface{}) {
	h.Logger.Printf(format, rest...)
}

func (h *Fetcher) GetConcurrentRequestCount(count int) int {
	if count > h.MaxConcurrentRequestsPerHost {
		return h.MaxConcurrentRequestsPerHost
	}
	return count
}

type Headers map[string]string

type HTTPError struct {
	StatusCode int
	Response   *http.Response
}

func (err *HTTPError) Error() string {
	req := err.Response.Request
	return fmt.Sprint(req.Method, " ", req.URL, " failed: ", err.StatusCode)
}

type FetchRequest struct {
	Url      string
	Filename string

	// Don't try to resume downloads if this is set.
	FullDownload   bool
	RequestHeaders Headers
}

func (req *FetchRequest) Host() (string, error) {
	reqUrl, err := url.Parse(req.Url)
	if err != nil {
		return "", err
	}
	return reqUrl.Host, nil
}

func (req *FetchRequest) String() string {
	return fmt.Sprint(req.Url, " -> ", req.Filename)
}

type FetchResult struct {
	Req          *FetchRequest
	Err          error
	DownloadSize int64
}

func fetchError(req *FetchRequest, err error) *FetchResult {
	return &FetchResult{req, err, 0}
}

func (headers Headers) AddHeaders(h *http.Header) {
	for header, value := range headers {
		h.Add(header, value)
	}
}

func (headers Headers) Copy() Headers {
	res := make(Headers)
	for k, v := range headers {
		res[k] = v
	}
	return res
}

func HeadersWith(headers Headers, newHeader, newValue string) Headers {
	headerCopy := headers.Copy()
	headerCopy[newHeader] = newValue
	return headerCopy
}

func (h *Fetcher) FileGetResponse(url string, headers Headers) (*http.Response, error) {
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	request.Header.Add("User-Agent", h.UserAgent)
	if headers != nil {
		headers.AddHeaders(&request.Header)
	}
	h.Logf("FileGetResponse[%s]: pre-connect", url)
	resp, err := h.HTTPClient.Do(request)
	h.Logf("FileGetResponse[%s]: connected: %v, %v", url, resp, err)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode >= 400 {
		resp.Body.Close()
		return nil, &HTTPError{resp.StatusCode, resp}
	}
	return resp, err
}

func (h *Fetcher) FetchFile(req *FetchRequest, complete chan<- *FetchResult) {
	h.Logf("FetchFile[%s] -> %s (full download: %v)", req.Url, req.Filename,
		req.FullDownload)
	if !req.FullDownload {
		finf, err := os.Stat(req.Filename)
		if err == nil && finf != nil && finf.Size() > 0 {
			h.Logf("FetchFile[%s]: resuming download for %s",
				req.Url, req.Filename)
			h.ResumeFileDownload(req, complete)
			return
		}
	}
	h.Logf("FetchFile[%s]: new file download %s", req.Url, req.Filename)
	h.NewFileDownload(req, complete)
}

func fileResumeHeaders(req *FetchRequest, file *os.File) (Headers, int64) {
	headers := req.RequestHeaders
	finf, err := file.Stat()
	resumePoint := int64(0)
	if err == nil && finf != nil {
		resumePoint = finf.Size()
		if headers == nil {
			headers = Headers{}
		} else {
			headers = headers.Copy()
		}
		headers["Range"] = fmt.Sprintf("bytes=%d-", resumePoint)
		headers["Accept-Encoding"] = ""
	}
	return headers, resumePoint
}

func (h *Fetcher) ResumeFileDownload(req *FetchRequest, complete chan<- *FetchResult) {
	h.Logf("ResumeFileDownload[%s] -> %s", req.Url, req.Filename)
	var err error
	handleError := func() {
		if err != nil && !h.Quiet {
			h.Logf("Download of %s failed: %s\n", req, err)
		}
		h.Logf("handleError[%s] -> %s, err: %v", req.Url, req.Filename, err)
		complete <- fetchError(req, err)
	}

	if !h.Quiet {
		h.Logf("ResumeFileDownload(%s)\n", req)
	}
	file, err := os.OpenFile(req.Filename,
		os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		handleError()
		return
	}
	defer file.Close()

	headers, resumePoint := fileResumeHeaders(req, file)
	resp, err := h.FileGetResponse(req.Url, headers)

	var copied int64 = 0
	if err != nil {
		httpErr, _ := err.(*HTTPError)
		if httpErr == nil || httpErr.StatusCode != http.StatusRequestedRangeNotSatisfiable {
			handleError()
			return
		}
		err = nil
	} else {
		defer resp.Body.Close()
		h.Logf("ResumeFileDownload[%s]: Copying bytes to %s from response",
			req.Url, req.Filename)

		copied, err = io.Copy(file, resp.Body)
	}
	if !h.Quiet {
		h.Logf("[DONE:%d] ResumeFileDownload (at %d) %s\n", copied, resumePoint, req)
	}
	h.Logf("ResumeFileDownload[%s] -> %s: completed, bytes copied: %v, err: %v",
		req.Url, req.Filename, copied, err)
	complete <- &FetchResult{req, err, copied}
}

func (h *Fetcher) NewFileDownload(req *FetchRequest, complete chan<- *FetchResult) {
	h.Logf("NewFileDownload[%s] -> %s", req.Url, req.Filename)
	if !h.Quiet {
		h.Logf("NewFileDownload ", req)
	}
	resp, err := h.FileGetResponse(req.Url, req.RequestHeaders)
	if err != nil {
		h.Logf("NewFileDownload[%s] -> %s: error: %v (http)", req.Url, req.Filename, err)
		complete <- fetchError(req, err)
		return
	}
	defer resp.Body.Close()

	file, err := os.Create(req.Filename)
	if err != nil {
		h.Logf("NewFileDownload[%s] -> %s: error: %v (fopen)", req.Url, req.Filename, err)
		complete <- fetchError(req, err)
		return
	}
	defer file.Close()

	h.Logf("NewFileDownload[%s] -> %s: copying bytes", req.Url, req.Filename)
	copied, err := io.Copy(file, resp.Body)
	if !h.Quiet {
		h.Logf("[DONE:%d] NewFileDownload %s\n", copied, req)
	}
	h.Logf("NewFileDownload[%s] -> %s: completed copy:%v, err:%v", req.Url, req.Filename, copied, err)
	complete <- &FetchResult{req, err, copied}
}

func groupFetchRequestsByHost(requests []*FetchRequest) map[string][]*FetchRequest {
	grouped := make(map[string][]*FetchRequest)
	for _, req := range requests {
		reqHost, _ := req.Host()
		grouped[reqHost] = append(grouped[reqHost], req)
	}
	return grouped
}

// QueueFetch enqueues the given download requests for asynchronous download.
func (h *Fetcher) QueueFetch(req []*FetchRequest) {
	log.Printf("QueueFetch: %d files\n", len(req))
	for host, reqs := range groupFetchRequestsByHost(req) {
		hostQueue := h.hostQueue(host)
		h.enqueueWaitGroup.Add(1)
		go h.enqueueRequests(hostQueue, reqs)
	}
}

// Shutdown gracefully shuts down the fetcher, cleaning up all
// background goroutines, and waiting for all outstanding downloads to
// end.
func (h *Fetcher) Shutdown() {
	h.enqueueWaitGroup.Wait()
	for host, queue := range h.hostQueues {
		close(queue)
		delete(h.hostQueues, host)
	}
	h.hostWaitGroup.Wait()
}

func (h *Fetcher) enqueueRequests(queue chan<- *FetchRequest, reqs []*FetchRequest) {
	for _, req := range reqs {
		queue <- req
	}
	h.enqueueWaitGroup.Done()
}

func (h *Fetcher) hostQueue(host string) chan<- *FetchRequest {
	queue := h.hostQueues[host]
	if queue == nil {
		h.hostWaitGroup.Add(1)
		newQueue := make(chan *FetchRequest)
		go h.monitorHostQueue(host, newQueue)
		h.hostQueues[host] = newQueue
		queue = newQueue
	}
	return queue
}

func (h *Fetcher) monitorHostQueue(host string, incoming <-chan *FetchRequest) {
	slaveResult := make(chan *FetchResult)
	slaveQueue := make(chan *FetchRequest)

	nSlaves := h.MaxConcurrentRequestsPerHost
	slaveWaitGroup := sync.WaitGroup{}
	slaveWaitGroup.Add(nSlaves)
	// Slaves lead uncomplicated lives:
	for i := 0; i < nSlaves; i++ {
		go func() {
			for req := range slaveQueue {
				h.FetchFile(req, slaveResult)
			}
			slaveWaitGroup.Done()
		}()
	}
	// And a goroutine to close the slaveResult channel when
	// everyone's done.
	go func() {
		slaveWaitGroup.Wait()
		log.Printf("Cleaning up host monitor for %s\n", host)
		close(slaveResult)
	}()

	queue := []*FetchRequest{}
	inProgress := map[string]bool{}
	reqKey := func(req *FetchRequest) string {
		return req.Url + " | " + req.Filename
	}

	queueRequest := func(req *FetchRequest) {
		// Suppress duplicate fetch requests:
		key := reqKey(req)
		if inProgress[key] {
			log.Printf("%s: ignoring duplicate download %s\n", host, req.Url)
			return
		}
		inProgress[key] = true
		queue = append(queue, req)
	}

	applyResult := func(res *FetchResult) {
		delete(inProgress, reqKey(res.Req))
		if res.Err != nil {
			log.Printf("ERR %s (%s)\n", res.Req, res.Err)
		} else {
			log.Printf("ok %s [%d]\n", res.Req, res.DownloadSize)
		}
	}

	firstItem := func() *FetchRequest {
		if len(queue) == 0 {
			return nil
		}
		return queue[0]
	}
	slaveQueueOrNil := func() chan<- *FetchRequest {
		if len(queue) == 0 {
			return nil
		}
		return slaveQueue
	}

	for incoming != nil || len(inProgress) > 0 {
		// Bi-modal select: if there are things in the queue, try to
		// feed them to the first slave who will listen. In all cases,
		// track incoming requests and slaves reporting results.
		select {
		case slaveQueueOrNil() <- firstItem():
			queue = queue[1:]
		case newRequest := <-incoming:
			if newRequest == nil {
				log.Printf("%s: Download queue shutting down\n", host)
				incoming = nil
				break
			}
			queueRequest(newRequest)
		case result := <-slaveResult:
			applyResult(result)
		}
	}

	// Exiting, clean up:
	close(slaveQueue)
	for res := range slaveResult {
		applyResult(res)
	}
	h.hostWaitGroup.Done()
}
<commit_msg>Don't report successful fetches that retrieved nothing.<commit_after>package httpfetch

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"
)

const DefaultUserAgent = "Sequell httpfetch/1.0"

type Fetcher struct {
	HTTPClient                   *http.Client
	Quiet                        bool
	ConnectTimeout               time.Duration
	ReadTimeout                  time.Duration
	UserAgent                    string
	MaxConcurrentRequestsPerHost int
	Logger                       *log.Logger
	logWriter                    Logger

	// Queues for each host, monitored by the service goroutine.
	hostQueues       map[string]chan<- *FetchRequest
	hostWaitGroup    sync.WaitGroup
	enqueueWaitGroup sync.WaitGroup
}

// New returns a new Fetcher for parallel downloads. Fetcher
// methods are not threadsafe.
func New() *Fetcher {
	writer := CreateLogger()
	return &Fetcher{
		HTTPClient:                   DefaultHTTPClient,
		ConnectTimeout:               DefaultConnectTimeout,
		ReadTimeout:                  DefaultReadTimeout,
		UserAgent:                    DefaultUserAgent,
		MaxConcurrentRequestsPerHost: 5,
		logWriter:                    writer,
		Logger:                       log.New(writer, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile),
		hostQueues:                   map[string]chan<- *FetchRequest{},
	}
}

var DefaultConnectTimeout = 10 * time.Second
var DefaultReadTimeout = 20 * time.Second
var DefaultHTTPTransport = http.Transport{
	Dial: dialer(DefaultConnectTimeout, DefaultReadTimeout),
	ResponseHeaderTimeout: DefaultConnectTimeout,
}
var DefaultHTTPClient = &http.Client{
	Transport: &DefaultHTTPTransport,
}

func (h *Fetcher) SetLogWriter(writer io.Writer) {
	h.logWriter.SetWriter(writer)
}

type unbufferedWriter struct {
	file *os.File
}

func (uw unbufferedWriter) Write(b []byte) (n int, err error) {
	n, err = uw.file.Write(b)
	if err != nil {
		_ = uw.file.Sync()
	}
	return
}

func (h *Fetcher) SetLogFile(file *os.File) {
	h.SetLogWriter(unbufferedWriter{file: file})
}

func (h *Fetcher) Logf(format string, rest ...interface{}) {
	h.Logger.Printf(format, rest...)
}

func (h *Fetcher) GetConcurrentRequestCount(count int) int {
	if count > h.MaxConcurrentRequestsPerHost {
		return h.MaxConcurrentRequestsPerHost
	}
	return count
}

type Headers map[string]string

type HTTPError struct {
	StatusCode int
	Response   *http.Response
}

func (err *HTTPError) Error() string {
	req := err.Response.Request
	return fmt.Sprint(req.Method, " ", req.URL, " failed: ", err.StatusCode)
}

type FetchRequest struct {
	Url      string
	Filename string

	// Don't try to resume downloads if this is set.
	FullDownload   bool
	RequestHeaders Headers
}

func (req *FetchRequest) Host() (string, error) {
	reqUrl, err := url.Parse(req.Url)
	if err != nil {
		return "", err
	}
	return reqUrl.Host, nil
}

func (req *FetchRequest) String() string {
	return fmt.Sprint(req.Url, " -> ", req.Filename)
}

type FetchResult struct {
	Req          *FetchRequest
	Err          error
	DownloadSize int64
}

func fetchError(req *FetchRequest, err error) *FetchResult {
	return &FetchResult{req, err, 0}
}

func (headers Headers) AddHeaders(h *http.Header) {
	for header, value := range headers {
		h.Add(header, value)
	}
}

func (headers Headers) Copy() Headers {
	res := make(Headers)
	for k, v := range headers {
		res[k] = v
	}
	return res
}

func HeadersWith(headers Headers, newHeader, newValue string) Headers {
	headerCopy := headers.Copy()
	headerCopy[newHeader] = newValue
	return headerCopy
}

func (h *Fetcher) FileGetResponse(url string, headers Headers) (*http.Response, error) {
	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	request.Header.Add("User-Agent", h.UserAgent)
	if headers != nil {
		headers.AddHeaders(&request.Header)
	}
	h.Logf("FileGetResponse[%s]: pre-connect", url)
	resp, err := h.HTTPClient.Do(request)
	h.Logf("FileGetResponse[%s]: connected: %v, %v", url, resp, err)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode >= 400 {
		resp.Body.Close()
		return nil, &HTTPError{resp.StatusCode, resp}
	}
	return resp, err
}

func (h *Fetcher) FetchFile(req *FetchRequest, complete chan<- *FetchResult) {
	h.Logf("FetchFile[%s] -> %s (full download: %v)", req.Url, req.Filename,
		req.FullDownload)
	if !req.FullDownload {
		finf, err := os.Stat(req.Filename)
		if err == nil && finf != nil && finf.Size() > 0 {
			h.Logf("FetchFile[%s]: resuming download for %s",
				req.Url, req.Filename)
			h.ResumeFileDownload(req, complete)
			return
		}
	}
	h.Logf("FetchFile[%s]: new file download %s", req.Url, req.Filename)
	h.NewFileDownload(req, complete)
}

func fileResumeHeaders(req *FetchRequest, file *os.File) (Headers, int64) {
	headers := req.RequestHeaders
	finf, err := file.Stat()
	resumePoint := int64(0)
	if err == nil && finf != nil {
		resumePoint = finf.Size()
		if headers == nil {
			headers = Headers{}
		} else {
			headers = headers.Copy()
		}
		headers["Range"] = fmt.Sprintf("bytes=%d-", resumePoint)
		headers["Accept-Encoding"] = ""
	}
	return headers, resumePoint
}

func (h *Fetcher) ResumeFileDownload(req *FetchRequest, complete chan<- *FetchResult) {
	h.Logf("ResumeFileDownload[%s] -> %s", req.Url, req.Filename)
	var err error
	handleError := func() {
		if err != nil && !h.Quiet {
			h.Logf("Download of %s failed: %s\n", req, err)
		}
		h.Logf("handleError[%s] -> %s, err: %v", req.Url, req.Filename, err)
		complete <- fetchError(req, err)
	}

	if !h.Quiet {
		h.Logf("ResumeFileDownload(%s)\n", req)
	}
	file, err := os.OpenFile(req.Filename,
		os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		handleError()
		return
	}
	defer file.Close()

	headers, resumePoint := fileResumeHeaders(req, file)
	resp, err := h.FileGetResponse(req.Url, headers)

	var copied int64 = 0
	if err != nil {
		httpErr, _ := err.(*HTTPError)
		if httpErr == nil || httpErr.StatusCode != http.StatusRequestedRangeNotSatisfiable {
			handleError()
			return
		}
		err = nil
	} else {
		defer resp.Body.Close()
		h.Logf("ResumeFileDownload[%s]: Copying bytes to %s from response",
			req.Url, req.Filename)

		copied, err = io.Copy(file, resp.Body)
	}
	if !h.Quiet {
		h.Logf("[DONE:%d] ResumeFileDownload (at %d) %s\n", copied, resumePoint, req)
	}
	h.Logf("ResumeFileDownload[%s] -> %s: completed, bytes copied: %v, err: %v",
		req.Url, req.Filename, copied, err)
	complete <- &FetchResult{req, err, copied}
}

func (h *Fetcher) NewFileDownload(req *FetchRequest, complete chan<- *FetchResult) {
	h.Logf("NewFileDownload[%s] -> %s", req.Url, req.Filename)
	if !h.Quiet {
		h.Logf("NewFileDownload ", req)
	}
	resp, err := h.FileGetResponse(req.Url, req.RequestHeaders)
	if err != nil {
		h.Logf("NewFileDownload[%s] -> %s: error: %v (http)", req.Url, req.Filename, err)
		complete <- fetchError(req, err)
		return
	}
	defer resp.Body.Close()

	file, err := os.Create(req.Filename)
	if err != nil {
		h.Logf("NewFileDownload[%s] -> %s: error: %v (fopen)", req.Url, req.Filename, err)
		complete <- fetchError(req, err)
		return
	}
	defer file.Close()

	h.Logf("NewFileDownload[%s] -> %s: copying bytes", req.Url, req.Filename)
	copied, err := io.Copy(file, resp.Body)
	if !h.Quiet {
		h.Logf("[DONE:%d] NewFileDownload %s\n", copied, req)
	}
	h.Logf("NewFileDownload[%s] -> %s: completed copy:%v, err:%v", req.Url, req.Filename, copied, err)
	complete <- &FetchResult{req, err, copied}
}

func groupFetchRequestsByHost(requests []*FetchRequest) map[string][]*FetchRequest {
	grouped := make(map[string][]*FetchRequest)
	for _, req := range requests {
		reqHost, _ := req.Host()
		grouped[reqHost] = append(grouped[reqHost], req)
	}
	return grouped
}

// QueueFetch enqueues the given download requests for asynchronous download.
func (h *Fetcher) QueueFetch(req []*FetchRequest) {
	log.Printf("QueueFetch: %d files\n", len(req))
	for host, reqs := range groupFetchRequestsByHost(req) {
		hostQueue := h.hostQueue(host)
		h.enqueueWaitGroup.Add(1)
		go h.enqueueRequests(hostQueue, reqs)
	}
}

// Shutdown gracefully shuts down the fetcher, cleaning up all
// background goroutines, and waiting for all outstanding downloads to
// end.
func (h *Fetcher) Shutdown() {
	h.enqueueWaitGroup.Wait()
	for host, queue := range h.hostQueues {
		close(queue)
		delete(h.hostQueues, host)
	}
	h.hostWaitGroup.Wait()
}

func (h *Fetcher) enqueueRequests(queue chan<- *FetchRequest, reqs []*FetchRequest) {
	for _, req := range reqs {
		queue <- req
	}
	h.enqueueWaitGroup.Done()
}

func (h *Fetcher) hostQueue(host string) chan<- *FetchRequest {
	queue := h.hostQueues[host]
	if queue == nil {
		h.hostWaitGroup.Add(1)
		newQueue := make(chan *FetchRequest)
		go h.monitorHostQueue(host, newQueue)
		h.hostQueues[host] = newQueue
		queue = newQueue
	}
	return queue
}

func (h *Fetcher) monitorHostQueue(host string, incoming <-chan *FetchRequest) {
	slaveResult := make(chan *FetchResult)
	slaveQueue := make(chan *FetchRequest)

	nSlaves := h.MaxConcurrentRequestsPerHost
	slaveWaitGroup := sync.WaitGroup{}
	slaveWaitGroup.Add(nSlaves)
	// Slaves lead uncomplicated lives:
	for i := 0; i < nSlaves; i++ {
		go func() {
			for req := range slaveQueue {
				h.FetchFile(req, slaveResult)
			}
			slaveWaitGroup.Done()
		}()
	}
	// And a goroutine to close the slaveResult channel when
	// everyone's done.
	go func() {
		slaveWaitGroup.Wait()
		log.Printf("Cleaning up host monitor for %s\n", host)
		close(slaveResult)
	}()

	queue := []*FetchRequest{}
	inProgress := map[string]bool{}
	reqKey := func(req *FetchRequest) string {
		return req.Url + " | " + req.Filename
	}

	queueRequest := func(req *FetchRequest) {
		// Suppress duplicate fetch requests:
		key := reqKey(req)
		if inProgress[key] {
			log.Printf("%s: ignoring duplicate download %s\n", host, req.Url)
			return
		}
		inProgress[key] = true
		queue = append(queue, req)
	}

	applyResult := func(res *FetchResult) {
		delete(inProgress, reqKey(res.Req))
		if res.Err != nil {
			log.Printf("ERR %s (%s)\n", res.Req, res.Err)
		} else if res.DownloadSize > 0 {
			log.Printf("ok %s [%d]\n", res.Req, res.DownloadSize)
		}
	}

	firstItem := func() *FetchRequest {
		if len(queue) == 0 {
			return nil
		}
		return queue[0]
	}
	slaveQueueOrNil := func() chan<- *FetchRequest {
		if len(queue) == 0 {
			return nil
		}
		return slaveQueue
	}

	for incoming != nil || len(inProgress) > 0 {
		// Bi-modal select: if there are things in the queue, try to
		// feed them to the first slave who will listen. In all cases,
		// track incoming requests and slaves reporting results.
		select {
		case slaveQueueOrNil() <- firstItem():
			queue = queue[1:]
		case newRequest := <-incoming:
			if newRequest == nil {
				log.Printf("%s: Download queue shutting down\n", host)
				incoming = nil
				break
			}
			queueRequest(newRequest)
		case result := <-slaveResult:
			applyResult(result)
		}
	}

	// Exiting, clean up:
	close(slaveQueue)
	for res := range slaveResult {
		applyResult(res)
	}
	h.hostWaitGroup.Done()
}
<|endoftext|>


<commit_before>package autopilot

import (
	"net"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/lightningnetwork/lnd/lnwire"
)

// Node node is an interface which represents n abstract vertex within the
// channel graph. All nodes should have at least a single edge to/from them
// within the graph.
//
// TODO(roasbeef): combine with routing.ChannelGraphSource
type Node interface {
	// PubKey is the identity public key of the node. This will be used to
	// attempt to target a node for channel opening by the main autopilot
	// agent.
	PubKey() *btcec.PublicKey

	// Addrs returns a slice of publicly reachable public TCP addresses
	// that the peer is known to be listening on.
	Addrs() []net.Addr

	// ForEachChannel is a higher-order function that will be used to
	// iterate through all edges emanating from/to the target node. For
	// each active channel, this function should be called with the
	// populated ChannelEdge that describes the active channel.
	ForEachChannel(func(ChannelEdge) error) error
}

// Channel is a simple struct which contains relevant details of a particular
// channel within the channel graph. The fields in this struct may be used a
// signals for various AttachmentHeuristic implementations.
type Channel struct {
	// ChanID is the short channel ID for this channel as defined within
	// BOLT-0007.
	ChanID lnwire.ShortChannelID

	// Capacity is the capacity of the channel expressed in satoshis.
	Capacity btcutil.Amount

	// FundedAmt is the amount the local node funded into the target
	// channel.
	//
	// TODO(roasbeef): need this?
	FundedAmt btcutil.Amount

	// Node is the peer that this channel has been established with.
	Node NodeID

	// TODO(roasbeef): also add other traits?
	//  * fee, timelock, etc
}

// ChannelEdge is a struct that holds details concerning a channel, but also
// contains a reference to the Node that this channel connects to as a directed
// edge within the graph. The existence of this reference to the connected node
// will allow callers to traverse the graph in an object-oriented manner.
type ChannelEdge struct {
	// Channel contains the attributes of this channel.
	Channel

	// Peer is the peer that this channel creates an edge to in the channel
	// graph.
	Peer Node
}

// ChannelGraph in an interface that represents a traversable channel graph.
// The autopilot agent will use this interface as its source of graph traits in
// order to make decisions concerning which channels should be opened, and to
// whom.
//
// TODO(roasbeef): abstract??
type ChannelGraph interface {
	// ForEachNode is a higher-order function that should be called once
	// for each connected node within the channel graph. If the passed
	// callback returns an error, then execution should be terminated.
	ForEachNode(func(Node) error) error
}

// AttachmentDirective describes a channel attachment proscribed by an
// AttachmentHeuristic. It details to which node a channel should be created
// to, and also the parameters which should be used in the channel creation.
type AttachmentDirective struct {
	// PeerKey is the target node for this attachment directive. It can be
	// identified by its public key, and therefore can be used along with
	// a ChannelOpener implementation to execute the directive.
	PeerKey *btcec.PublicKey

	// ChanAmt is the size of the channel that should be opened, expressed
	// in satoshis.
	ChanAmt btcutil.Amount

	// Addrs is a list of addresses that the target peer may be reachable
	// at.
	Addrs []net.Addr
}

// AttachmentHeuristic is one of the primary interfaces within this package.
// Implementations of this interface will be used to implement a control system
// which automatically regulates channels of a particular agent, attempting to
// optimize channels opened/closed based on various heuristics.  The purpose of
// the interface is to allow an auto-pilot agent to decide if it needs more
// channels, and if so, which exact channels should be opened.
type AttachmentHeuristic interface {
	// NeedMoreChans is a predicate that should return true if, given the
	// passed parameters, and its internal state, more channels should be
	// opened within the channel graph. If the heuristic decides that we do
	// indeed need more channels, then the second argument returned will
	// represent the amount of additional funds to be used towards creating
	// channels. This method should also return the exact *number* of
	// additional channels that are needed in order to converge towards our
	// ideal state.
	NeedMoreChans(chans []Channel, balance btcutil.Amount) (btcutil.Amount, uint32, bool)

	// Select is a method that given the current state of the channel
	// graph, a set of nodes to ignore, and an amount of available funds,
	// should return a set of attachment directives which describe which
	// additional channels should be opened within the graph to push the
	// heuristic back towards its equilibrium state. The numNewChans
	// argument represents the additional number of channels that should be
	// open.
	Select(self *btcec.PublicKey, graph ChannelGraph,
		amtToUse btcutil.Amount, numNewChans uint32,
		skipNodes map[NodeID]struct{}) ([]AttachmentDirective, error)
}

// ChannelController is a simple interface that allows an auto-pilot agent to
// open a channel within the graph to a target peer, close targeted channels,
// or add/remove funds from existing channels via a splice in/out mechanisms.
type ChannelController interface {
	// OpenChannel opens a channel to a target peer, with a capacity of the
	// specified amount. This function should un-block immediately after
	// the funding transaction that marks the channel open has been
	// broadcast.
	OpenChannel(target *btcec.PublicKey, amt btcutil.Amount,
		addrs []net.Addr) error

	// CloseChannel attempts to close out the target channel.
	//
	// TODO(roasbeef): add force option?
	CloseChannel(chanPoint *wire.OutPoint) error

	// SpliceIn attempts to add additional funds to the target channel via
	// a splice in mechanism. The new channel with an updated capacity
	// should be returned.
	SpliceIn(chanPoint *wire.OutPoint, amt btcutil.Amount) (*Channel, error)

	// SpliceOut attempts to remove funds from an existing channels using a
	// splice out mechanism. The removed funds from the channel should be
	// returned to an output under the control of the backing wallet.
	SpliceOut(chanPoint *wire.OutPoint, amt btcutil.Amount) (*Channel, error)
}
<commit_msg>autopilot: modify OpenChannel to no longer take in set of addrs<commit_after>package autopilot

import (
	"net"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/lightningnetwork/lnd/lnwire"
)

// Node node is an interface which represents n abstract vertex within the
// channel graph. All nodes should have at least a single edge to/from them
// within the graph.
//
// TODO(roasbeef): combine with routing.ChannelGraphSource
type Node interface {
	// PubKey is the identity public key of the node. This will be used to
	// attempt to target a node for channel opening by the main autopilot
	// agent.
	PubKey() *btcec.PublicKey

	// Addrs returns a slice of publicly reachable public TCP addresses
	// that the peer is known to be listening on.
	Addrs() []net.Addr

	// ForEachChannel is a higher-order function that will be used to
	// iterate through all edges emanating from/to the target node. For
	// each active channel, this function should be called with the
	// populated ChannelEdge that describes the active channel.
	ForEachChannel(func(ChannelEdge) error) error
}

// Channel is a simple struct which contains relevant details of a particular
// channel within the channel graph. The fields in this struct may be used a
// signals for various AttachmentHeuristic implementations.
type Channel struct {
	// ChanID is the short channel ID for this channel as defined within
	// BOLT-0007.
	ChanID lnwire.ShortChannelID

	// Capacity is the capacity of the channel expressed in satoshis.
	Capacity btcutil.Amount

	// FundedAmt is the amount the local node funded into the target
	// channel.
	//
	// TODO(roasbeef): need this?
	FundedAmt btcutil.Amount

	// Node is the peer that this channel has been established with.
	Node NodeID

	// TODO(roasbeef): also add other traits?
	//  * fee, timelock, etc
}

// ChannelEdge is a struct that holds details concerning a channel, but also
// contains a reference to the Node that this channel connects to as a directed
// edge within the graph. The existence of this reference to the connected node
// will allow callers to traverse the graph in an object-oriented manner.
type ChannelEdge struct {
	// Channel contains the attributes of this channel.
	Channel

	// Peer is the peer that this channel creates an edge to in the channel
	// graph.
	Peer Node
}

// ChannelGraph in an interface that represents a traversable channel graph.
// The autopilot agent will use this interface as its source of graph traits in
// order to make decisions concerning which channels should be opened, and to
// whom.
//
// TODO(roasbeef): abstract??
type ChannelGraph interface {
	// ForEachNode is a higher-order function that should be called once
	// for each connected node within the channel graph. If the passed
	// callback returns an error, then execution should be terminated.
	ForEachNode(func(Node) error) error
}

// AttachmentDirective describes a channel attachment proscribed by an
// AttachmentHeuristic. It details to which node a channel should be created
// to, and also the parameters which should be used in the channel creation.
type AttachmentDirective struct {
	// PeerKey is the target node for this attachment directive. It can be
	// identified by its public key, and therefore can be used along with
	// a ChannelOpener implementation to execute the directive.
	PeerKey *btcec.PublicKey

	// ChanAmt is the size of the channel that should be opened, expressed
	// in satoshis.
	ChanAmt btcutil.Amount

	// Addrs is a list of addresses that the target peer may be reachable
	// at.
	Addrs []net.Addr
}

// AttachmentHeuristic is one of the primary interfaces within this package.
// Implementations of this interface will be used to implement a control system
// which automatically regulates channels of a particular agent, attempting to
// optimize channels opened/closed based on various heuristics.  The purpose of
// the interface is to allow an auto-pilot agent to decide if it needs more
// channels, and if so, which exact channels should be opened.
type AttachmentHeuristic interface {
	// NeedMoreChans is a predicate that should return true if, given the
	// passed parameters, and its internal state, more channels should be
	// opened within the channel graph. If the heuristic decides that we do
	// indeed need more channels, then the second argument returned will
	// represent the amount of additional funds to be used towards creating
	// channels. This method should also return the exact *number* of
	// additional channels that are needed in order to converge towards our
	// ideal state.
	NeedMoreChans(chans []Channel, balance btcutil.Amount) (btcutil.Amount, uint32, bool)

	// Select is a method that given the current state of the channel
	// graph, a set of nodes to ignore, and an amount of available funds,
	// should return a set of attachment directives which describe which
	// additional channels should be opened within the graph to push the
	// heuristic back towards its equilibrium state. The numNewChans
	// argument represents the additional number of channels that should be
	// open.
	Select(self *btcec.PublicKey, graph ChannelGraph,
		amtToUse btcutil.Amount, numNewChans uint32,
		skipNodes map[NodeID]struct{}) ([]AttachmentDirective, error)
}

// ChannelController is a simple interface that allows an auto-pilot agent to
// open a channel within the graph to a target peer, close targeted channels,
// or add/remove funds from existing channels via a splice in/out mechanisms.
type ChannelController interface {
	// OpenChannel opens a channel to a target peer, with a capacity of the
	// specified amount. This function should un-block immediately after
	// the funding transaction that marks the channel open has been
	// broadcast.
	OpenChannel(target *btcec.PublicKey, amt btcutil.Amount) error

	// CloseChannel attempts to close out the target channel.
	//
	// TODO(roasbeef): add force option?
	CloseChannel(chanPoint *wire.OutPoint) error

	// SpliceIn attempts to add additional funds to the target channel via
	// a splice in mechanism. The new channel with an updated capacity
	// should be returned.
	SpliceIn(chanPoint *wire.OutPoint, amt btcutil.Amount) (*Channel, error)

	// SpliceOut attempts to remove funds from an existing channels using a
	// splice out mechanism. The removed funds from the channel should be
	// returned to an output under the control of the backing wallet.
	SpliceOut(chanPoint *wire.OutPoint, amt btcutil.Amount) (*Channel, error)
}
<|endoftext|>


<commit_before>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 11, 8, 20, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<commit_msg>Go's 8th anniversary<commit_after>// clock counts down to or up from a target time.
package main

import (
	"fmt"
	"os"
	"time"
)

func main() {
	target := time.Date(2017, 11, 10, 0, 0, 0, 0, time.Local)
	motto := "Just Go"
	printTargetTime(target, motto)
	exitOnEnterKey()

	var previous time.Time
	for {
		now := time.Now().Truncate(time.Second)
		if now != previous {
			previous = now
			countdown := now.Sub(target) // Negative times are before the target
			printCountdown(now, countdown)
		}
		time.Sleep(100 * time.Millisecond)
	}
}

func exitOnEnterKey() {
	go func() {
		buf := make([]byte, 1)
		_, _ = os.Stdin.Read(buf)
		os.Exit(0)
	}()
}

const (
	indent         = "\t"
	highlightStart = "\x1b[1;35m"
	highlightEnd   = "\x1b[0m"
)

func printTargetTime(target time.Time, motto string) {
	fmt.Print(indent, highlightStart, motto, highlightEnd, "\n")
	fmt.Print(indent, target.Format(time.UnixDate), "\n")
}

func printCountdown(now time.Time, countdown time.Duration) {
	var sign string
	if countdown >= 0 {
		sign = "+"
	} else {
		sign = "-"
		countdown = -countdown
	}

	days := int(countdown / (24 * time.Hour))
	countdown = countdown % (24 * time.Hour)

	fmt.Print(indent, now.Format(time.UnixDate), "  ", sign)
	if days > 0 {
		fmt.Print(days, "d")
	}
	fmt.Print(countdown, "          \r")
	os.Stdout.Sync()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ProfOak/cmdji/go-bindings"
	"io/ioutil"
	"os/user"
	"strconv"
	"strings"
	"time"
)

func main() {
	// get home dir
	usr, err := user.Current()
	if err != nil {
		fmt.Println(err)
		// no user, no root, no service
		return
	}

	// .kanji_today in home dir
	kanji_today_filename := usr.HomeDir + "/.kanji_today"

	// file to store today's kanji
	// prevents multiple lookups on the server
	contents, err := ioutil.ReadFile(kanji_today_filename)

	var k cmdji.KanjiADay

	u2d, err := upToDate(contents)
	// first time running, or not up to date
	if err != nil || !u2d || err != nil {
		// file not found/make file
		k = cmdji.Kanji()
		err = k.Update()

		if err != nil {
			fmt.Println("Cannot retrieve new information from server")
			return
		}

		ioutil.WriteFile(kanji_today_filename, k.RawJson(), 0744)

	} else {
		// already up to date contents in file
		k = cmdji.OpenKanji(contents)
	}

	// ready to use kanjiaday variable
	err = k.UnJson()

	// nicer printing
	str := k.KanjiCharacter() + "\n"
	str += strings.Join(k.Meanings(), ", ") + " "
	str += "JLPT: " + strconv.Itoa(k.Jlpt()) + "\n"

	if len(k.Kunyomis()) > 0 {
		str += "Kun'yomi: "
		str += strings.Join(k.Kunyomis(), ",")
	}
	if len(k.Kunyomis()) > 0 && len(k.Onyomis()) > 0 {
		str += " | "
	}
	if len(k.Onyomis()) > 0 {
		str += "On'yomi: "
		str += strings.Join(k.Onyomis(), ",")
	}

	fmt.Println(str)
	//k.Print()
}

func upToDate(contents []byte) (bool, error) {

	// check json blob from .kanji_today file
	past_kanji := cmdji.OpenKanji(contents)

	fmt.Println("fail")
	err := past_kanji.UnJson()
	if err != nil {
		// malformed json blob in file
		return false, err
	}

	date_raw := past_kanji.Date()

	// date_raw    = []string{date, time}
	// date_raw[0] = []string{year, month, day}
	old_date := strings.Split(strings.Split(date_raw, "T")[0], "-")

	old_year, _ := strconv.Atoi(old_date[0])
	old_month, _ := strconv.Atoi(old_date[1])
	old_day, _ := strconv.Atoi(old_date[2])

	// kanji a day uses UTC time
	// updates at midnight
	year, month, day := time.Now().UTC().Date()

	if year != old_year || int(month) != old_month || day != old_day {
		return false, err
	}
	return true, nil
}
<commit_msg>fixed error on update from server without internet connection<commit_after>package main

import (
	"fmt"
	"github.com/ProfOak/cmdji/go-bindings"
	"io/ioutil"
	"os/user"
	"strconv"
	"strings"
	"time"
)

func main() {
	// get home dir
	usr, err := user.Current()
	if err != nil {
		fmt.Println(err)
		// no user, no root, no service
		return
	}

	// .kanji_today in home dir
	kanji_today_filename := usr.HomeDir + "/.kanji_today"

	// file to store today's kanji
	// prevents multiple lookups on the server
	contents, err := ioutil.ReadFile(kanji_today_filename)

	var k cmdji.KanjiADay

	u2d, err := upToDate(contents)
	// first time running, or not up to date
	if err != nil || !u2d {
		// file not found/make file
		fmt.Println("Attempting to update from server...")
		k = cmdji.Kanji()
		err = k.Update()

		if err != nil {
			fmt.Println("Failed to retrieve new information from server")
			return
		} else {
			fmt.Println("Update successful")
		}

		ioutil.WriteFile(kanji_today_filename, k.RawJson(), 0744)

	} else {
		// already up to date contents in file
		k = cmdji.OpenKanji(contents)
	}

	// ready to use kanjiaday variable
	err = k.UnJson()

	// nicer printing
	str := k.KanjiCharacter() + "\n"
	str += strings.Join(k.Meanings(), ", ") + " "
	str += "JLPT: " + strconv.Itoa(k.Jlpt()) + "\n"

	if len(k.Kunyomis()) > 0 {
		str += "Kun'yomi: "
		str += strings.Join(k.Kunyomis(), ",")
	}
	if len(k.Kunyomis()) > 0 && len(k.Onyomis()) > 0 {
		str += " | "
	}
	if len(k.Onyomis()) > 0 {
		str += "On'yomi: "
		str += strings.Join(k.Onyomis(), ",")
	}

	fmt.Println(str)
	//k.Print()
}

func upToDate(contents []byte) (bool, error) {

	// check json blob from .kanji_today file
	past_kanji := cmdji.OpenKanji(contents)

	err := past_kanji.UnJson()
	if err != nil {
		// malformed json blob in file
		return false, err
	}

	date_raw := past_kanji.Date()

	// date_raw    = []string{date, time}
	// date_raw[0] = []string{year, month, day}
	old_date := strings.Split(strings.Split(date_raw, "T")[0], "-")

	old_year, _ := strconv.Atoi(old_date[0])
	old_month, _ := strconv.Atoi(old_date[1])
	old_day, _ := strconv.Atoi(old_date[2])

	// kanji a day uses UTC time
	// updates at midnight
	year, month, day := time.Now().UTC().Date()

	if year != old_year || int(month) != old_month || day != old_day {
		return false, err
	}
	return true, nil
}
<|endoftext|>


<commit_before>package tarantool

/*
Copy-Paste from https://github.com/tarantool/go-tarantool/blob/master/const.go
*/

const (
	BadRequest       = byte(0)
	SelectRequest    = byte(1)
	InsertRequest    = byte(2)
	ReplaceRequest   = byte(3)
	UpdateRequest    = byte(4)
	DeleteRequest    = byte(5)
	CallRequest      = byte(6)
	AuthRequest      = byte(7)
	EvalRequest      = byte(8)
	UpsertRequest    = byte(9)
	PingRequest      = byte(64)
	SubscribeRequest = byte(66)

	KeyCode         = 0x00
	KeySync         = 0x01
	KeySchemaID     = 0x05
	KeySpaceNo      = 0x10
	KeyIndexNo      = 0x11
	KeyLimit        = 0x12
	KeyOffset       = 0x13
	KeyIterator     = 0x14
	KeyKey          = 0x20
	KeyTuple        = 0x21
	KeyFunctionName = 0x22
	KeyUserName     = 0x23
	KeyExpression   = 0x27
	KeyDefTuple     = 0x28
	KeyData         = 0x30
	KeyError        = 0x31

	// https://github.com/fl00r/go-tarantool-1.6/issues/2
	IterEq            = uint8(0) // key == x ASC order
	IterReq           = uint8(1) // key == x DESC order
	IterAll           = uint8(2) // all tuples
	IterLt            = uint8(3) // key < x
	IterLe            = uint8(4) // key <= x
	IterGe            = uint8(5) // key > x
	IterGt            = uint8(6) // key >= x
	IterBitsAllSet    = uint8(7) // all bits from x are set in key
	IterBitsAnySet    = uint8(8) // at least one x's bit is set
	IterBitsAllNotSet = uint8(9) // all bits are not set

	OkCode            = uint32(0)
	PacketLengthBytes = 5
	ErrorFlag         = 0x8000
)

const (
	SpaceSchema  = 272
	SpaceSpace   = 280
	ViewSpace    = 281
	SpaceIndex   = 288
	ViewIndex    = 289
	SpaceFunc    = 296
	SpaceUser    = 304
	SpacePriv    = 312
	SpaceCluster = 320
)

// Tarantool server error codes
const (
	ErrUnknown                       = iota // Unknown error
	ErrIllegalParams                 = iota // Illegal parameters, %s
	ErrMemoryIssue                   = iota // Failed to allocate %u bytes in %s for %s
	ErrTupleFound                    = iota // Duplicate key exists in unique index '%s' in space '%s'
	ErrTupleNotFound                 = iota // Tuple doesn't exist in index '%s' in space '%s'
	ErrUnsupported                   = iota // %s does not support %s
	ErrNonmaster                     = iota // Can't modify data on a replication slave. My master is: %s
	ErrReadonly                      = iota // Can't modify data because this server is in read-only mode.
	ErrInjection                     = iota // Error injection '%s'
	ErrCreateSpace                   = iota // Failed to create space '%s': %s
	ErrSpaceExists                   = iota // Space '%s' already exists
	ErrDropSpace                     = iota // Can't drop space '%s': %s
	ErrAlterSpace                    = iota // Can't modify space '%s': %s
	ErrIndexType                     = iota // Unsupported index type supplied for index '%s' in space '%s'
	ErrModifyIndex                   = iota // Can't create or modify index '%s' in space '%s': %s
	ErrLastDrop                      = iota // Can't drop the primary key in a system space, space '%s'
	ErrTupleFormatLimit              = iota // Tuple format limit reached: %u
	ErrDropPrimaryKey                = iota // Can't drop primary key in space '%s' while secondary keys exist
	ErrKeyPartType                   = iota // Supplied key type of part %u does not match index part type: expected %s
	ErrExactMatch                    = iota // Invalid key part count in an exact match (expected %u, got %u)
	ErrInvalidMsgpack                = iota // Invalid MsgPack - %s
	ErrProcRet                       = iota // msgpack.encode: can not encode Lua type '%s'
	ErrTupleNotArray                 = iota // Tuple/Key must be MsgPack array
	ErrFieldType                     = iota // Tuple field %u type does not match one required by operation: expected %s
	ErrFieldTypeMismatch             = iota // Ambiguous field type in index '%s', key part %u. Requested type is %s but the field has previously been defined as %s
	ErrSplice                        = iota // SPLICE error on field %u: %s
	ErrArgType                       = iota // Argument type in operation '%c' on field %u does not match field type: expected a %s
	ErrTupleIsTooLong                = iota // Tuple is too long %u
	ErrUnknownUpdateOp               = iota // Unknown UPDATE operation
	ErrUpdateField                   = iota // Field %u UPDATE error: %s
	ErrFiberStack                    = iota // Can not create a new fiber: recursion limit reached
	ErrKeyPartCount                  = iota // Invalid key part count (expected [0..%u], got %u)
	ErrProcLua                       = iota // %s
	ErrNoSuchProc                    = iota // Procedure '%.*s' is not defined
	ErrNoSuchTrigger                 = iota // Trigger is not found
	ErrNoSuchIndex                   = iota // No index #%u is defined in space '%s'
	ErrNoSuchSpace                   = iota // Space '%s' does not exist
	ErrNoSuchField                   = iota // Field %d was not found in the tuple
	ErrSpaceFieldCount               = iota // Tuple field count %u does not match space '%s' field count %u
	ErrIndexFieldCount               = iota // Tuple field count %u is less than required by a defined index (expected %u)
	ErrWalIo                         = iota // Failed to write to disk
	ErrMoreThanOneTuple              = iota // More than one tuple found by get()
	ErrAccessDenied                  = iota // %s access denied for user '%s'
	ErrCreateUser                    = iota // Failed to create user '%s': %s
	ErrDropUser                      = iota // Failed to drop user '%s': %s
	ErrNoSuchUser                    = iota // User '%s' is not found
	ErrUserExists                    = iota // User '%s' already exists
	ErrPasswordMismatch              = iota // Incorrect password supplied for user '%s'
	ErrUnknownRequestType            = iota // Unknown request type %u
	ErrUnknownSchemaObject           = iota // Unknown object type '%s'
	ErrCreateFunction                = iota // Failed to create function '%s': %s
	ErrNoSuchFunction                = iota // Function '%s' does not exist
	ErrFunctionExists                = iota // Function '%s' already exists
	ErrFunctionAccessDenied          = iota // %s access denied for user '%s' to function '%s'
	ErrFunctionMax                   = iota // A limit on the total number of functions has been reached: %u
	ErrSpaceAccessDenied             = iota // %s access denied for user '%s' to space '%s'
	ErrUserMax                       = iota // A limit on the total number of users has been reached: %u
	ErrNoSuchEngine                  = iota // Space engine '%s' does not exist
	ErrReloadCfg                     = iota // Can't set option '%s' dynamically
	ErrCfg                           = iota // Incorrect value for option '%s': %s
	ErrSophia                        = iota // %s
	ErrLocalServerIsNotActive        = iota // Local server is not active
	ErrUnknownServer                 = iota // Server %s is not registered with the cluster
	ErrClusterIdMismatch             = iota // Cluster id of the replica %s doesn't match cluster id of the master %s
	ErrInvalidUuid                   = iota // Invalid UUID: %s
	ErrClusterIdIsRo                 = iota // Can't reset cluster id: it is already assigned
	ErrReserved66                    = iota // Reserved66
	ErrServerIdIsReserved            = iota // Can't initialize server id with a reserved value %u
	ErrInvalidOrder                  = iota // Invalid LSN order for server %u: previous LSN = %llu, new lsn = %llu
	ErrMissingRequestField           = iota // Missing mandatory field '%s' in request
	ErrIdentifier                    = iota // Invalid identifier '%s' (expected letters, digits or an underscore)
	ErrDropFunction                  = iota // Can't drop function %u: %s
	ErrIteratorType                  = iota // Unknown iterator type '%s'
	ErrReplicaMax                    = iota // Replica count limit reached: %u
	ErrInvalidXlog                   = iota // Failed to read xlog: %lld
	ErrInvalidXlogName               = iota // Invalid xlog name: expected %lld got %lld
	ErrInvalidXlogOrder              = iota // Invalid xlog order: %lld and %lld
	ErrNoConnection                  = iota // Connection is not established
	ErrTimeout                       = iota // Timeout exceeded
	ErrActiveTransaction             = iota // Operation is not permitted when there is an active transaction
	ErrNoActiveTransaction           = iota // Operation is not permitted when there is no active transaction
	ErrCrossEngineTransaction        = iota // A multi-statement transaction can not use multiple storage engines
	ErrNoSuchRole                    = iota // Role '%s' is not found
	ErrRoleExists                    = iota // Role '%s' already exists
	ErrCreateRole                    = iota // Failed to create role '%s': %s
	ErrIndexExists                   = iota // Index '%s' already exists
	ErrTupleRefOverflow              = iota // Tuple reference counter overflow
	ErrRoleLoop                      = iota // Granting role '%s' to role '%s' would create a loop
	ErrGrant                         = iota // Incorrect grant arguments: %s
	ErrPrivGranted                   = iota // User '%s' already has %s access on %s '%s'
	ErrRoleGranted                   = iota // User '%s' already has role '%s'
	ErrPrivNotGranted                = iota // User '%s' does not have %s access on %s '%s'
	ErrRoleNotGranted                = iota // User '%s' does not have role '%s'
	ErrMissingSnapshot               = iota // Can't find snapshot
	ErrCantUpdatePrimaryKey          = iota // Attempt to modify a tuple field which is part of index '%s' in space '%s'
	ErrUpdateIntegerOverflow         = iota // Integer overflow when performing '%c' operation on field %u
	ErrGuestUserPassword             = iota // Setting password for guest user has no effect
	ErrTransactionConflict           = iota // Transaction has been aborted by conflict
	ErrUnsupportedRolePriv           = iota // Unsupported role privilege '%s'
	ErrLoadFunction                  = iota // Failed to dynamically load function '%s': %s
	ErrFunctionLanguage              = iota // Unsupported language '%s' specified for function '%s'
	ErrRtreeRect                     = iota // RTree: %s must be an array with %u (point) or %u (rectangle/box) numeric coordinates
	ErrProcC                         = iota // ???
	ErrUnknownRtreeIndexDistanceType = iota //Unknown RTREE index distance type %s
	ErrProtocol                      = iota // %s
	ErrUpsertUniqueSecondaryKey      = iota // Space %s has a unique secondary index and does not support UPSERT
	ErrWrongIndexRecord              = iota // Wrong record in _index space: got {%s}, expected {%s}
	ErrWrongIndexParts               = iota // Wrong index parts (field %u): %s; expected field1 id (number), field1 type (string), ...
	ErrWrongIndexOptions             = iota // Wrong index options (field %u): %s
	ErrWrongSchemaVaersion           = iota // Wrong schema version, current: %d, in request: %u
	ErrSlabAllocMax                  = iota // Failed to allocate %u bytes for tuple in the slab allocator: tuple is too large. Check 'slab_alloc_maximal' configuration option.
)
<commit_msg>Fix comment<commit_after>package tarantool

/*
Copy-Paste from https://github.com/tarantool/go-tarantool/blob/master/const.go
*/

const (
	BadRequest       = byte(0)
	SelectRequest    = byte(1)
	InsertRequest    = byte(2)
	ReplaceRequest   = byte(3)
	UpdateRequest    = byte(4)
	DeleteRequest    = byte(5)
	CallRequest      = byte(6)
	AuthRequest      = byte(7)
	EvalRequest      = byte(8)
	UpsertRequest    = byte(9)
	PingRequest      = byte(64)
	SubscribeRequest = byte(66)

	KeyCode         = 0x00
	KeySync         = 0x01
	KeySchemaID     = 0x05
	KeySpaceNo      = 0x10
	KeyIndexNo      = 0x11
	KeyLimit        = 0x12
	KeyOffset       = 0x13
	KeyIterator     = 0x14
	KeyKey          = 0x20
	KeyTuple        = 0x21
	KeyFunctionName = 0x22
	KeyUserName     = 0x23
	KeyExpression   = 0x27
	KeyDefTuple     = 0x28
	KeyData         = 0x30
	KeyError        = 0x31

	// https://github.com/fl00r/go-tarantool-1.6/issues/2
	IterEq            = uint8(0) // key == x ASC order
	IterReq           = uint8(1) // key == x DESC order
	IterAll           = uint8(2) // all tuples
	IterLt            = uint8(3) // key < x
	IterLe            = uint8(4) // key <= x
	IterGe            = uint8(5) // key >= x
	IterGt            = uint8(6) // key > x
	IterBitsAllSet    = uint8(7) // all bits from x are set in key
	IterBitsAnySet    = uint8(8) // at least one x's bit is set
	IterBitsAllNotSet = uint8(9) // all bits are not set

	OkCode            = uint32(0)
	PacketLengthBytes = 5
	ErrorFlag         = 0x8000
)

const (
	SpaceSchema  = 272
	SpaceSpace   = 280
	ViewSpace    = 281
	SpaceIndex   = 288
	ViewIndex    = 289
	SpaceFunc    = 296
	SpaceUser    = 304
	SpacePriv    = 312
	SpaceCluster = 320
)

// Tarantool server error codes
const (
	ErrUnknown                       = iota // Unknown error
	ErrIllegalParams                 = iota // Illegal parameters, %s
	ErrMemoryIssue                   = iota // Failed to allocate %u bytes in %s for %s
	ErrTupleFound                    = iota // Duplicate key exists in unique index '%s' in space '%s'
	ErrTupleNotFound                 = iota // Tuple doesn't exist in index '%s' in space '%s'
	ErrUnsupported                   = iota // %s does not support %s
	ErrNonmaster                     = iota // Can't modify data on a replication slave. My master is: %s
	ErrReadonly                      = iota // Can't modify data because this server is in read-only mode.
	ErrInjection                     = iota // Error injection '%s'
	ErrCreateSpace                   = iota // Failed to create space '%s': %s
	ErrSpaceExists                   = iota // Space '%s' already exists
	ErrDropSpace                     = iota // Can't drop space '%s': %s
	ErrAlterSpace                    = iota // Can't modify space '%s': %s
	ErrIndexType                     = iota // Unsupported index type supplied for index '%s' in space '%s'
	ErrModifyIndex                   = iota // Can't create or modify index '%s' in space '%s': %s
	ErrLastDrop                      = iota // Can't drop the primary key in a system space, space '%s'
	ErrTupleFormatLimit              = iota // Tuple format limit reached: %u
	ErrDropPrimaryKey                = iota // Can't drop primary key in space '%s' while secondary keys exist
	ErrKeyPartType                   = iota // Supplied key type of part %u does not match index part type: expected %s
	ErrExactMatch                    = iota // Invalid key part count in an exact match (expected %u, got %u)
	ErrInvalidMsgpack                = iota // Invalid MsgPack - %s
	ErrProcRet                       = iota // msgpack.encode: can not encode Lua type '%s'
	ErrTupleNotArray                 = iota // Tuple/Key must be MsgPack array
	ErrFieldType                     = iota // Tuple field %u type does not match one required by operation: expected %s
	ErrFieldTypeMismatch             = iota // Ambiguous field type in index '%s', key part %u. Requested type is %s but the field has previously been defined as %s
	ErrSplice                        = iota // SPLICE error on field %u: %s
	ErrArgType                       = iota // Argument type in operation '%c' on field %u does not match field type: expected a %s
	ErrTupleIsTooLong                = iota // Tuple is too long %u
	ErrUnknownUpdateOp               = iota // Unknown UPDATE operation
	ErrUpdateField                   = iota // Field %u UPDATE error: %s
	ErrFiberStack                    = iota // Can not create a new fiber: recursion limit reached
	ErrKeyPartCount                  = iota // Invalid key part count (expected [0..%u], got %u)
	ErrProcLua                       = iota // %s
	ErrNoSuchProc                    = iota // Procedure '%.*s' is not defined
	ErrNoSuchTrigger                 = iota // Trigger is not found
	ErrNoSuchIndex                   = iota // No index #%u is defined in space '%s'
	ErrNoSuchSpace                   = iota // Space '%s' does not exist
	ErrNoSuchField                   = iota // Field %d was not found in the tuple
	ErrSpaceFieldCount               = iota // Tuple field count %u does not match space '%s' field count %u
	ErrIndexFieldCount               = iota // Tuple field count %u is less than required by a defined index (expected %u)
	ErrWalIo                         = iota // Failed to write to disk
	ErrMoreThanOneTuple              = iota // More than one tuple found by get()
	ErrAccessDenied                  = iota // %s access denied for user '%s'
	ErrCreateUser                    = iota // Failed to create user '%s': %s
	ErrDropUser                      = iota // Failed to drop user '%s': %s
	ErrNoSuchUser                    = iota // User '%s' is not found
	ErrUserExists                    = iota // User '%s' already exists
	ErrPasswordMismatch              = iota // Incorrect password supplied for user '%s'
	ErrUnknownRequestType            = iota // Unknown request type %u
	ErrUnknownSchemaObject           = iota // Unknown object type '%s'
	ErrCreateFunction                = iota // Failed to create function '%s': %s
	ErrNoSuchFunction                = iota // Function '%s' does not exist
	ErrFunctionExists                = iota // Function '%s' already exists
	ErrFunctionAccessDenied          = iota // %s access denied for user '%s' to function '%s'
	ErrFunctionMax                   = iota // A limit on the total number of functions has been reached: %u
	ErrSpaceAccessDenied             = iota // %s access denied for user '%s' to space '%s'
	ErrUserMax                       = iota // A limit on the total number of users has been reached: %u
	ErrNoSuchEngine                  = iota // Space engine '%s' does not exist
	ErrReloadCfg                     = iota // Can't set option '%s' dynamically
	ErrCfg                           = iota // Incorrect value for option '%s': %s
	ErrSophia                        = iota // %s
	ErrLocalServerIsNotActive        = iota // Local server is not active
	ErrUnknownServer                 = iota // Server %s is not registered with the cluster
	ErrClusterIdMismatch             = iota // Cluster id of the replica %s doesn't match cluster id of the master %s
	ErrInvalidUuid                   = iota // Invalid UUID: %s
	ErrClusterIdIsRo                 = iota // Can't reset cluster id: it is already assigned
	ErrReserved66                    = iota // Reserved66
	ErrServerIdIsReserved            = iota // Can't initialize server id with a reserved value %u
	ErrInvalidOrder                  = iota // Invalid LSN order for server %u: previous LSN = %llu, new lsn = %llu
	ErrMissingRequestField           = iota // Missing mandatory field '%s' in request
	ErrIdentifier                    = iota // Invalid identifier '%s' (expected letters, digits or an underscore)
	ErrDropFunction                  = iota // Can't drop function %u: %s
	ErrIteratorType                  = iota // Unknown iterator type '%s'
	ErrReplicaMax                    = iota // Replica count limit reached: %u
	ErrInvalidXlog                   = iota // Failed to read xlog: %lld
	ErrInvalidXlogName               = iota // Invalid xlog name: expected %lld got %lld
	ErrInvalidXlogOrder              = iota // Invalid xlog order: %lld and %lld
	ErrNoConnection                  = iota // Connection is not established
	ErrTimeout                       = iota // Timeout exceeded
	ErrActiveTransaction             = iota // Operation is not permitted when there is an active transaction
	ErrNoActiveTransaction           = iota // Operation is not permitted when there is no active transaction
	ErrCrossEngineTransaction        = iota // A multi-statement transaction can not use multiple storage engines
	ErrNoSuchRole                    = iota // Role '%s' is not found
	ErrRoleExists                    = iota // Role '%s' already exists
	ErrCreateRole                    = iota // Failed to create role '%s': %s
	ErrIndexExists                   = iota // Index '%s' already exists
	ErrTupleRefOverflow              = iota // Tuple reference counter overflow
	ErrRoleLoop                      = iota // Granting role '%s' to role '%s' would create a loop
	ErrGrant                         = iota // Incorrect grant arguments: %s
	ErrPrivGranted                   = iota // User '%s' already has %s access on %s '%s'
	ErrRoleGranted                   = iota // User '%s' already has role '%s'
	ErrPrivNotGranted                = iota // User '%s' does not have %s access on %s '%s'
	ErrRoleNotGranted                = iota // User '%s' does not have role '%s'
	ErrMissingSnapshot               = iota // Can't find snapshot
	ErrCantUpdatePrimaryKey          = iota // Attempt to modify a tuple field which is part of index '%s' in space '%s'
	ErrUpdateIntegerOverflow         = iota // Integer overflow when performing '%c' operation on field %u
	ErrGuestUserPassword             = iota // Setting password for guest user has no effect
	ErrTransactionConflict           = iota // Transaction has been aborted by conflict
	ErrUnsupportedRolePriv           = iota // Unsupported role privilege '%s'
	ErrLoadFunction                  = iota // Failed to dynamically load function '%s': %s
	ErrFunctionLanguage              = iota // Unsupported language '%s' specified for function '%s'
	ErrRtreeRect                     = iota // RTree: %s must be an array with %u (point) or %u (rectangle/box) numeric coordinates
	ErrProcC                         = iota // ???
	ErrUnknownRtreeIndexDistanceType = iota //Unknown RTREE index distance type %s
	ErrProtocol                      = iota // %s
	ErrUpsertUniqueSecondaryKey      = iota // Space %s has a unique secondary index and does not support UPSERT
	ErrWrongIndexRecord              = iota // Wrong record in _index space: got {%s}, expected {%s}
	ErrWrongIndexParts               = iota // Wrong index parts (field %u): %s; expected field1 id (number), field1 type (string), ...
	ErrWrongIndexOptions             = iota // Wrong index options (field %u): %s
	ErrWrongSchemaVaersion           = iota // Wrong schema version, current: %d, in request: %u
	ErrSlabAllocMax                  = iota // Failed to allocate %u bytes for tuple in the slab allocator: tuple is too large. Check 'slab_alloc_maximal' configuration option.
)
<|endoftext|>


<commit_before>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora

const (
	// The driver name registered with the database/sql package.
	Name string = "ora"

	// The driver version sent to an Oracle server and visible in
	// V$SESSION_CONNECT_INFO or GV$SESSION_CONNECT_INFO.
	Version string = "v1.02"
)

// ColumnGoType defines the Go type returned from a sql select column.
type GoColumnType uint

// go column types
const (
	// D defines a sql select column based on its default mapping.
	D GoColumnType = iota
	// I64 defines a sql select column as a Go int64.
	I64
	// I32 defines a sql select column as a Go int32.
	I32
	// I16 defines a sql select column as a Go int16.
	I16
	// I8 defines a sql select column as a Go int8.
	I8
	// U64 defines a sql select column as a Go uint64.
	U64
	// U32 defines a sql select column as a Go uint32.
	U32
	// U16 defines a sql select column as a Go uint16.
	U16
	// U8 defines a sql select column as a Go uint8.
	U8
	// F64 defines a sql select column as a Go float64.
	F64
	// F32 defines a sql select column as a Go float32.
	F32
	// OraI64 defines a sql select column as a nullable Go ora.Int64.
	OraI64
	// OraI32 defines a sql select column as a nullable Go ora.Int32.
	OraI32
	// OraI16 defines a sql select column as a nullable Go ora.Int16.
	OraI16
	// OraI8 defines a sql select column as a nullable Go ora.Int8.
	OraI8
	// OraU64 defines a sql select column as a nullable Go ora.Uint64.
	OraU64
	// OraU32 defines a sql select column as a nullable Go ora.Uint32.
	OraU32
	// OraU16 defines a sql select column as a nullable Go ora.Uint16.
	OraU16
	// OraU8 defines a sql select column as a nullable Go ora.Uint8.
	OraU8
	// OraF64 defines a sql select column as a nullable Go ora.Float64.
	OraF64
	// OraF32 defines a sql select column as a nullable Go ora.Float32.
	OraF32
	// T defines a sql select column as a Go time.Time.
	T
	// OraT defines a sql select column as a nullable Go ora.Time.
	OraT
	// S defines a sql select column as a Go string.
	S
	// OraS defines a sql select column as a nullable Go ora.String.
	OraS
	// B defines a sql select column as a Go bool.
	B
	// OraB defines a sql select column as a nullable Go ora.Bool.
	OraB
	// Bin defines a sql select column or bind parmeter as a Go byte slice.
	Bin
	// OraBin defines a sql select column as a nullable Go ora.Binary.
	OraBin
)

// bind pool indexes
const (
	bndIdxInt64 int = iota
	bndIdxInt32
	bndIdxInt16
	bndIdxInt8
	bndIdxUint64
	bndIdxUint32
	bndIdxUint16
	bndIdxUint8
	bndIdxFloat64
	bndIdxFloat32

	bndIdxInt64Ptr
	bndIdxInt32Ptr
	bndIdxInt16Ptr
	bndIdxInt8Ptr
	bndIdxUint64Ptr
	bndIdxUint32Ptr
	bndIdxUint16Ptr
	bndIdxUint8Ptr
	bndIdxFloat64Ptr
	bndIdxFloat32Ptr

	bndIdxInt64Slice
	bndIdxInt32Slice
	bndIdxInt16Slice
	bndIdxInt8Slice
	bndIdxUint64Slice
	bndIdxUint32Slice
	bndIdxUint16Slice
	bndIdxUint8Slice
	bndIdxFloat64Slice
	bndIdxFloat32Slice

	bndIdxTime
	bndIdxTimePtr
	bndIdxTimeSlice

	bndIdxString
	bndIdxStringPtr
	bndIdxStringSlice

	bndIdxBool
	bndIdxBoolPtr
	bndIdxBoolSlice

	bndIdxBin
	bndIdxBinSlice
	bndIdxLob
	bndIdxLobPtr
	bndIdxLobSlice

	bndIdxIntervalYM
	bndIdxIntervalYMSlice
	bndIdxIntervalDS
	bndIdxIntervalDSSlice

	bndIdxBfile
	bndIdxRset
	bndIdxNil
)

// define pool indexes
const (
	defIdxInt64 int = iota
	defIdxInt32
	defIdxInt16
	defIdxInt8
	defIdxUint64
	defIdxUint32
	defIdxUint16
	defIdxUint8
	defIdxFloat64
	defIdxFloat32

	defIdxTime
	defIdxString
	defIdxBool

	defIdxLob
	defIdxRaw
	defIdxLongRaw

	defIdxIntervalYM
	defIdxIntervalDS
	defIdxBfile
	defIdxRowid
)
<commit_msg>removed const Name, Version. Moved to ora.go<commit_after>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora

// ColumnGoType defines the Go type returned from a sql select column.
type GoColumnType uint

// go column types
const (
	// D defines a sql select column based on its default mapping.
	D GoColumnType = iota
	// I64 defines a sql select column as a Go int64.
	I64
	// I32 defines a sql select column as a Go int32.
	I32
	// I16 defines a sql select column as a Go int16.
	I16
	// I8 defines a sql select column as a Go int8.
	I8
	// U64 defines a sql select column as a Go uint64.
	U64
	// U32 defines a sql select column as a Go uint32.
	U32
	// U16 defines a sql select column as a Go uint16.
	U16
	// U8 defines a sql select column as a Go uint8.
	U8
	// F64 defines a sql select column as a Go float64.
	F64
	// F32 defines a sql select column as a Go float32.
	F32
	// OraI64 defines a sql select column as a nullable Go ora.Int64.
	OraI64
	// OraI32 defines a sql select column as a nullable Go ora.Int32.
	OraI32
	// OraI16 defines a sql select column as a nullable Go ora.Int16.
	OraI16
	// OraI8 defines a sql select column as a nullable Go ora.Int8.
	OraI8
	// OraU64 defines a sql select column as a nullable Go ora.Uint64.
	OraU64
	// OraU32 defines a sql select column as a nullable Go ora.Uint32.
	OraU32
	// OraU16 defines a sql select column as a nullable Go ora.Uint16.
	OraU16
	// OraU8 defines a sql select column as a nullable Go ora.Uint8.
	OraU8
	// OraF64 defines a sql select column as a nullable Go ora.Float64.
	OraF64
	// OraF32 defines a sql select column as a nullable Go ora.Float32.
	OraF32
	// T defines a sql select column as a Go time.Time.
	T
	// OraT defines a sql select column as a nullable Go ora.Time.
	OraT
	// S defines a sql select column as a Go string.
	S
	// OraS defines a sql select column as a nullable Go ora.String.
	OraS
	// B defines a sql select column as a Go bool.
	B
	// OraB defines a sql select column as a nullable Go ora.Bool.
	OraB
	// Bin defines a sql select column or bind parmeter as a Go byte slice.
	Bin
	// OraBin defines a sql select column as a nullable Go ora.Binary.
	OraBin
)

// bind pool indexes
const (
	bndIdxInt64 int = iota
	bndIdxInt32
	bndIdxInt16
	bndIdxInt8
	bndIdxUint64
	bndIdxUint32
	bndIdxUint16
	bndIdxUint8
	bndIdxFloat64
	bndIdxFloat32

	bndIdxInt64Ptr
	bndIdxInt32Ptr
	bndIdxInt16Ptr
	bndIdxInt8Ptr
	bndIdxUint64Ptr
	bndIdxUint32Ptr
	bndIdxUint16Ptr
	bndIdxUint8Ptr
	bndIdxFloat64Ptr
	bndIdxFloat32Ptr

	bndIdxInt64Slice
	bndIdxInt32Slice
	bndIdxInt16Slice
	bndIdxInt8Slice
	bndIdxUint64Slice
	bndIdxUint32Slice
	bndIdxUint16Slice
	bndIdxUint8Slice
	bndIdxFloat64Slice
	bndIdxFloat32Slice

	bndIdxTime
	bndIdxTimePtr
	bndIdxTimeSlice

	bndIdxString
	bndIdxStringPtr
	bndIdxStringSlice

	bndIdxBool
	bndIdxBoolPtr
	bndIdxBoolSlice

	bndIdxBin
	bndIdxBinSlice
	bndIdxLob
	bndIdxLobPtr
	bndIdxLobSlice

	bndIdxIntervalYM
	bndIdxIntervalYMSlice
	bndIdxIntervalDS
	bndIdxIntervalDSSlice

	bndIdxBfile
	bndIdxRset
	bndIdxNil
)

// define pool indexes
const (
	defIdxInt64 int = iota
	defIdxInt32
	defIdxInt16
	defIdxInt8
	defIdxUint64
	defIdxUint32
	defIdxUint16
	defIdxUint8
	defIdxFloat64
	defIdxFloat32

	defIdxTime
	defIdxString
	defIdxBool

	defIdxLob
	defIdxRaw
	defIdxLongRaw

	defIdxIntervalYM
	defIdxIntervalDS
	defIdxBfile
	defIdxRowid
)
<|endoftext|>


<commit_before>package pin

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// PostsService is the service for accessing Post-related calls from the
// Pinboard API.
type PostsService struct {
	client *Client
}

// Post represents a post stored in Pinboard. Fields are transformed from the
// actual response to be a bit more sane. For example, description from the
// response is renamed to Title and the extended field is renamed to
// Description.
type Post struct {
	Title       string
	Description string
	Hash        string
	URL         string
	Tags        []string
	ToRead      bool
}

func newPostFromPostResp(presp *postResp) *Post {
	var toRead bool
	if presp.ToRead == "yes" {
		toRead = true
	}

	return &Post{
		Title:       presp.Title,
		Description: presp.Description,
		Hash:        presp.Hash,
		URL:         presp.URL,
		Tags:        strings.Split(presp.Tag, " "),
		ToRead:      toRead,
	}
}

type postResp struct {
	Title       string `xml:"description,attr"`
	Description string `xml:"extended,attr"`
	Hash        string `xml:"hash,attr"`
	URL         string `xml:"href,attr"`
	Tag         string `xml:"tag,attr"`
	ToRead      string `xml:"toread,attr"`
}

// Add creates a new Post for the authenticated account. urlStr and title are
// required.
//
// https://pinboard.in/api/#posts_add
func (s *PostsService) Add(urlStr, title, description string, tags []string,
	creationTime *time.Time, replace, shared,
	toread bool) (*http.Response, error) {
	var strTime string
	if creationTime != nil {
		strTime = creationTime.String()
	}

	params := &url.Values{
		"url":         {urlStr},
		"description": {title},
		"extended":    {description},
		"tags":        tags,
		"dt":          {strTime},
		"replace":     {fmt.Sprintf("%t", replace)},
		"shared":      {fmt.Sprintf("%t", shared)},
		"toread":      {fmt.Sprintf("%t", toread)},
	}

	req, err := s.client.NewRequest("posts/add", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Delete deletes the specified Post from the authenticated account where
// urlStr is the URL of the Post to delete.
//
// https://pinboard.in/api/#posts_delete
func (s *PostsService) Delete(urlStr string) (*http.Response, error) {
	params := &url.Values{"url": {urlStr}}
	req, err := s.client.NewRequest("posts/delete", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Get returns one or more posts on a single day matching the arguments.
// If no date or url is given, date of most recent bookmark will be used.
//
// https://pinboard.in/api#posts_get
func (s *PostsService) Get(tags []string, creationTime *time.Time, urlStr string) ([]*Post, *http.Response, error) {

	params := &url.Values{}

	if creationTime != nil {
		params.Add("dt", creationTime.String())
	}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil {
		params.Add("tags", strings.Join(tags, " "))
	}

	if len(urlStr) > 0 {
		params.Add("url", urlStr)
	}

	req, err := s.client.NewRequest("posts/get", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// TODO
//
// https://pinboard.in/api#posts_update
func (s *PostsService) LastTimeUpdated() {
}

// TODO
//
// https://pinboard.in/api#posts_dates
func (s *PostsService) Dates() {
}

// Recent fetches the most recent Posts for the authenticated account, filtered
// by tag. Up to 3 tags can be specified to filter by. The max count is 100. If
// a negative count is passed, then the default number of posts (15) is
// returned.
//
// https://pinboard.in/api/#posts_recent
func (s *PostsService) Recent(tags []string, count int) ([]*Post,
	*http.Response, error) {
	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	}
	if count > 100 {
		return nil, nil, errors.New("count must be below 100")
	}
	if count < 0 {
		count = 15
	}

	req, err := s.client.NewRequest("posts/recent", &url.Values{
		"tag":   tags,
		"count": {strconv.Itoa(count)},
	})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// TODO
//
// https://pinboard.in/api#posts_all
func (s *PostsService) All() {
}

// TODO
//
// https://pinboard.in/api#posts_suggest
func (s *PostsService) Suggest() {
}
<commit_msg>Add the time field within the posts returned<commit_after>package pin

import (
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// PostsService is the service for accessing Post-related calls from the
// Pinboard API.
type PostsService struct {
	client *Client
}

// Post represents a post stored in Pinboard. Fields are transformed from the
// actual response to be a bit more sane. For example, description from the
// response is renamed to Title and the extended field is renamed to
// Description.
type Post struct {
	Title       string
	Description string
	Hash        string
	URL         string
	Tags        []string
	ToRead      bool
	Time        *time.Time
}

func newPostFromPostResp(presp *postResp) *Post {
	var toRead bool
	if presp.ToRead == "yes" {
		toRead = true
	}

	dt, _ := time.Parse("2006-01-02T15:04:05Z", presp.Time)

	return &Post{
		Title:       presp.Title,
		Description: presp.Description,
		Hash:        presp.Hash,
		URL:         presp.URL,
		Tags:        strings.Split(presp.Tag, " "),
		ToRead:      toRead,
		Time:        &dt,
	}
}

type postResp struct {
	Title       string `xml:"description,attr"`
	Description string `xml:"extended,attr"`
	Hash        string `xml:"hash,attr"`
	URL         string `xml:"href,attr"`
	Tag         string `xml:"tag,attr"`
	ToRead      string `xml:"toread,attr"`
	Time        string `xml:"time,attr"`
}

// Add creates a new Post for the authenticated account. urlStr and title are
// required.
//
// https://pinboard.in/api/#posts_add
func (s *PostsService) Add(urlStr, title, description string, tags []string,
	creationTime *time.Time, replace, shared,
	toread bool) (*http.Response, error) {
	var strTime string
	if creationTime != nil {
		strTime = creationTime.String()
	}

	params := &url.Values{
		"url":         {urlStr},
		"description": {title},
		"extended":    {description},
		"tags":        tags,
		"dt":          {strTime},
		"replace":     {fmt.Sprintf("%t", replace)},
		"shared":      {fmt.Sprintf("%t", shared)},
		"toread":      {fmt.Sprintf("%t", toread)},
	}

	req, err := s.client.NewRequest("posts/add", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Delete deletes the specified Post from the authenticated account where
// urlStr is the URL of the Post to delete.
//
// https://pinboard.in/api/#posts_delete
func (s *PostsService) Delete(urlStr string) (*http.Response, error) {
	params := &url.Values{"url": {urlStr}}
	req, err := s.client.NewRequest("posts/delete", params)
	if err != nil {
		return nil, err
	}

	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}

	return resp, nil
}

// Get returns one or more posts on a single day matching the arguments.
// If no date or url is given, date of most recent bookmark will be used.
//
// https://pinboard.in/api#posts_get
func (s *PostsService) Get(tags []string, creationTime *time.Time, urlStr string) ([]*Post, *http.Response, error) {

	params := &url.Values{}

	if creationTime != nil {
		params.Add("dt", creationTime.String())
	}

	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	} else if tags != nil {
		params.Add("tags", strings.Join(tags, " "))
	}

	if len(urlStr) > 0 {
		params.Add("url", urlStr)
	}

	req, err := s.client.NewRequest("posts/get", params)
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// TODO
//
// https://pinboard.in/api#posts_update
func (s *PostsService) LastTimeUpdated() {
}

// TODO
//
// https://pinboard.in/api#posts_dates
func (s *PostsService) Dates() {
}

// Recent fetches the most recent Posts for the authenticated account, filtered
// by tag. Up to 3 tags can be specified to filter by. The max count is 100. If
// a negative count is passed, then the default number of posts (15) is
// returned.
//
// https://pinboard.in/api/#posts_recent
func (s *PostsService) Recent(tags []string, count int) ([]*Post,
	*http.Response, error) {
	if tags != nil && len(tags) > 3 {
		return nil, nil, errors.New("too many tags (max is 3)")
	}
	if count > 100 {
		return nil, nil, errors.New("count must be below 100")
	}
	if count < 0 {
		count = 15
	}

	req, err := s.client.NewRequest("posts/recent", &url.Values{
		"tag":   tags,
		"count": {strconv.Itoa(count)},
	})
	if err != nil {
		return nil, nil, err
	}

	var result struct {
		Posts []*postResp `xml:"post"`
	}

	resp, err := s.client.Do(req, &result)
	if err != nil {
		return nil, resp, err
	}

	posts := make([]*Post, len(result.Posts))
	for i, v := range result.Posts {
		posts[i] = newPostFromPostResp(v)
	}

	return posts, resp, nil
}

// TODO
//
// https://pinboard.in/api#posts_all
func (s *PostsService) All() {
}

// TODO
//
// https://pinboard.in/api#posts_suggest
func (s *PostsService) Suggest() {
}
<|endoftext|>


<commit_before>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

var (
	mu sync.Mutex // Protect access to dup

	dup = map[string]bool{} // Duplicates table

	ingestionSet []Document // ingestion data TODO make non global

	badLinks []string // bad links TODO make non global
)

// Crawl function that will take a url string and start firing out some crawling functions
// it will return true/false based on the url root it starts with.
func Crawl(settings Settings, url string, u url.URL) ([]Document, bool) {
	// Create the muxer
	mux := fetchbot.NewMux()

	// Handle all errors the same
	mux.HandleErrors(fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	}))

	// Handle GET requests for html responses, to parse the body and enqueue all links as HEAD
	// requests.
	mux.Response().Method("GET").ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			// Process the body to find the links
			doc, err := goquery.NewDocumentFromResponse(res)
			if err != nil {
				// find the bad links in the documents
				badLinks = append(badLinks, ctx.Cmd.URL().String())
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
				return
			}
			// Enqueue all links as HEAD requests
			enqueueLinks(ctx, doc)
		}))

	// Handle HEAD requests for html responses coming from the source host - we don't want
	// to crawl links from other hosts.
	mux.Response().Method("HEAD").Host(u.Host).ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			if _, err := ctx.Q.SendStringGet(ctx.Cmd.URL().String()); err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			}
		}))

	// Create the Fetcher, handle the logging first, then dispatch to the Muxer
	h := scrapeHandler(mux)
	if settings.StopAtURL != "" || settings.CancelAtURL != "" {
		stopURL := settings.StopAtURL
		if settings.CancelAtURL != "" {
			stopURL = settings.CancelAtURL
		}
		h = stopHandler(stopURL, settings.CancelAtURL != "", scrapeHandler(mux))
	}
	f := fetchbot.New(h)

	// set the fetchbots settings from flag parameters
	f.UserAgent = settings.UserAgent
	f.CrawlDelay = settings.CrawlDelay * time.Second
	f.WorkerIdleTTL = settings.WorkerIdleTTL * time.Second

	// First mem stat print must be right after creating the fetchbot
	if settings.MemStatsInterval > 0 {
		// Print starting stats
		printMemStats(nil)
		// Run at regular intervals
		runMemStats(f, settings.MemStatsInterval)
		// On exit, print ending stats after a GC
		defer func() {
			runtime.GC()
			printMemStats(nil)
		}()
	}

	// Start processing
	q := f.Start()

	// if a stop or cancel is requested after some duration, launch the goroutine
	// that will stop or cancel.
	if settings.StopDuration > 0 || settings.CancelDuration > 0 {
		after := settings.StopDuration
		stopFunc := q.Close
		if settings.CancelDuration != 0 {
			after = settings.CancelDuration
			stopFunc = q.Cancel
		}

		go func() {
			c := time.After(after)
			<-c
			stopFunc()
		}()
	}

	// Enqueue the seed, which is the first entry in the dup map
	dup[url] = true
	_, err := q.SendStringGet(url)
	if err != nil {
		fmt.Printf("[ERR] GET %s - %s\n", url, err)
	}
	q.Block()

	return ingestionSet, true
}

// stopHandler stops the fetcher if the stopurl is reached. Otherwise it dispatches
// the call to the wrapped Handler.
func stopHandler(stopurl string, cancel bool, wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if ctx.Cmd.URL().String() == stopurl {
			fmt.Printf(">>>>> STOP URL %s\n", ctx.Cmd.URL())
			// generally not a good idea to stop/block from a handler goroutine
			// so do it in a separate goroutine
			go func() {
				if cancel {
					ctx.Q.Cancel()
				} else {
					ctx.Q.Close()
				}
			}()
			return
		}
		wrapped.Handle(ctx, res, err)
	})
}

// scrapeHandler will fire a scraper function on the page if successful response,
// append the scraped document stored for index ingestion
// and dispatches the call to the wrapped Handler.
func scrapeHandler(wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			if res.StatusCode == 200 {
				url := ctx.Cmd.URL().String()
				response := Scrape(url)
				// TODO: store/log the bad sites with null fields
				if response.Content != "" && response.Title != "" && response.Link != "" {
					ingestionSet = append(ingestionSet, response)
					fmt.Println("Total Pages Scraped Successfully: ", len(ingestionSet))
				}
			} else {
				fmt.Println("scrapeHandler bad links +1 ========== status code: ", res.StatusCode)
				badLinks = append(badLinks, ctx.Cmd.URL().String())
			}
			// fmt.Printf("[%d] %s %s - %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
		wrapped.Handle(ctx, res, err)
	})
}

// enqueueLinks will make sure we are adding links to the queue to be processed for crawling/scraping
// this will pull all the href attributes on pages, check for duplicates and add them to the queue
func enqueueLinks(ctx *fetchbot.Context, doc *goquery.Document) {
	mu.Lock()
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, _ := s.Attr("href")
		// Resolve address
		u, err := ctx.Cmd.URL().Parse(val)
		if err != nil {
			fmt.Printf("error: resolve URL %s - %s\n", val, err)
			return
		}
		// catch the duplicate urls here before trying to add them to the queue
		if !dup[u.String()] {
			if _, err := ctx.Q.SendStringHead(u.String()); err != nil {
				fmt.Printf("error: enqueue head %s - %s\n", u, err)
			} else {
				dup[u.String()] = true
			}
		}
	})
	mu.Unlock()
}
<commit_msg>changed the stop/cancel durations to minutes<commit_after>package hermes

import (
	"fmt"
	"net/http"
	"net/url"
	"runtime"
	"sync"
	"time"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

var (
	mu sync.Mutex // Protect access to dup

	dup = map[string]bool{} // Duplicates table

	ingestionSet []Document // ingestion data TODO make non global

	badLinks []string // bad links TODO make non global
)

// Crawl function that will take a url string and start firing out some crawling functions
// it will return true/false based on the url root it starts with.
func Crawl(settings Settings, url string, u url.URL) ([]Document, bool) {
	// Create the muxer
	mux := fetchbot.NewMux()

	// Handle all errors the same
	mux.HandleErrors(fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	}))

	// Handle GET requests for html responses, to parse the body and enqueue all links as HEAD
	// requests.
	mux.Response().Method("GET").ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			// Process the body to find the links
			doc, err := goquery.NewDocumentFromResponse(res)
			if err != nil {
				// find the bad links in the documents
				badLinks = append(badLinks, ctx.Cmd.URL().String())
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
				return
			}
			// Enqueue all links as HEAD requests
			enqueueLinks(ctx, doc)
		}))

	// Handle HEAD requests for html responses coming from the source host - we don't want
	// to crawl links from other hosts.
	mux.Response().Method("HEAD").Host(u.Host).ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			if _, err := ctx.Q.SendStringGet(ctx.Cmd.URL().String()); err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			}
		}))

	// Create the Fetcher, handle the logging first, then dispatch to the Muxer
	h := scrapeHandler(mux)
	if settings.StopAtURL != "" || settings.CancelAtURL != "" {
		stopURL := settings.StopAtURL
		if settings.CancelAtURL != "" {
			stopURL = settings.CancelAtURL
		}
		h = stopHandler(stopURL, settings.CancelAtURL != "", scrapeHandler(mux))
	}
	f := fetchbot.New(h)

	// set the fetchbots settings from flag parameters
	f.UserAgent = settings.UserAgent
	f.CrawlDelay = settings.CrawlDelay * time.Second
	f.WorkerIdleTTL = settings.WorkerIdleTTL * time.Second

	// First mem stat print must be right after creating the fetchbot
	if settings.MemStatsInterval > 0 {
		// Print starting stats
		printMemStats(nil)
		// Run at regular intervals
		runMemStats(f, settings.MemStatsInterval)
		// On exit, print ending stats after a GC
		defer func() {
			runtime.GC()
			printMemStats(nil)
		}()
	}

	// Start processing
	q := f.Start()

	// if a stop or cancel is requested after some duration, launch the goroutine
	// that will stop or cancel.
	if settings.StopDuration*time.Minute > 0 || settings.CancelDuration*time.Minute > 0 {
		after := settings.StopDuration * time.Minute
		stopFunc := q.Close
		if settings.CancelDuration != 0 {
			after = settings.CancelDuration * time.Minute
			stopFunc = q.Cancel
		}

		go func() {
			c := time.After(after)
			<-c
			stopFunc()
		}()
	}

	// Enqueue the seed, which is the first entry in the dup map
	dup[url] = true
	_, err := q.SendStringGet(url)
	if err != nil {
		fmt.Printf("[ERR] GET %s - %s\n", url, err)
	}
	q.Block()

	return ingestionSet, true
}

// stopHandler stops the fetcher if the stopurl is reached. Otherwise it dispatches
// the call to the wrapped Handler.
func stopHandler(stopurl string, cancel bool, wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if ctx.Cmd.URL().String() == stopurl {
			fmt.Printf(">>>>> STOP URL %s\n", ctx.Cmd.URL())
			// generally not a good idea to stop/block from a handler goroutine
			// so do it in a separate goroutine
			go func() {
				if cancel {
					ctx.Q.Cancel()
				} else {
					ctx.Q.Close()
				}
			}()
			return
		}
		wrapped.Handle(ctx, res, err)
	})
}

// scrapeHandler will fire a scraper function on the page if successful response,
// append the scraped document stored for index ingestion
// and dispatches the call to the wrapped Handler.
func scrapeHandler(wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			if res.StatusCode == 200 {
				url := ctx.Cmd.URL().String()
				response := Scrape(url)
				// TODO: store/log the bad sites with null fields
				if response.Content != "" && response.Title != "" && response.Link != "" {
					ingestionSet = append(ingestionSet, response)
					fmt.Println("Total Pages Scraped Successfully: ", len(ingestionSet))
				}
			} else {
				fmt.Println("scrapeHandler bad links +1 ========== status code: ", res.StatusCode)
				badLinks = append(badLinks, ctx.Cmd.URL().String())
			}
			// fmt.Printf("[%d] %s %s - %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
		wrapped.Handle(ctx, res, err)
	})
}

// enqueueLinks will make sure we are adding links to the queue to be processed for crawling/scraping
// this will pull all the href attributes on pages, check for duplicates and add them to the queue
func enqueueLinks(ctx *fetchbot.Context, doc *goquery.Document) {
	mu.Lock()
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, _ := s.Attr("href")
		// Resolve address
		u, err := ctx.Cmd.URL().Parse(val)
		if err != nil {
			fmt.Printf("error: resolve URL %s - %s\n", val, err)
			return
		}
		// catch the duplicate urls here before trying to add them to the queue
		if !dup[u.String()] {
			if _, err := ctx.Q.SendStringHead(u.String()); err != nil {
				fmt.Printf("error: enqueue head %s - %s\n", u, err)
			} else {
				dup[u.String()] = true
			}
		}
	})
	mu.Unlock()
}
<|endoftext|>