<commit_before>package httpfile

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"

	"github.com/go-errors/errors"
	"github.com/itchio/httpkit/retrycontext"
	uuid "github.com/satori/go.uuid"
)

// A GetURLFunc returns a URL we can download the resource from.
// It's handy to have this as a function rather than a constant for signed expiring URLs
type GetURLFunc func() (urlString string, err error)

// A NeedsRenewalFunc analyzes an HTTP response and returns true if it needs to be renewed
type NeedsRenewalFunc func(res *http.Response, body []byte) bool

// A LogFunc prints debug message
type LogFunc func(msg string)

// amount we're willing to download and throw away
const maxDiscard int64 = 1 * 1024 * 1024 // 1MB

var ErrNotFound = errors.New("HTTP file not found on server")

type HTTPFile struct {
	getURL        GetURLFunc
	needsRenewal  NeedsRenewalFunc
	client        *http.Client
	retrySettings *retrycontext.Settings

	Log LogFunc

	name   string
	size   int64
	offset int64 // for io.ReadSeeker

	ReaderStaleThreshold time.Duration

	closed bool

	readers      map[string]*httpReader
	readersMutex sync.Mutex

	currentURL string
	urlMutex   sync.Mutex
}

type httpReader struct {
	file      *HTTPFile
	id        string
	touchedAt time.Time
	offset    int64
	body      io.ReadCloser
	reader    *bufio.Reader
}

const DefaultReaderStaleThreshold = time.Second * time.Duration(10)

func (hr *httpReader) Stale() bool {
	return time.Since(hr.touchedAt) > hr.file.ReaderStaleThreshold
}

func (hr *httpReader) Read(data []byte) (int, error) {
	hr.touchedAt = time.Now()
	readBytes, err := hr.reader.Read(data)
	hr.offset += int64(readBytes)

	if err != nil {
		return readBytes, err
	}
	return readBytes, nil
}

func (hr *httpReader) Discard(n int) (int, error) {
	hr.touchedAt = time.Now()
	discarded, err := hr.reader.Discard(n)
	hr.offset += int64(discarded)

	if err != nil {
		return discarded, err
	}
	return discarded, nil
}

func (hr *httpReader) Connect() error {
	if hr.body != nil {
		err := hr.body.Close()
		if err != nil {
			return err
		}

		hr.body = nil
		hr.reader = nil
	}

	tryUrl := func(urlStr string) (bool, error) {
		req, err := http.NewRequest("GET", urlStr, nil)
		if err != nil {
			return false, err
		}

		byteRange := fmt.Sprintf("bytes=%d-", hr.offset)
		req.Header.Set("Range", byteRange)

		res, err := hr.file.client.Do(req)
		if err != nil {
			return false, err
		}
		hr.file.log("did request, status %d", res.StatusCode)

		if res.StatusCode == 200 && hr.offset > 0 {
			defer res.Body.Close()

			err = fmt.Errorf("HTTP Range header not supported by %s, bailing out", req.Host)
			return false, err
		}

		if res.StatusCode/100 != 2 {
			defer res.Body.Close()

			body, err := ioutil.ReadAll(res.Body)
			if err != nil {
				body = []byte("could not read error body")
				err = nil
			}

			if hr.file.needsRenewal(res, body) {
				return true, nil
			}

			err = fmt.Errorf("HTTP %d returned by %s (%s), bailing out", res.StatusCode, req.Host, string(body))
			return false, err
		}

		hr.reader = bufio.NewReaderSize(res.Body, int(maxDiscard))
		hr.body = res.Body
		return false, nil
	}

	urlStr := hr.file.getCurrentURL()
	shouldRenew, err := tryUrl(urlStr)
	if err != nil {
		return err
	}

	if shouldRenew {
		urlStr, err = hr.file.renewURL()
		if err != nil {
			return err
		}

		shouldRenew, err = tryUrl(urlStr)
		if err != nil {
			return err
		}

		if shouldRenew {
			return fmt.Errorf("getting expired URLs from URL source (timezone issue?)")
		}
	}

	return nil
}

func (hr *httpReader) Close() error {
	err := hr.body.Close()

	if err != nil {
		return err
	}
	return nil
}

var _ io.Seeker = (*HTTPFile)(nil)
var _ io.Reader = (*HTTPFile)(nil)
var _ io.ReaderAt = (*HTTPFile)(nil)
var _ io.Closer = (*HTTPFile)(nil)

type Settings struct {
	Client        *http.Client
	RetrySettings *retrycontext.Settings
}

func New(getURL GetURLFunc, needsRenewal NeedsRenewalFunc, settings *Settings) (*HTTPFile, error) {
	client := settings.Client
	if client == nil {
		client = http.DefaultClient
	}

	retryCtx := retrycontext.NewDefault()
	if settings.RetrySettings != nil {
		retryCtx.Settings = *settings.RetrySettings
	}

	for retryCtx.ShouldTry() {
		urlStr, err := getURL()
		if err != nil {
			// this assumes getURL does its own retrying
			return nil, err
		}

		parsedUrl, err := url.Parse(urlStr)
		if err != nil {
			// can't recover from a bad url
			return nil, err
		}

		req, err := http.NewRequest("HEAD", urlStr, nil)
		if err != nil {
			// internal error
			return nil, err
		}

		res, err := client.Do(req)
		if err != nil {
			// we can recover from some client errors
			// (example: temporarily offline, DNS failure, etc.)
			retryCtx.Retry(err.Error())
			continue
		}

		if res.StatusCode != 200 {
			if res.StatusCode == 404 {
				// no need to retry - it's not coming back
				return nil, errors.Wrap(ErrNotFound, 1)
			}

			body, _ := ioutil.ReadAll(res.Body)
			if needsRenewal(res, body) {
				retryCtx.Retry(fmt.Sprintf("HTTP %d (needs renewal)", res.StatusCode))
				continue
			}

			if res.StatusCode == 429 || res.StatusCode/100 == 5 {
				retryCtx.Retry(fmt.Sprintf("HTTP %d (retrying)", res.StatusCode))
				continue
			}

			return nil, fmt.Errorf("Expected HTTP 200, got HTTP %d, not retrying", res.StatusCode)
		}

		hf := &HTTPFile{
			currentURL:    urlStr,
			getURL:        getURL,
			retrySettings: &retryCtx.Settings,
			needsRenewal:  needsRenewal,
			client:        client,

			name:    parsedUrl.Path,
			size:    res.ContentLength,
			readers: make(map[string]*httpReader),

			ReaderStaleThreshold: DefaultReaderStaleThreshold,
		}
		return hf, nil
	}

	return nil, fmt.Errorf("Could not access remote file. Last error: %s", retryCtx.LastMessage)
}

func (hf *HTTPFile) NumReaders() int {
	return len(hf.readers)
}

func (hf *HTTPFile) borrowReader(offset int64) (*httpReader, error) {
	hf.readersMutex.Lock()
	defer hf.readersMutex.Unlock()

	var bestReader string
	var bestDiff int64 = math.MaxInt64

	for _, reader := range hf.readers {
		if reader.Stale() {
			delete(hf.readers, reader.id)

			err := reader.Close()
			if err != nil {
				return nil, err
			}
			continue
		}

		diff := offset - reader.offset
		if diff >= 0 && diff < maxDiscard {
			if diff < bestDiff {
				bestReader = reader.id
				bestDiff = diff
			}
		}
	}

	if bestReader != "" {
		// re-use!
		reader := hf.readers[bestReader]
		delete(hf.readers, bestReader)

		// discard if needed
		if bestDiff > 0 {
			hf.log("borrow: for %d, re-using %d by discarding %d bytes", offset, reader.offset, bestDiff)

			// XXX: not int64-clean
			_, err := reader.Discard(int(bestDiff))
			if err != nil {
				if shouldRetry(err) {
					hf.log("borrow: for %d, discard failed because of retriable error, reconnecting", offset)
					reader.offset = offset
					err = reader.Connect()
					if err != nil {
						return nil, err
					}
				} else {
					return nil, err
				}
			}
		}

		return reader, nil
	}

	// provision a new reader
	hf.log("borrow: making fresh for offset %d", offset)

	reader := &httpReader{
		file:      hf,
		id:        uuid.NewV4().String(),
		touchedAt: time.Now(),
		offset:    offset,
	}

	err := reader.Connect()
	if err != nil {
		return nil, err
	}

	return reader, nil
}

func (hf *HTTPFile) returnReader(reader *httpReader) {
	hf.readersMutex.Lock()
	defer hf.readersMutex.Unlock()

	// TODO: enforce max idle readers ?

	reader.touchedAt = time.Now()
	hf.readers[reader.id] = reader
}

func (hf *HTTPFile) getCurrentURL() string {
	hf.urlMutex.Lock()
	defer hf.urlMutex.Unlock()

	return hf.currentURL
}

func (hf *HTTPFile) renewURL() (string, error) {
	hf.urlMutex.Lock()
	defer hf.urlMutex.Unlock()

	urlStr, err := hf.getURL()
	if err != nil {
		return "", err
	}

	hf.currentURL = urlStr
	return hf.currentURL, nil
}

func (hf *HTTPFile) Stat() (os.FileInfo, error) {
	return &httpFileInfo{hf}, nil
}

func (hf *HTTPFile) Seek(offset int64, whence int) (int64, error) {
	var newOffset int64

	switch whence {
	case os.SEEK_SET:
		newOffset = offset
	case os.SEEK_END:
		newOffset = hf.size + offset
	case os.SEEK_CUR:
		newOffset = hf.offset + offset
	default:
		return hf.offset, fmt.Errorf("invalid whence value %d", whence)
	}

	if newOffset < 0 {
		newOffset = 0
	}

	if newOffset > hf.size {
		newOffset = hf.size
	}

	hf.offset = newOffset
	return hf.offset, nil
}

func (hf *HTTPFile) Read(data []byte) (int, error) {
	hf.log("Read(%d)", len(data))
	bytesRead, err := hf.readAt(data, hf.offset)
	hf.offset += int64(bytesRead)
	return bytesRead, err
}

func (hf *HTTPFile) ReadAt(data []byte, offset int64) (int, error) {
	hf.log("ReadAt(%d, %d)", len(data), offset)
	return hf.readAt(data, offset)
}

func (hf *HTTPFile) readAt(data []byte, offset int64) (int, error) {
	reader, err := hf.borrowReader(offset)
	if err != nil {
		return 0, err
	}

	defer hf.returnReader(reader)

	totalBytesRead := 0
	bytesToRead := len(data)

	for totalBytesRead < bytesToRead {
		bytesRead, err := reader.Read(data[totalBytesRead:])
		totalBytesRead += bytesRead

		if err != nil {
			if shouldRetry(err) {
				hf.log("Got %s, retrying", err.Error())
				err = reader.Connect()
				if err != nil {
					return totalBytesRead, err
				}
			} else {
				return totalBytesRead, err
			}
		}
	}

	return totalBytesRead, nil
}

func shouldRetry(err error) bool {
	if errors.Is(err, io.ErrUnexpectedEOF) {
		return true
	} else if opError, ok := err.(*net.OpError); ok {
		if opError.Timeout() || opError.Temporary() {
			return true
		} else {
			return false
		}
	} else {
		return false
	}
}

func (hf *HTTPFile) closeAllReaders() error {
	hf.readersMutex.Lock()
	defer hf.readersMutex.Unlock()

	for id, reader := range hf.readers {
		err := reader.Close()
		if err != nil {
			return err
		}

		delete(hf.readers, id)
	}

	return nil
}

func (hf *HTTPFile) Close() error {
	if hf.closed {
		return nil
	}

	err := hf.closeAllReaders()
	if err != nil {
		return err
	}

	hf.closed = true

	return nil
}

func (hf *HTTPFile) log(format string, args ...interface{}) {
	if hf.Log == nil {
		return
	}

	hf.Log(fmt.Sprintf(format, args...))
}
<commit_msg>Retry on EOF in url.Error (can happen when a Dial timeout occurs)<commit_after>package httpfile

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"net/url"
	"os"
	"sync"
	"time"

	"github.com/go-errors/errors"
	"github.com/itchio/httpkit/retrycontext"
	uuid "github.com/satori/go.uuid"
)

// A GetURLFunc returns a URL we can download the resource from.
// It's handy to have this as a function rather than a constant for signed expiring URLs
type GetURLFunc func() (urlString string, err error)

// A NeedsRenewalFunc analyzes an HTTP response and returns true if it needs to be renewed
type NeedsRenewalFunc func(res *http.Response, body []byte) bool

// A LogFunc prints debug message
type LogFunc func(msg string)

// amount we're willing to download and throw away
const maxDiscard int64 = 1 * 1024 * 1024 // 1MB

// ErrNotFound is returned when the HTTP server returns 404 - it's not considered a temporary error
var ErrNotFound = errors.New("HTTP file not found on server")

// HTTPFile allows accessing a file served by an HTTP server as if it was local
// (for random-access reading purposes, not writing)
type HTTPFile struct {
	getURL        GetURLFunc
	needsRenewal  NeedsRenewalFunc
	client        *http.Client
	retrySettings *retrycontext.Settings

	Log LogFunc

	name   string
	size   int64
	offset int64 // for io.ReadSeeker

	ReaderStaleThreshold time.Duration

	closed bool

	readers      map[string]*httpReader
	readersMutex sync.Mutex

	currentURL string
	urlMutex   sync.Mutex
}

type httpReader struct {
	file      *HTTPFile
	id        string
	touchedAt time.Time
	offset    int64
	body      io.ReadCloser
	reader    *bufio.Reader
}

// DefaultReaderStaleThreshold is the duration after which HTTPFile's readers
// are considered stale, and are closed instead of reused. It's set to 10 seconds.
const DefaultReaderStaleThreshold = time.Second * time.Duration(10)

func (hr *httpReader) Stale() bool {
	return time.Since(hr.touchedAt) > hr.file.ReaderStaleThreshold
}

func (hr *httpReader) Read(data []byte) (int, error) {
	hr.touchedAt = time.Now()
	readBytes, err := hr.reader.Read(data)
	hr.offset += int64(readBytes)

	if err != nil {
		return readBytes, err
	}
	return readBytes, nil
}

func (hr *httpReader) Discard(n int) (int, error) {
	hr.touchedAt = time.Now()
	discarded, err := hr.reader.Discard(n)
	hr.offset += int64(discarded)

	if err != nil {
		return discarded, err
	}
	return discarded, nil
}

func (hr *httpReader) Connect() error {
	if hr.body != nil {
		err := hr.body.Close()
		if err != nil {
			return err
		}

		hr.body = nil
		hr.reader = nil
	}

	tryURL := func(urlStr string) (bool, error) {
		req, err := http.NewRequest("GET", urlStr, nil)
		if err != nil {
			return false, err
		}

		byteRange := fmt.Sprintf("bytes=%d-", hr.offset)
		req.Header.Set("Range", byteRange)

		res, err := hr.file.client.Do(req)
		if err != nil {
			if shouldRetry(err) {
				hr.file.log("Connect: got Do() error %s, retrying", err.Error())
				return true, nil
			}
			return false, err
		}
		hr.file.log("Connect: HTTP %d", res.StatusCode)

		if res.StatusCode == 200 && hr.offset > 0 {
			defer res.Body.Close()

			err = fmt.Errorf("HTTP Range header not supported by %s, bailing out", req.Host)
			return false, err
		}

		if res.StatusCode/100 != 2 {
			defer res.Body.Close()

			body, err := ioutil.ReadAll(res.Body)
			if err != nil {
				body = []byte("could not read error body")
				err = nil
			}

			if hr.file.needsRenewal(res, body) {
				return true, nil
			}

			err = fmt.Errorf("HTTP %d returned by %s (%s), bailing out", res.StatusCode, req.Host, string(body))
			return false, err
		}

		hr.reader = bufio.NewReaderSize(res.Body, int(maxDiscard))
		hr.body = res.Body
		return false, nil
	}

	urlStr := hr.file.getCurrentURL()
	shouldRenew, err := tryURL(urlStr)
	if err != nil {
		return err
	}

	if shouldRenew {
		urlStr, err = hr.file.renewURL()
		if err != nil {
			return err
		}

		shouldRenew, err = tryURL(urlStr)
		if err != nil {
			return err
		}

		if shouldRenew {
			return fmt.Errorf("getting expired URLs from URL source (timezone issue?)")
		}
	}

	return nil
}

func (hr *httpReader) Close() error {
	err := hr.body.Close()

	if err != nil {
		return err
	}
	return nil
}

var _ io.Seeker = (*HTTPFile)(nil)
var _ io.Reader = (*HTTPFile)(nil)
var _ io.ReaderAt = (*HTTPFile)(nil)
var _ io.Closer = (*HTTPFile)(nil)

// Settings allows passing additional settings to an HTTPFile
type Settings struct {
	Client        *http.Client
	RetrySettings *retrycontext.Settings
}

// New returns a new HTTPFile. Note that it differs from os.Open in that it does a first request
// to determine the remote file's size. If that fails (after retries), an error will be returned.
func New(getURL GetURLFunc, needsRenewal NeedsRenewalFunc, settings *Settings) (*HTTPFile, error) {
	client := settings.Client
	if client == nil {
		client = http.DefaultClient
	}

	retryCtx := retrycontext.NewDefault()
	if settings.RetrySettings != nil {
		retryCtx.Settings = *settings.RetrySettings
	}

	for retryCtx.ShouldTry() {
		urlStr, err := getURL()
		if err != nil {
			// this assumes getURL does its own retrying
			return nil, err
		}

		parsedURL, err := url.Parse(urlStr)
		if err != nil {
			// can't recover from a bad url
			return nil, err
		}

		req, err := http.NewRequest("HEAD", urlStr, nil)
		if err != nil {
			// internal error
			return nil, err
		}

		res, err := client.Do(req)
		if err != nil {
			// we can recover from some client errors
			// (example: temporarily offline, DNS failure, etc.)
			retryCtx.Retry(err.Error())
			continue
		}

		if res.StatusCode != 200 {
			if res.StatusCode == 404 {
				// no need to retry - it's not coming back
				return nil, errors.Wrap(ErrNotFound, 1)
			}

			body, _ := ioutil.ReadAll(res.Body)
			if needsRenewal(res, body) {
				retryCtx.Retry(fmt.Sprintf("HTTP %d (needs renewal)", res.StatusCode))
				continue
			}

			if res.StatusCode == 429 || res.StatusCode/100 == 5 {
				retryCtx.Retry(fmt.Sprintf("HTTP %d (retrying)", res.StatusCode))
				continue
			}

			return nil, fmt.Errorf("Expected HTTP 200, got HTTP %d, not retrying", res.StatusCode)
		}

		hf := &HTTPFile{
			currentURL:    urlStr,
			getURL:        getURL,
			retrySettings: &retryCtx.Settings,
			needsRenewal:  needsRenewal,
			client:        client,

			name:    parsedURL.Path,
			size:    res.ContentLength,
			readers: make(map[string]*httpReader),

			ReaderStaleThreshold: DefaultReaderStaleThreshold,
		}
		return hf, nil
	}

	return nil, fmt.Errorf("Could not access remote file. Last error: %s", retryCtx.LastMessage)
}

// NumReaders returns the number of connections currently used by the httpfile
// to serve ReadAt calls
func (hf *HTTPFile) NumReaders() int {
	return len(hf.readers)
}

func (hf *HTTPFile) borrowReader(offset int64) (*httpReader, error) {
	hf.readersMutex.Lock()
	defer hf.readersMutex.Unlock()

	var bestReader string
	var bestDiff int64 = math.MaxInt64

	for _, reader := range hf.readers {
		if reader.Stale() {
			delete(hf.readers, reader.id)

			err := reader.Close()
			if err != nil {
				return nil, err
			}
			continue
		}

		diff := offset - reader.offset
		if diff >= 0 && diff < maxDiscard {
			if diff < bestDiff {
				bestReader = reader.id
				bestDiff = diff
			}
		}
	}

	if bestReader != "" {
		// re-use!
		reader := hf.readers[bestReader]
		delete(hf.readers, bestReader)

		// discard if needed
		if bestDiff > 0 {
			hf.log("borrow: for %d, re-using %d by discarding %d bytes", offset, reader.offset, bestDiff)

			// XXX: not int64-clean
			_, err := reader.Discard(int(bestDiff))
			if err != nil {
				if shouldRetry(err) {
					hf.log("borrow: for %d, discard failed because of retriable error, reconnecting", offset)
					reader.offset = offset
					err = reader.Connect()
					if err != nil {
						return nil, err
					}
				} else {
					return nil, err
				}
			}
		}

		return reader, nil
	}

	// provision a new reader
	hf.log("borrow: making fresh for offset %d", offset)

	reader := &httpReader{
		file:      hf,
		id:        uuid.NewV4().String(),
		touchedAt: time.Now(),
		offset:    offset,
	}

	err := reader.Connect()
	if err != nil {
		return nil, err
	}

	return reader, nil
}

func (hf *HTTPFile) returnReader(reader *httpReader) {
	hf.readersMutex.Lock()
	defer hf.readersMutex.Unlock()

	// TODO: enforce max idle readers ?

	reader.touchedAt = time.Now()
	hf.readers[reader.id] = reader
}

func (hf *HTTPFile) getCurrentURL() string {
	hf.urlMutex.Lock()
	defer hf.urlMutex.Unlock()

	return hf.currentURL
}

func (hf *HTTPFile) renewURL() (string, error) {
	hf.urlMutex.Lock()
	defer hf.urlMutex.Unlock()

	urlStr, err := hf.getURL()
	if err != nil {
		return "", err
	}

	hf.currentURL = urlStr
	return hf.currentURL, nil
}

// Stat returns an os.FileInfo for this particular file. Only the Size()
// method is useful, the rest is default values.
func (hf *HTTPFile) Stat() (os.FileInfo, error) {
	return &httpFileInfo{hf}, nil
}

// Seek the read head within the file - it's instant and never returns an
// error, except if whence is one of os.SEEK_SET, os.SEEK_END, or os.SEEK_CUR.
// If an invalid offset is given, it will be truncated to a valid one, between
// [0,size).
func (hf *HTTPFile) Seek(offset int64, whence int) (int64, error) {
	var newOffset int64

	switch whence {
	case os.SEEK_SET:
		newOffset = offset
	case os.SEEK_END:
		newOffset = hf.size + offset
	case os.SEEK_CUR:
		newOffset = hf.offset + offset
	default:
		return hf.offset, fmt.Errorf("invalid whence value %d", whence)
	}

	if newOffset < 0 {
		newOffset = 0
	}

	if newOffset > hf.size {
		newOffset = hf.size
	}

	hf.offset = newOffset
	return hf.offset, nil
}

func (hf *HTTPFile) Read(data []byte) (int, error) {
	hf.log("Read(%d)", len(data))
	bytesRead, err := hf.readAt(data, hf.offset)
	hf.offset += int64(bytesRead)
	return bytesRead, err
}

// ReadAt reads len(data) byte from the remote file at offset.
// It returns the number of bytes read, and an error. In case of temporary
// network errors or timeouts, it will retry with truncated exponential backoff
// according to RetrySettings
func (hf *HTTPFile) ReadAt(data []byte, offset int64) (int, error) {
	hf.log("ReadAt(%d, %d)", len(data), offset)
	return hf.readAt(data, offset)
}

func (hf *HTTPFile) readAt(data []byte, offset int64) (int, error) {
	reader, err := hf.borrowReader(offset)
	if err != nil {
		return 0, err
	}

	defer hf.returnReader(reader)

	totalBytesRead := 0
	bytesToRead := len(data)

	for totalBytesRead < bytesToRead {
		bytesRead, err := reader.Read(data[totalBytesRead:])
		totalBytesRead += bytesRead

		if err != nil {
			if shouldRetry(err) {
				hf.log("Got %s, retrying", err.Error())
				err = reader.Connect()
				if err != nil {
					return totalBytesRead, err
				}
			} else {
				return totalBytesRead, err
			}
		}
	}

	return totalBytesRead, nil
}

func shouldRetry(err error) bool {
	if errors.Is(err, io.ErrUnexpectedEOF) {
		return true
	} else if opError, ok := err.(*net.OpError); ok {
		if opError.Timeout() || opError.Temporary() {
			return true
		}
	} else if urlError, ok := err.(*url.Error); ok {
		if urlError.Timeout() || urlError.Temporary() || errors.Is(urlError.Err, io.EOF) {
			return true
		}
	} else {
		return false
	}
	return false
}

func (hf *HTTPFile) closeAllReaders() error {
	hf.readersMutex.Lock()
	defer hf.readersMutex.Unlock()

	for id, reader := range hf.readers {
		err := reader.Close()
		if err != nil {
			return err
		}

		delete(hf.readers, id)
	}

	return nil
}

// Close closes all connections to the distant http server used by this HTTPFile
func (hf *HTTPFile) Close() error {
	if hf.closed {
		return nil
	}

	err := hf.closeAllReaders()
	if err != nil {
		return err
	}

	hf.closed = true

	return nil
}

func (hf *HTTPFile) log(format string, args ...interface{}) {
	if hf.Log == nil {
		return
	}

	hf.Log(fmt.Sprintf(format, args...))
}
<|endoftext|>


<commit_before>package ethchain

import (
	"github.com/ethereum/eth-go/ethutil"
	"github.com/obscuren/secp256k1-go"
	"math/big"
)

var ContractAddr = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

type Transaction struct {
	Nonce     uint64
	Recipient []byte
	Value     *big.Int
	Gas       *big.Int
	GasPrice  *big.Int
	Data      []byte
	Init      []byte
	v         byte
	r, s      []byte

	// Indicates whether this tx is a contract creation transaction
	contractCreation bool
}

func NewContractCreationTx(value, gas, gasPrice *big.Int, script []byte, init []byte) *Transaction {
	return &Transaction{Value: value, Gas: gas, GasPrice: gasPrice, Data: script, Init: init, contractCreation: true}
}

func NewTransactionMessage(to []byte, value, gas, gasPrice *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: to, Value: value, GasPrice: gasPrice, Gas: gas, Data: data}
}

func NewTransactionFromBytes(data []byte) *Transaction {
	tx := &Transaction{}
	tx.RlpDecode(data)

	return tx
}

func NewTransactionFromValue(val *ethutil.Value) *Transaction {
	tx := &Transaction{}
	tx.RlpValueDecode(val)

	return tx
}

func (tx *Transaction) Hash() []byte {
	data := []interface{}{tx.Nonce, tx.Value, tx.GasPrice, tx.Gas, tx.Recipient, tx.Data}

	if tx.contractCreation {
		data = append(data, tx.Init)
	}

	return ethutil.Sha3Bin(ethutil.NewValue(data).Encode())
}

func (tx *Transaction) IsContract() bool {
	return tx.contractCreation
}

func (tx *Transaction) CreationAddress() []byte {
	return ethutil.Sha3Bin(ethutil.NewValue([]interface{}{tx.Sender(), tx.Nonce}).Encode())[12:]
}

func (tx *Transaction) Signature(key []byte) []byte {
	hash := tx.Hash()

	sig, _ := secp256k1.Sign(hash, key)

	return sig
}

func (tx *Transaction) PublicKey() []byte {
	hash := tx.Hash()

	// If we don't make a copy we will overwrite the existing underlying array
	dst := make([]byte, len(tx.r))
	copy(dst, tx.r)

	sig := append(dst, tx.s...)
	sig = append(sig, tx.v-27)

	pubkey, _ := secp256k1.RecoverPubkey(hash, sig)

	return pubkey
}

func (tx *Transaction) Sender() []byte {
	pubkey := tx.PublicKey()

	// Validate the returned key.
	// Return nil if public key isn't in full format
	if pubkey[0] != 4 {
		return nil
	}

	return ethutil.Sha3Bin(pubkey[1:])[12:]
}

func (tx *Transaction) Sign(privk []byte) error {

	sig := tx.Signature(privk)

	tx.r = sig[:32]
	tx.s = sig[32:64]
	tx.v = sig[64] + 27

	return nil
}

// [ NONCE, VALUE, GASPRICE, GAS, TO, DATA, V, R, S ]
// [ NONCE, VALUE, GASPRICE, GAS, 0, CODE, INIT, V, R, S ]
func (tx *Transaction) RlpData() interface{} {
	data := []interface{}{tx.Nonce, tx.Value, tx.GasPrice, tx.Gas, tx.Recipient, tx.Data}

	if tx.contractCreation {
		data = append(data, tx.Init)
	}

	return append(data, tx.v, tx.r, tx.s)
}

func (tx *Transaction) RlpValue() *ethutil.Value {
	return ethutil.NewValue(tx.RlpData())
}

func (tx *Transaction) RlpEncode() []byte {
	return tx.RlpValue().Encode()
}

func (tx *Transaction) RlpDecode(data []byte) {
	tx.RlpValueDecode(ethutil.NewValueFromBytes(data))
}

func (tx *Transaction) RlpValueDecode(decoder *ethutil.Value) {
	tx.Nonce = decoder.Get(0).Uint()
	tx.Value = decoder.Get(1).BigInt()
	tx.GasPrice = decoder.Get(2).BigInt()
	tx.Gas = decoder.Get(3).BigInt()
	tx.Recipient = decoder.Get(4).Bytes()
	tx.Data = decoder.Get(5).Bytes()

	// If the list is of length 10 it's a contract creation tx
	if decoder.Len() == 10 {
		tx.contractCreation = true
		tx.Init = decoder.Get(6).Bytes()

		tx.v = byte(decoder.Get(7).Uint())
		tx.r = decoder.Get(8).Bytes()
		tx.s = decoder.Get(9).Bytes()
	} else {
		tx.v = byte(decoder.Get(6).Uint())
		tx.r = decoder.Get(7).Bytes()
		tx.s = decoder.Get(8).Bytes()
	}
}
<commit_msg>Rearranged according to YP<commit_after>package ethchain

import (
	"github.com/ethereum/eth-go/ethutil"
	"github.com/obscuren/secp256k1-go"
	"math/big"
)

var ContractAddr = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}

type Transaction struct {
	Nonce     uint64
	Recipient []byte
	Value     *big.Int
	Gas       *big.Int
	GasPrice  *big.Int
	Data      []byte
	Init      []byte
	v         byte
	r, s      []byte

	// Indicates whether this tx is a contract creation transaction
	contractCreation bool
}

func NewContractCreationTx(value, gas, gasPrice *big.Int, script []byte, init []byte) *Transaction {
	return &Transaction{Value: value, Gas: gas, GasPrice: gasPrice, Data: script, Init: init, contractCreation: true}
}

func NewTransactionMessage(to []byte, value, gas, gasPrice *big.Int, data []byte) *Transaction {
	return &Transaction{Recipient: to, Value: value, GasPrice: gasPrice, Gas: gas, Data: data}
}

func NewTransactionFromBytes(data []byte) *Transaction {
	tx := &Transaction{}
	tx.RlpDecode(data)

	return tx
}

func NewTransactionFromValue(val *ethutil.Value) *Transaction {
	tx := &Transaction{}
	tx.RlpValueDecode(val)

	return tx
}

func (tx *Transaction) Hash() []byte {
	data := []interface{}{tx.Nonce, tx.Value, tx.GasPrice, tx.Gas, tx.Recipient, tx.Data}

	if tx.contractCreation {
		data = append(data, tx.Init)
	}

	return ethutil.Sha3Bin(ethutil.NewValue(data).Encode())
}

func (tx *Transaction) IsContract() bool {
	return tx.contractCreation
}

func (tx *Transaction) CreationAddress() []byte {
	return ethutil.Sha3Bin(ethutil.NewValue([]interface{}{tx.Sender(), tx.Nonce}).Encode())[12:]
}

func (tx *Transaction) Signature(key []byte) []byte {
	hash := tx.Hash()

	sig, _ := secp256k1.Sign(hash, key)

	return sig
}

func (tx *Transaction) PublicKey() []byte {
	hash := tx.Hash()

	// If we don't make a copy we will overwrite the existing underlying array
	dst := make([]byte, len(tx.r))
	copy(dst, tx.r)

	sig := append(dst, tx.s...)
	sig = append(sig, tx.v-27)

	pubkey, _ := secp256k1.RecoverPubkey(hash, sig)

	return pubkey
}

func (tx *Transaction) Sender() []byte {
	pubkey := tx.PublicKey()

	// Validate the returned key.
	// Return nil if public key isn't in full format
	if pubkey[0] != 4 {
		return nil
	}

	return ethutil.Sha3Bin(pubkey[1:])[12:]
}

func (tx *Transaction) Sign(privk []byte) error {

	sig := tx.Signature(privk)

	tx.r = sig[:32]
	tx.s = sig[32:64]
	tx.v = sig[64] + 27

	return nil
}

func (tx *Transaction) RlpData() interface{} {
	data := []interface{}{tx.Nonce, tx.GasPrice, tx.Gas, tx.Recipient, tx.Value, tx.Data}

	if tx.contractCreation {
		data = append(data, tx.Init)
	}

	return append(data, tx.v, tx.r, tx.s)
}

func (tx *Transaction) RlpValue() *ethutil.Value {
	return ethutil.NewValue(tx.RlpData())
}

func (tx *Transaction) RlpEncode() []byte {
	return tx.RlpValue().Encode()
}

func (tx *Transaction) RlpDecode(data []byte) {
	tx.RlpValueDecode(ethutil.NewValueFromBytes(data))
}

func (tx *Transaction) RlpValueDecode(decoder *ethutil.Value) {
	tx.Nonce = decoder.Get(0).Uint()
	tx.GasPrice = decoder.Get(1).BigInt()
	tx.Gas = decoder.Get(2).BigInt()
	tx.Recipient = decoder.Get(3).Bytes()
	tx.Value = decoder.Get(4).BigInt()
	tx.Data = decoder.Get(5).Bytes()

	// If the list is of length 10 it's a contract creation tx
	if decoder.Len() == 10 {
		tx.contractCreation = true
		tx.Init = decoder.Get(6).Bytes()

		tx.v = byte(decoder.Get(7).Uint())
		tx.r = decoder.Get(8).Bytes()
		tx.s = decoder.Get(9).Bytes()
	} else {
		tx.v = byte(decoder.Get(6).Uint())
		tx.r = decoder.Get(7).Bytes()
		tx.s = decoder.Get(8).Bytes()
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/cheggaaa/pb"
	"github.com/mattn/go-isatty"
	expirable "github.com/odeke-em/cache"
	"github.com/odeke-em/drive/config"
	"github.com/odeke-em/log"
)

var (
	ErrNoContext = errors.New("not in a drive context")
)

const (
	DriveIgnoreSuffix = ".driveignore"
)

type Options struct {
	// Depth is the number of pages/ listing recursion depth
	Depth int `cli:"depth"`
	// Exports contains the formats to export your Google Docs + Sheets to
	// e.g ["csv" "txt"]
	Exports []string `cli:"exports"`
	// ExportsDir is the directory to put the exported Google Docs + Sheets.
	// If not provided, will export them to the same dir as the source files are
	ExportsDir string `cli:"exports-dir"`
	// Force once set always converts NoChange into an Addition
	Force bool `cli:"force"`
	// Hidden discovers hidden paths if set
	Hidden       bool `cli:"hidden"`
	IgnoreRegexp *regexp.Regexp
	// IgnoreChecksum when set avoids the step
	// of comparing checksums as a final check.
	IgnoreChecksum bool `cli:"ignore"`
	// IgnoreConflict when set turns off the conflict resolution safety.
	IgnoreConflict bool `cli:"ignore-conflict"`
	// Allows listing of content in trash
	InTrash bool `cli:"in-trash"`
	Meta    *map[string][]string
	Mount   *config.Mount
	// NoClobber when set prevents overwriting of stale content
	NoClobber bool `cli:"no-clobber"`
	// NoPrompt overwrites any prompt pauses
	NoPrompt bool `cli:"no-prompt"`
	Path     string
	// PageSize determines the number of results returned per API call
	PageSize  int64 `cli:"pagesize"`
	Recursive bool  `cli:"recursive"`
	// Sources is a of list all paths that are
	// within the scope/path of the current gd context
	Sources []string `cli:"sources"`
	// TypeMask contains the result of setting different type bits e.g
	// Folder to search only for folders etc.
	TypeMask int `cli:"typeMask"`
	// Piped when set means to infer content to or from stdin
	Piped bool `cli:"piped"`
	// Quiet when set toggles only logging of errors to stderrs as
	// well as reading from stdin in this case stdout is not logged to
	Quiet             bool      `cli:"quiet"`
	StdoutIsTty       bool      `cli:"istty"`
	IgnoreNameClashes bool      `cli:"ignoreNameClashes"`
	ExcludeCrudMask   CrudValue `cli:"excludeCrudMask"`
	ExplicitlyExport  bool      `cli:"explicitlyExport"`
	Md5sum            bool      `cli:"md5sum"`
	indexingOnly      bool      `cli:"indexingOnly"`
	Verbose           bool      `cli:"verbose"`
	FixClashes        bool
}

type Commands struct {
	context *config.Context
	rem     *Remote
	opts    *Options
	rcOpts  *Options
	log     *log.Logger

	progress      *pb.ProgressBar
	mkdirAllCache *expirable.OperationCache
}

func (opts *Options) canPrompt() bool {
	if opts == nil || !opts.StdoutIsTty {
		return false
	}
	if opts.Quiet {
		return false
	}
	return !opts.NoPrompt
}

func rcPathChecker(absDir string) (string, error) {
	p := rcPath(absDir)
	statInfo, err := os.Stat(p)
	if err != nil && !os.IsNotExist(err) {
		return "", err
	}

	if statInfo == nil {
		return "", os.ErrNotExist
	}
	return p, nil
}

func (opts *Options) rcPath() (string, error) {
	lastCurPath := ""
	for curPath := opts.Path; curPath != ""; curPath = path.Dir(curPath) {
		localRCP, err := rcPathChecker(curPath)
		if err == nil && localRCP != "" {
			return localRCP, nil
		}

		if err != nil && !os.IsNotExist(err) {
			return "", err
		}

		if lastCurPath == curPath { // Avoid getting a stalemate incase path.Dir cannot progress
			break
		}

		lastCurPath = curPath
	}

	return rcPathChecker(FsHomeDir)
}

func New(context *config.Context, opts *Options) *Commands {
	var r *Remote
	if context != nil {
		r = NewRemoteContext(context)
	}

	stdin, stdout, stderr := os.Stdin, os.Stdout, os.Stderr

	logger := log.New(stdin, stdout, stderr)

	if opts != nil {

		rcP, rcErr := opts.rcPath()
		if rcErr == nil {
			rcOpt, err := rcFileToOptions(rcP)
			if true { // DEBUG
				fmt.Println("kvmap", rcOpt, "err", err)
			}
		}

		// should always start with /
		opts.Path = path.Clean(path.Join("/", opts.Path))

		if !opts.Force {
			ignoresPath := filepath.Join(context.AbsPath, DriveIgnoreSuffix)
			ignoreRegexp, regErr := combineIgnores(ignoresPath)

			if regErr != nil {
				logger.LogErrf("combining ignores from path %s and internally: %v\n", ignoresPath, regErr)
			}

			opts.IgnoreRegexp = ignoreRegexp
		}

		opts.StdoutIsTty = isatty.IsTerminal(stdout.Fd())

		if opts.Quiet {
			stdout = nil
		}
	}

	return &Commands{
		context:       context,
		rem:           r,
		opts:          opts,
		log:           logger,
		mkdirAllCache: expirable.New(),
	}
}

func combineIgnores(ignoresPath string) (*regexp.Regexp, error) {
	clauses, err := readCommentedFile(ignoresPath, "#")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	clauses = append(clauses, internalIgnores()...)
	if len(clauses) < 1 {
		return nil, nil
	}

	regExComp, regErr := regexp.Compile(strings.Join(clauses, "|"))
	if regErr != nil {
		return nil, regErr
	}
	return regExComp, nil
}

func (g *Commands) taskStart(tasks int64) {
	if tasks > 0 {
		g.progress = newProgressBar(tasks)
	}
}

func newProgressBar(total int64) *pb.ProgressBar {
	pbf := pb.New64(total)
	pbf.Start()
	return pbf
}

func (g *Commands) taskAdd(n int64) {
	if g.progress != nil {
		g.progress.Add64(n)
	}
}

func (g *Commands) taskFinish() {
	if g.progress != nil {
		g.progress.Finish()
	}
}
<commit_msg>rc-file: ignore not found rc-related errors for now<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package drive

import (
	"errors"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/cheggaaa/pb"
	"github.com/mattn/go-isatty"
	expirable "github.com/odeke-em/cache"
	"github.com/odeke-em/drive/config"
	"github.com/odeke-em/log"
)

var (
	ErrNoContext = errors.New("not in a drive context")
)

const (
	DriveIgnoreSuffix = ".driveignore"
)

type Options struct {
	// Depth is the number of pages/ listing recursion depth
	Depth int `cli:"depth"`
	// Exports contains the formats to export your Google Docs + Sheets to
	// e.g ["csv" "txt"]
	Exports []string `cli:"exports"`
	// ExportsDir is the directory to put the exported Google Docs + Sheets.
	// If not provided, will export them to the same dir as the source files are
	ExportsDir string `cli:"exports-dir"`
	// Force once set always converts NoChange into an Addition
	Force bool `cli:"force"`
	// Hidden discovers hidden paths if set
	Hidden       bool `cli:"hidden"`
	IgnoreRegexp *regexp.Regexp
	// IgnoreChecksum when set avoids the step
	// of comparing checksums as a final check.
	IgnoreChecksum bool `cli:"ignore"`
	// IgnoreConflict when set turns off the conflict resolution safety.
	IgnoreConflict bool `cli:"ignore-conflict"`
	// Allows listing of content in trash
	InTrash bool `cli:"in-trash"`
	Meta    *map[string][]string
	Mount   *config.Mount
	// NoClobber when set prevents overwriting of stale content
	NoClobber bool `cli:"no-clobber"`
	// NoPrompt overwrites any prompt pauses
	NoPrompt bool `cli:"no-prompt"`
	Path     string
	// PageSize determines the number of results returned per API call
	PageSize  int64 `cli:"pagesize"`
	Recursive bool  `cli:"recursive"`
	// Sources is a of list all paths that are
	// within the scope/path of the current gd context
	Sources []string `cli:"sources"`
	// TypeMask contains the result of setting different type bits e.g
	// Folder to search only for folders etc.
	TypeMask int `cli:"typeMask"`
	// Piped when set means to infer content to or from stdin
	Piped bool `cli:"piped"`
	// Quiet when set toggles only logging of errors to stderrs as
	// well as reading from stdin in this case stdout is not logged to
	Quiet             bool      `cli:"quiet"`
	StdoutIsTty       bool      `cli:"istty"`
	IgnoreNameClashes bool      `cli:"ignoreNameClashes"`
	ExcludeCrudMask   CrudValue `cli:"excludeCrudMask"`
	ExplicitlyExport  bool      `cli:"explicitlyExport"`
	Md5sum            bool      `cli:"md5sum"`
	indexingOnly      bool      `cli:"indexingOnly"`
	Verbose           bool      `cli:"verbose"`
	FixClashes        bool
}

type Commands struct {
	context *config.Context
	rem     *Remote
	opts    *Options
	rcOpts  *Options
	log     *log.Logger

	progress      *pb.ProgressBar
	mkdirAllCache *expirable.OperationCache
}

func (opts *Options) canPrompt() bool {
	if opts == nil || !opts.StdoutIsTty {
		return false
	}
	if opts.Quiet {
		return false
	}
	return !opts.NoPrompt
}

func rcPathChecker(absDir string) (string, error) {
	p := rcPath(absDir)
	statInfo, err := os.Stat(p)
	if err != nil && !os.IsNotExist(err) {
		return "", err
	}

	if statInfo == nil {
		return "", os.ErrNotExist
	}
	return p, nil
}

func (opts *Options) rcPath() (string, error) {
	lastCurPath := ""
	for curPath := opts.Path; curPath != ""; curPath = path.Dir(curPath) {
		localRCP, err := rcPathChecker(curPath)
		if err == nil && localRCP != "" {
			return localRCP, nil
		}

		if false && err != nil && !os.IsNotExist(err) {
			return "", err
		}

		if lastCurPath == curPath { // Avoid getting a stalemate incase path.Dir cannot progress
			break
		}

		lastCurPath = curPath
	}

	return rcPathChecker(FsHomeDir)
}

func New(context *config.Context, opts *Options) *Commands {
	var r *Remote
	if context != nil {
		r = NewRemoteContext(context)
	}

	stdin, stdout, stderr := os.Stdin, os.Stdout, os.Stderr

	logger := log.New(stdin, stdout, stderr)

	if opts != nil {

		rcP, rcErr := opts.rcPath()
		if rcErr == nil {
			rcOpt, err := rcFileToOptions(rcP)
			if true { // DEBUG
				fmt.Println("kvmap", rcOpt, "err", err)
			}
		}

		// should always start with /
		opts.Path = path.Clean(path.Join("/", opts.Path))

		if !opts.Force {
			ignoresPath := filepath.Join(context.AbsPath, DriveIgnoreSuffix)
			ignoreRegexp, regErr := combineIgnores(ignoresPath)

			if regErr != nil {
				logger.LogErrf("combining ignores from path %s and internally: %v\n", ignoresPath, regErr)
			}

			opts.IgnoreRegexp = ignoreRegexp
		}

		opts.StdoutIsTty = isatty.IsTerminal(stdout.Fd())

		if opts.Quiet {
			stdout = nil
		}
	}

	return &Commands{
		context:       context,
		rem:           r,
		opts:          opts,
		log:           logger,
		mkdirAllCache: expirable.New(),
	}
}

func combineIgnores(ignoresPath string) (*regexp.Regexp, error) {
	clauses, err := readCommentedFile(ignoresPath, "#")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	clauses = append(clauses, internalIgnores()...)
	if len(clauses) < 1 {
		return nil, nil
	}

	regExComp, regErr := regexp.Compile(strings.Join(clauses, "|"))
	if regErr != nil {
		return nil, regErr
	}
	return regExComp, nil
}

func (g *Commands) taskStart(tasks int64) {
	if tasks > 0 {
		g.progress = newProgressBar(tasks)
	}
}

func newProgressBar(total int64) *pb.ProgressBar {
	pbf := pb.New64(total)
	pbf.Start()
	return pbf
}

func (g *Commands) taskAdd(n int64) {
	if g.progress != nil {
		g.progress.Add64(n)
	}
}

func (g *Commands) taskFinish() {
	if g.progress != nil {
		g.progress.Finish()
	}
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"fmt"
	"sync"

	"github.com/keybase/client/go/logger"
	"golang.org/x/net/context"
)

// CtxCRTagKey is the type used for unique context tags related to
// conflict resolution
type CtxCRTagKey int

const (
	// CtxCRIDKey is the type of the tag for unique operation IDs
	// related to conflict resolution
	CtxCRIDKey CtxCRTagKey = iota
)

// CtxCROpID is the display name for the unique operation
// conflict resolution ID tag.
const CtxCROpID = "CRID"

type conflictInput struct {
	unmerged MetadataRevision
	merged   MetadataRevision
}

// ConflictResolver is responsible for resolving conflicts in the
// background.
type ConflictResolver struct {
	config     Config
	fbo        *FolderBranchOps
	inputChan  chan conflictInput
	inputGroup sync.WaitGroup
	log        logger.Logger

	currInput conflictInput
	inputLock sync.Mutex
}

// NewConflictResolver constructs a new ConflictResolver (and launches
// any necessary background goroutines).
func NewConflictResolver(
	config Config, fbo *FolderBranchOps) *ConflictResolver {
	// make a logger with an appropriate module name
	branchSuffix := ""
	if fbo.branch() != MasterBranch {
		branchSuffix = " " + string(fbo.branch())
	}
	tlfStringFull := fbo.id().String()
	log := config.MakeLogger(fmt.Sprintf("CR %s%s", tlfStringFull[:8],
		branchSuffix))

	cr := &ConflictResolver{
		config:    config,
		fbo:       fbo,
		inputChan: make(chan conflictInput),
		log:       log,
		currInput: conflictInput{
			unmerged: MetadataRevisionUninitialized,
			merged:   MetadataRevisionUninitialized,
		},
	}

	go cr.inputProcessor()
	return cr
}

func (cr *ConflictResolver) inputProcessor() {
	logTags := make(logger.CtxLogTags)
	logTags[CtxCRIDKey] = CtxCROpID
	ctx := logger.NewContextWithLogTags(context.Background(), logTags)

	var cancel func()
	defer func() {
		if cancel != nil {
			cancel()
		}
	}()
	for ci := range cr.inputChan {
		ctx := ctx
		id, err := MakeRandomRequestID()
		if err != nil {
			cr.log.Warning("Couldn't generate a random request ID: %v", err)
		} else {
			ctx = context.WithValue(ctx, CtxCRIDKey, id)
		}

		valid := func() bool {
			cr.inputLock.Lock()
			defer cr.inputLock.Unlock()
			// The input is only interesting if one of the revisions
			// is greater than what we've looked at to date.
			if ci.unmerged <= cr.currInput.unmerged &&
				ci.merged <= cr.currInput.merged {
				return false
			}
			cr.log.CDebugf(ctx, "New conflict input %v following old "+
				"input %v", ci, cr.currInput)
			cr.currInput = ci
			// cancel the existing conflict resolution (if any)
			if cancel != nil {
				cancel()
			}
			return true
		}()
		if !valid {
			cr.log.CDebugf(ctx, "Ignoring uninteresting input: %v", ci)
			cr.inputGroup.Done()
			continue
		}

		ctx, cancel = context.WithCancel(ctx)
		go cr.doResolve(ctx, ci)
	}
}

// Resolve takes the latest known unmerged and merged revision
// numbers, and kicks off the resolution process.
func (cr *ConflictResolver) Resolve(unmerged MetadataRevision,
	merged MetadataRevision) {
	cr.inputGroup.Add(1)
	cr.inputChan <- conflictInput{unmerged, merged}
}

// Wait blocks until the current set of submitted resolutions are
// complete (though not necessarily successful), or until the given
// context is canceled.
func (cr *ConflictResolver) Wait(ctx context.Context) error {
	c := make(chan struct{}, 1)
	go func() {
		cr.inputGroup.Wait()
		c <- struct{}{}
	}()

	select {
	case <-c:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Shutdown cancels any ongoing resolutions and stops any background
// goroutines.
func (cr *ConflictResolver) Shutdown() {
	close(cr.inputChan)
}

func (cr *ConflictResolver) checkDone(ctx context.Context) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
		return nil
	}
}

func (cr *ConflictResolver) getMDs(ctx context.Context) (
	unmerged []*RootMetadata, merged []*RootMetadata, err error) {
	// first get all outstanding unmerged MDs for this device
	branchPoint, unmerged, err := cr.fbo.getUnmergedMDUpdates(ctx)
	if err != nil {
		return nil, nil, err
	}

	// now get all the merged MDs, starting from after the branch point
	merged, err = getMergedMDUpdates(ctx, cr.fbo.config, cr.fbo.id(),
		branchPoint+1)
	return unmerged, merged, err
}

func (cr *ConflictResolver) updateCurrInput(ctx context.Context,
	unmerged []*RootMetadata, merged []*RootMetadata) error {
	cr.inputLock.Lock()
	defer cr.inputLock.Unlock()
	// check done while holding the lock, so we know for sure if
	// we've already been canceled and replaced by a new input.
	err := cr.checkDone(ctx)
	if err != nil {
		return err
	}

	if len(unmerged) > 0 {
		cr.currInput.unmerged = unmerged[len(unmerged)-1].Revision
	}
	if len(merged) > 0 {
		cr.currInput.merged = merged[len(merged)-1].Revision
	}
	return nil
}

func (cr *ConflictResolver) doResolve(ctx context.Context, ci conflictInput) {
	cr.log.CDebugf(ctx, "Starting conflict resolution with input %v", ci)
	var err error
	defer cr.inputGroup.Done()
	defer cr.log.CDebugf(ctx, "Finished conflict resolution: %v", err)

	// Canceled before we even got started?
	err = cr.checkDone(ctx)
	if err != nil {
		return
	}

	// Fetch the merged and unmerged MDs
	unmerged, merged, err := cr.getMDs(ctx)
	if err != nil {
		return
	}

	// Update the current input to reflect the MDs we'll actually be
	// working with.
	err = cr.updateCurrInput(ctx, unmerged, merged)
	if err != nil {
		return
	}
}
<commit_msg>cr: fix a race by avoiding writes to a shutdown channel<commit_after>package libkbfs

import (
	"fmt"
	"sync"

	"github.com/keybase/client/go/logger"
	"golang.org/x/net/context"
)

// CtxCRTagKey is the type used for unique context tags related to
// conflict resolution
type CtxCRTagKey int

const (
	// CtxCRIDKey is the type of the tag for unique operation IDs
	// related to conflict resolution
	CtxCRIDKey CtxCRTagKey = iota
)

// CtxCROpID is the display name for the unique operation
// conflict resolution ID tag.
const CtxCROpID = "CRID"

type conflictInput struct {
	unmerged MetadataRevision
	merged   MetadataRevision
}

// ConflictResolver is responsible for resolving conflicts in the
// background.
type ConflictResolver struct {
	config     Config
	fbo        *FolderBranchOps
	inputChan  chan conflictInput
	inputGroup sync.WaitGroup
	log        logger.Logger

	shutdown     bool
	shutdownLock sync.RWMutex

	currInput conflictInput
	inputLock sync.Mutex
}

// NewConflictResolver constructs a new ConflictResolver (and launches
// any necessary background goroutines).
func NewConflictResolver(
	config Config, fbo *FolderBranchOps) *ConflictResolver {
	// make a logger with an appropriate module name
	branchSuffix := ""
	if fbo.branch() != MasterBranch {
		branchSuffix = " " + string(fbo.branch())
	}
	tlfStringFull := fbo.id().String()
	log := config.MakeLogger(fmt.Sprintf("CR %s%s", tlfStringFull[:8],
		branchSuffix))

	cr := &ConflictResolver{
		config:    config,
		fbo:       fbo,
		inputChan: make(chan conflictInput),
		log:       log,
		currInput: conflictInput{
			unmerged: MetadataRevisionUninitialized,
			merged:   MetadataRevisionUninitialized,
		},
	}

	go cr.inputProcessor()
	return cr
}

func (cr *ConflictResolver) inputProcessor() {
	logTags := make(logger.CtxLogTags)
	logTags[CtxCRIDKey] = CtxCROpID
	ctx := logger.NewContextWithLogTags(context.Background(), logTags)

	var cancel func()
	defer func() {
		if cancel != nil {
			cancel()
		}
	}()
	for ci := range cr.inputChan {
		ctx := ctx
		id, err := MakeRandomRequestID()
		if err != nil {
			cr.log.Warning("Couldn't generate a random request ID: %v", err)
		} else {
			ctx = context.WithValue(ctx, CtxCRIDKey, id)
		}

		valid := func() bool {
			cr.inputLock.Lock()
			defer cr.inputLock.Unlock()
			// The input is only interesting if one of the revisions
			// is greater than what we've looked at to date.
			if ci.unmerged <= cr.currInput.unmerged &&
				ci.merged <= cr.currInput.merged {
				return false
			}
			cr.log.CDebugf(ctx, "New conflict input %v following old "+
				"input %v", ci, cr.currInput)
			cr.currInput = ci
			// cancel the existing conflict resolution (if any)
			if cancel != nil {
				cancel()
			}
			return true
		}()
		if !valid {
			cr.log.CDebugf(ctx, "Ignoring uninteresting input: %v", ci)
			cr.inputGroup.Done()
			continue
		}

		ctx, cancel = context.WithCancel(ctx)
		go cr.doResolve(ctx, ci)
	}
}

// Resolve takes the latest known unmerged and merged revision
// numbers, and kicks off the resolution process.
func (cr *ConflictResolver) Resolve(unmerged MetadataRevision,
	merged MetadataRevision) {
	cr.shutdownLock.RLock()
	defer cr.shutdownLock.RUnlock()
	if cr.shutdown {
		return
	}

	cr.inputGroup.Add(1)
	cr.inputChan <- conflictInput{unmerged, merged}
}

// Wait blocks until the current set of submitted resolutions are
// complete (though not necessarily successful), or until the given
// context is canceled.
func (cr *ConflictResolver) Wait(ctx context.Context) error {
	c := make(chan struct{}, 1)
	go func() {
		cr.inputGroup.Wait()
		c <- struct{}{}
	}()

	select {
	case <-c:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Shutdown cancels any ongoing resolutions and stops any background
// goroutines.
func (cr *ConflictResolver) Shutdown() {
	cr.shutdownLock.Lock()
	defer cr.shutdownLock.Unlock()
	cr.shutdown = true
	close(cr.inputChan)
}

func (cr *ConflictResolver) checkDone(ctx context.Context) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	default:
		return nil
	}
}

func (cr *ConflictResolver) getMDs(ctx context.Context) (
	unmerged []*RootMetadata, merged []*RootMetadata, err error) {
	// first get all outstanding unmerged MDs for this device
	branchPoint, unmerged, err := cr.fbo.getUnmergedMDUpdates(ctx)
	if err != nil {
		return nil, nil, err
	}

	// now get all the merged MDs, starting from after the branch point
	merged, err = getMergedMDUpdates(ctx, cr.fbo.config, cr.fbo.id(),
		branchPoint+1)
	return unmerged, merged, err
}

func (cr *ConflictResolver) updateCurrInput(ctx context.Context,
	unmerged []*RootMetadata, merged []*RootMetadata) error {
	cr.inputLock.Lock()
	defer cr.inputLock.Unlock()
	// check done while holding the lock, so we know for sure if
	// we've already been canceled and replaced by a new input.
	err := cr.checkDone(ctx)
	if err != nil {
		return err
	}

	if len(unmerged) > 0 {
		cr.currInput.unmerged = unmerged[len(unmerged)-1].Revision
	}
	if len(merged) > 0 {
		cr.currInput.merged = merged[len(merged)-1].Revision
	}
	return nil
}

func (cr *ConflictResolver) doResolve(ctx context.Context, ci conflictInput) {
	cr.log.CDebugf(ctx, "Starting conflict resolution with input %v", ci)
	var err error
	defer cr.inputGroup.Done()
	defer cr.log.CDebugf(ctx, "Finished conflict resolution: %v", err)

	// Canceled before we even got started?
	err = cr.checkDone(ctx)
	if err != nil {
		return
	}

	// Fetch the merged and unmerged MDs
	unmerged, merged, err := cr.getMDs(ctx)
	if err != nil {
		return
	}

	// Update the current input to reflect the MDs we'll actually be
	// working with.
	err = cr.updateCurrInput(ctx, unmerged, merged)
	if err != nil {
		return
	}
}
<|endoftext|>


<commit_before>package mackerelplugin

import (
	"math"
	"testing"
	"time"
)

func TestCalcDiff(t *testing.T) {
	var mp MackerelPlugin

	val1 := 10.0
	val2 := 0.0
	now := time.Now()
	last := time.Unix(now.Unix()-10, 0)

	diff, err := mp.calcDiff(val1, now, val2, last)
	if diff != 60 {
		t.Errorf("calcDiff: %f should be %f", diff, 60.0)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}

func TestCalcDiffWithUInt32OverflowWithSigned(t *testing.T) {
	var mp MackerelPlugin

	val := 10.0
	now := time.Now()
	lastval := (math.MaxUint32) - 10.0
	last := time.Unix(now.Unix()-60, 0)

	diff, err := mp.calcDiff(val, now, lastval, last)
	if diff > 0 {
		t.Errorf("calcDiff: last: %f, now: %f, %f should be negative", val, lastval, diff)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}

func TestCalcDiffWithUInt32Overflow(t *testing.T) {
	var mp MackerelPlugin

	val := uint32(10)
	now := time.Now()
	lastval := math.MaxUint32 - uint32(10)
	last := time.Unix(now.Unix()-60, 0)

	diff, err := mp.calcDiffUint32(val, now, lastval, last)
	if diff != 21.0 {
		t.Errorf("calcDiff: last: %d, now: %d, %f should be %f", val, lastval, diff, 21.0)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}

func TestCalcDiffWithUInt64Overflow(t *testing.T) {
	var mp MackerelPlugin

	val := uint64(10)
	now := time.Now()
	lastval := math.MaxUint64 - uint64(10)
	last := time.Unix(now.Unix()-60, 0)

	diff, err := mp.calcDiffUint64(val, now, lastval, last)
	if diff != 21.0 {
		t.Errorf("calcDiff: last: %d, now: %d, %f should be %f", val, lastval, diff, 21.0)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}
<commit_msg>fix uint32 with reset test<commit_after>package mackerelplugin

import (
	"math"
	"testing"
	"time"
)

func TestCalcDiff(t *testing.T) {
	var mp MackerelPlugin

	val1 := 10.0
	val2 := 0.0
	now := time.Now()
	last := time.Unix(now.Unix()-10, 0)

	diff, err := mp.calcDiff(val1, now, val2, last)
	if diff != 60 {
		t.Errorf("calcDiff: %f should be %f", diff, 60.0)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}

func TestCalcDiffWithUInt32WithReset(t *testing.T) {
	var mp MackerelPlugin

	val := uint32(10)
	now := time.Now()
	lastval := math.MaxUint32 - uint32(10)
	last := time.Unix(now.Unix()-60, 0)

	diff, err := mp.calcDiffUint32(val, now, lastval, last)
	if diff > 0 {
		t.Errorf("calcDiff: last: %f, now: %f, %f should be negative", val, lastval, diff)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}

func TestCalcDiffWithUInt32Overflow(t *testing.T) {
	var mp MackerelPlugin

	val := uint32(10)
	now := time.Now()
	lastval := math.MaxUint32 - uint32(10)
	last := time.Unix(now.Unix()-60, 0)

	diff, err := mp.calcDiffUint32(val, now, lastval, last)
	if diff != 21.0 {
		t.Errorf("calcDiff: last: %d, now: %d, %f should be %f", val, lastval, diff, 21.0)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}

func TestCalcDiffWithUInt64Overflow(t *testing.T) {
	var mp MackerelPlugin

	val := uint64(10)
	now := time.Now()
	lastval := math.MaxUint64 - uint64(10)
	last := time.Unix(now.Unix()-60, 0)

	diff, err := mp.calcDiffUint64(val, now, lastval, last)
	if diff != 21.0 {
		t.Errorf("calcDiff: last: %d, now: %d, %f should be %f", val, lastval, diff, 21.0)
	}
	if err != nil {
		t.Error("calcDiff causes an error")
	}
}
<|endoftext|>


<commit_before>//   Copyright 2016 DigitalOcean
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package collectors

import (
	"encoding/json"
	"log"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	cephNamespace = "ceph"
)

// A ClusterUsageCollector is used to gather all the global stats about a given
// ceph cluster. It is sometimes essential to know how fast the cluster is growing
// or shrinking as a whole in order to zero in on the cause. The pool specific
// stats are provided separately.
type ClusterUsageCollector struct {
	conn Conn

	// GlobalCapacity displays the total storage capacity of the cluster. This
	// information is based on the actual no. of objects that are allocated. It
	// does not take overcommittment into consideration.
	GlobalCapacity prometheus.Gauge

	// UsedCapacity shows the storage under use.
	UsedCapacity prometheus.Gauge

	// AvailableCapacity shows the remaining capacity of the cluster that is left unallocated.
	AvailableCapacity prometheus.Gauge

	// Objects show the total no. of RADOS objects that are currently allocated.
	Objects prometheus.Gauge
}

// NewClusterUsageCollector creates and returns the reference to ClusterUsageCollector
// and internally defines each metric that display cluster stats.
func NewClusterUsageCollector(conn Conn) *ClusterUsageCollector {
	return &ClusterUsageCollector{
		conn: conn,

		GlobalCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_capacity_bytes",
			Help:      "Total capacity of the cluster",
		}),
		UsedCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_used_bytes",
			Help:      "Capacity of the cluster currently in use",
		}),
		AvailableCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_available_bytes",
			Help:      "Available space within the cluster",
		}),
		Objects: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_objects",
			Help:      "No. of rados objects within the cluster",
		}),
	}
}

func (c *ClusterUsageCollector) metricsList() []prometheus.Metric {
	return []prometheus.Metric{
		c.GlobalCapacity,
		c.UsedCapacity,
		c.AvailableCapacity,
		c.Objects,
	}
}

type cephClusterStats struct {
	Stats struct {
		TotalBytes      json.Number `json:"total_bytes"`
		TotalUsedBytes  json.Number `json:"total_used_bytes"`
		TotalAvailBytes json.Number `json:"total_avail_bytes"`
		TotalObjects    json.Number `json:"total_objects"`
	} `json:"stats"`
}

func (c *ClusterUsageCollector) collect() error {
	cmd := c.cephUsageCommand()
	buf, _, err := c.conn.MonCommand(cmd)
	if err != nil {
		return err
	}

	stats := &cephClusterStats{}
	if err := json.Unmarshal(buf, stats); err != nil {
		return err
	}

	var totBytes, usedBytes, availBytes, totObjects float64

	totBytes, err = stats.Stats.TotalBytes.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract total bytes:", err)
	}

	usedBytes, err = stats.Stats.TotalUsedBytes.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract used bytes:", err)
	}

	availBytes, err = stats.Stats.TotalAvailBytes.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract available bytes:", err)
	}

	totObjects, err = stats.Stats.TotalObjects.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract total objects:", err)
	}

	c.GlobalCapacity.Set(totBytes)
	c.UsedCapacity.Set(usedBytes)
	c.AvailableCapacity.Set(availBytes)
	c.Objects.Set(totObjects)

	return nil
}

func (c *ClusterUsageCollector) cephUsageCommand() []byte {
	cmd, err := json.Marshal(map[string]interface{}{
		"prefix": "df",
		"detail": "detail",
		"format": "json",
	})
	if err != nil {
		// panic! because ideally in no world this hard-coded input
		// should fail.
		panic(err)
	}
	return cmd
}

// Describe sends the descriptors of each metric over to the provided channel.
// The corresponding metric values are sent separately.
func (c *ClusterUsageCollector) Describe(ch chan<- *prometheus.Desc) {
	for _, metric := range c.metricsList() {
		ch <- metric.Desc()
	}
}

// Collect sends the metric values for each metric pertaining to the global
// cluster usage over to the provided prometheus Metric channel.
func (c *ClusterUsageCollector) Collect(ch chan<- prometheus.Metric) {
	if err := c.collect(); err != nil {
		log.Println("[ERROR] failed collecting cluster usage metrics:", err)
		return
	}

	for _, metric := range c.metricsList() {
		ch <- metric
	}
}
<commit_msg>Fix comment typo<commit_after>//   Copyright 2016 DigitalOcean
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package collectors

import (
	"encoding/json"
	"log"

	"github.com/prometheus/client_golang/prometheus"
)

const (
	cephNamespace = "ceph"
)

// A ClusterUsageCollector is used to gather all the global stats about a given
// ceph cluster. It is sometimes essential to know how fast the cluster is growing
// or shrinking as a whole in order to zero in on the cause. The pool specific
// stats are provided separately.
type ClusterUsageCollector struct {
	conn Conn

	// GlobalCapacity displays the total storage capacity of the cluster. This
	// information is based on the actual no. of objects that are allocated. It
	// does not take overcommitment into consideration.
	GlobalCapacity prometheus.Gauge

	// UsedCapacity shows the storage under use.
	UsedCapacity prometheus.Gauge

	// AvailableCapacity shows the remaining capacity of the cluster that is left unallocated.
	AvailableCapacity prometheus.Gauge

	// Objects show the total no. of RADOS objects that are currently allocated.
	Objects prometheus.Gauge
}

// NewClusterUsageCollector creates and returns the reference to ClusterUsageCollector
// and internally defines each metric that display cluster stats.
func NewClusterUsageCollector(conn Conn) *ClusterUsageCollector {
	return &ClusterUsageCollector{
		conn: conn,

		GlobalCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_capacity_bytes",
			Help:      "Total capacity of the cluster",
		}),
		UsedCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_used_bytes",
			Help:      "Capacity of the cluster currently in use",
		}),
		AvailableCapacity: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_available_bytes",
			Help:      "Available space within the cluster",
		}),
		Objects: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: cephNamespace,
			Name:      "cluster_objects",
			Help:      "No. of rados objects within the cluster",
		}),
	}
}

func (c *ClusterUsageCollector) metricsList() []prometheus.Metric {
	return []prometheus.Metric{
		c.GlobalCapacity,
		c.UsedCapacity,
		c.AvailableCapacity,
		c.Objects,
	}
}

type cephClusterStats struct {
	Stats struct {
		TotalBytes      json.Number `json:"total_bytes"`
		TotalUsedBytes  json.Number `json:"total_used_bytes"`
		TotalAvailBytes json.Number `json:"total_avail_bytes"`
		TotalObjects    json.Number `json:"total_objects"`
	} `json:"stats"`
}

func (c *ClusterUsageCollector) collect() error {
	cmd := c.cephUsageCommand()
	buf, _, err := c.conn.MonCommand(cmd)
	if err != nil {
		return err
	}

	stats := &cephClusterStats{}
	if err := json.Unmarshal(buf, stats); err != nil {
		return err
	}

	var totBytes, usedBytes, availBytes, totObjects float64

	totBytes, err = stats.Stats.TotalBytes.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract total bytes:", err)
	}

	usedBytes, err = stats.Stats.TotalUsedBytes.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract used bytes:", err)
	}

	availBytes, err = stats.Stats.TotalAvailBytes.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract available bytes:", err)
	}

	totObjects, err = stats.Stats.TotalObjects.Float64()
	if err != nil {
		log.Println("[ERROR] cannot extract total objects:", err)
	}

	c.GlobalCapacity.Set(totBytes)
	c.UsedCapacity.Set(usedBytes)
	c.AvailableCapacity.Set(availBytes)
	c.Objects.Set(totObjects)

	return nil
}

func (c *ClusterUsageCollector) cephUsageCommand() []byte {
	cmd, err := json.Marshal(map[string]interface{}{
		"prefix": "df",
		"detail": "detail",
		"format": "json",
	})
	if err != nil {
		// panic! because ideally in no world this hard-coded input
		// should fail.
		panic(err)
	}
	return cmd
}

// Describe sends the descriptors of each metric over to the provided channel.
// The corresponding metric values are sent separately.
func (c *ClusterUsageCollector) Describe(ch chan<- *prometheus.Desc) {
	for _, metric := range c.metricsList() {
		ch <- metric.Desc()
	}
}

// Collect sends the metric values for each metric pertaining to the global
// cluster usage over to the provided prometheus Metric channel.
func (c *ClusterUsageCollector) Collect(ch chan<- prometheus.Metric) {
	if err := c.collect(); err != nil {
		log.Println("[ERROR] failed collecting cluster usage metrics:", err)
		return
	}

	for _, metric := range c.metricsList() {
		ch <- metric
	}
}
<|endoftext|>


<commit_before>package asset

// DefaultImages are the defualt images bootkube components use.
var DefaultImages = ImageVersions{
	Etcd:            "quay.io/coreos/etcd:v3.3.12",
	Flannel:         "quay.io/coreos/flannel:v0.10.0-amd64",
	FlannelCNI:      "quay.io/coreos/flannel-cni:v0.3.0",
	Calico:          "quay.io/calico/node:v3.0.3",
	CalicoCNI:       "quay.io/calico/cni:v2.0.0",
	CoreDNS:         "k8s.gcr.io/coredns:1.2.6",
	Hyperkube:       "k8s.gcr.io/hyperkube:v1.16.2",
	PodCheckpointer: "quay.io/coreos/pod-checkpointer:83e25e5968391b9eb342042c435d1b3eeddb2be1",
}
<commit_msg>updates flannel<commit_after>package asset

// DefaultImages are the defualt images bootkube components use.
var DefaultImages = ImageVersions{
	Etcd:            "quay.io/coreos/etcd:v3.3.12",
	Flannel:         "quay.io/coreos/flannel:v0.11.0-amd64",
	FlannelCNI:      "quay.io/coreos/flannel-cni:v0.3.0",
	Calico:          "quay.io/calico/node:v3.0.3",
	CalicoCNI:       "quay.io/calico/cni:v2.0.0",
	CoreDNS:         "k8s.gcr.io/coredns:1.2.6",
	Hyperkube:       "k8s.gcr.io/hyperkube:v1.16.2",
	PodCheckpointer: "quay.io/coreos/pod-checkpointer:83e25e5968391b9eb342042c435d1b3eeddb2be1",
}
<|endoftext|>


<commit_before>package watchdog

import (
	"testing"
	"time"
)

func drainExecutions(execs *[]*Execution, execCh <- chan *Execution, done chan <- bool) {
	var tmp []*Execution
	for e := range execCh {
		tmp = append(*execs, e)
		*execs = tmp
	}
	done <- true
}

func drainStalls(stalls *[]*Stall, stallCh <- chan *Stall, done chan <- bool) {
	var tmp []*Stall
	for s := range stallCh {
		tmp = append(*stalls, s)
		*stalls = tmp
	}
	done <- true
}

// true if b happened within d of a
func within(a, b time.Time, delta time.Duration) bool {
	return a.Before(b) && b.Sub(a) < delta
}

func TestRun(t *testing.T) {
	count := 0
	freq := 10 * time.Millisecond
	expectedRuntime := (3 * time.Millisecond) // generous padding here
	task := &Task{freq, func(time.Time) error { count += 1; return nil }, time.Hour}
	start := time.Now()
	w := Watch(task)
	var executions []*Execution
	var stalls []*Stall
	done := make(chan bool)
	go drainExecutions(&executions, w.Executions(), done)
	go drainStalls(&stalls, w.Stalls(), done)
	<- time.After(25 * time.Millisecond)
	w.Stop()
	<- done
	<- done
	if count != 2 {
		t.Errorf("expected task to trigger 2 times; got %d", count)
	}
	if execCount := len(executions); execCount != 2 {
		t.Errorf("expected task to send 2 executions; got %d", execCount)
	}
	for i, exec := range executions {
		if exec.Task != task {
			t.Errorf("expected execution %d task to be %p; got %p", i, task, exec.Task)
		}
		padding := 1 * time.Millisecond
		stepDelay := time.Duration(i + 1) * freq
		if expected := start.Add(stepDelay); !within(expected, exec.StartedAt, padding) {
			t.Errorf("expected execution %d start to be within %v of schedule; got within %v",
				i, padding, exec.StartedAt.Sub(expected))
		}
		if exec.FinishedAt.Before(exec.StartedAt) {
			t.Errorf("expected execution %d end not to shear time; started at %v, finished at %v",
				i, exec.StartedAt, exec.FinishedAt)
		}
		if exec.FinishedAt.Sub(exec.StartedAt) > expectedRuntime {
			t.Errorf("expected execution %d end to run for no more than %v; got %v",
				i, expectedRuntime, exec.FinishedAt.Sub(exec.StartedAt))
		}
		if err := exec.Error; err != nil {
			t.Errorf("expected execution %d success; got err %v", i, err)
		}
	}
	if stallCount := len(stalls); stallCount != 0 {
		t.Errorf("expected task to send 0 stalls; got %d", stallCount)
	}
	<- time.After(20 * time.Millisecond)
	if count > 2 {
		t.Errorf("expected task to stop when watchdog stopped")
	}
}

// - doesn't send duplicate stall notifications
// - sn

// check
//  + simple execution
//  + multiple tasks
//  - stalls
//    - no duplicate stalls ?
//  - recovery
//  - stalls with multiple tasks
//  - recovery with multiple tasks
//  - multiple stalls with multiple tasks
//  - multiple recoveries with multiple tasks

<commit_msg>Redo tests as table-driven<commit_after>package watchdog

import (
	"testing"
	"time"
)

type taskInfo struct {
	Schedule time.Duration
	Timeout time.Duration
	Executions []execInfo
}

type execInfo struct {
	Error error
	Duration time.Duration
}

// check
//  + simple execution
//  + multiple tasks
//  - stalls
//    - no duplicate stalls ?
//  - recovery
//  - stalls with multiple tasks
//  - recovery with multiple tasks
//  - multiple stalls with multiple tasks
//  - multiple recoveries with multiple tasks

var workloads = []struct{
	WatchDuration time.Duration
	Tasks []taskInfo
}{
	{
		Tasks: []taskInfo{
			{
				Schedule: 10 * time.Millisecond,
				Timeout: 1 * time.Hour,
				Executions: []execInfo{
					{Error: nil, Duration: time.Millisecond},
					{Error: nil, Duration: time.Millisecond},
				},
			},
		},
	},
}

func drainExecutions(execs map[*Task][]*Execution, execCh <- chan *Execution, done chan <- bool) {
	for e := range execCh {
		execs[e.Task] = append(execs[e.Task], e)
	}
	done <- true
}

func drainStalls(stalls map[*Task][]*Stall, stallCh <- chan *Stall, done chan <- bool) {
	for s := range stallCh {
		stalls[s.Task] = append(stalls[s.Task], s)
	}
	done <- true
}

// true if b happened within d of a
func within(a, b time.Time, delta time.Duration) bool {
	return a.Before(b) && b.Sub(a) < delta
}

func TestScheduling(t *testing.T) {
	for i, workload := range workloads {
		execCounts := make(map[*Task]int)
		tasks := make([]*Task, len(workload.Tasks))
		taskMap := make(map[*Task]*taskInfo)
		for j, proto := range workload.Tasks {
			task := &Task{
				Schedule: proto.Schedule,
				Timeout: proto.Timeout,
			}
			// N.B.: Can't assign inline, since it
			// references task itself
			task.Command = func(t time.Time) error {
				execCount := execCounts[task]
				exec := proto.Executions[execCount]
				time.Sleep(exec.Duration)
				execCounts[task] += 1
				return exec.Error
			}

			tasks[j] = task
			taskMap[task] = &proto
		}
		execMap := make(map[*Task][]*Execution)
		stallMap := make(map[*Task][]*Stall)

		start := time.Now()
		w := Watch(tasks...)

		done := make(chan bool)
		go drainExecutions(execMap, w.Executions(), done)
		go drainStalls(stallMap, w.Stalls(), done)

		duration := workload.WatchDuration
		if duration == 0 {
			// Figure out duration if one was not provided
			for _, proto := range workload.Tasks {
				// pad each task by half its schedule
				currDuration := proto.Schedule / 2
				for _, exec := range proto.Executions {
					currDuration += exec.Duration
				}
				if currDuration > duration {
					duration = currDuration
				}
			}
		}
		<- time.After(duration)
		w.Stop()
		<- done
		<- done

		for task, actual := range execCounts {
			proto := taskMap[task]
			if expected := len(proto.Executions); expected != actual {
				t.Errorf("workload %d task %v: expected %d invocations; got %d",
					i, task, expected, actual)
			}
		}

		for task, execs := range execMap {
			proto := taskMap[task]
			if expected, actual := len(proto.Executions), len(execs); expected != actual {
				t.Errorf("workload %d task %v: expected %d executions; got %d",
					i, task, expected, actual)
			}
			for j, exec := range execs {
				if exec.Task != task {
					t.Errorf("workload %d task %v: expected execution %d task reference to match; got %v",
						i, task, j, exec.Task)
				}
				slack := 1 * time.Millisecond
				stepDelay := time.Duration(j + 1) * task.Schedule
				if expected := start.Add(stepDelay); !within(expected, exec.StartedAt, slack) {
					t.Errorf("workload %d task %v: expected execution %d start to be within %v of schedule; got within %v",
						i, task, j, slack, exec.StartedAt.Sub(expected))
				}
				if exec.FinishedAt.Before(exec.StartedAt) {
					t.Errorf("workload %d task %v: expected execution %d not to shear time; started at %v, finished at %v",
						i, task, j, exec.StartedAt, exec.FinishedAt)
				}

				if expected, duration := proto.Executions[j].Duration, exec.FinishedAt.Sub(exec.StartedAt); duration > expected + slack || duration < expected - slack {
					t.Errorf("workload %d task %v: expected execution %d to run for %v±%v; got %v",
						i, task, j, expected, slack, exec.FinishedAt.Sub(exec.StartedAt))
				}
				if expected := proto.Executions[j].Error; exec.Error != expected {
					t.Errorf("workload %d task %v: expected execution %d error to be %v; got %v",
						i, task, j, expected, exec.Error)
				}
			}
		}
		for task, stalls := range stallMap {
			proto := taskMap[task]
			expectedStalls := 0
			for _, exec := range proto.Executions {
				if exec.Duration > proto.Timeout {
					expectedStalls += 1
				}
			}
			if stallCount := len(stalls); stallCount != expectedStalls {
				t.Errorf("workload %d task %v: expected %v stalls; got %d",
					i, task, expectedStalls, stallCount)
			}
		}
		// Now let's sleep for another cycle to make sure we
		// don't have any stray goroutines that will cause
		// panics after task execution stops
		maxFreq := 0 * time.Millisecond
		for _, task := range workload.Tasks {
			if task.Schedule > maxFreq {
				maxFreq = task.Schedule
			}
		}
		<- time.After(maxFreq)
	}
}

<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ml

import (
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/utl"
)

// PlotterReg defines a plotter to plot results from regression models
type PlotterReg struct {

	// input
	data   *Data      // data
	params *ParamsReg // parameters
	model  Regression // model
	mapper DataMapper // mapper

	// constants
	YlineNpts int // number of points for ModelY()
	MgridNpts int // nubmer of poitns for meshgrid (for contours)

	// arguments
	ArgsYdata    *plt.A         // arguments for x-y data plots
	ArgsYmodel   *plt.A         // arguments for x-y model line
	ArgsYclasses map[int]*plt.A // maps y classes [0, 1, 2, ...] to plot arguments
	ArgsYbinary  map[int]*plt.A // maps y classes [0 or 1] to plot arguments
	ArgsCcost    *plt.A         // arguments for ContourCost
	ArgsCcostMdl *plt.A         // arguments for the model parameters in ContourCost
	ArgsCmodel   *plt.A         // arguments for ContourModel
}

// NewPlotterReg returns a new ploter
//   mapper -- data mapper [may be nil]
func NewPlotterReg(data *Data, params *ParamsReg, reg Regression, mapper DataMapper) (o *PlotterReg) {

	// input
	o = new(PlotterReg)
	o.data = data
	o.params = params
	o.model = reg
	o.mapper = mapper

	// constants
	o.YlineNpts = 21
	o.MgridNpts = 21

	// arguments
	o.ArgsYdata = &plt.A{C: plt.C(2, 0), M: plt.M(0, 0), Ls: "None", NoClip: true}
	o.ArgsYmodel = &plt.A{C: plt.C(0, 0), M: "None", Ls: "-", NoClip: true}
	o.ArgsYclasses = make(map[int]*plt.A)
	o.ArgsYbinary = map[int]*plt.A{
		0: &plt.A{C: plt.C(0, 0), M: "o", Ls: "None", NoClip: true},
		1: &plt.A{C: plt.C(2, 0), M: "*", Ls: "None", NoClip: true, Mec: plt.C(2, 0), Ms: 8},
	}
	nMaxClassesIni := 10
	for k := 0; k < nMaxClassesIni; k++ {
		o.ArgsYclasses[k] = &plt.A{C: plt.C(k, 0), M: plt.M(k, 0), NoClip: true}
	}
	o.ArgsCcost = &plt.A{}
	o.ArgsCcostMdl = &plt.A{C: "yellow", M: "o"}
	o.ArgsCmodel = &plt.A{Colors: []string{plt.C(1, 0)}, Levels: []float64{0.5}}
	return
}

// DataY plots data x[iFeature] versus data y values
func (o *PlotterReg) DataY(iFeature int) {
	u := o.data.X.GetCol(iFeature)
	v := o.data.Y
	plt.Plot(u, v, o.ArgsYdata)
	plt.HideTRborders()
	plt.Gll(io.Sf("$x_{%d}$", iFeature), "$y$", nil)
}

// ModelY plots model y values
func (o *PlotterReg) ModelY(iFeature int, xmin, xmax float64) {
	x := la.NewVector(o.data.Nfeatures)
	u := utl.LinSpace(xmin, xmax, o.YlineNpts)
	v := utl.GetMapped(u, func(xi float64) float64 {
		x[iFeature] = xi
		return o.model.Predict(x)
	})
	plt.Plot(u, v, o.ArgsYmodel)
}

// ContourCost plots a contour of Cost for many parameters values
//  iParam, jParam -- selected parameters [use -1 for bias]
func (o *PlotterReg) ContourCost(iParam, jParam int, pimin, pimax, pjmin, pjmax float64) {

	// create meshgrid
	o.params.Backup()
	U, V, W := utl.MeshGrid2dF(pimin, pimax, pjmin, pjmax, o.MgridNpts, o.MgridNpts, func(s, t float64) (w float64) {
		o.params.Restore(true)
		o.params.SetParam(iParam, s)
		o.params.SetParam(jParam, t)
		w = o.model.Cost()
		return
	})
	o.params.Restore(false)

	// plot contour
	plt.ContourF(U, V, W, o.ArgsCcost)

	// plot optimal solution
	o.params.Restore(true)
	plt.PlotOne(o.params.GetParam(iParam), o.params.GetParam(jParam), o.ArgsCcostMdl)

	// set labels
	stri := "$b$"
	strj := "$b$"
	if iParam >= 0 {
		stri = io.Sf("$\\theta_{%d}$", iParam)
	}
	if jParam >= 0 {
		strj = io.Sf("$\\theta_{%d}$", jParam)
	}
	plt.SetXlabel(stri, nil)
	plt.SetYlabel(strj, nil)
}

// for classification /////////////////////////////////////////////////////////////////////////////

// DataClass plots data classes; e.g. for classification
func (o *PlotterReg) DataClass(iFeature, jFeature int, binary bool) {
	argsmap := o.ArgsYclasses
	if binary {
		argsmap = o.ArgsYbinary
	}
	for iSample := 0; iSample < o.data.Nsamples; iSample++ {
		k := int(o.data.Y[iSample]) % len(argsmap)
		args := argsmap[k]
		ui := o.data.X.Get(iSample, iFeature)
		vi := o.data.X.Get(iSample, jFeature)
		plt.PlotOne(ui, vi, args)
	}
	plt.HideTRborders()
	plt.Gll(io.Sf("$x_{%d}$", iFeature), io.Sf("$y_{%d}$", jFeature), nil)
}

// ContourModel plots contour of model points; e.g. for classification
func (o *PlotterReg) ContourModel(iFeature, jFeature int, level float64, ximin, ximax, xjmin, xjmax float64) {

	// x vectors
	x := la.NewVector(o.data.Nfeatures) // TODO: set x with xmean
	var xRaw la.Vector
	if o.mapper != nil {
		xRaw = la.NewVector(o.mapper.NumOriginalFeatures())
	}

	// create meshgrid
	U, V, W := utl.MeshGrid2dF(ximin, ximax, xjmin, xjmax, o.MgridNpts, o.MgridNpts, func(s, t float64) (w float64) {
		if o.mapper == nil {
			x[iFeature] = s
			x[jFeature] = t
		} else {
			xRaw[iFeature] = s
			xRaw[jFeature] = t
			o.mapper.Map(x, xRaw)
		}
		w = o.model.Predict(x)
		return
	})

	// plot contour
	plt.ContourL(U, V, W, o.ArgsCmodel)
}
<commit_msg>Improve code with gofmt<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ml

import (
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/utl"
)

// PlotterReg defines a plotter to plot results from regression models
type PlotterReg struct {

	// input
	data   *Data      // data
	params *ParamsReg // parameters
	model  Regression // model
	mapper DataMapper // mapper

	// constants
	YlineNpts int // number of points for ModelY()
	MgridNpts int // nubmer of poitns for meshgrid (for contours)

	// arguments
	ArgsYdata    *plt.A         // arguments for x-y data plots
	ArgsYmodel   *plt.A         // arguments for x-y model line
	ArgsYclasses map[int]*plt.A // maps y classes [0, 1, 2, ...] to plot arguments
	ArgsYbinary  map[int]*plt.A // maps y classes [0 or 1] to plot arguments
	ArgsCcost    *plt.A         // arguments for ContourCost
	ArgsCcostMdl *plt.A         // arguments for the model parameters in ContourCost
	ArgsCmodel   *plt.A         // arguments for ContourModel
}

// NewPlotterReg returns a new ploter
//   mapper -- data mapper [may be nil]
func NewPlotterReg(data *Data, params *ParamsReg, reg Regression, mapper DataMapper) (o *PlotterReg) {

	// input
	o = new(PlotterReg)
	o.data = data
	o.params = params
	o.model = reg
	o.mapper = mapper

	// constants
	o.YlineNpts = 21
	o.MgridNpts = 21

	// arguments
	o.ArgsYdata = &plt.A{C: plt.C(2, 0), M: plt.M(0, 0), Ls: "None", NoClip: true}
	o.ArgsYmodel = &plt.A{C: plt.C(0, 0), M: "None", Ls: "-", NoClip: true}
	o.ArgsYclasses = make(map[int]*plt.A)
	o.ArgsYbinary = map[int]*plt.A{
		0: {C: plt.C(0, 0), M: "o", Ls: "None", NoClip: true},
		1: {C: plt.C(2, 0), M: "*", Ls: "None", NoClip: true, Mec: plt.C(2, 0), Ms: 8},
	}
	nMaxClassesIni := 10
	for k := 0; k < nMaxClassesIni; k++ {
		o.ArgsYclasses[k] = &plt.A{C: plt.C(k, 0), M: plt.M(k, 0), NoClip: true}
	}
	o.ArgsCcost = &plt.A{}
	o.ArgsCcostMdl = &plt.A{C: "yellow", M: "o"}
	o.ArgsCmodel = &plt.A{Colors: []string{plt.C(1, 0)}, Levels: []float64{0.5}}
	return
}

// DataY plots data x[iFeature] versus data y values
func (o *PlotterReg) DataY(iFeature int) {
	u := o.data.X.GetCol(iFeature)
	v := o.data.Y
	plt.Plot(u, v, o.ArgsYdata)
	plt.HideTRborders()
	plt.Gll(io.Sf("$x_{%d}$", iFeature), "$y$", nil)
}

// ModelY plots model y values
func (o *PlotterReg) ModelY(iFeature int, xmin, xmax float64) {
	x := la.NewVector(o.data.Nfeatures)
	u := utl.LinSpace(xmin, xmax, o.YlineNpts)
	v := utl.GetMapped(u, func(xi float64) float64 {
		x[iFeature] = xi
		return o.model.Predict(x)
	})
	plt.Plot(u, v, o.ArgsYmodel)
}

// ContourCost plots a contour of Cost for many parameters values
//  iParam, jParam -- selected parameters [use -1 for bias]
func (o *PlotterReg) ContourCost(iParam, jParam int, pimin, pimax, pjmin, pjmax float64) {

	// create meshgrid
	o.params.Backup()
	U, V, W := utl.MeshGrid2dF(pimin, pimax, pjmin, pjmax, o.MgridNpts, o.MgridNpts, func(s, t float64) (w float64) {
		o.params.Restore(true)
		o.params.SetParam(iParam, s)
		o.params.SetParam(jParam, t)
		w = o.model.Cost()
		return
	})
	o.params.Restore(false)

	// plot contour
	plt.ContourF(U, V, W, o.ArgsCcost)

	// plot optimal solution
	o.params.Restore(true)
	plt.PlotOne(o.params.GetParam(iParam), o.params.GetParam(jParam), o.ArgsCcostMdl)

	// set labels
	stri := "$b$"
	strj := "$b$"
	if iParam >= 0 {
		stri = io.Sf("$\\theta_{%d}$", iParam)
	}
	if jParam >= 0 {
		strj = io.Sf("$\\theta_{%d}$", jParam)
	}
	plt.SetXlabel(stri, nil)
	plt.SetYlabel(strj, nil)
}

// for classification /////////////////////////////////////////////////////////////////////////////

// DataClass plots data classes; e.g. for classification
func (o *PlotterReg) DataClass(iFeature, jFeature int, binary bool) {
	argsmap := o.ArgsYclasses
	if binary {
		argsmap = o.ArgsYbinary
	}
	for iSample := 0; iSample < o.data.Nsamples; iSample++ {
		k := int(o.data.Y[iSample]) % len(argsmap)
		args := argsmap[k]
		ui := o.data.X.Get(iSample, iFeature)
		vi := o.data.X.Get(iSample, jFeature)
		plt.PlotOne(ui, vi, args)
	}
	plt.HideTRborders()
	plt.Gll(io.Sf("$x_{%d}$", iFeature), io.Sf("$y_{%d}$", jFeature), nil)
}

// ContourModel plots contour of model points; e.g. for classification
func (o *PlotterReg) ContourModel(iFeature, jFeature int, level float64, ximin, ximax, xjmin, xjmax float64) {

	// x vectors
	x := la.NewVector(o.data.Nfeatures) // TODO: set x with xmean
	var xRaw la.Vector
	if o.mapper != nil {
		xRaw = la.NewVector(o.mapper.NumOriginalFeatures())
	}

	// create meshgrid
	U, V, W := utl.MeshGrid2dF(ximin, ximax, xjmin, xjmax, o.MgridNpts, o.MgridNpts, func(s, t float64) (w float64) {
		if o.mapper == nil {
			x[iFeature] = s
			x[jFeature] = t
		} else {
			xRaw[iFeature] = s
			xRaw[jFeature] = t
			o.mapper.Map(x, xRaw)
		}
		w = o.model.Predict(x)
		return
	})

	// plot contour
	plt.ContourL(U, V, W, o.ArgsCmodel)
}
<|endoftext|>