<commit_before>package logdb

import (
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/errwrap"
)

// A chunk is one memory-mapped file.
type chunk struct {
	path string

	bytes []byte
	mmapf *os.File

	// One past the ending addresses of entries in the 'bytes'
	// slice.
	//
	// This choice is because starting addresses can always be
	// calculated from ending addresses, as the first entry starts
	// at offset 0 (and there are no gaps). Ending addresses
	// cannot be calculated from starting addresses, unless the
	// ending address of the final entry is stored as well.
	ends []int32

	oldest uint64

	next uint64

	newFrom int
	delete  bool
}

// Delete the files associated with a chunk.
func (c *chunk) closeAndRemove() error {
	if err := closeAndRemove(c.mmapf); err != nil {
		return err
	}
	return os.Remove(c.metaFilePath())
}

const (
	chunkPrefix      = "chunk"
	metaSuffix       = "meta"
	sep              = "_"
	initialChunkFile = chunkPrefix + sep + "0" + sep + "1"
)

// Get the meta file path associated with a chunk data file path.
func metaFilePath(dataFilePath string) string {
	return dataFilePath + sep + metaSuffix
}

// Get the meta file path associated with a chunk.
func (c *chunk) metaFilePath() string {
	return metaFilePath(c.path)
}

// Check if a file basename is a chunk data file.
//
// A valid chunk filename consists of the chunkPrefix followed by one
// or more digits, with no leading zeroes.
func isBasenameChunkDataFile(basename string) bool {
	bits := strings.Split(basename, chunkPrefix+sep)
	// In the form chunkPrefix[.+]
	if len(bits) != 2 || len(bits[0]) != 0 || len(bits[1]) == 0 {
		return false
	}

	bits = strings.Split(bits[1], sep)

	if len(bits) != 2 {
		return false
	}

	// Must be [0-9]+_[0-9]+]
	if len(bits[0]) == 0 || len(bits[1]) == 0 {
		return false
	}
	first, err := strconv.ParseUint(bits[0], 10, 0)
	if err != nil {
		return false
	}
	if _, err := strconv.ParseUint(bits[1], 10, 0); err != nil {
		return false
	}

	// Leading zeroes are disallowed.
	if bits[0][0] == '0' {
		return first == 0
	}

	return true
}

// Given a chunk, get the filename of the next chunk.
//
// This function panics if the chunk path is invalid. This should
// never happen unless openChunkSliceDB or isChunkDataFile is broken.
func (c *chunk) nextDataFileName(oldest uint64) string {
	// If there are directories in the path, correctly identify the basename.
	firstSep := chunkPrefix + sep
	if strings.ContainsRune(c.path, '/') {
		firstSep = "/" + firstSep
	}

	bits := strings.Split(c.path, firstSep)
	if len(bits) < 2 {
		panic("malformed chunk file name: " + c.path)
	}

	bits = strings.Split(bits[len(bits)-1], sep)
	if len(bits) != 2 {
		panic("malformed chunk file name: " + c.path)
	}

	num, err := strconv.ParseUint(bits[0], 10, 0)
	if err != nil {
		panic("malformed chunk file name: " + c.path)
	}

	return fmt.Sprintf("%s%s%v%s%v", chunkPrefix, sep, num+1, sep, oldest)
}

// Create the files for a new chunk. As an empty chunk is not allowed,
// it is assumed that an entry will be immediately written.
func createChunkFiles(dataFilePath string, chunkSize uint32, oldest uint64) error {
	// Create the chunk files.
	if err := createFile(dataFilePath, chunkSize); err != nil {
		return err
	}
	file, err := os.OpenFile(metaFilePath(dataFilePath), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	file.Close()
	return err
}

// Open a chunk file
func openChunkFile(basedir string, fi os.FileInfo, priorChunk *chunk, chunkSize uint32) (chunk, error) {
	chunk := chunk{path: basedir + "/" + fi.Name()}
	// Get the oldest ID from the file name
	if !isBasenameChunkDataFile(fi.Name()) {
		return chunk, fmt.Errorf("invalid chunk data file name: %s", fi.Name())
	}
	// This does no validation because isBasenameChunkDataFile took care of that.
	nameBits := strings.Split(fi.Name(), sep)
	oldnum, _ := strconv.ParseUint(nameBits[2], 10, 0)
	chunk.oldest = uint64(oldnum)

	// mmap the data file
	mmapf, bytes, err := mmap(chunk.path)
	if err != nil {
		return chunk, &ReadError{err}
	}
	if uint32(len(bytes)) != chunkSize {
		return chunk, &FormatError{
			FilePath: chunk.path,
			Err:      fmt.Errorf("incorrect file size (expected %v got %v)", chunkSize, uint32(len(bytes))),
		}
	}
	chunk.bytes = bytes
	chunk.mmapf = mmapf

	// read the ending address metadata
	mfile, err := os.Open((&chunk).metaFilePath())
	if err != nil {
		return chunk, &ReadError{err}
	}
	defer mfile.Close()
	ends, err := readMetadata(mfile)
	if err != nil {
		return chunk, &FormatError{
			FilePath: (&chunk).metaFilePath(),
			Err:      err,
		}
	}
	chunk.ends = ends

	// Chunk oldest/next IDs must match: there can be no gaps!
	if priorChunk != nil && chunk.oldest != priorChunk.next {
		return chunk, &FormatError{
			FilePath: (&chunk).metaFilePath(),
			Err:      fmt.Errorf("discontinuity in entry IDs (expected %v got %v)", priorChunk.next, chunk.oldest),
		}
	}

	chunk.next = chunk.oldest + uint64(len(chunk.ends))
	return chunk, nil
}

// Write a chunk to disk.
func (c *chunk) sync() error {
	// To ensure ACID, sync the data first and only then the
	// metadata. This means that if there is a failure between the
	// two syncs, even if the newly-written data is corrupt, there
	// will be no metadata referring to it, and so it will be
	// invisible to the database when next opened.
	if err := fsync(c.mmapf); err != nil {
		return err
	}

	// Write the new end points.
	for i := c.newFrom; i < len(c.ends); i++ {
		if err := appendFile(c.metaFilePath(), int32(i)); err != nil {
			return err
		}
		if err := appendFile(c.metaFilePath(), c.ends[i]); err != nil {
			return err
		}
	}
	c.newFrom = len(c.ends)

	return nil
}

// Read a chunk metadata file.
//
// metadata is in the format [index int32][end int32], it ends at EOF.
// If the indices go backwards, that means entries have been rolled back
func readMetadata(r io.Reader) ([]int32, error) {
	var ends []int32
	var idx, this int32

	for {
		// Read the index into the ends slice.
		if err := binary.Read(r, binary.LittleEndian, &idx); err != nil {
			if err == io.EOF {
				break
			}
			return ends, errwrap.Wrapf("unexpected error reading chunk metadata: {{err}}", err)
		}
		if idx > int32(len(ends)) {
			return ends, fmt.Errorf("entry index too large (expected <=%v, got %v)", len(ends), idx)
		}

		// Read the offset. If this fails, it means that syncing failed between the two writes.
		if err := binary.Read(r, binary.LittleEndian, &this); err != nil {
			return ends, errwrap.Wrapf("unexpected error reading chunk metadata: {{err}}", err)
		}

		// Check the offset is geq the prior offset.
		if idx > 0 && this < ends[idx-1] {
			return ends, fmt.Errorf("entry ending positions are not monotonically increasing (prior %v got %v)", ends[idx-1], this)
		}

		// Pop entries from the "ends" slice so that the current index is one past the end, and append it.
		ends = ends[0:idx]
		ends = append(ends, this)
	}

	return ends, nil
}
<commit_msg>Use a single file append for metadata.<commit_after>package logdb

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/errwrap"
)

// A chunk is one memory-mapped file.
type chunk struct {
	path string

	bytes []byte
	mmapf *os.File

	// One past the ending addresses of entries in the 'bytes'
	// slice.
	//
	// This choice is because starting addresses can always be
	// calculated from ending addresses, as the first entry starts
	// at offset 0 (and there are no gaps). Ending addresses
	// cannot be calculated from starting addresses, unless the
	// ending address of the final entry is stored as well.
	ends []int32

	oldest uint64

	next uint64

	newFrom int
	delete  bool
}

// Delete the files associated with a chunk.
func (c *chunk) closeAndRemove() error {
	if err := closeAndRemove(c.mmapf); err != nil {
		return err
	}
	return os.Remove(c.metaFilePath())
}

const (
	chunkPrefix      = "chunk"
	metaSuffix       = "meta"
	sep              = "_"
	initialChunkFile = chunkPrefix + sep + "0" + sep + "1"
)

// Get the meta file path associated with a chunk data file path.
func metaFilePath(dataFilePath string) string {
	return dataFilePath + sep + metaSuffix
}

// Get the meta file path associated with a chunk.
func (c *chunk) metaFilePath() string {
	return metaFilePath(c.path)
}

// Check if a file basename is a chunk data file.
//
// A valid chunk filename consists of the chunkPrefix followed by one
// or more digits, with no leading zeroes.
func isBasenameChunkDataFile(basename string) bool {
	bits := strings.Split(basename, chunkPrefix+sep)
	// In the form chunkPrefix[.+]
	if len(bits) != 2 || len(bits[0]) != 0 || len(bits[1]) == 0 {
		return false
	}

	bits = strings.Split(bits[1], sep)

	if len(bits) != 2 {
		return false
	}

	// Must be [0-9]+_[0-9]+]
	if len(bits[0]) == 0 || len(bits[1]) == 0 {
		return false
	}
	first, err := strconv.ParseUint(bits[0], 10, 0)
	if err != nil {
		return false
	}
	if _, err := strconv.ParseUint(bits[1], 10, 0); err != nil {
		return false
	}

	// Leading zeroes are disallowed.
	if bits[0][0] == '0' {
		return first == 0
	}

	return true
}

// Given a chunk, get the filename of the next chunk.
//
// This function panics if the chunk path is invalid. This should
// never happen unless openChunkSliceDB or isChunkDataFile is broken.
func (c *chunk) nextDataFileName(oldest uint64) string {
	// If there are directories in the path, correctly identify the basename.
	firstSep := chunkPrefix + sep
	if strings.ContainsRune(c.path, '/') {
		firstSep = "/" + firstSep
	}

	bits := strings.Split(c.path, firstSep)
	if len(bits) < 2 {
		panic("malformed chunk file name: " + c.path)
	}

	bits = strings.Split(bits[len(bits)-1], sep)
	if len(bits) != 2 {
		panic("malformed chunk file name: " + c.path)
	}

	num, err := strconv.ParseUint(bits[0], 10, 0)
	if err != nil {
		panic("malformed chunk file name: " + c.path)
	}

	return fmt.Sprintf("%s%s%v%s%v", chunkPrefix, sep, num+1, sep, oldest)
}

// Create the files for a new chunk. As an empty chunk is not allowed,
// it is assumed that an entry will be immediately written.
func createChunkFiles(dataFilePath string, chunkSize uint32, oldest uint64) error {
	// Create the chunk files.
	if err := createFile(dataFilePath, chunkSize); err != nil {
		return err
	}
	file, err := os.OpenFile(metaFilePath(dataFilePath), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	file.Close()
	return err
}

// Open a chunk file
func openChunkFile(basedir string, fi os.FileInfo, priorChunk *chunk, chunkSize uint32) (chunk, error) {
	chunk := chunk{path: basedir + "/" + fi.Name()}
	// Get the oldest ID from the file name
	if !isBasenameChunkDataFile(fi.Name()) {
		return chunk, fmt.Errorf("invalid chunk data file name: %s", fi.Name())
	}
	// This does no validation because isBasenameChunkDataFile took care of that.
	nameBits := strings.Split(fi.Name(), sep)
	oldnum, _ := strconv.ParseUint(nameBits[2], 10, 0)
	chunk.oldest = uint64(oldnum)

	// mmap the data file
	mmapf, bytes, err := mmap(chunk.path)
	if err != nil {
		return chunk, &ReadError{err}
	}
	if uint32(len(bytes)) != chunkSize {
		return chunk, &FormatError{
			FilePath: chunk.path,
			Err:      fmt.Errorf("incorrect file size (expected %v got %v)", chunkSize, uint32(len(bytes))),
		}
	}
	chunk.bytes = bytes
	chunk.mmapf = mmapf

	// read the ending address metadata
	mfile, err := os.Open((&chunk).metaFilePath())
	if err != nil {
		return chunk, &ReadError{err}
	}
	defer mfile.Close()
	ends, err := readMetadata(mfile)
	if err != nil {
		return chunk, &FormatError{
			FilePath: (&chunk).metaFilePath(),
			Err:      err,
		}
	}
	chunk.ends = ends

	// Chunk oldest/next IDs must match: there can be no gaps!
	if priorChunk != nil && chunk.oldest != priorChunk.next {
		return chunk, &FormatError{
			FilePath: (&chunk).metaFilePath(),
			Err:      fmt.Errorf("discontinuity in entry IDs (expected %v got %v)", priorChunk.next, chunk.oldest),
		}
	}

	chunk.next = chunk.oldest + uint64(len(chunk.ends))
	return chunk, nil
}

// Write a chunk to disk.
func (c *chunk) sync() error {
	// To ensure ACID, sync the data first and only then the metadata. This means that if there is a failure
	// between the two syncs, even if the newly-written data is corrupt, there will be no metadata referring
	// to it, and so it will be invisible to the database when next opened.
	if err := fsync(c.mmapf); err != nil {
		return err
	}

	// Construct the metadata as a buffer. This is done rather than appending to the output file directly
	// because individual "write" syscalls with a small enough buffer (which this will be for any reasonable
	// syncing period) are atomic. Multiple appends would have the possibility of failure in the middle.
	buf := new(bytes.Buffer)
	for i := c.newFrom; i < len(c.ends); i++ {
		if err := binary.Write(buf, binary.LittleEndian, int32(i)); err != nil {
			return err
		}
		if err := binary.Write(buf, binary.LittleEndian, c.ends[i]); err != nil {
			return err
		}
	}

	// Write the new end points.
	if err := appendFile(c.metaFilePath(), buf.Bytes()); err != nil {
		return err
	}
	c.newFrom = len(c.ends)

	return nil
}

// Read a chunk metadata file.
//
// metadata is in the format [index int32][end int32], it ends at EOF.
// If the indices go backwards, that means entries have been rolled back
func readMetadata(r io.Reader) ([]int32, error) {
	var ends []int32
	var idx, this int32

	for {
		// Read the index into the ends slice.
		if err := binary.Read(r, binary.LittleEndian, &idx); err != nil {
			if err == io.EOF {
				break
			}
			return ends, errwrap.Wrapf("unexpected error reading chunk metadata: {{err}}", err)
		}
		if idx > int32(len(ends)) {
			return ends, fmt.Errorf("entry index too large (expected <=%v, got %v)", len(ends), idx)
		}

		// Read the offset. If this fails, it means that syncing failed between the two writes.
		if err := binary.Read(r, binary.LittleEndian, &this); err != nil {
			return ends, errwrap.Wrapf("unexpected error reading chunk metadata: {{err}}", err)
		}

		// Check the offset is geq the prior offset.
		if idx > 0 && this < ends[idx-1] {
			return ends, fmt.Errorf("entry ending positions are not monotonically increasing (prior %v got %v)", ends[idx-1], this)
		}

		// Pop entries from the "ends" slice so that the current index is one past the end, and append it.
		ends = ends[0:idx]
		ends = append(ends, this)
	}

	return ends, nil
}
<|endoftext|>


<commit_before>// Analyse churn rate of a set of consensuses.
package main

import (
	"fmt"
	"log"
	"math"
	"sync"

	tor "git.torproject.org/user/phw/zoossh.git"
)

const (
	movingAvgWinSize = 24
)

// Churn represents a churn value.
type Churn float64

// MovingAverage represents a simple moving average.
type MovingAverage struct {
	WindowIndex int
	WindowSize  int
	WindowFill  int
	Window      []Churn
}

// NewMovingAverage allocates and returns a new moving average struct.
func NewMovingAverage(windowSize int) *MovingAverage {

	return &MovingAverage{WindowIndex: 0, WindowSize: movingAvgWinSize, Window: make([]Churn, movingAvgWinSize)}
}

// CalcAvg determines and returns the mean of the moving average window.
func (ma *MovingAverage) CalcAvg() Churn {

	var total Churn
	for i := 0; i < ma.WindowSize; i++ {
		total += ma.Window[i]
	}
	return total / Churn(ma.WindowSize)
}

// AddValue returns a churn value to the moving average window.
func (ma *MovingAverage) AddValue(val Churn) {

	if ma.WindowFill < ma.WindowSize {
		ma.WindowFill++
	}
	ma.Window[ma.WindowIndex] = val
	ma.WindowIndex = (ma.WindowIndex + 1) % ma.WindowSize
}

// IsWindowFull returns true if the moving average's window is full.
func (ma *MovingAverage) IsWindowFull() bool {

	return ma.WindowFill == ma.WindowSize
}

// dumpChurnRelays dumps the given relays to stdout for manual analysis.
func dumpChurnRelays(goneRelays, newRelays *tor.Consensus) {

	// Sort relays by nickname.
	nickname := func(relay1, relay2 tor.GetStatus) bool {
		return relay1().Nickname < relay2().Nickname
	}

	gone := goneRelays.ToSlice()
	By(nickname).Sort(gone)
	fresh := newRelays.ToSlice()
	By(nickname).Sort(fresh)

	for _, getStatus := range gone {
		log.Printf("- <https://atlas.torproject.org/#details/%s> %s\n",
			getStatus().Fingerprint, getStatus().Nickname)
	}
	for _, getStatus := range fresh {
		log.Printf("+ <https://atlas.torproject.org/#details/%s> %s\n",
			getStatus().Fingerprint, getStatus().Nickname)
	}
}

// determineChurn determines and returns the churn rate of the two given
// subsequent consensuses.  For the churn rate, we adapt the formula shown in
// Section 2.1 of: <http://www.cs.berkeley.edu/~istoica/papers/2006/churn.pdf>.
func determineChurn(prevConsensus, newConsensus *tor.Consensus) (Churn, Churn) {

	goneRelays := prevConsensus.Subtract(newConsensus)
	newRelays := newConsensus.Subtract(prevConsensus)

	max := math.Max(float64(prevConsensus.Length()), float64(newConsensus.Length()))
	newChurn := (float64(newRelays.Length()) / max)
	goneChurn := (float64(goneRelays.Length()) / max)

	return Churn(newChurn), Churn(goneChurn)
}

// AnalyseChurn determines the churn rates of a set of consecutive consensuses.
// If the churn rate exceeds the given threshold, all new and disappeared
// relays are dumped to stdout.
func AnalyseChurn(channel chan tor.ObjectSet, params *CmdLineParams, group *sync.WaitGroup) {

	defer group.Done()

	var newConsensus, prevConsensus *tor.Consensus

	log.Printf("Threshold for churn analysis is %.5f.\n", params.Threshold)
	fmt.Println("date,newchurn,gonechurn,avgnewchurn,avggonechurn")

	movingAvgNew := NewMovingAverage(movingAvgWinSize)
	movingAvgGone := NewMovingAverage(movingAvgWinSize)

	// Every loop iteration processes one consensus.  We compare consensus t
	// with consensus t - 1.
	for objects := range channel {

		switch obj := objects.(type) {
		case *tor.Consensus:
			newConsensus = obj
		default:
			log.Fatalln("Only router status files are supported for churn analysis.")
		}

		if prevConsensus == nil {
			prevConsensus = newConsensus
			continue
		}
		newChurn, goneChurn := determineChurn(prevConsensus, newConsensus)

		// Update moving averages.
		movingAvgNew.AddValue(newChurn)
		movingAvgGone.AddValue(goneChurn)

		newAvg := movingAvgNew.CalcAvg()
		goneAvg := movingAvgGone.CalcAvg()

		// Print raw numbers to stdout for later analysis.
		if movingAvgNew.IsWindowFull() {
			timeStr := newConsensus.ValidAfter.Format("2006-01-02T15:04:05Z")
			fmt.Printf("%s,%.5f,%.5f,%.5f,%.5f\n", timeStr, newChurn, goneChurn, newAvg, goneAvg)
		}

		if (newChurn >= Churn(params.Threshold)) || (goneChurn >= Churn(params.Threshold)) {
			goneRelays := prevConsensus.Subtract(newConsensus)
			newRelays := newConsensus.Subtract(prevConsensus)
			dumpChurnRelays(goneRelays, newRelays)
		}

		prevConsensus = newConsensus
	}
}
<commit_msg>Ignore missing consensuses.<commit_after>// Analyse churn rate of a set of consensuses.
package main

import (
	"fmt"
	"log"
	"math"
	"sync"
	"time"

	tor "git.torproject.org/user/phw/zoossh.git"
)

const (
	movingAvgWinSize = 24
)

// Churn represents a churn value.
type Churn float64

// MovingAverage represents a simple moving average.
type MovingAverage struct {
	WindowIndex int
	WindowSize  int
	WindowFill  int
	Window      []Churn
}

// NewMovingAverage allocates and returns a new moving average struct.
func NewMovingAverage(windowSize int) *MovingAverage {

	return &MovingAverage{WindowIndex: 0, WindowSize: movingAvgWinSize, Window: make([]Churn, movingAvgWinSize)}
}

// CalcAvg determines and returns the mean of the moving average window.
func (ma *MovingAverage) CalcAvg() Churn {

	var total Churn
	for i := 0; i < ma.WindowSize; i++ {
		total += ma.Window[i]
	}
	return total / Churn(ma.WindowSize)
}

// AddValue returns a churn value to the moving average window.
func (ma *MovingAverage) AddValue(val Churn) {

	if ma.WindowFill < ma.WindowSize {
		ma.WindowFill++
	}
	ma.Window[ma.WindowIndex] = val
	ma.WindowIndex = (ma.WindowIndex + 1) % ma.WindowSize
}

// IsWindowFull returns true if the moving average's window is full.
func (ma *MovingAverage) IsWindowFull() bool {

	return ma.WindowFill == ma.WindowSize
}

// dumpChurnRelays dumps the given relays to stdout for manual analysis.
func dumpChurnRelays(goneRelays, newRelays *tor.Consensus) {

	// Sort relays by nickname.
	nickname := func(relay1, relay2 tor.GetStatus) bool {
		return relay1().Nickname < relay2().Nickname
	}

	gone := goneRelays.ToSlice()
	By(nickname).Sort(gone)
	fresh := newRelays.ToSlice()
	By(nickname).Sort(fresh)

	for _, getStatus := range gone {
		log.Printf("- <https://atlas.torproject.org/#details/%s> %s\n",
			getStatus().Fingerprint, getStatus().Nickname)
	}
	for _, getStatus := range fresh {
		log.Printf("+ <https://atlas.torproject.org/#details/%s> %s\n",
			getStatus().Fingerprint, getStatus().Nickname)
	}
}

// determineChurn determines and returns the churn rate of the two given
// subsequent consensuses.  For the churn rate, we adapt the formula shown in
// Section 2.1 of: <http://www.cs.berkeley.edu/~istoica/papers/2006/churn.pdf>.
func determineChurn(prevConsensus, newConsensus *tor.Consensus) (Churn, Churn) {

	goneRelays := prevConsensus.Subtract(newConsensus)
	newRelays := newConsensus.Subtract(prevConsensus)

	max := math.Max(float64(prevConsensus.Length()), float64(newConsensus.Length()))
	newChurn := (float64(newRelays.Length()) / max)
	goneChurn := (float64(goneRelays.Length()) / max)

	return Churn(newChurn), Churn(goneChurn)
}

// AnalyseChurn determines the churn rates of a set of consecutive consensuses.
// If the churn rate exceeds the given threshold, all new and disappeared
// relays are dumped to stdout.
func AnalyseChurn(channel chan tor.ObjectSet, params *CmdLineParams, group *sync.WaitGroup) {

	defer group.Done()

	var newConsensus, prevConsensus *tor.Consensus

	log.Printf("Threshold for churn analysis is %.5f.\n", params.Threshold)
	fmt.Println("date,newchurn,gonechurn,avgnewchurn,avggonechurn")

	movingAvgNew := NewMovingAverage(movingAvgWinSize)
	movingAvgGone := NewMovingAverage(movingAvgWinSize)

	// Every loop iteration processes one consensus.  We compare consensus t
	// with consensus t - 1.
	for objects := range channel {

		switch obj := objects.(type) {
		case *tor.Consensus:
			newConsensus = obj
		default:
			log.Fatalln("Only router status files are supported for churn analysis.")
		}

		if prevConsensus == nil {
			prevConsensus = newConsensus
			continue
		}

		// Are we missing consensuses?
		if prevConsensus.ValidAfter.Add(time.Hour) != newConsensus.ValidAfter {
			log.Printf("Missing consensuses between %s and %s.\n",
				prevConsensus.ValidAfter.Format(time.RFC3339),
				newConsensus.ValidAfter.Format(time.RFC3339))
			prevConsensus = newConsensus
			continue
		}

		newChurn, goneChurn := determineChurn(prevConsensus, newConsensus)

		// Update moving averages.
		movingAvgNew.AddValue(newChurn)
		movingAvgGone.AddValue(goneChurn)

		newAvg := movingAvgNew.CalcAvg()
		goneAvg := movingAvgGone.CalcAvg()

		// Set averaged values to -1 if we our window isn't full yet.
		if !movingAvgNew.IsWindowFull() {
			newAvg = -1
			goneAvg = -1
		}
		timeStr := newConsensus.ValidAfter.Format("2006-01-02T15:04:05Z")
		fmt.Printf("%s,%.5f,%.5f,%.5f,%.5f\n", timeStr, newChurn, goneChurn, newAvg, goneAvg)

		if (newChurn >= Churn(params.Threshold)) || (goneChurn >= Churn(params.Threshold)) {
			goneRelays := prevConsensus.Subtract(newConsensus)
			newRelays := newConsensus.Subtract(prevConsensus)
			dumpChurnRelays(goneRelays, newRelays)
		}

		prevConsensus = newConsensus
	}
}
<|endoftext|>


<commit_before>// Package cielo is used to communicate with Cielo API
package cielo

import (
	"bytes"
	"encoding/json"
	"net/http"

	"github.com/satori/go.uuid"
)

// CieloEcommerce is the interface used to communicate with Cielo v3 API
type Ecommerce interface {
	CreateSale(sale Sale) (Sale, error)
}

type cieloEcommerce struct {
	merchant Merchant
	env      environment
}

func (c *cieloEcommerce) CreateSale(s Sale) (sale Sale, err error) {
	body, err := json.Marshal(s)
	if err != nil {
		return
	}
	req, err := http.NewRequest(http.MethodPost, c.env.APIURL+"/1/sales", bytes.NewBuffer(body))
	if err != nil {
		return
	}
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Accept-Encoding", "gzip")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("User-Agent", "CieloEcommerce/3.0 Golang client")
	req.Header.Add("MerchantId", c.merchant.ID)
	req.Header.Add("MerchantKey", c.merchant.Key)
	req.Header.Add("RequestId", uuid.NewV4().String())

	response, err := http.DefaultClient.Do(req)

	if err != nil {
		return
	}

	err = json.NewDecoder(response.Body).Decode(&sale)
	return
}

// NewEcommerce create a Cielo v3 API client
func NewEcommerce(m Merchant, env environment) Ecommerce {
	return &cieloEcommerce{
		merchant: m,
		env:      env,
	}
}
<commit_msg>Fix doc<commit_after>// Package cielo is used to communicate with Cielo API
package cielo

import (
	"bytes"
	"encoding/json"
	"net/http"

	"github.com/satori/go.uuid"
)

// Ecommerce is the interface used to communicate with Cielo v3 API
type Ecommerce interface {
	CreateSale(sale Sale) (Sale, error)
}

type cieloEcommerce struct {
	merchant Merchant
	env      environment
}

func (c *cieloEcommerce) CreateSale(s Sale) (sale Sale, err error) {
	body, err := json.Marshal(s)
	if err != nil {
		return
	}
	req, err := http.NewRequest(http.MethodPost, c.env.APIURL+"/1/sales", bytes.NewBuffer(body))
	if err != nil {
		return
	}
	req.Header.Add("Accept", "application/json")
	req.Header.Add("Accept-Encoding", "gzip")
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("User-Agent", "CieloEcommerce/3.0 Golang client")
	req.Header.Add("MerchantId", c.merchant.ID)
	req.Header.Add("MerchantKey", c.merchant.Key)
	req.Header.Add("RequestId", uuid.NewV4().String())

	response, err := http.DefaultClient.Do(req)

	if err != nil {
		return
	}

	err = json.NewDecoder(response.Body).Decode(&sale)
	return
}

// NewEcommerce create a Cielo v3 API client
func NewEcommerce(m Merchant, env environment) Ecommerce {
	return &cieloEcommerce{
		merchant: m,
		env:      env,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
)

type ConstantPoolItem interface {
	isConstantPoolItem()
}

type AccessFlags uint16

const (
	Public     AccessFlags = 0x0001
	Static                 = 0x0008
	Final                  = 0x0010
	Super                  = 0x0020
	Native                 = 0x0100
	Interface              = 0x0200
	Abstract               = 0x0400
	Synthetic              = 0x1000
	Annotation             = 0x2000
	Enum                   = 0x4000
)

type Code struct {
	maxStack  uint16
	maxLocals uint16
	code      []byte
}

type Method struct {
	accessFlags     AccessFlags
	nameIndex       uint16
	descriptorIndex uint16
	code            Code
}

type RawClass struct {
	magic             uint32
	minorVersion      uint16
	majorVersion      uint16
	constantPoolItems []ConstantPoolItem
	accessFlags       AccessFlags
	thisClass         uint16
	superClass        uint16
	interfaces        []uint16
	fields            []uint16
	methods           []Method
}

type Class struct {
	Name string
}

func fromRawClass(raw RawClass) (class Class) {
	classInfo := raw.constantPoolItems[raw.thisClass-1].(ClassInfo)
	class.Name = raw.constantPoolItems[classInfo.NameIndex-1].(UTF8String).Contents
	return
}

type NameAndType struct {
	NameIndex       uint16
	DescriptorIndex uint16
}

func (_ NameAndType) isConstantPoolItem() {}

type UTF8String struct {
	Contents string
}

func (_ UTF8String) isConstantPoolItem() {}

type ClassInfo struct {
	NameIndex uint16
}

func (_ ClassInfo) isConstantPoolItem() {}

type MethodRef struct {
	ClassIndex       uint16
	NameAndTypeIndex uint16
}

func (_ MethodRef) isConstantPoolItem() {}

type FieldRef struct {
	ClassIndex       uint16
	NameAndTypeIndex uint16
}

func (_ FieldRef) isConstantPoolItem() {}

type StringConstant struct {
	UTF8Index uint16
}

func (_ StringConstant) isConstantPoolItem() {}

func parseMethodRef(buf *bytes.Reader) (m MethodRef, err error) {
	err = binary.Read(buf, binary.BigEndian, &m)
	return
}

func parseFieldRef(buf *bytes.Reader) (f FieldRef, err error) {
	err = binary.Read(buf, binary.BigEndian, &f)
	return
}

func parseStringConstant(buf *bytes.Reader) (s StringConstant, err error) {
	err = binary.Read(buf, binary.BigEndian, &s)
	return
}

func parseClassInfo(buf *bytes.Reader) (c ClassInfo, err error) {
	err = binary.Read(buf, binary.BigEndian, &c)
	return
}

func parseNameAndType(buf *bytes.Reader) (n NameAndType, err error) {
	err = binary.Read(buf, binary.BigEndian, &n)
	return
}

func parseUTF8String(buf *bytes.Reader) (s UTF8String, err error) {
	var length uint16
	err = binary.Read(buf, binary.BigEndian, &length)
	if err != nil {
		return
	}
	bytes := make([]byte, length)
	var i uint16
	for i = 0; i < length; i++ {
		bytes[i], err = buf.ReadByte()
		if err != nil {
			return
		}
	}
	s.Contents = string(bytes)
	return
}

type ConstantPoolTag uint8

const (
	CP_UTF8String  ConstantPoolTag = 1
	CP_ClassInfo                   = 7
	CP_String                      = 8
	CP_Field                       = 9
	CP_Method                      = 10
	CP_NameAndType                 = 12
)

func parseConstantPoolItem(buf *bytes.Reader) (ConstantPoolItem, error) {
	var tag ConstantPoolTag
	err := binary.Read(buf, binary.BigEndian, &tag)
	if err != nil {
		panic("Could not read constant pool tag")
	}
	switch tag {
	case CP_UTF8String:
		return parseUTF8String(buf)
	case CP_ClassInfo:
		return parseClassInfo(buf)
	case CP_String:
		return parseStringConstant(buf)
	case CP_Field:
		return parseFieldRef(buf)
	case CP_Method:
		return parseMethodRef(buf)
	case CP_NameAndType:
		return parseNameAndType(buf)
	default:
		panic("Unknown constant pool item")
	}
}

func parseCode(buf *bytes.Reader, length uint32) (c Code) {
	binary.Read(buf, binary.BigEndian, &c.maxStack)
	binary.Read(buf, binary.BigEndian, &c.maxLocals)
	var codeLength uint32
	binary.Read(buf, binary.BigEndian, &codeLength)
	c.code = make([]byte, codeLength)
	for k := 0; k < len(c.code); k++ {
		binary.Read(buf, binary.BigEndian, &c.code[k])
	}
	for k := uint32(8) + codeLength; k < length; k++ {
		var bytes uint8
		if binary.Read(buf, binary.BigEndian, &bytes) != nil {
			panic("Error reading code block")
		}
	}
	return
}

func parse(b []byte) (c RawClass, err error) {
	buf := bytes.NewReader(b)
	err = binary.Read(buf, binary.BigEndian, &c.magic)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.minorVersion)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.majorVersion)
	if err != nil {
		return
	}
	var constantPoolCount uint16
	err = binary.Read(buf, binary.BigEndian, &constantPoolCount)
	if err != nil {
		return
	}
	constantPoolCount -= 1
	c.constantPoolItems = make([]ConstantPoolItem, constantPoolCount)
	var i uint16
	for i = 0; i < constantPoolCount; i++ {
		c.constantPoolItems[i], err = parseConstantPoolItem(buf)
		if err != nil {
			return
		}
	}
	err = binary.Read(buf, binary.BigEndian, &c.accessFlags)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.thisClass)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.superClass)
	if err != nil {
		return
	}
	var interfacesCount uint16
	err = binary.Read(buf, binary.BigEndian, &interfacesCount)
	if err != nil {
		return
	}
	c.interfaces = make([]uint16, interfacesCount)

	var fieldsCount uint16
	err = binary.Read(buf, binary.BigEndian, &fieldsCount)
	if err != nil {
		return
	}
	c.fields = make([]uint16, fieldsCount)

	var methodsCount uint16
	err = binary.Read(buf, binary.BigEndian, &methodsCount)
	if err != nil {
		return
	}
	c.methods = make([]Method, methodsCount)
	for i = 0; i < methodsCount; i++ {
		err = binary.Read(buf, binary.BigEndian, &c.methods[i].accessFlags)
		err = binary.Read(buf, binary.BigEndian, &c.methods[i].nameIndex)
		err = binary.Read(buf, binary.BigEndian, &c.methods[i].descriptorIndex)
		var attrCount uint16
		err = binary.Read(buf, binary.BigEndian, &attrCount)
		if err != nil {
			return
		}
		for j := uint16(0); j < attrCount; j++ {
			var name uint16
			var length uint32
			err = binary.Read(buf, binary.BigEndian, &name)
			err = binary.Read(buf, binary.BigEndian, &length)
			actualName := (c.constantPoolItems[name-1]).(UTF8String)
			if actualName.Contents == "Code" {
				c.methods[i].code = parseCode(buf, length)
			} else {
				for k := uint32(0); k < length; k++ {
					var bytes uint8
					err = binary.Read(buf, binary.BigEndian, &bytes)
				}
			}
		}
	}
	var attrCount uint16
	err = binary.Read(buf, binary.BigEndian, &attrCount)
	if err != nil {
		return
	}
	for j := uint16(0); j < attrCount; j++ {
		var name uint16
		var length uint32
		err = binary.Read(buf, binary.BigEndian, &name)
		err = binary.Read(buf, binary.BigEndian, &length)
		for k := uint32(0); k < length; k++ {
			var bytes uint8
			err = binary.Read(buf, binary.BigEndian, &bytes)
		}
	}

	return
}
<commit_msg>replace switch with array dispatch for constant pool items<commit_after>package main

import (
	"bytes"
	"encoding/binary"
)

type ConstantPoolItem interface {
	isConstantPoolItem()
}

type AccessFlags uint16

const (
	Public     AccessFlags = 0x0001
	Static                 = 0x0008
	Final                  = 0x0010
	Super                  = 0x0020
	Native                 = 0x0100
	Interface              = 0x0200
	Abstract               = 0x0400
	Synthetic              = 0x1000
	Annotation             = 0x2000
	Enum                   = 0x4000
)

type Code struct {
	maxStack  uint16
	maxLocals uint16
	code      []byte
}

type Method struct {
	accessFlags     AccessFlags
	nameIndex       uint16
	descriptorIndex uint16
	code            Code
}

type RawClass struct {
	magic             uint32
	minorVersion      uint16
	majorVersion      uint16
	constantPoolItems []ConstantPoolItem
	accessFlags       AccessFlags
	thisClass         uint16
	superClass        uint16
	interfaces        []uint16
	fields            []uint16
	methods           []Method
}

type Class struct {
	Name string
}

func fromRawClass(raw RawClass) (class Class) {
	classInfo := raw.constantPoolItems[raw.thisClass-1].(ClassInfo)
	class.Name = raw.constantPoolItems[classInfo.NameIndex-1].(UTF8String).Contents
	return
}

type NameAndType struct {
	NameIndex       uint16
	DescriptorIndex uint16
}

func (_ NameAndType) isConstantPoolItem() {}

type UTF8String struct {
	Contents string
}

func (_ UTF8String) isConstantPoolItem() {}

type ClassInfo struct {
	NameIndex uint16
}

func (_ ClassInfo) isConstantPoolItem() {}

type MethodRef struct {
	ClassIndex       uint16
	NameAndTypeIndex uint16
}

func (_ MethodRef) isConstantPoolItem() {}

type FieldRef struct {
	ClassIndex       uint16
	NameAndTypeIndex uint16
}

func (_ FieldRef) isConstantPoolItem() {}

type StringConstant struct {
	UTF8Index uint16
}

func (_ StringConstant) isConstantPoolItem() {}

func parseMethodRef(buf *bytes.Reader) ConstantPoolItem {
	var m MethodRef
	binary.Read(buf, binary.BigEndian, &m)
	return m
}

func parseFieldRef(buf *bytes.Reader) ConstantPoolItem {
	var f FieldRef
	binary.Read(buf, binary.BigEndian, &f)
	return f
}

func parseStringConstant(buf *bytes.Reader) ConstantPoolItem {
	var s StringConstant
	binary.Read(buf, binary.BigEndian, &s)
	return s
}

func parseClassInfo(buf *bytes.Reader) ConstantPoolItem {
	var c ClassInfo
	binary.Read(buf, binary.BigEndian, &c)
	return c
}

func parseNameAndType(buf *bytes.Reader) ConstantPoolItem {
	var n NameAndType
	binary.Read(buf, binary.BigEndian, &n)
	return n
}

func parseUTF8String(buf *bytes.Reader) ConstantPoolItem {
	var length uint16
	err := binary.Read(buf, binary.BigEndian, &length)
	if err != nil {
		panic("Could not parse UTF8 String")
	}
	bytes := make([]byte, length)
	var i uint16
	for i = 0; i < length; i++ {
		bytes[i], err = buf.ReadByte()
		if err != nil {
			panic("Could not parse UTF8 String")
		}
	}
	return UTF8String{string(bytes)}
}

func unknownConstantPoolItem(_ *bytes.Reader) ConstantPoolItem {
	panic("Unknown constant pool item")
}

func parseConstantPoolItem(buf *bytes.Reader) ConstantPoolItem {
	parsers := []func(*bytes.Reader) ConstantPoolItem{
		unknownConstantPoolItem,
		parseUTF8String,
		unknownConstantPoolItem,
		unknownConstantPoolItem,
		unknownConstantPoolItem,
		unknownConstantPoolItem,
		unknownConstantPoolItem,
		parseClassInfo,
		parseStringConstant,
		parseFieldRef,
		parseMethodRef,
		unknownConstantPoolItem,
		parseNameAndType,
	}
	var tag uint8
	err := binary.Read(buf, binary.BigEndian, &tag)
	if err != nil {
		panic("Could not read constant pool tag")
	}
	return parsers[tag](buf)
}

func parseCode(buf *bytes.Reader, length uint32) (c Code) {
	binary.Read(buf, binary.BigEndian, &c.maxStack)
	binary.Read(buf, binary.BigEndian, &c.maxLocals)
	var codeLength uint32
	binary.Read(buf, binary.BigEndian, &codeLength)
	c.code = make([]byte, codeLength)
	for k := 0; k < len(c.code); k++ {
		binary.Read(buf, binary.BigEndian, &c.code[k])
	}
	for k := uint32(8) + codeLength; k < length; k++ {
		var bytes uint8
		if binary.Read(buf, binary.BigEndian, &bytes) != nil {
			panic("Error reading code block")
		}
	}
	return
}

func parse(b []byte) (c RawClass, err error) {
	buf := bytes.NewReader(b)
	err = binary.Read(buf, binary.BigEndian, &c.magic)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.minorVersion)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.majorVersion)
	if err != nil {
		return
	}
	var constantPoolCount uint16
	err = binary.Read(buf, binary.BigEndian, &constantPoolCount)
	if err != nil {
		return
	}
	constantPoolCount -= 1
	c.constantPoolItems = make([]ConstantPoolItem, constantPoolCount)
	var i uint16
	for i = 0; i < constantPoolCount; i++ {
		c.constantPoolItems[i] = parseConstantPoolItem(buf)
	}
	err = binary.Read(buf, binary.BigEndian, &c.accessFlags)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.thisClass)
	if err != nil {
		return
	}
	err = binary.Read(buf, binary.BigEndian, &c.superClass)
	if err != nil {
		return
	}
	var interfacesCount uint16
	err = binary.Read(buf, binary.BigEndian, &interfacesCount)
	if err != nil {
		return
	}
	c.interfaces = make([]uint16, interfacesCount)

	var fieldsCount uint16
	err = binary.Read(buf, binary.BigEndian, &fieldsCount)
	if err != nil {
		return
	}
	c.fields = make([]uint16, fieldsCount)

	var methodsCount uint16
	err = binary.Read(buf, binary.BigEndian, &methodsCount)
	if err != nil {
		return
	}
	c.methods = make([]Method, methodsCount)
	for i = 0; i < methodsCount; i++ {
		err = binary.Read(buf, binary.BigEndian, &c.methods[i].accessFlags)
		err = binary.Read(buf, binary.BigEndian, &c.methods[i].nameIndex)
		err = binary.Read(buf, binary.BigEndian, &c.methods[i].descriptorIndex)
		var attrCount uint16
		err = binary.Read(buf, binary.BigEndian, &attrCount)
		if err != nil {
			return
		}
		for j := uint16(0); j < attrCount; j++ {
			var name uint16
			var length uint32
			err = binary.Read(buf, binary.BigEndian, &name)
			err = binary.Read(buf, binary.BigEndian, &length)
			actualName := (c.constantPoolItems[name-1]).(UTF8String)
			if actualName.Contents == "Code" {
				c.methods[i].code = parseCode(buf, length)
			} else {
				for k := uint32(0); k < length; k++ {
					var bytes uint8
					err = binary.Read(buf, binary.BigEndian, &bytes)
				}
			}
		}
	}
	var attrCount uint16
	err = binary.Read(buf, binary.BigEndian, &attrCount)
	if err != nil {
		return
	}
	for j := uint16(0); j < attrCount; j++ {
		var name uint16
		var length uint32
		err = binary.Read(buf, binary.BigEndian, &name)
		err = binary.Read(buf, binary.BigEndian, &length)
		for k := uint32(0); k < length; k++ {
			var bytes uint8
			err = binary.Read(buf, binary.BigEndian, &bytes)
		}
	}

	return
}
<|endoftext|>


<commit_before>// Package pkgcloud allows you to talk to the packagecloud API.
// See https://packagecloud.io/docs/api
package pkgcloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	"github.com/mlafeldt/pkgcloud/upload"
)

//go:generate bash -c "./gendistros.py supportedDistros | gofmt > distros.go"

// ServiceURL is the URL of packagecloud's API.
const ServiceURL = "https://packagecloud.io/api/v1"

// A Client is a packagecloud client.
type Client struct {
	token string
}

// NewClient creates a packagecloud client. API requests are authenticated
// using an API token. If no token is passed, it will be read from the
// PACKAGECLOUD_TOKEN environment variable.
func NewClient(token string) (*Client, error) {
	if token == "" {
		token = os.Getenv("PACKAGECLOUD_TOKEN")
		if token == "" {
			return nil, errors.New("PACKAGECLOUD_TOKEN unset")
		}
	}
	return &Client{token}, nil
}

func decodeResponse(status int, body []byte) error {
	switch status {
	case http.StatusOK, http.StatusCreated:
		return nil
	case http.StatusUnauthorized, http.StatusNotFound:
		return fmt.Errorf("HTTP status: %s", http.StatusText(status))
	case 422: // Unprocessable Entity
		var v map[string][]string
		if err := json.Unmarshal(body, &v); err != nil {
			return err
		}
		for _, messages := range v {
			for _, msg := range messages {
				// Only return the very first error message
				return errors.New(msg)
			}
			break
		}
		return fmt.Errorf("invalid HTTP body: %s", body)
	default:
		return fmt.Errorf("unexpected HTTP status: %d", status)
	}
}

// CreatePackage pushes a new package to packagecloud.
func (c Client) CreatePackage(repo, distro, pkgFile string) error {
	var extraParams map[string]string
	if distro != "" {
		distID, ok := supportedDistros[distro]
		if !ok {
			return fmt.Errorf("invalid distro name: %s", distro)
		}
		extraParams = map[string]string{
			"package[distro_version_id]": strconv.Itoa(distID),
		}
	}

	endpoint := fmt.Sprintf("%s/repos/%s/packages.json", ServiceURL, repo)
	request, err := upload.NewRequest(endpoint, extraParams, "package[package_file]", pkgFile)
	if err != nil {
		return err
	}
	request.SetBasicAuth(c.token, "")
	request.Header.Add("User-Agent", "pkgcloud Go client")

	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	return decodeResponse(resp.StatusCode, body)
}
<commit_msg>Add All() and Destroy() methods.<commit_after>// Package pkgcloud allows you to talk to the packagecloud API.
// See https://packagecloud.io/docs/api
package pkgcloud

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"strconv"

	"github.com/mlafeldt/pkgcloud/upload"
)

//go:generate bash -c "./gendistros.py supportedDistros | gofmt > distros.go"

// ServiceURL is the URL of packagecloud's API.
const ServiceURL = "https://packagecloud.io/api/v1"

const UserAgent = "pkgcloud Go client"

// A Client is a packagecloud client.
type Client struct {
	token string
}

// NewClient creates a packagecloud client. API requests are authenticated
// using an API token. If no token is passed, it will be read from the
// PACKAGECLOUD_TOKEN environment variable.
func NewClient(token string) (*Client, error) {
	if token == "" {
		token = os.Getenv("PACKAGECLOUD_TOKEN")
		if token == "" {
			return nil, errors.New("PACKAGECLOUD_TOKEN unset")
		}
	}
	return &Client{token}, nil
}

// decodeResponse checks http status code and tries to decode json body
func decodeResponse(resp *http.Response, respJson interface{}) error {
	switch resp.StatusCode {
	case http.StatusOK, http.StatusCreated:
		return json.NewDecoder(resp.Body).Decode(respJson)
	case http.StatusUnauthorized, http.StatusNotFound:
		return fmt.Errorf("HTTP status: %s", http.StatusText(resp.StatusCode))
	case 422: // Unprocessable Entity
		var v map[string][]string
		if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
			return err
		}
		for _, messages := range v {
			for _, msg := range messages {
				// Only return the very first error message
				return errors.New(msg)
			}
			break
		}
		return fmt.Errorf("invalid HTTP body")
	default:
		return fmt.Errorf("unexpected HTTP status: %d", resp.StatusCode)
	}
}

// CreatePackage pushes a new package to packagecloud.
func (c Client) CreatePackage(repo, distro, pkgFile string) error {
	var extraParams map[string]string
	if distro != "" {
		distID, ok := supportedDistros[distro]
		if !ok {
			return fmt.Errorf("invalid distro name: %s", distro)
		}
		extraParams = map[string]string{
			"package[distro_version_id]": strconv.Itoa(distID),
		}
	}

	endpoint := fmt.Sprintf("%s/repos/%s/packages.json", ServiceURL, repo)
	request, err := upload.NewRequest(endpoint, extraParams, "package[package_file]", pkgFile)
	if err != nil {
		return err
	}
	request.SetBasicAuth(c.token, "")
	request.Header.Add("User-Agent", UserAgent)

	client := &http.Client{}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return decodeResponse(resp, &struct{}{})
}

type Package struct {
	Name           string `json:"name"`
	Filename       string `json:"filename"`
	DistroVersion  string `json:"distro_version"`
	Version        string `json:"version"`
	Release        string `json:"release"`
	Type           string `json:"type"`
	PackageUrl     string `json:"package_url"`
	PackageHtmlUrl string `json:"package_html_url"`
}

// All list all packages in repository
func (c Client) All(repo string) ([]Package, error) {
	endpoint := fmt.Sprintf("%s/repos/%s/packages.json", ServiceURL, repo)
	req, err := http.NewRequest("GET", endpoint, nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(c.token, "")
	req.Header.Add("User-Agent", UserAgent)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var packages []Package
	err = decodeResponse(resp, &packages)
	return packages, err
}

// Destroy removes package from repository.
//
// repo should be full path to repository
// (e.g. youruser/repository/ubuntu/xenial).
func (c Client) Destroy(repo, packageFilename string) error {
	endpoint := fmt.Sprintf("%s/repos/%s/%s", ServiceURL, repo, packageFilename)

	req, err := http.NewRequest("DELETE", endpoint, nil)
	if err != nil {
		return err
	}

	req.SetBasicAuth(c.token, "")
	req.Header.Add("User-Agent", UserAgent)

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return decodeResponse(resp, &struct{}{})
}
<|endoftext|>


<commit_before>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/utl"
)

// TwoVarsFunc_t defines a function to plot contours (len(x)==2)
type TwoVarsFunc_t func(x []float64) float64

// TwoVarsTrans_t defines a tranformation x → y (len(x)==len(y)==2)
type TwoVarsTrans_t func(x []float64) (y []float64)

// PlotTwoVarsContour plots contour for two variables problem. len(x) == 2
//  Input:
//   dirout  -- directory to save files
//   fnkey   -- file name key for eps figure
//   pop0    -- initial population. can be <nil> if individuals are not to be plotted
//   pop1    -- final population. can be <nil> if individuals are not to be plotted
//   best    -- best individual. can be <nil>
//   np      -- number of points for contour
//   extra   -- called just before saving figure
//   axequal -- axis.equal
//   vmin    -- min 0 values
//   vmax    -- max 1 values
//   istrans -- vmin, vmax and individuals are transformed y-values; otherwise they are x-values
//   tplot   -- plot transformed plot; needs T and Ti.
//   T       -- transformation: x → y
//   Ti      -- transformation: y → x
//   f       -- function to plot filled contour. can be <nil>
//   gs      -- functions to plot contour @ level 0. can be <nil>
//  Note: g(x) operates on original x values
func PlotTwoVarsContour(dirout, fnkey string, pop0, pop1 Population, best *Individual, np int, extra func(), axequal bool,
	vmin, vmax []float64, istrans, tplot bool, T, Ti TwoVarsTrans_t, f TwoVarsFunc_t, gs ...TwoVarsFunc_t) {
	if fnkey == "" {
		return
	}
	chk.IntAssert(len(vmin), 2)
	chk.IntAssert(len(vmax), 2)
	V0, V1 := utl.MeshGrid2D(vmin[0], vmax[0], vmin[1], vmax[1], np, np)
	var Zf [][]float64
	var Zg [][][]float64
	if f != nil {
		Zf = la.MatAlloc(np, np)
	}
	if len(gs) > 0 {
		Zg = utl.Deep3alloc(len(gs), np, np)
	}
	dotrans := !istrans && tplot // do transform
	untrans := istrans && !tplot // un-transform
	x := make([]float64, 2)
	for i := 0; i < np; i++ {
		for j := 0; j < np; j++ {
			if istrans {
				x = Ti([]float64{V0[i][j], V1[i][j]}) // x ← T⁻¹(y)
				if !tplot {
					V0[i][j], V1[i][j] = x[0], x[1]
				}
			} else {
				x[0], x[1] = V0[i][j], V1[i][j]
				if tplot {
					y := T(x) // v ← y = T(x)
					V0[i][j], V1[i][j] = y[0], y[1]
				}
			}
			if f != nil {
				Zf[i][j] = f(x)
			}
			for k, g := range gs {
				Zg[k][i][j] = g(x)
			}
		}
	}
	plt.Reset()
	plt.SetForEps(0.8, 350)
	if f != nil {
		cmapidx := 0
		if tplot {
			cmapidx = 4
		}
		plt.Contour(V0, V1, Zf, io.Sf("fsz=7, cmapidx=%d", cmapidx))
	}
	for k, _ := range gs {
		plt.ContourSimple(V0, V1, Zg[k], "zorder=5, levels=[0], colors=['yellow'], linewidths=[2], clip_on=0")
	}
	get_v := func(ind *Individual) (v []float64) {
		v = ind.GetFloats()
		if dotrans {
			y := T(v)
			v[0], v[1] = y[0], y[1]
		}
		if untrans {
			x := Ti(v)
			v[0], v[1] = x[0], x[1]
		}
		return
	}
	if pop0 != nil {
		for i, ind := range pop0 {
			l := ""
			if i == 0 {
				l = "initial population"
			}
			v := get_v(ind)
			plt.PlotOne(v[0], v[1], io.Sf("'k.', zorder=20, clip_on=0, label='%s'", l))
		}
	}
	if pop1 != nil {
		for i, ind := range pop1 {
			l := ""
			if i == 0 {
				l = "final population"
			}
			v := get_v(ind)
			plt.PlotOne(v[0], v[1], io.Sf("'ko', ms=6, zorder=30, clip_on=0, label='%s', markerfacecolor='none'", l))
		}
	}
	if extra != nil {
		extra()
	}
	if best != nil {
		v := get_v(best)
		plt.PlotOne(v[0], v[1], "'m*', zorder=50, clip_on=0, label='best', markeredgecolor='m'")
	}
	if dirout == "" {
		dirout = "."
	}
	plt.Cross("clr='grey'")
	plt.SetXnticks(11)
	plt.SetYnticks(11)
	if axequal {
		plt.Equal()
	}
	umin, umax := vmin, vmax
	if istrans && !tplot {
		xmin := Ti(vmin)
		xmax := Ti(vmax)
		umin = []float64{xmin[0], xmin[1]}
		umax = []float64{xmax[0], xmax[1]}
	}
	if !istrans && tplot {
		ymin := T(vmin)
		ymax := T(vmax)
		umin = []float64{ymin[0], ymin[1]}
		umax = []float64{ymax[0], ymax[1]}
	}
	plt.AxisRange(umin[0], umax[0], umin[1], umax[1])
	args := "leg_out='1', leg_ncol=4, leg_hlen=1.5"
	if tplot {
		plt.Gll("$y_0$", "$y_1$", args)
	} else {
		plt.Gll("$x_0$", "$x_1$", args)
	}
	plt.SaveD(dirout, fnkey+".eps")
}

// PlotOvs plots objective values versus time
func PlotOvs(isl *Island, ext, args string, t0, tf int, withtxt bool, numfmt string, first, last bool) {
	if isl.C.DoPlot == false || isl.C.FnKey == "" {
		return
	}
	if first {
		plt.SetForEps(0.75, 250)
	}
	var y []float64
	if tf == -1 {
		y = isl.OVA[t0:]
		tf = len(isl.OVA)
	} else {
		y = isl.OVA[t0:tf]
	}
	n := len(y)
	T := utl.LinSpace(float64(t0), float64(tf), n)
	me := (tf-t0)/10 + isl.Id
	if me < 1 {
		me = 1
	}
	if len(args) > 0 {
		args += ","
	}
	plt.Plot(T, y, io.Sf("%s marker='%s', markersize=%d, markevery=%d, zorder=10, clip_on=0", args, get_marker(isl.Id), get_mrksz(isl.Id), me))
	if withtxt {
		plt.Text(T[0], y[0], io.Sf(numfmt, y[0]), "ha='left'")
		plt.Text(T[n-1], y[n-1], io.Sf(numfmt, y[n-1]), "ha='right'")
	}
	if last {
		plt.Gll("time", "objective value", "")
		plt.SaveD(isl.C.DirOut, isl.C.FnKey+"_ova"+ext)
	}
}

// PlotOor plots out-of-range values versus time
func PlotOor(isl *Island, ext, args string, t0, tf int, withtxt bool, numfmt string, first, last bool) {
	if isl.C.DoPlot == false || isl.C.FnKey == "" {
		return
	}
	if first {
		plt.SetForEps(0.75, 250)
	}
	var y []float64
	if tf == -1 {
		y = isl.OOR[t0:]
		tf = len(isl.OOR)
	} else {
		y = isl.OOR[t0:tf]
	}
	n := len(y)
	T := utl.LinSpace(float64(t0), float64(tf), n)
	me := (tf-t0)/10 + isl.Id
	if me < 1 {
		me = 1
	}
	if len(args) > 0 {
		args += ","
	}
	plt.Plot(T, y, io.Sf("%s marker='%s', markersize=%d, markevery=%d, zorder=10, clip_on=0", args, get_marker(isl.Id), get_mrksz(isl.Id), me))
	if withtxt {
		plt.Text(T[0], y[0], io.Sf(numfmt, y[0]), "ha='left'")
		plt.Text(T[n-1], y[n-1], io.Sf(numfmt, y[n-1]), "ha='right'")
	}
	if last {
		plt.Gll("time", "out-of-range value", "")
		plt.SaveD(isl.C.DirOut, isl.C.FnKey+"_oor"+ext)
	}
}

// get_marker returns a marker for graphs
func get_marker(i int) string {
	pool := []string{"", "+", ".", "x", "s", "o", "*"}
	return pool[i%len(pool)]
}

// get_mrksz returns a marker size for graphs
func get_mrksz(i int) int {
	pool := []int{6, 6, 6, 3, 6, 6, 6}
	return pool[i%len(pool)]
}
<commit_msg>check for invalid transformation added to plot contour<commit_after>// Copyright 2015 Dorival de Moraes Pedroso. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package goga

import (
	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
	"github.com/cpmech/gosl/plt"
	"github.com/cpmech/gosl/utl"
)

// TwoVarsFunc_t defines a function to plot contours (len(x)==2)
type TwoVarsFunc_t func(x []float64) float64

// TwoVarsTrans_t defines a tranformation x → y (len(x)==len(y)==2)
type TwoVarsTrans_t func(x []float64) (y []float64, invalid bool)

// PlotTwoVarsContour plots contour for two variables problem. len(x) == 2
//  Input:
//   dirout  -- directory to save files
//   fnkey   -- file name key for eps figure
//   pop0    -- initial population. can be <nil> if individuals are not to be plotted
//   pop1    -- final population. can be <nil> if individuals are not to be plotted
//   best    -- best individual. can be <nil>
//   np      -- number of points for contour
//   extra   -- called just before saving figure
//   axequal -- axis.equal
//   vmin    -- min 0 values
//   vmax    -- max 1 values
//   istrans -- vmin, vmax and individuals are transformed y-values; otherwise they are x-values
//   tplot   -- plot transformed plot; needs T and Ti.
//   T       -- transformation: x → y
//   Ti      -- transformation: y → x
//   f       -- function to plot filled contour. can be <nil>
//   gs      -- functions to plot contour @ level 0. can be <nil>
//  Note: g(x) operates on original x values
func PlotTwoVarsContour(dirout, fnkey string, pop0, pop1 Population, best *Individual, np int, extra func(), axequal bool,
	vmin, vmax []float64, istrans, tplot bool, T, Ti TwoVarsTrans_t, f TwoVarsFunc_t, gs ...TwoVarsFunc_t) {
	if fnkey == "" {
		return
	}
	chk.IntAssert(len(vmin), 2)
	chk.IntAssert(len(vmax), 2)
	V0, V1 := utl.MeshGrid2D(vmin[0], vmax[0], vmin[1], vmax[1], np, np)
	var Zf [][]float64
	var Zg [][][]float64
	if f != nil {
		Zf = la.MatAlloc(np, np)
	}
	if len(gs) > 0 {
		Zg = utl.Deep3alloc(len(gs), np, np)
	}
	dotrans := !istrans && tplot // do transform
	untrans := istrans && !tplot // un-transform
	x := make([]float64, 2)
	for i := 0; i < np; i++ {
		for j := 0; j < np; j++ {
			if istrans {
				x, invalid := Ti([]float64{V0[i][j], V1[i][j]}) // x ← T⁻¹(y)
				if invalid {
					chk.Panic("cannot plot contour due to invalid transformation")
				}
				if !tplot {
					V0[i][j], V1[i][j] = x[0], x[1]
				}
			} else {
				x[0], x[1] = V0[i][j], V1[i][j]
				if tplot {
					y, invalid := T(x) // v ← y = T(x)
					if invalid {
						chk.Panic("cannot plot contour due to invalid transformation")
					}
					V0[i][j], V1[i][j] = y[0], y[1]
				}
			}
			if f != nil {
				Zf[i][j] = f(x)
			}
			for k, g := range gs {
				Zg[k][i][j] = g(x)
			}
		}
	}
	plt.Reset()
	plt.SetForEps(0.8, 350)
	if f != nil {
		cmapidx := 0
		if tplot {
			cmapidx = 4
		}
		plt.Contour(V0, V1, Zf, io.Sf("fsz=7, cmapidx=%d", cmapidx))
	}
	for k, _ := range gs {
		plt.ContourSimple(V0, V1, Zg[k], "zorder=5, levels=[0], colors=['yellow'], linewidths=[2], clip_on=0")
	}
	get_v := func(ind *Individual) (v []float64) {
		v = ind.GetFloats()
		if dotrans {
			y, invalid := T(v)
			if invalid {
				chk.Panic("cannot plot contour due to invalid transformation")
			}
			v[0], v[1] = y[0], y[1]
		}
		if untrans {
			x, invalid := Ti(v)
			if invalid {
				chk.Panic("cannot plot contour due to invalid transformation")
			}
			v[0], v[1] = x[0], x[1]
		}
		return
	}
	if pop0 != nil {
		for i, ind := range pop0 {
			l := ""
			if i == 0 {
				l = "initial population"
			}
			v := get_v(ind)
			plt.PlotOne(v[0], v[1], io.Sf("'k.', zorder=20, clip_on=0, label='%s'", l))
		}
	}
	if pop1 != nil {
		for i, ind := range pop1 {
			l := ""
			if i == 0 {
				l = "final population"
			}
			v := get_v(ind)
			plt.PlotOne(v[0], v[1], io.Sf("'ko', ms=6, zorder=30, clip_on=0, label='%s', markerfacecolor='none'", l))
		}
	}
	if extra != nil {
		extra()
	}
	if best != nil {
		v := get_v(best)
		plt.PlotOne(v[0], v[1], "'m*', zorder=50, clip_on=0, label='best', markeredgecolor='m'")
	}
	if dirout == "" {
		dirout = "."
	}
	plt.Cross("clr='grey'")
	plt.SetXnticks(11)
	plt.SetYnticks(11)
	if axequal {
		plt.Equal()
	}
	umin, umax := vmin, vmax
	if istrans && !tplot {
		xmin, invalid := Ti(vmin)
		if invalid {
			chk.Panic("cannot plot contour due to invalid transformation")
		}
		xmax, invalid := Ti(vmax)
		if invalid {
			chk.Panic("cannot plot contour due to invalid transformation")
		}
		umin = []float64{xmin[0], xmin[1]}
		umax = []float64{xmax[0], xmax[1]}
	}
	if !istrans && tplot {
		ymin, invalid := T(vmin)
		if invalid {
			chk.Panic("cannot plot contour due to invalid transformation")
		}
		ymax, invalid := T(vmax)
		if invalid {
			chk.Panic("cannot plot contour due to invalid transformation")
		}
		umin = []float64{ymin[0], ymin[1]}
		umax = []float64{ymax[0], ymax[1]}
	}
	plt.AxisRange(umin[0], umax[0], umin[1], umax[1])
	args := "leg_out='1', leg_ncol=4, leg_hlen=1.5"
	if tplot {
		plt.Gll("$y_0$", "$y_1$", args)
	} else {
		plt.Gll("$x_0$", "$x_1$", args)
	}
	plt.SaveD(dirout, fnkey+".eps")
}

// PlotOvs plots objective values versus time
func PlotOvs(isl *Island, ext, args string, t0, tf int, withtxt bool, numfmt string, first, last bool) {
	if isl.C.DoPlot == false || isl.C.FnKey == "" {
		return
	}
	if first {
		plt.SetForEps(0.75, 250)
	}
	var y []float64
	if tf == -1 {
		y = isl.OVA[t0:]
		tf = len(isl.OVA)
	} else {
		y = isl.OVA[t0:tf]
	}
	n := len(y)
	T := utl.LinSpace(float64(t0), float64(tf), n)
	me := (tf-t0)/10 + isl.Id
	if me < 1 {
		me = 1
	}
	if len(args) > 0 {
		args += ","
	}
	plt.Plot(T, y, io.Sf("%s marker='%s', markersize=%d, markevery=%d, zorder=10, clip_on=0", args, get_marker(isl.Id), get_mrksz(isl.Id), me))
	if withtxt {
		plt.Text(T[0], y[0], io.Sf(numfmt, y[0]), "ha='left'")
		plt.Text(T[n-1], y[n-1], io.Sf(numfmt, y[n-1]), "ha='right'")
	}
	if last {
		plt.Gll("time", "objective value", "")
		plt.SaveD(isl.C.DirOut, isl.C.FnKey+"_ova"+ext)
	}
}

// PlotOor plots out-of-range values versus time
func PlotOor(isl *Island, ext, args string, t0, tf int, withtxt bool, numfmt string, first, last bool) {
	if isl.C.DoPlot == false || isl.C.FnKey == "" {
		return
	}
	if first {
		plt.SetForEps(0.75, 250)
	}
	var y []float64
	if tf == -1 {
		y = isl.OOR[t0:]
		tf = len(isl.OOR)
	} else {
		y = isl.OOR[t0:tf]
	}
	n := len(y)
	T := utl.LinSpace(float64(t0), float64(tf), n)
	me := (tf-t0)/10 + isl.Id
	if me < 1 {
		me = 1
	}
	if len(args) > 0 {
		args += ","
	}
	plt.Plot(T, y, io.Sf("%s marker='%s', markersize=%d, markevery=%d, zorder=10, clip_on=0", args, get_marker(isl.Id), get_mrksz(isl.Id), me))
	if withtxt {
		plt.Text(T[0], y[0], io.Sf(numfmt, y[0]), "ha='left'")
		plt.Text(T[n-1], y[n-1], io.Sf(numfmt, y[n-1]), "ha='right'")
	}
	if last {
		plt.Gll("time", "out-of-range value", "")
		plt.SaveD(isl.C.DirOut, isl.C.FnKey+"_oor"+ext)
	}
}

// get_marker returns a marker for graphs
func get_marker(i int) string {
	pool := []string{"", "+", ".", "x", "s", "o", "*"}
	return pool[i%len(pool)]
}

// get_mrksz returns a marker size for graphs
func get_mrksz(i int) int {
	pool := []int{6, 6, 6, 3, 6, 6, 6}
	return pool[i%len(pool)]
}
<|endoftext|>


<commit_before>package recurrent

import "time"

type (
	clock interface {
		After(duration time.Duration) <-chan time.Time
		NewTicker(duration time.Duration) ticker
	}

	ticker interface {
		Chan() <-chan time.Time
		Stop()
	}

	realClock  struct{}
	realTicker struct{ *time.Ticker }
)

func (rc *realClock) After(duration time.Duration) <-chan time.Time {
	return time.After(duration)
}

func (rc *realClock) NewTicker(duration time.Duration) ticker {
	return &realTicker{time.NewTicker(duration)}
}

func (rt *realTicker) Chan() <-chan time.Time {
	return rt.C
}

func (rt *realTicker) Stop() {
	rt.Stop()
}
<commit_msg>(Closes #2) Fix accidental infinite recursion.<commit_after>package recurrent

import "time"

type (
	clock interface {
		After(duration time.Duration) <-chan time.Time
		NewTicker(duration time.Duration) ticker
	}

	ticker interface {
		Chan() <-chan time.Time
		Stop()
	}

	realClock  struct{}
	realTicker struct {
		ticker *time.Ticker
	}
)

func (rc *realClock) After(duration time.Duration) <-chan time.Time {
	return time.After(duration)
}

func (rc *realClock) NewTicker(duration time.Duration) ticker {
	return &realTicker{
		ticker: time.NewTicker(duration),
	}
}

func (rt *realTicker) Chan() <-chan time.Time {
	return rt.ticker.C
}

func (rt *realTicker) Stop() {
	rt.ticker.Stop()
}
<|endoftext|>


<commit_before>package source

import (
	"encoding/json"
	"strconv"
	"strings"

	"github.com/containerd/containerd/reference"
	"github.com/moby/buildkit/client"
	"github.com/moby/buildkit/solver/pb"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

var (
	errInvalid  = errors.New("invalid")
	errNotFound = errors.New("not found")
)

type ResolveMode int

const (
	ResolveModeDefault ResolveMode = iota
	ResolveModeForcePull
	ResolveModePreferLocal
)

const (
	DockerImageScheme = "docker-image"
	GitScheme         = "git"
	LocalScheme       = "local"
	HttpScheme        = "http"
	HttpsScheme       = "https"
)

type Identifier interface {
	ID() string // until sources are in process this string comparison could be avoided
}

func FromString(s string) (Identifier, error) {
	// TODO: improve this
	parts := strings.SplitN(s, "://", 2)
	if len(parts) != 2 {
		return nil, errors.Wrapf(errInvalid, "failed to parse %s", s)
	}

	switch parts[0] {
	case DockerImageScheme:
		return NewImageIdentifier(parts[1])
	case GitScheme:
		return NewGitIdentifier(parts[1])
	case LocalScheme:
		return NewLocalIdentifier(parts[1])
	case HttpsScheme:
		return NewHttpIdentifier(parts[1], true)
	case HttpScheme:
		return NewHttpIdentifier(parts[1], false)
	default:
		return nil, errors.Wrapf(errNotFound, "unknown schema %s", parts[0])
	}
}

func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
	id, err := FromString(op.Source.Identifier)
	if err != nil {
		return nil, err
	}

	if id, ok := id.(*ImageIdentifier); ok {
		if platform != nil {
			id.Platform = &specs.Platform{
				OS:           platform.OS,
				Architecture: platform.Architecture,
				Variant:      platform.Variant,
				OSVersion:    platform.OSVersion,
				OSFeatures:   platform.OSFeatures,
			}
		}
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrImageResolveMode:
				rm, err := ParseImageResolveMode(v)
				if err != nil {
					return nil, err
				}
				id.ResolveMode = rm
			case pb.AttrImageRecordType:
				rt, err := parseImageRecordType(v)
				if err != nil {
					return nil, err
				}
				id.RecordType = rt
			}
		}
	}
	if id, ok := id.(*GitIdentifier); ok {
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrKeepGitDir:
				if v == "true" {
					id.KeepGitDir = true
				}
			case pb.AttrFullRemoteURL:
				id.Remote = v
			}
		}
	}
	if id, ok := id.(*LocalIdentifier); ok {
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrLocalSessionID:
				id.SessionID = v
				if p := strings.SplitN(v, ":", 2); len(p) == 2 {
					id.Name = p[0] + "-" + id.Name
					id.SessionID = p[1]
				}
			case pb.AttrIncludePatterns:
				var patterns []string
				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
					return nil, err
				}
				id.IncludePatterns = patterns
			case pb.AttrExcludePatterns:
				var patterns []string
				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
					return nil, err
				}
				id.ExcludePatterns = patterns
			case pb.AttrFollowPaths:
				var paths []string
				if err := json.Unmarshal([]byte(v), &paths); err != nil {
					return nil, err
				}
				id.FollowPaths = paths
			case pb.AttrSharedKeyHint:
				id.SharedKeyHint = v
			}
		}
	}
	if id, ok := id.(*HttpIdentifier); ok {
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrHTTPChecksum:
				dgst, err := digest.Parse(v)
				if err != nil {
					return nil, err
				}
				id.Checksum = dgst
			case pb.AttrHTTPFilename:
				id.Filename = v
			case pb.AttrHTTPPerm:
				i, err := strconv.ParseInt(v, 0, 64)
				if err != nil {
					return nil, err
				}
				id.Perm = int(i)
			case pb.AttrHTTPUID:
				i, err := strconv.ParseInt(v, 0, 64)
				if err != nil {
					return nil, err
				}
				id.UID = int(i)
			case pb.AttrHTTPGID:
				i, err := strconv.ParseInt(v, 0, 64)
				if err != nil {
					return nil, err
				}
				id.GID = int(i)
			}
		}
	}
	return id, nil
}

type ImageIdentifier struct {
	Reference   reference.Spec
	Platform    *specs.Platform
	ResolveMode ResolveMode
	RecordType  client.UsageRecordType
}

func NewImageIdentifier(str string) (*ImageIdentifier, error) {
	ref, err := reference.Parse(str)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	if ref.Object == "" {
		return nil, errors.WithStack(reference.ErrObjectRequired)
	}
	return &ImageIdentifier{Reference: ref}, nil
}

func (_ *ImageIdentifier) ID() string {
	return DockerImageScheme
}

type LocalIdentifier struct {
	Name            string
	SessionID       string
	IncludePatterns []string
	ExcludePatterns []string
	FollowPaths     []string
	SharedKeyHint   string
}

func NewLocalIdentifier(str string) (*LocalIdentifier, error) {
	return &LocalIdentifier{Name: str}, nil
}

func (*LocalIdentifier) ID() string {
	return LocalScheme
}

func NewHttpIdentifier(str string, tls bool) (*HttpIdentifier, error) {
	proto := "https://"
	if !tls {
		proto = "http://"
	}
	return &HttpIdentifier{TLS: tls, URL: proto + str}, nil
}

type HttpIdentifier struct {
	TLS      bool
	URL      string
	Checksum digest.Digest
	Filename string
	Perm     int
	UID      int
	GID      int
}

func (_ *HttpIdentifier) ID() string {
	return HttpsScheme
}

func ParseImageResolveMode(v string) (ResolveMode, error) {
	switch v {
	case pb.AttrImageResolveModeDefault, "":
		return ResolveModeDefault, nil
	case pb.AttrImageResolveModeForcePull:
		return ResolveModeForcePull, nil
	case pb.AttrImageResolveModePreferLocal:
		return ResolveModePreferLocal, nil
	default:
		return 0, errors.Errorf("invalid resolvemode: %s", v)
	}
}

func parseImageRecordType(v string) (client.UsageRecordType, error) {
	switch client.UsageRecordType(v) {
	case "", client.UsageRecordTypeRegular:
		return client.UsageRecordTypeRegular, nil
	case client.UsageRecordTypeInternal:
		return client.UsageRecordTypeInternal, nil
	case client.UsageRecordTypeFrontend:
		return client.UsageRecordTypeFrontend, nil
	default:
		return "", errors.Errorf("invalid record type %s", v)
	}
}
<commit_msg>source: Have ResolveMode implement fmt.Stringer interface<commit_after>package source

import (
	"encoding/json"
	"strconv"
	"strings"

	"github.com/containerd/containerd/reference"
	"github.com/moby/buildkit/client"
	"github.com/moby/buildkit/solver/pb"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

var (
	errInvalid  = errors.New("invalid")
	errNotFound = errors.New("not found")
)

type ResolveMode int

const (
	ResolveModeDefault ResolveMode = iota
	ResolveModeForcePull
	ResolveModePreferLocal
)

const (
	DockerImageScheme = "docker-image"
	GitScheme         = "git"
	LocalScheme       = "local"
	HttpScheme        = "http"
	HttpsScheme       = "https"
)

type Identifier interface {
	ID() string // until sources are in process this string comparison could be avoided
}

func FromString(s string) (Identifier, error) {
	// TODO: improve this
	parts := strings.SplitN(s, "://", 2)
	if len(parts) != 2 {
		return nil, errors.Wrapf(errInvalid, "failed to parse %s", s)
	}

	switch parts[0] {
	case DockerImageScheme:
		return NewImageIdentifier(parts[1])
	case GitScheme:
		return NewGitIdentifier(parts[1])
	case LocalScheme:
		return NewLocalIdentifier(parts[1])
	case HttpsScheme:
		return NewHttpIdentifier(parts[1], true)
	case HttpScheme:
		return NewHttpIdentifier(parts[1], false)
	default:
		return nil, errors.Wrapf(errNotFound, "unknown schema %s", parts[0])
	}
}

func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) {
	id, err := FromString(op.Source.Identifier)
	if err != nil {
		return nil, err
	}

	if id, ok := id.(*ImageIdentifier); ok {
		if platform != nil {
			id.Platform = &specs.Platform{
				OS:           platform.OS,
				Architecture: platform.Architecture,
				Variant:      platform.Variant,
				OSVersion:    platform.OSVersion,
				OSFeatures:   platform.OSFeatures,
			}
		}
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrImageResolveMode:
				rm, err := ParseImageResolveMode(v)
				if err != nil {
					return nil, err
				}
				id.ResolveMode = rm
			case pb.AttrImageRecordType:
				rt, err := parseImageRecordType(v)
				if err != nil {
					return nil, err
				}
				id.RecordType = rt
			}
		}
	}
	if id, ok := id.(*GitIdentifier); ok {
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrKeepGitDir:
				if v == "true" {
					id.KeepGitDir = true
				}
			case pb.AttrFullRemoteURL:
				id.Remote = v
			}
		}
	}
	if id, ok := id.(*LocalIdentifier); ok {
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrLocalSessionID:
				id.SessionID = v
				if p := strings.SplitN(v, ":", 2); len(p) == 2 {
					id.Name = p[0] + "-" + id.Name
					id.SessionID = p[1]
				}
			case pb.AttrIncludePatterns:
				var patterns []string
				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
					return nil, err
				}
				id.IncludePatterns = patterns
			case pb.AttrExcludePatterns:
				var patterns []string
				if err := json.Unmarshal([]byte(v), &patterns); err != nil {
					return nil, err
				}
				id.ExcludePatterns = patterns
			case pb.AttrFollowPaths:
				var paths []string
				if err := json.Unmarshal([]byte(v), &paths); err != nil {
					return nil, err
				}
				id.FollowPaths = paths
			case pb.AttrSharedKeyHint:
				id.SharedKeyHint = v
			}
		}
	}
	if id, ok := id.(*HttpIdentifier); ok {
		for k, v := range op.Source.Attrs {
			switch k {
			case pb.AttrHTTPChecksum:
				dgst, err := digest.Parse(v)
				if err != nil {
					return nil, err
				}
				id.Checksum = dgst
			case pb.AttrHTTPFilename:
				id.Filename = v
			case pb.AttrHTTPPerm:
				i, err := strconv.ParseInt(v, 0, 64)
				if err != nil {
					return nil, err
				}
				id.Perm = int(i)
			case pb.AttrHTTPUID:
				i, err := strconv.ParseInt(v, 0, 64)
				if err != nil {
					return nil, err
				}
				id.UID = int(i)
			case pb.AttrHTTPGID:
				i, err := strconv.ParseInt(v, 0, 64)
				if err != nil {
					return nil, err
				}
				id.GID = int(i)
			}
		}
	}
	return id, nil
}

type ImageIdentifier struct {
	Reference   reference.Spec
	Platform    *specs.Platform
	ResolveMode ResolveMode
	RecordType  client.UsageRecordType
}

func NewImageIdentifier(str string) (*ImageIdentifier, error) {
	ref, err := reference.Parse(str)
	if err != nil {
		return nil, errors.WithStack(err)
	}

	if ref.Object == "" {
		return nil, errors.WithStack(reference.ErrObjectRequired)
	}
	return &ImageIdentifier{Reference: ref}, nil
}

func (_ *ImageIdentifier) ID() string {
	return DockerImageScheme
}

type LocalIdentifier struct {
	Name            string
	SessionID       string
	IncludePatterns []string
	ExcludePatterns []string
	FollowPaths     []string
	SharedKeyHint   string
}

func NewLocalIdentifier(str string) (*LocalIdentifier, error) {
	return &LocalIdentifier{Name: str}, nil
}

func (*LocalIdentifier) ID() string {
	return LocalScheme
}

func NewHttpIdentifier(str string, tls bool) (*HttpIdentifier, error) {
	proto := "https://"
	if !tls {
		proto = "http://"
	}
	return &HttpIdentifier{TLS: tls, URL: proto + str}, nil
}

type HttpIdentifier struct {
	TLS      bool
	URL      string
	Checksum digest.Digest
	Filename string
	Perm     int
	UID      int
	GID      int
}

func (_ *HttpIdentifier) ID() string {
	return HttpsScheme
}

func (r ResolveMode) String() string {
	switch r {
	case ResolveModeDefault:
		return pb.AttrImageResolveModeDefault
	case ResolveModeForcePull:
		return pb.AttrImageResolveModeForcePull
	case ResolveModePreferLocal:
		return pb.AttrImageResolveModePreferLocal
	default:
		return ""
	}
}

func ParseImageResolveMode(v string) (ResolveMode, error) {
	switch v {
	case pb.AttrImageResolveModeDefault, "":
		return ResolveModeDefault, nil
	case pb.AttrImageResolveModeForcePull:
		return ResolveModeForcePull, nil
	case pb.AttrImageResolveModePreferLocal:
		return ResolveModePreferLocal, nil
	default:
		return 0, errors.Errorf("invalid resolvemode: %s", v)
	}
}

func parseImageRecordType(v string) (client.UsageRecordType, error) {
	switch client.UsageRecordType(v) {
	case "", client.UsageRecordTypeRegular:
		return client.UsageRecordTypeRegular, nil
	case client.UsageRecordTypeInternal:
		return client.UsageRecordTypeInternal, nil
	case client.UsageRecordTypeFrontend:
		return client.UsageRecordTypeFrontend, nil
	default:
		return "", errors.Errorf("invalid record type %s", v)
	}
}
<|endoftext|>


<commit_before>// Package colly implements a HTTP scraping framework
package colly

import (
	"bytes"
	"errors"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
)

// Collector provides the scraper instance for a scraping job
type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// `0` means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize       int
	CacheDir          string
	visitedURLs       []string
	htmlCallbacks     map[string]HTMLCallback
	requestCallbacks  []RequestCallback
	responseCallbacks []ResponseCallback
	backend           *httpBackend
	wg                *sync.WaitGroup
	lock              *sync.Mutex
}

// Request is the representation of a HTTP request made by a Collector
type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of this request
	Depth     int
	collector *Collector
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
	// Headers contains the Response's HTTP headers
	Headers *http.Header
}

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// Context provides a tiny layer for passing data between callbacks
type Context struct {
	ContextMap map[string]string
	lock       *sync.Mutex
}

// RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// NewCollector creates a new Collector instance with default configuration
func NewCollector() *Collector {
	c := &Collector{}
	c.Init()
	return c
}

// NewContext initializes a new Context instance
func NewContext() *Context {
	return &Context{
		ContextMap: make(map[string]string),
		lock:       &sync.Mutex{},
	}
}

// Init initializes the Collector's private variables and sets default
// configuration for the Collector
func (c *Collector) Init() {
	c.UserAgent = "colly - https://github.com/asciimoo/colly"
	c.MaxDepth = 0
	c.visitedURLs = make([]string, 0, 8)
	c.htmlCallbacks = make(map[string]HTMLCallback, 0)
	c.requestCallbacks = make([]RequestCallback, 0, 8)
	c.responseCallbacks = make([]ResponseCallback, 0, 8)
	c.MaxBodySize = 10 * 1024 * 1024
	c.backend = &httpBackend{}
	c.backend.Init()
	c.wg = &sync.WaitGroup{}
	c.lock = &sync.Mutex{}
}

// Visit starts Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Visit(URL string) error {
	return c.scrape(URL, "GET", 1, nil)
}

// Post starts collecting job by creating a POST
// request.
// Post also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Post(URL string, requestData map[string]string) error {
	return c.scrape(URL, "POST", 1, requestData)
}

func (c *Collector) scrape(u, method string, depth int, requestData map[string]string) error {
	c.wg.Add(1)
	defer c.wg.Done()
	if u == "" {
		return errors.New("Missing URL")
	}
	if c.MaxDepth > 0 && c.MaxDepth < depth {
		return errors.New("Max depth limit reached")
	}
	if !c.AllowURLRevisit {
		visited := false
		for _, u2 := range c.visitedURLs {
			if u2 == u {
				visited = true
				break
			}
		}
		if visited {
			return errors.New("URL already visited")
		}
	}
	parsedURL, err := url.Parse(u)
	if err != nil {
		return err
	}
	allowed := false
	if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
		allowed = true
	} else {
		for _, d := range c.AllowedDomains {
			if d == parsedURL.Host {
				allowed = true
				break
			}
		}
	}
	if !allowed {
		return errors.New("Forbidden domain")
	}
	if !c.AllowURLRevisit {
		c.lock.Lock()
		c.visitedURLs = append(c.visitedURLs, u)
		c.lock.Unlock()
	}
	var form url.Values
	if method == "POST" {
		form = url.Values{}
		for k, v := range requestData {
			form.Add(k, v)
		}
	}
	req, err := http.NewRequest(method, u, strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if method == "POST" {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	ctx := NewContext()
	request := &Request{
		URL:       parsedURL,
		Headers:   &req.Header,
		Ctx:       ctx,
		Depth:     depth,
		collector: c,
	}
	if len(c.requestCallbacks) > 0 {
		c.handleOnRequest(request)
	}
	response, err := c.backend.Cache(req, c.MaxBodySize, c.CacheDir)
	// TODO add OnError callback to handle these cases
	if err != nil {
		return err
	}
	response.Ctx = ctx
	response.Request = request
	if strings.Index(strings.ToLower(response.Headers.Get("Content-Type")), "html") > -1 {
		c.handleOnHTML(response.Body, request, response)
	}
	if len(c.responseCallbacks) > 0 {
		c.handleOnResponse(response)
	}
	return nil
}

// Wait returns when the collector jobs are finished
func (c *Collector) Wait() {
	c.wg.Wait()
}

// OnRequest registers a function. Function will be executed on every
// request made by the Collector
func (c *Collector) OnRequest(f RequestCallback) {
	c.lock.Lock()
	c.requestCallbacks = append(c.requestCallbacks, f)
	c.lock.Unlock()
}

// OnResponse registers a function. Function will be executed on every response
func (c *Collector) OnResponse(f ResponseCallback) {
	c.lock.Lock()
	c.responseCallbacks = append(c.responseCallbacks, f)
	c.lock.Unlock()
}

// OnHTML registers a function. Function will be executed on every HTML
// element matched by the `goquerySelector` parameter.
// `goquerySelector` is a selector used by https://github.com/PuerkitoBio/goquery
func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
	c.lock.Lock()
	c.htmlCallbacks[goquerySelector] = f
	c.lock.Unlock()
}

// WithTransport allows you to set a custom http.Transport for this collector.
func (c *Collector) WithTransport(transport *http.Transport) {
	c.backend.Client.Transport = transport
}

// DisableCookies turns off cookie handling for this collector
func (c *Collector) DisableCookies() {
	c.backend.Client.Jar = nil
}

// SetRequestTimeout overrides the default timeout (10 seconds) for this collector
func (c *Collector) SetRequestTimeout(timeout time.Duration) {
	c.backend.Client.Timeout = timeout
}

func (c *Collector) handleOnRequest(r *Request) {
	for _, f := range c.requestCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnResponse(r *Response) {
	for _, f := range c.responseCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnHTML(body []byte, req *Request, resp *Response) {
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
	if err != nil {
		return
	}
	for expr, f := range c.htmlCallbacks {
		doc.Find(expr).Each(func(i int, s *goquery.Selection) {
			for _, n := range s.Nodes {
				e := &HTMLElement{
					Name:       n.Data,
					Request:    req,
					Response:   resp,
					Text:       goquery.NewDocumentFromNode(n).Text(),
					DOM:        s,
					attributes: n.Attr,
				}
				f(e)
			}
		})
	}
}

// Limit adds a new `LimitRule` to the collector
func (c *Collector) Limit(rule *LimitRule) error {
	return c.backend.Limit(rule)
}

// Limits adds new `LimitRule`s to the collector
func (c *Collector) Limits(rules []*LimitRule) error {
	return c.backend.Limits(rules)
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// AbsoluteURL returns with the resolved absolute URL of an URL chunk.
// AbsoluteURL returns empty string if the URL chunk is a fragment or
// could not be parsed
func (r *Request) AbsoluteURL(u string) string {
	if strings.HasPrefix(u, "#") {
		return ""
	}
	absURL, err := r.URL.Parse(u)
	if err != nil {
		return ""
	}
	absURL.Fragment = ""
	if absURL.Scheme == "//" {
		absURL.Scheme = r.URL.Scheme
	}
	return absURL.String()
}

// Visit continues Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (r *Request) Visit(URL string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, nil)
}

// Post continues a collector job by creating a POST request.
// Post also calls the previously provided OnRequest, OnResponse, OnHTML callbacks
func (r *Request) Post(URL string, requestData map[string]string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "POST", r.Depth+1, requestData)
}

// Put stores a value in Context
func (c *Context) Put(key, value string) {
	c.lock.Lock()
	c.ContextMap[key] = value
	c.lock.Unlock()
}

// Get retrieves a value from Context. If no value found for `k`
// Get returns an empty string if key not found
func (c *Context) Get(key string) string {
	if v, ok := c.ContextMap[key]; ok {
		return v
	}
	return ""
}
<commit_msg>docstring describing CacheDir<commit_after>// Package colly implements a HTTP scraping framework
package colly

import (
	"bytes"
	"errors"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/html"

	"github.com/PuerkitoBio/goquery"
)

// Collector provides the scraper instance for a scraping job
type Collector struct {
	// UserAgent is the User-Agent string used by HTTP requests
	UserAgent string
	// MaxDepth limits the recursion depth of visited URLs.
	// Set it to 0 for infinite recursion (default).
	MaxDepth int
	// AllowedDomains is a domain whitelist.
	// Leave it blank to allow any domains to be visited
	AllowedDomains []string
	// AllowURLRevisit allows multiple downloads of the same URL
	AllowURLRevisit bool
	// MaxBodySize is the limit of the retrieved response body in bytes.
	// `0` means unlimited.
	// The default value for MaxBodySize is 10MB (10 * 1024 * 1024 bytes).
	MaxBodySize int
	// CacheDir specifies a location where GET requests are cached as files.
	// When it's not defined, caching is disabled.
	CacheDir          string
	visitedURLs       []string
	htmlCallbacks     map[string]HTMLCallback
	requestCallbacks  []RequestCallback
	responseCallbacks []ResponseCallback
	backend           *httpBackend
	wg                *sync.WaitGroup
	lock              *sync.Mutex
}

// Request is the representation of a HTTP request made by a Collector
type Request struct {
	// URL is the parsed URL of the HTTP request
	URL *url.URL
	// Headers contains the Request's HTTP headers
	Headers *http.Header
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Depth is the number of the parents of this request
	Depth     int
	collector *Collector
}

// Response is the representation of a HTTP response made by a Collector
type Response struct {
	// StatusCode is the status code of the Response
	StatusCode int
	// Body is the content of the Response
	Body []byte
	// Ctx is a context between a Request and a Response
	Ctx *Context
	// Request is the Request object of the response
	Request *Request
	// Headers contains the Response's HTTP headers
	Headers *http.Header
}

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// Context provides a tiny layer for passing data between callbacks
type Context struct {
	ContextMap map[string]string
	lock       *sync.Mutex
}

// RequestCallback is a type alias for OnRequest callback functions
type RequestCallback func(*Request)

// ResponseCallback is a type alias for OnResponse callback functions
type ResponseCallback func(*Response)

// HTMLCallback is a type alias for OnHTML callback functions
type HTMLCallback func(*HTMLElement)

// NewCollector creates a new Collector instance with default configuration
func NewCollector() *Collector {
	c := &Collector{}
	c.Init()
	return c
}

// NewContext initializes a new Context instance
func NewContext() *Context {
	return &Context{
		ContextMap: make(map[string]string),
		lock:       &sync.Mutex{},
	}
}

// Init initializes the Collector's private variables and sets default
// configuration for the Collector
func (c *Collector) Init() {
	c.UserAgent = "colly - https://github.com/asciimoo/colly"
	c.MaxDepth = 0
	c.visitedURLs = make([]string, 0, 8)
	c.htmlCallbacks = make(map[string]HTMLCallback, 0)
	c.requestCallbacks = make([]RequestCallback, 0, 8)
	c.responseCallbacks = make([]ResponseCallback, 0, 8)
	c.MaxBodySize = 10 * 1024 * 1024
	c.backend = &httpBackend{}
	c.backend.Init()
	c.wg = &sync.WaitGroup{}
	c.lock = &sync.Mutex{}
}

// Visit starts Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Visit(URL string) error {
	return c.scrape(URL, "GET", 1, nil)
}

// Post starts collecting job by creating a POST
// request.
// Post also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (c *Collector) Post(URL string, requestData map[string]string) error {
	return c.scrape(URL, "POST", 1, requestData)
}

func (c *Collector) scrape(u, method string, depth int, requestData map[string]string) error {
	c.wg.Add(1)
	defer c.wg.Done()
	if u == "" {
		return errors.New("Missing URL")
	}
	if c.MaxDepth > 0 && c.MaxDepth < depth {
		return errors.New("Max depth limit reached")
	}
	if !c.AllowURLRevisit {
		visited := false
		for _, u2 := range c.visitedURLs {
			if u2 == u {
				visited = true
				break
			}
		}
		if visited {
			return errors.New("URL already visited")
		}
	}
	parsedURL, err := url.Parse(u)
	if err != nil {
		return err
	}
	allowed := false
	if c.AllowedDomains == nil || len(c.AllowedDomains) == 0 {
		allowed = true
	} else {
		for _, d := range c.AllowedDomains {
			if d == parsedURL.Host {
				allowed = true
				break
			}
		}
	}
	if !allowed {
		return errors.New("Forbidden domain")
	}
	if !c.AllowURLRevisit {
		c.lock.Lock()
		c.visitedURLs = append(c.visitedURLs, u)
		c.lock.Unlock()
	}
	var form url.Values
	if method == "POST" {
		form = url.Values{}
		for k, v := range requestData {
			form.Add(k, v)
		}
	}
	req, err := http.NewRequest(method, u, strings.NewReader(form.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if method == "POST" {
		req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	}
	ctx := NewContext()
	request := &Request{
		URL:       parsedURL,
		Headers:   &req.Header,
		Ctx:       ctx,
		Depth:     depth,
		collector: c,
	}
	if len(c.requestCallbacks) > 0 {
		c.handleOnRequest(request)
	}
	response, err := c.backend.Cache(req, c.MaxBodySize, c.CacheDir)
	// TODO add OnError callback to handle these cases
	if err != nil {
		return err
	}
	response.Ctx = ctx
	response.Request = request
	if strings.Index(strings.ToLower(response.Headers.Get("Content-Type")), "html") > -1 {
		c.handleOnHTML(response.Body, request, response)
	}
	if len(c.responseCallbacks) > 0 {
		c.handleOnResponse(response)
	}
	return nil
}

// Wait returns when the collector jobs are finished
func (c *Collector) Wait() {
	c.wg.Wait()
}

// OnRequest registers a function. Function will be executed on every
// request made by the Collector
func (c *Collector) OnRequest(f RequestCallback) {
	c.lock.Lock()
	c.requestCallbacks = append(c.requestCallbacks, f)
	c.lock.Unlock()
}

// OnResponse registers a function. Function will be executed on every response
func (c *Collector) OnResponse(f ResponseCallback) {
	c.lock.Lock()
	c.responseCallbacks = append(c.responseCallbacks, f)
	c.lock.Unlock()
}

// OnHTML registers a function. Function will be executed on every HTML
// element matched by the `goquerySelector` parameter.
// `goquerySelector` is a selector used by https://github.com/PuerkitoBio/goquery
func (c *Collector) OnHTML(goquerySelector string, f HTMLCallback) {
	c.lock.Lock()
	c.htmlCallbacks[goquerySelector] = f
	c.lock.Unlock()
}

// WithTransport allows you to set a custom http.Transport for this collector.
func (c *Collector) WithTransport(transport *http.Transport) {
	c.backend.Client.Transport = transport
}

// DisableCookies turns off cookie handling for this collector
func (c *Collector) DisableCookies() {
	c.backend.Client.Jar = nil
}

// SetRequestTimeout overrides the default timeout (10 seconds) for this collector
func (c *Collector) SetRequestTimeout(timeout time.Duration) {
	c.backend.Client.Timeout = timeout
}

func (c *Collector) handleOnRequest(r *Request) {
	for _, f := range c.requestCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnResponse(r *Response) {
	for _, f := range c.responseCallbacks {
		f(r)
	}
}

func (c *Collector) handleOnHTML(body []byte, req *Request, resp *Response) {
	doc, err := goquery.NewDocumentFromReader(bytes.NewBuffer(body))
	if err != nil {
		return
	}
	for expr, f := range c.htmlCallbacks {
		doc.Find(expr).Each(func(i int, s *goquery.Selection) {
			for _, n := range s.Nodes {
				e := &HTMLElement{
					Name:       n.Data,
					Request:    req,
					Response:   resp,
					Text:       goquery.NewDocumentFromNode(n).Text(),
					DOM:        s,
					attributes: n.Attr,
				}
				f(e)
			}
		})
	}
}

// Limit adds a new `LimitRule` to the collector
func (c *Collector) Limit(rule *LimitRule) error {
	return c.backend.Limit(rule)
}

// Limits adds new `LimitRule`s to the collector
func (c *Collector) Limits(rules []*LimitRule) error {
	return c.backend.Limits(rules)
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// AbsoluteURL returns with the resolved absolute URL of an URL chunk.
// AbsoluteURL returns empty string if the URL chunk is a fragment or
// could not be parsed
func (r *Request) AbsoluteURL(u string) string {
	if strings.HasPrefix(u, "#") {
		return ""
	}
	absURL, err := r.URL.Parse(u)
	if err != nil {
		return ""
	}
	absURL.Fragment = ""
	if absURL.Scheme == "//" {
		absURL.Scheme = r.URL.Scheme
	}
	return absURL.String()
}

// Visit continues Collector's collecting job by creating a
// request to the URL specified in parameter.
// Visit also calls the previously provided OnRequest,
// OnResponse, OnHTML callbacks
func (r *Request) Visit(URL string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "GET", r.Depth+1, nil)
}

// Post continues a collector job by creating a POST request.
// Post also calls the previously provided OnRequest, OnResponse, OnHTML callbacks
func (r *Request) Post(URL string, requestData map[string]string) error {
	return r.collector.scrape(r.AbsoluteURL(URL), "POST", r.Depth+1, requestData)
}

// Put stores a value in Context
func (c *Context) Put(key, value string) {
	c.lock.Lock()
	c.ContextMap[key] = value
	c.lock.Unlock()
}

// Get retrieves a value from Context. If no value found for `k`
// Get returns an empty string if key not found
func (c *Context) Get(key string) string {
	if v, ok := c.ContextMap[key]; ok {
		return v
	}
	return ""
}
<|endoftext|>