<commit_before>package dotquote

import (
	"github.com/reiver/go-dotquote/detect"

	"fmt"
)

// Decoder
//
// Example usage:
//
//	decoder := Decoder{
//		Bytes: p,
//	}
//
//	for decoder.Next() {
//		keyBegin, keyEnd, err := decoder.Key()
//		if nil != err {
//			return err
//		}
//
//		key := p[keyBegin:keyEnd]
//
//		//@TODO: do something with the key
//
//		values := decoder.Values()
//		for values.Next() {
//			valueBegin, valueEnd, err := values.Value()
//			if nil != err {
//				return err
//			}
//
//			value := p[valueBegin:valueEnd]
//
//			/?@TOD: do something with the value.
//		}
//		if err := values.Err(); nil != err {
//			return err
//		}
//	}
//	if err := decoder.Err(); nil != err {
//		return nil
//	}
type Decoder struct {
	Bytes []byte
	Logger Logger

	iterationCount int

	index int
	err error

	hasBegun bool

	keyBegin int
	keyEnd   int

	values []struct{
		valueBegin int
		valueEnd   int
	}
}

func (decoder Decoder) Err() error {
	return decoder.err
}

func (decoder *Decoder) Next() bool {
	if nil == decoder {
		panic(errNilReceiver)
	}

	logger := decoder.Logger
	if nil == logger {
		logger = internalDiscardLogger{}
	}

	decoder.hasBegun = true
	if nil != decoder.err {
		return false
	}
	decoder.iterationCount++
	logger.Debugf("[NEXT] ITERATION COUNT #%d", decoder.iterationCount)

	b := decoder.Bytes
	if nil == b {
		decoder.err = errNilBytes
		return false
	}
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Bytes: |||||%s|||||", string(b))
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatWhitespace()
	if nil != decoder.err {
		return false
	}
	//logger.Trace("[NEXT] After eating whitespace.")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	{
		index := decoder.index

		p := b[index:]

		if 0 >= len(p) {
			return false
		}
	}

	{
		index := decoder.index
		if 0 > index {
			decoder.err = errInternalError
			return false
		}

		p := b[index:]

		begin, end, err := dotquotedetect.DetectKey(p)
		if nil != err {
			decoder.err = fmt.Errorf("Problem detecting dotquote key #%d: %q; when looking at index %d of |||||%s||||| ==> |||||%s|||||", decoder.iterationCount, err, index, string(b), string(p))
			return false
		}

		decoder.keyBegin = index + begin
		decoder.keyEnd   = index + end

		decoder.index += end
	}
	//logger.Trace("[NEXT] Key Parsed")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatWhitespace()
	if nil != decoder.err {
		return false
	}
	//logger.Trace("[NEXT] After eating whitespace.")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatEqualsSign()
	if err := decoder.err; nil != err {
		index := decoder.index

		decoder.err = fmt.Errorf("Problem detecting equals sign (=) #%d: %q; when looking at index %d of |||||%s|||||", decoder.iterationCount, err, index, string(b))
		return false
	}
	//logger.Trace("[NEXT] After eating equals sign (=)")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatWhitespace()
	if nil != decoder.err {
		return false
	}
	//logger.Trace("[NEXT] After eating whitespace.")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	{
		index := decoder.index

		detectValues := dotquotedetect.DetectValues{
			Bytes: b[index:],
		}
		for detectValues.Next() {

			begin, end, err := detectValues.Detect()
			if nil != err {
				decoder.err = err
				return false
			}

			value := struct{
				valueBegin int
				valueEnd   int
			}{
				valueBegin: index+begin,
				valueEnd:   index+end,
			}

			decoder.values = append(decoder.values, value)

		}
		if err := detectValues.Err(); nil != err {
			decoder.err = err
			return false
		}
		decoder.index += detectValues.EndIndex()
	}
	//logger.Trace("[NEXT] Values Parsed")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	return true
}
<commit_msg>values<commit_after>package dotquote

import (
	"github.com/reiver/go-dotquote/detect"

	"fmt"
)

// Decoder
//
// Example usage:
//
//	decoder := Decoder{
//		Bytes: p,
//	}
//
//	for decoder.Next() {
//		keyBegin, keyEnd, err := decoder.Key()
//		if nil != err {
//			return err
//		}
//
//		key := p[keyBegin:keyEnd]
//
//		//@TODO: do something with the key
//
//		values := decoder.Values()
//		for values.Next() {
//			valueBegin, valueEnd, err := values.Value()
//			if nil != err {
//				return err
//			}
//
//			value := p[valueBegin:valueEnd]
//
//			/?@TOD: do something with the value.
//		}
//		if err := values.Err(); nil != err {
//			return err
//		}
//	}
//	if err := decoder.Err(); nil != err {
//		return nil
//	}
type Decoder struct {
	Bytes []byte
	Logger Logger

	iterationCount int

	index int
	err error

	hasBegun bool

	keyBegin int
	keyEnd   int

	values []struct{
		valueBegin int
		valueEnd   int
	}
}

func (decoder Decoder) Err() error {
	return decoder.err
}

func (decoder *Decoder) Next() bool {
	if nil == decoder {
		panic(errNilReceiver)
	}

	logger := decoder.Logger
	if nil == logger {
		logger = internalDiscardLogger{}
	}

	decoder.hasBegun = true
	if nil != decoder.err {
		return false
	}
	decoder.iterationCount++
	logger.Debugf("[NEXT] ITERATION COUNT #%d", decoder.iterationCount)

	b := decoder.Bytes
	if nil == b {
		decoder.err = errNilBytes
		return false
	}
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Bytes: |||||%s|||||", string(b))
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatWhitespace()
	if nil != decoder.err {
		return false
	}
	//logger.Trace("[NEXT] After eating whitespace.")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	{
		index := decoder.index

		p := b[index:]

		if 0 >= len(p) {
			return false
		}
	}

	{
		index := decoder.index
		if 0 > index {
			decoder.err = errInternalError
			return false
		}

		p := b[index:]

		begin, end, err := dotquotedetect.DetectKey(p)
		if nil != err {
			decoder.err = fmt.Errorf("Problem detecting dotquote key #%d: %q; when looking at index %d of |||||%s||||| ==> |||||%s|||||", decoder.iterationCount, err, index, string(b), string(p))
			return false
		}

		decoder.keyBegin = index + begin
		decoder.keyEnd   = index + end

		decoder.index += end
	}
	//logger.Trace("[NEXT] Key Parsed")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatWhitespace()
	if nil != decoder.err {
		return false
	}
	//logger.Trace("[NEXT] After eating whitespace.")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatEqualsSign()
	if err := decoder.err; nil != err {
		index := decoder.index

		decoder.err = fmt.Errorf("Problem detecting equals sign (=) #%d: %q; when looking at index %d of |||||%s|||||", decoder.iterationCount, err, index, string(b))
		return false
	}
	//logger.Trace("[NEXT] After eating equals sign (=)")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.eatWhitespace()
	if nil != decoder.err {
		return false
	}
	//logger.Trace("[NEXT] After eating whitespace.")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	decoder.values = nil
	{
		index := decoder.index

		detectValues := dotquotedetect.DetectValues{
			Bytes: b[index:],
		}
		for detectValues.Next() {

			begin, end, err := detectValues.Detect()
			if nil != err {
				decoder.err = err
				return false
			}

			value := struct{
				valueBegin int
				valueEnd   int
			}{
				valueBegin: index+begin,
				valueEnd:   index+end,
			}

			decoder.values = append(decoder.values, value)

		}
		if err := detectValues.Err(); nil != err {
			decoder.err = err
			return false
		}
		decoder.index += detectValues.EndIndex()
	}
	//logger.Trace("[NEXT] Values Parsed")
	//logger.Tracef("[NEXT] index: %d", decoder.index)
	//logger.Tracef("[NEXT] Substring: |||||%s|||||", string(b[decoder.index:]))

	return true
}
<|endoftext|>


<commit_before>/*
	Repeatr is focused on telling a story about formulas: when you put the same
	things in, you should get the same things out.

	Formulas describe a piece of computation, its inputs, and how to collect
	its outputs.  After that, repeatr can help you make sure your formula
	produces the same thing time and time again.

	We call the parts of the formula that should be deterministic the "conjecture".
	We'll use this word consistently throughout the documentation.
	Anything that is part of the conjecture is hashed when processing the formula,
	and any output marked as part of the conjecture is hashed after the formula's task
	is executed.  (You can choose which of the outputs are part of your conjecture!
	But everything about your inputs must be part of your conjecture, because
	if the inputs change, output consistency is impossible -- except stuff like
	the network locations of data is skipped from the conjecture, since
	that can change without changing the meaning of your formula.)

	### Mathwise:

	Given a Formula j, and the []Output v, and some hash h:

	h(j.Inputs||j.Action||filter(j.Outputs, where Conjecture=true)) -> h(v)

	should be an onto relationship.

	In other words, a Formula should define a "pure" function.  And we'll let you know if it doesn't.

	### Misc docs:

	- The root filesystem of your execution engine is just another `Input` with the rest, with Location="/".
	Exactly one input with the root location is required at runtime.

	- Formula.SchedulingInfo, since it's *not* included in the 'conjecture',
	is expected not to have a major impact on your execution correctness.
*/
package def

import (
	"io"

	"github.com/spacemonkeygo/errors"
	"polydawn.net/repeatr/lib/streamer"
)

/*
	Formula describes `(inputs, computation) -> (outputs)`.

	Values may be mutated during final validation if missing,
	i.e. the special `Output` that describes stdout and stderr is required
	and will be supplied for you if not already specifically configured.
*/
type Formula struct {
	Inputs  []Input  `json:"inputs"`  // total set of inputs.  sorted order.  included in the conjecture.
	Action  Action   `json:"action"`  // description of the computation to be performed.  included in the conjecture.
	Outputs []Output `json:"outputs"` // set of expected outputs.  sorted order.  conditionally included in the conjecture (configurable per output).
	//SchedulingInfo interface{} // configures what execution framework is used and impl-specific additional parameters to that (minimum node memory, etc).  not considered part of the conjecture.
}

/*
	Input specifies a data source to feed into the beginning of a computation.

	Inputs can be one of many different `Type`s of data source.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	Inputs must specify both a `Hash` and a `URI`.
	`Input.Hash` is the content identity descriptor and will always be verified for all types of data source.
	`Input.Hash` is both identifies the data and provides integrity (in other words,
	all repeatr's input types will use a cryptographically strong hash here,
	so given a hash even an untrusted datastore is safe to use).
	Repeatr requires this to be accurate because if the inputs change, output
	consistency is impossible -- so even for plain http downloads, this is enforced.

	`Input.URI` is a secondary content lookup descriptor, like where on
	the filesystem or network information can be fetched from.
	`Input.URI` might contain extra description to answer questions like
	"which git remote url should i fetch from" or
	"which ipfs swarm do i coordinate with".

	The `URI` is *not* included in the conjecture, because repeatr understands
	that your data will be mobile -- that's why we have the `Input.Hash` take the leading role
	and why the `Input.Hash` should be sufficient to identify the information.
	Changes in the `Input.URI` field may make or break whether the data is accessible,
	but should never actually change the content of the data -- it's just supposed to talk about
	transport details; and content itself is still checked by `Input.Hash`.
*/
type Input struct {
	Type     string `json:"type"`  // implementation name (repeatr-internal).  included in the conjecture.
	Hash     string `json:"hash"`  // identifying hash of input data.  included in the conjecture.
	URI      string `json:"silo"`  // secondary content lookup descriptor.  not considered part of the conjecture.
	Location string `json:"mount"` // filepath where this input should be mounted in the execution context.  included in the conjecture.
}

/*
	Action describes the computation to be run once the inputs have been set up.
	All content is part of the conjecture.
*/
type Action struct {
	Entrypoint []string          `json:"command"` // executable to invoke as the task.  included in the conjecture.
	Cwd        string            `json:"cwd"`     // working directory to set when invoking the executable.  if not set, will be defaulted to "/".
	Env        map[string]string `json:"env"`     // environment variables.  included in the conjecture.
}

/*
	Output describes where we intend to pick up data after a task completes.

	Outputs can be one of many different `Type`s of data sink.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	`Output.Location` states where we should collect information from the
	task execution environment.  Repeatr executors will make sure this
	path exists and is owned&writable by the task before starting.
	After the task completes, repeatr will pick up this data, ship it off
	to storage, and also calculate a checksum of the data so we can see
	whether it matches any prior (or future) runs of this `Formula`.

	Outputs must specify a `URI`; repeatr will ship your data to this address.
	`Output.URI` has similar properties to `Input.URI` (and also similarly,
	is not included in the conjecture, because repeatr understands that
	your data can be mobile).

	The `Output.Hash` field will be filled in with a value computed
	from the data present in `Output.Location` after the task has completed.
	As with `Input.Hash`, the `Output.Hash` in repeatr will always be a
	cryptographically strong hash, which means it precisely describes your
	data, and makes it virtually impossible to accidentally get the same
	`Hash` as other data -- any changes to your output will always result
	in a very different `Hash` value.

	(In a content-addressable data store, repeatr may just lift the data store's
	address to use as `Output.Hash`, which is super efficient for everyone involved.
	For other more legacy-oriented systems, this may be a hash of the
	of the working filesystem right before before export.)

	Whether or not to include an `Output` in the overall `Formula`'s conjecture
	is up to you!  Many things in the world are not deterministic; repeatr
	is here to help you with the ones that should be, and stay out of the way
	for the ones that aren't.  Just set the `Output.Conjecture` boolean.

	Some examples of using `Conjecture` conditionally: if you have a job
	which does a bunch of calculations and should spit out a consistent result,
	but also does a lot of progress logging, gather those in two separate outputs.
	Mark the output of your computation in one output and set that to be
	included in the conjecture so repeatr can help you check your algorithm's
	correctness.  Now, since you may want to keep your logs for later, mark
	those as another output, and since these probably contain timestamps and
	other info that isn't *supposed* to be repeated exactly on another run,
	just set `Conjecture=false` on this one so repeatr knows not to check.

	`Output.Filters` may also be used to do a clean up pass on output files
	before committing them to storage or doing repeatr's consistency checks.
	(One typical example, which is engaged by default for you when an output
	is configured to be included in the conjecture, is setting all the file
	modification times to a standard value.)
*/
type Output struct {
	Type     string `json:"type"`           // implementation name (repeatr-internal).  included in the conjecture (iff the whole output is).
	Hash     string `json:"hash"`           // identifying hash of output data.  generated by the output handling implementation during data export when a task is complete.  included in the conjecture (iff the whole output is).
	URI      string `json:"silo,omitempty"` // where to ship the output data.  not considered part of the conjecture.
	Location string `json:"mount"`          // filepath where this output will be yanked from the job when it reaches completion.  included in the conjecture (iff the whole output is).
	// list of filter implementation names to apply to the output after the task has completeled and just before committing the data to storage.  included in the conjecture (iff the whole output is).
	// The filter name and its arguments are space-separated, whitespace cannot
	// be quoted.
	Filters    []string `json:"filters,omitempty"`
	Conjecture bool     `json:"cnj,omitempty"` // whether or not this output is expected to contain the same result, every time, when given the same set of `Input` items.
}

/*
	Job is an interface for observing actively running tasks.
	It contains progress reporting interfaces, streams that get realtime
	stdout/stderr, wait for finish, return exit codes, etc.

	All of the data available from `Job` should also be accessible as
	some form of `Output`s	after the execution is complete, but `Job` can
	provide them live.
*/
type Job interface {
	// question the first: provide readables, or accept writables for stdout?
	// probably provide.  the downside of course is this often forces a byte copy somewhere.
	// however, we're going to store these streams anyway.  so the most useful thing to do actually turns out to be log outputs immediately, and just reexpose that readers to that stream.

	Id() JobID // the ID of this job

	/*
		Returns a new reader that delivers the combined stdout and
		stderr of a command, from the beginning of execution.

		Shorthand for `Outputs().Reader(1, 2)`.
	*/
	OutputReader() io.Reader

	/*
		Returns a mux of readable streams.  Numbering is as typical
		unix convention (1=stdout, 2=stderr, etc).
	*/
	Outputs() streamer.ROMux

	/*
		Waits for completion if necessary, then returns the job's results
	*/
	Wait() JobResult
}

type JobID string // type def just to make it hard to accidentally get ids crossed.

/*
	Very much a work-in-progress.

	Holds all information you might want from a completed Job.
*/
type JobResult struct {
	ID JobID

	Error *errors.Error // if the executor experienced a problem running this job. REVIEW: type discussion? semantics?

	ExitCode int // The return code of this job

	Outputs []Output //The hashed outputs from this job

	// More?
}
<commit_msg>Docs fix.<commit_after>/*
	Repeatr is focused on telling a story about formulas: when you put the same
	things in, you should get the same things out.

	Formulas describe a piece of computation, its inputs, and how to collect
	its outputs.  After that, repeatr can help you make sure your formula
	produces the same thing time and time again.

	We call the parts of the formula that should be deterministic the "conjecture".
	We'll use this word consistently throughout the documentation.
	Anything that is part of the conjecture is hashed when processing the formula,
	and any output marked as part of the conjecture is hashed after the formula's task
	is executed.  (You can choose which of the outputs are part of your conjecture!
	But everything about your inputs must be part of your conjecture, because
	if the inputs change, output consistency is impossible -- except stuff like
	the network locations of data is skipped from the conjecture, since
	that can change without changing the meaning of your formula.)

	### Mathwise:

	Given a Formula j, and the []Output v, and some hash h:

	h(j.Inputs||j.Action||filter(j.Outputs, where Conjecture=true)) -> h(v)

	should be an onto relationship.

	In other words, a Formula should define a "pure" function.  And we'll let you know if it doesn't.

	### Misc docs:

	- The root filesystem of your execution engine is just another `Input` with the rest, with Location="/".
	Exactly one input with the root location is required at runtime.

	- Formula.SchedulingInfo, since it's *not* included in the 'conjecture',
	is expected not to have a major impact on your execution correctness.
*/
package def

import (
	"io"

	"github.com/spacemonkeygo/errors"
	"polydawn.net/repeatr/lib/streamer"
)

/*
	Formula describes `(inputs, computation) -> (outputs)`.

	Values may be mutated during final validation if missing,
	i.e. the special `Output` that describes stdout and stderr is required
	and will be supplied for you if not already specifically configured.
*/
type Formula struct {
	Inputs  []Input  `json:"inputs"`  // total set of inputs.  sorted order.  included in the conjecture.
	Action  Action   `json:"action"`  // description of the computation to be performed.  included in the conjecture.
	Outputs []Output `json:"outputs"` // set of expected outputs.  sorted order.  conditionally included in the conjecture (configurable per output).
	//SchedulingInfo interface{} // configures what execution framework is used and impl-specific additional parameters to that (minimum node memory, etc).  not considered part of the conjecture.
}

/*
	Input specifies a data source to feed into the beginning of a computation.

	Inputs can be one of many different `Type`s of data source.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	Inputs must specify both a `Hash` and a `URI`.
	`Input.Hash` is the content identity descriptor and will always be verified for all types of data source.
	`Input.Hash` is both identifies the data and provides integrity (in other words,
	all repeatr's input types will use a cryptographically strong hash here,
	so given a hash even an untrusted datastore is safe to use).
	Repeatr requires this to be accurate because if the inputs change, output
	consistency is impossible -- so even for plain http downloads, this is enforced.

	`Input.URI` is a secondary content lookup descriptor, like where on
	the filesystem or network information can be fetched from.
	`Input.URI` might contain extra description to answer questions like
	"which git remote url should i fetch from" or
	"which ipfs swarm do i coordinate with".

	The `URI` is *not* included in the conjecture, because repeatr understands
	that your data will be mobile -- that's why we have the `Input.Hash` take the leading role
	and why the `Input.Hash` should be sufficient to identify the information.
	Changes in the `Input.URI` field may make or break whether the data is accessible,
	but should never actually change the content of the data -- it's just supposed to talk about
	transport details; and content itself is still checked by `Input.Hash`.
*/
type Input struct {
	Type     string `json:"type"`  // implementation name (repeatr-internal).  included in the conjecture.
	Hash     string `json:"hash"`  // identifying hash of input data.  included in the conjecture.
	URI      string `json:"silo"`  // secondary content lookup descriptor.  not considered part of the conjecture.
	Location string `json:"mount"` // filepath where this input should be mounted in the execution context.  included in the conjecture.
}

/*
	Action describes the computation to be run once the inputs have been set up.
	All content is part of the conjecture.
*/
type Action struct {
	Entrypoint []string          `json:"command"` // executable to invoke as the task.  included in the conjecture.
	Cwd        string            `json:"cwd"`     // working directory to set when invoking the executable.  if not set, will be defaulted to "/".
	Env        map[string]string `json:"env"`     // environment variables.  included in the conjecture.
}

/*
	Output describes where we intend to pick up data after a task completes.

	Outputs can be one of many different `Type`s of data sink.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	`Output.Location` states where we should collect information from the
	task execution environment.
	After the task completes, repeatr will pick up this data, ship it off
	to storage, and also calculate a checksum of the data so we can see
	whether it matches any prior (or future) runs of this `Formula`.

	Outputs must specify a `URI`; repeatr will ship your data to this address.
	`Output.URI` has similar properties to `Input.URI` (and also similarly,
	is not included in the conjecture, because repeatr understands that
	your data can be mobile).

	The `Output.Hash` field will be filled in with a value computed
	from the data present in `Output.Location` after the task has completed.
	As with `Input.Hash`, the `Output.Hash` in repeatr will always be a
	cryptographically strong hash, which means it precisely describes your
	data, and makes it virtually impossible to accidentally get the same
	`Hash` as other data -- any changes to your output will always result
	in a very different `Hash` value.

	(In a content-addressable data store, repeatr may just lift the data store's
	address to use as `Output.Hash`, which is super efficient for everyone involved.
	For other more legacy-oriented systems, this may be a hash of the
	of the working filesystem right before before export.)

	Whether or not to include an `Output` in the overall `Formula`'s conjecture
	is up to you!  Many things in the world are not deterministic; repeatr
	is here to help you with the ones that should be, and stay out of the way
	for the ones that aren't.  Just set the `Output.Conjecture` boolean.

	Some examples of using `Conjecture` conditionally: if you have a job
	which does a bunch of calculations and should spit out a consistent result,
	but also does a lot of progress logging, gather those in two separate outputs.
	Mark the output of your computation in one output and set that to be
	included in the conjecture so repeatr can help you check your algorithm's
	correctness.  Now, since you may want to keep your logs for later, mark
	those as another output, and since these probably contain timestamps and
	other info that isn't *supposed* to be repeated exactly on another run,
	just set `Conjecture=false` on this one so repeatr knows not to check.

	`Output.Filters` may also be used to do a clean up pass on output files
	before committing them to storage or doing repeatr's consistency checks.
	(One typical example, which is engaged by default for you when an output
	is configured to be included in the conjecture, is setting all the file
	modification times to a standard value.)
*/
type Output struct {
	Type     string `json:"type"`           // implementation name (repeatr-internal).  included in the conjecture (iff the whole output is).
	Hash     string `json:"hash"`           // identifying hash of output data.  generated by the output handling implementation during data export when a task is complete.  included in the conjecture (iff the whole output is).
	URI      string `json:"silo,omitempty"` // where to ship the output data.  not considered part of the conjecture.
	Location string `json:"mount"`          // filepath where this output will be yanked from the job when it reaches completion.  included in the conjecture (iff the whole output is).
	// list of filter implementation names to apply to the output after the task has completeled and just before committing the data to storage.  included in the conjecture (iff the whole output is).
	// The filter name and its arguments are space-separated, whitespace cannot
	// be quoted.
	Filters    []string `json:"filters,omitempty"`
	Conjecture bool     `json:"cnj,omitempty"` // whether or not this output is expected to contain the same result, every time, when given the same set of `Input` items.
}

/*
	Job is an interface for observing actively running tasks.
	It contains progress reporting interfaces, streams that get realtime
	stdout/stderr, wait for finish, return exit codes, etc.

	All of the data available from `Job` should also be accessible as
	some form of `Output`s	after the execution is complete, but `Job` can
	provide them live.
*/
type Job interface {
	// question the first: provide readables, or accept writables for stdout?
	// probably provide.  the downside of course is this often forces a byte copy somewhere.
	// however, we're going to store these streams anyway.  so the most useful thing to do actually turns out to be log outputs immediately, and just reexpose that readers to that stream.

	Id() JobID // the ID of this job

	/*
		Returns a new reader that delivers the combined stdout and
		stderr of a command, from the beginning of execution.

		Shorthand for `Outputs().Reader(1, 2)`.
	*/
	OutputReader() io.Reader

	/*
		Returns a mux of readable streams.  Numbering is as typical
		unix convention (1=stdout, 2=stderr, etc).
	*/
	Outputs() streamer.ROMux

	/*
		Waits for completion if necessary, then returns the job's results
	*/
	Wait() JobResult
}

type JobID string // type def just to make it hard to accidentally get ids crossed.

/*
	Very much a work-in-progress.

	Holds all information you might want from a completed Job.
*/
type JobResult struct {
	ID JobID

	Error *errors.Error // if the executor experienced a problem running this job. REVIEW: type discussion? semantics?

	ExitCode int // The return code of this job

	Outputs []Output //The hashed outputs from this job

	// More?
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "os"
    "regexp"

    customCar    "github.com/jtyr/gbt/gbt/cars/custom"
    dirCar       "github.com/jtyr/gbt/gbt/cars/dir"
    exectimeCar  "github.com/jtyr/gbt/gbt/cars/exectime"
    gitCar       "github.com/jtyr/gbt/gbt/cars/git"
    hostnameCar  "github.com/jtyr/gbt/gbt/cars/hostname"
    osCar        "github.com/jtyr/gbt/gbt/cars/os"
    pyvirtenvCar "github.com/jtyr/gbt/gbt/cars/pyvirtenv"
    signCar      "github.com/jtyr/gbt/gbt/cars/sign"
    statusCar    "github.com/jtyr/gbt/gbt/cars/status"
    timeCar      "github.com/jtyr/gbt/gbt/cars/time"

    "github.com/jtyr/gbt/gbt/core/car"
    "github.com/jtyr/gbt/gbt/core/utils"
)

// Cars interface for methods from the core.car package.
type Cars interface {
    Init()
    Format() string
    SetParamStr(string, string)
    GetColor(string, bool) string
    DecorateElement(element, bg, fg, fm, text string) string
    GetModel() map[string]car.ModelElement
    GetDisplay() bool
    GetSep() string
    GetWrap() bool
}

const version = "1.1.3"

func printCars(cars []Cars, right bool) {
    prevBg := "\000"
    prevDisplay := true
    fakeCar := car.Car{}
    defaultSeparator := utils.GetEnv("GBT_SEPARATOR", "")

    if right {
        defaultSeparator = utils.GetEnv("GBT_RSEPARATOR", "")
    }

    if ! right && utils.GetEnv("GBT_BEGINNING_TEXT", "") != "" {
        fmt.Print(
            fakeCar.DecorateElement(
                "",
                fakeCar.GetColor(utils.GetEnv("GBT_BEGINNING_BG", "default"), false),
                fakeCar.GetColor(utils.GetEnv("GBT_BEGINNING_FG", "default"), true),
                fakeCar.GetColor(utils.GetEnv("GBT_BEGINNING_FM", "none"), false),
                utils.GetEnv("GBT_BEGINNING_TEXT", "")))
    }

    for _, c := range cars {
        c.Init()

        cModel := c.GetModel()
        cDisplay := c.GetDisplay()
        cSep := c.GetSep()
        cWrap := c.GetWrap()

        separator := defaultSeparator

        if cSep != "\000" {
            separator = cSep
        }

        if cDisplay {
            fmt.Print(fakeCar.GetColor("RESETALL", false))

            if prevBg != "\000" && prevDisplay {
                bg := c.GetColor(cModel["root"].Bg, false)
                fg := c.GetColor(cModel["root"].Bg, true)

                if cWrap {
                    bg = c.GetColor("default", false)
                    fg = c.GetColor("default", true)
                }

                if right {
                    fmt.Print(
                        c.DecorateElement(
                            "",
                            c.GetColor(prevBg, false),
                            fg,
                            "",
                            separator))
                } else {
                    fmt.Print(
                        c.DecorateElement(
                            "",
                            bg,
                            c.GetColor(prevBg, true),
                            "",
                            separator))
                }

                if cWrap {
                    fmt.Print("\n")
                }
            }

            prevBg = cModel["root"].Bg
            prevDisplay = cDisplay

            fmt.Print(c.Format())
        }
    }

    fmt.Print(fakeCar.GetColor("RESETALL", false))
}

// For the test
var run = flag.PrintDefaults

func main() {
    var argsHelp, argsVersion, argsRight bool

    flag.BoolVar(&argsHelp, "help", false, "show this help message and exit")
    flag.BoolVar(&argsVersion, "version", false, "show version and exit")
    flag.BoolVar(&argsRight, "right", false, "compose right hand site prompt")
    flag.Parse()

    if argsHelp {
        fmt.Printf("Usage of %s:\n", os.Args[0])
        run()
        os.Exit(0)
    }
    if argsVersion {
        fmt.Printf("GBT v%s\n", version)
        os.Exit(0)
    }

    carsStr := utils.GetEnv("GBT_CARS", "Status, Os, Hostname, Dir, Git, Sign")

    if argsRight {
        carsStr = utils.GetEnv("GBT_RCARS", "Time")
    }

    reCarSplit := regexp.MustCompile(`\s*,\s*`)
    carsNames := reCarSplit.Split(carsStr, -1)
    carsFactory := map[string]Cars{
        "Custom":    &customCar.Car{},
        "Dir":       &dirCar.Car{},
        "ExecTime":  &exectimeCar.Car{},
        "Git":       &gitCar.Car{},
        "Hostname":  &hostnameCar.Car{},
        "Os":        &osCar.Car{},
        "PyVirtEnv": &pyvirtenvCar.Car{},
        "Sign":      &signCar.Car{},
        "Status":    &statusCar.Car{},
        "Time":      &timeCar.Car{},
    }
    cars := []Cars{}

    for _, cn := range carsNames {
        custom := "\000"

        if len(cn) >= 6 && cn[:6] == "Custom" {
            custom = cn[6:]
            cn = cn[:6]
        }

        if val, ok := carsFactory[cn]; ok {
            if cn == "Status" && len(flag.Args()) > 0 {
                val.SetParamStr("args", flag.Args()[0])
            } else if custom != "\000" {
                val = &customCar.Car{}
                val.SetParamStr("name", custom)
            }

            cars = append(cars, val)
        }
    }

    printCars(cars, argsRight)
}
<commit_msg>Version bump to v1.1.4<commit_after>package main

import (
    "flag"
    "fmt"
    "os"
    "regexp"

    customCar    "github.com/jtyr/gbt/gbt/cars/custom"
    dirCar       "github.com/jtyr/gbt/gbt/cars/dir"
    exectimeCar  "github.com/jtyr/gbt/gbt/cars/exectime"
    gitCar       "github.com/jtyr/gbt/gbt/cars/git"
    hostnameCar  "github.com/jtyr/gbt/gbt/cars/hostname"
    osCar        "github.com/jtyr/gbt/gbt/cars/os"
    pyvirtenvCar "github.com/jtyr/gbt/gbt/cars/pyvirtenv"
    signCar      "github.com/jtyr/gbt/gbt/cars/sign"
    statusCar    "github.com/jtyr/gbt/gbt/cars/status"
    timeCar      "github.com/jtyr/gbt/gbt/cars/time"

    "github.com/jtyr/gbt/gbt/core/car"
    "github.com/jtyr/gbt/gbt/core/utils"
)

// Cars interface for methods from the core.car package.
type Cars interface {
    Init()
    Format() string
    SetParamStr(string, string)
    GetColor(string, bool) string
    DecorateElement(element, bg, fg, fm, text string) string
    GetModel() map[string]car.ModelElement
    GetDisplay() bool
    GetSep() string
    GetWrap() bool
}

const version = "1.1.4"

func printCars(cars []Cars, right bool) {
    prevBg := "\000"
    prevDisplay := true
    fakeCar := car.Car{}
    defaultSeparator := utils.GetEnv("GBT_SEPARATOR", "")

    if right {
        defaultSeparator = utils.GetEnv("GBT_RSEPARATOR", "")
    }

    if ! right && utils.GetEnv("GBT_BEGINNING_TEXT", "") != "" {
        fmt.Print(
            fakeCar.DecorateElement(
                "",
                fakeCar.GetColor(utils.GetEnv("GBT_BEGINNING_BG", "default"), false),
                fakeCar.GetColor(utils.GetEnv("GBT_BEGINNING_FG", "default"), true),
                fakeCar.GetColor(utils.GetEnv("GBT_BEGINNING_FM", "none"), false),
                utils.GetEnv("GBT_BEGINNING_TEXT", "")))
    }

    for _, c := range cars {
        c.Init()

        cModel := c.GetModel()
        cDisplay := c.GetDisplay()
        cSep := c.GetSep()
        cWrap := c.GetWrap()

        separator := defaultSeparator

        if cSep != "\000" {
            separator = cSep
        }

        if cDisplay {
            fmt.Print(fakeCar.GetColor("RESETALL", false))

            if prevBg != "\000" && prevDisplay {
                bg := c.GetColor(cModel["root"].Bg, false)
                fg := c.GetColor(cModel["root"].Bg, true)

                if cWrap {
                    bg = c.GetColor("default", false)
                    fg = c.GetColor("default", true)
                }

                if right {
                    fmt.Print(
                        c.DecorateElement(
                            "",
                            c.GetColor(prevBg, false),
                            fg,
                            "",
                            separator))
                } else {
                    fmt.Print(
                        c.DecorateElement(
                            "",
                            bg,
                            c.GetColor(prevBg, true),
                            "",
                            separator))
                }

                if cWrap {
                    fmt.Print("\n")
                }
            }

            prevBg = cModel["root"].Bg
            prevDisplay = cDisplay

            fmt.Print(c.Format())
        }
    }

    fmt.Print(fakeCar.GetColor("RESETALL", false))
}

// For the test
var run = flag.PrintDefaults

func main() {
    var argsHelp, argsVersion, argsRight bool

    flag.BoolVar(&argsHelp, "help", false, "show this help message and exit")
    flag.BoolVar(&argsVersion, "version", false, "show version and exit")
    flag.BoolVar(&argsRight, "right", false, "compose right hand site prompt")
    flag.Parse()

    if argsHelp {
        fmt.Printf("Usage of %s:\n", os.Args[0])
        run()
        os.Exit(0)
    }
    if argsVersion {
        fmt.Printf("GBT v%s\n", version)
        os.Exit(0)
    }

    carsStr := utils.GetEnv("GBT_CARS", "Status, Os, Hostname, Dir, Git, Sign")

    if argsRight {
        carsStr = utils.GetEnv("GBT_RCARS", "Time")
    }

    reCarSplit := regexp.MustCompile(`\s*,\s*`)
    carsNames := reCarSplit.Split(carsStr, -1)
    carsFactory := map[string]Cars{
        "Custom":    &customCar.Car{},
        "Dir":       &dirCar.Car{},
        "ExecTime":  &exectimeCar.Car{},
        "Git":       &gitCar.Car{},
        "Hostname":  &hostnameCar.Car{},
        "Os":        &osCar.Car{},
        "PyVirtEnv": &pyvirtenvCar.Car{},
        "Sign":      &signCar.Car{},
        "Status":    &statusCar.Car{},
        "Time":      &timeCar.Car{},
    }
    cars := []Cars{}

    for _, cn := range carsNames {
        custom := "\000"

        if len(cn) >= 6 && cn[:6] == "Custom" {
            custom = cn[6:]
            cn = cn[:6]
        }

        if val, ok := carsFactory[cn]; ok {
            if cn == "Status" && len(flag.Args()) > 0 {
                val.SetParamStr("args", flag.Args()[0])
            } else if custom != "\000" {
                val = &customCar.Car{}
                val.SetParamStr("name", custom)
            }

            cars = append(cars, val)
        }
    }

    printCars(cars, argsRight)
}
<|endoftext|>


<commit_before>package gitbase

import (
	"io"
	"os"

	errors "gopkg.in/src-d/go-errors.v1"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
	"gopkg.in/src-d/go-git.v4/utils/ioutil"

	billy "gopkg.in/src-d/go-billy.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/objfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
)

type packRepository struct {
	packs map[plumbing.Hash]packfile.Index
}

func repositoryPackfiles(repo repository) (*dotgit.DotGit, []plumbing.Hash, error) {
	fs, err := repo.FS()
	if err != nil {
		return nil, nil, err
	}

	fs, err = findDotGit(fs)
	if err != nil {
		return nil, nil, err
	}

	dot := dotgit.New(fs)
	packfiles, err := dot.ObjectPacks()
	if err != nil {
		return nil, nil, err
	}

	return dot, packfiles, nil
}

type packfileIndex struct {
	packfile plumbing.Hash
	idx      *packfile.Index
}

type repositoryIndex struct {
	dir     *dotgit.DotGit
	indexes []*packfileIndex
}

func newRepositoryIndex(repo repository) (*repositoryIndex, error) {
	dot, packfiles, err := repositoryPackfiles(repo)
	if err != nil {
		return nil, err
	}

	ri := &repositoryIndex{dir: dot}
	for _, p := range packfiles {
		idx, err := openPackfileIndex(dot, p)
		if err != nil {
			return nil, err
		}

		ri.indexes = append(ri.indexes, &packfileIndex{p, idx})
	}

	return ri, nil
}

func openPackfileIndex(
	dotGit *dotgit.DotGit,
	hash plumbing.Hash,
) (*packfile.Index, error) {
	f, err := dotGit.ObjectPackIdx(hash)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	idx := idxfile.NewIdxfile()
	if err := idxfile.NewDecoder(f).Decode(idx); err != nil {
		return nil, err
	}

	return packfile.NewIndexFromIdxFile(idx), nil
}

var errHashNotInIndex = errors.NewKind("object hash %s is not in repository")

func (i *repositoryIndex) find(hash plumbing.Hash) (int64, plumbing.Hash, error) {
	for _, idx := range i.indexes {
		if entry, ok := idx.idx.LookupHash(hash); ok {
			return int64(entry.Offset), idx.packfile, nil
		}
	}

	ok, err := i.isUnpacked(hash)
	if err != nil || ok {
		return -1, plumbing.ZeroHash, err
	}

	return -1, plumbing.ZeroHash, errHashNotInIndex.New(hash)
}

func (i *repositoryIndex) isUnpacked(hash plumbing.Hash) (bool, error) {
	f, err := i.dir.Object(hash)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}

		return false, err
	}
	if err := f.Close(); err != nil {
		return false, err
	}

	return true, nil
}

func findDotGit(fs billy.Filesystem) (billy.Filesystem, error) {
	fi, err := fs.Stat(".git")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if fi != nil && fi.IsDir() {
		return fs.Chroot(".git")
	}

	return fs, nil
}

func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, err error) {
	fs, err := repo.FS()
	if err != nil {
		return nil, err
	}

	fs, err = findDotGit(fs)
	if err != nil {
		return nil, err
	}

	dot := dotgit.New(fs)

	f, err := dot.Object(hash)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, plumbing.ErrObjectNotFound
		}

		return nil, err
	}

	defer ioutil.CheckClose(f, &err)

	storage, err := filesystem.NewStorage(fs)
	if err != nil {
		return nil, err
	}

	obj := storage.NewEncodedObject()
	r, err := objfile.NewReader(f)
	if err != nil {
		return nil, err
	}

	defer ioutil.CheckClose(r, &err)

	t, size, err := r.Header()
	if err != nil {
		return nil, err
	}

	obj.SetType(t)
	obj.SetSize(size)
	w, err := obj.Writer()
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(w, r)

	o, err = object.DecodeObject(storage, obj)
	return
}

type repoObjectDecoder struct {
	repo     string
	packfile plumbing.Hash
	decoder  *packfile.Decoder
	storage  storer.EncodedObjectStorer
}

func newRepoObjectDecoder(
	repo repository,
	hash plumbing.Hash,
) (*repoObjectDecoder, error) {
	fs, err := repo.FS()
	if err != nil {
		return nil, err
	}

	fs, err = findDotGit(fs)
	if err != nil {
		return nil, err
	}

	dot := dotgit.New(fs)
	packf, err := dot.ObjectPack(hash)
	if err != nil {
		return nil, err
	}

	storage, err := filesystem.NewStorage(fs)
	if err != nil {
		_ = packf.Close()
		return nil, err
	}

	decoder, err := packfile.NewDecoder(packfile.NewScanner(packf), storage)
	if err != nil {
		_ = packf.Close()
		return nil, err
	}

	return &repoObjectDecoder{
		repo:     repo.Path(),
		packfile: hash,
		decoder:  decoder,
		storage:  storage,
	}, nil
}

func (d *repoObjectDecoder) equals(repo string, packfile plumbing.Hash) bool {
	return d.repo == repo && d.packfile == packfile
}

func (d *repoObjectDecoder) get(offset int64) (object.Object, error) {
	encodedObj, err := d.decoder.DecodeObjectAt(offset)
	if err != nil {
		return nil, err
	}

	return object.DecodeObject(d.storage, encodedObj)
}

func (d *repoObjectDecoder) Close() error {
	return d.decoder.Close()
}

type objectDecoder struct {
	pool    *RepositoryPool
	decoder *repoObjectDecoder
}

func newObjectDecoder(pool *RepositoryPool) *objectDecoder {
	return &objectDecoder{pool: pool}
}

func (d *objectDecoder) decode(
	repository string,
	packfile plumbing.Hash,
	offset int64,
	hash plumbing.Hash,
) (object.Object, error) {
	if offset >= 0 {
		if d.decoder == nil || !d.decoder.equals(repository, packfile) {
			if d.decoder != nil {
				if err := d.decoder.Close(); err != nil {
					return nil, err
				}
			}

			var err error
			d.decoder, err = newRepoObjectDecoder(d.pool.repositories[repository], packfile)
			if err != nil {
				return nil, err
			}
		}

		return d.decoder.get(offset)
	}

	return getUnpackedObject(d.pool.repositories[repository], hash)
}

func (d *objectDecoder) Close() error {
	if d.decoder != nil {
		return d.decoder.Close()
	}
	return nil
}
<commit_msg>Compare repos by ID.<commit_after>package gitbase

import (
	"io"
	"os"

	errors "gopkg.in/src-d/go-errors.v1"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"
	"gopkg.in/src-d/go-git.v4/storage/filesystem/dotgit"
	"gopkg.in/src-d/go-git.v4/utils/ioutil"

	billy "gopkg.in/src-d/go-billy.v4"
	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/format/idxfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/objfile"
	"gopkg.in/src-d/go-git.v4/plumbing/format/packfile"
)

type packRepository struct {
	packs map[plumbing.Hash]packfile.Index
}

func repositoryPackfiles(repo repository) (*dotgit.DotGit, []plumbing.Hash, error) {
	fs, err := repo.FS()
	if err != nil {
		return nil, nil, err
	}

	fs, err = findDotGit(fs)
	if err != nil {
		return nil, nil, err
	}

	dot := dotgit.New(fs)
	packfiles, err := dot.ObjectPacks()
	if err != nil {
		return nil, nil, err
	}

	return dot, packfiles, nil
}

type packfileIndex struct {
	packfile plumbing.Hash
	idx      *packfile.Index
}

type repositoryIndex struct {
	dir     *dotgit.DotGit
	indexes []*packfileIndex
}

func newRepositoryIndex(repo repository) (*repositoryIndex, error) {
	dot, packfiles, err := repositoryPackfiles(repo)
	if err != nil {
		return nil, err
	}

	ri := &repositoryIndex{dir: dot}
	for _, p := range packfiles {
		idx, err := openPackfileIndex(dot, p)
		if err != nil {
			return nil, err
		}

		ri.indexes = append(ri.indexes, &packfileIndex{p, idx})
	}

	return ri, nil
}

func openPackfileIndex(
	dotGit *dotgit.DotGit,
	hash plumbing.Hash,
) (*packfile.Index, error) {
	f, err := dotGit.ObjectPackIdx(hash)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	idx := idxfile.NewIdxfile()
	if err := idxfile.NewDecoder(f).Decode(idx); err != nil {
		return nil, err
	}

	return packfile.NewIndexFromIdxFile(idx), nil
}

var errHashNotInIndex = errors.NewKind("object hash %s is not in repository")

func (i *repositoryIndex) find(hash plumbing.Hash) (int64, plumbing.Hash, error) {
	for _, idx := range i.indexes {
		if entry, ok := idx.idx.LookupHash(hash); ok {
			return int64(entry.Offset), idx.packfile, nil
		}
	}

	ok, err := i.isUnpacked(hash)
	if err != nil || ok {
		return -1, plumbing.ZeroHash, err
	}

	return -1, plumbing.ZeroHash, errHashNotInIndex.New(hash)
}

func (i *repositoryIndex) isUnpacked(hash plumbing.Hash) (bool, error) {
	f, err := i.dir.Object(hash)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}

		return false, err
	}
	if err := f.Close(); err != nil {
		return false, err
	}

	return true, nil
}

func findDotGit(fs billy.Filesystem) (billy.Filesystem, error) {
	fi, err := fs.Stat(".git")
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}

	if fi != nil && fi.IsDir() {
		return fs.Chroot(".git")
	}

	return fs, nil
}

func getUnpackedObject(repo repository, hash plumbing.Hash) (o object.Object, err error) {
	fs, err := repo.FS()
	if err != nil {
		return nil, err
	}

	fs, err = findDotGit(fs)
	if err != nil {
		return nil, err
	}

	dot := dotgit.New(fs)

	f, err := dot.Object(hash)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, plumbing.ErrObjectNotFound
		}

		return nil, err
	}

	defer ioutil.CheckClose(f, &err)

	storage, err := filesystem.NewStorage(fs)
	if err != nil {
		return nil, err
	}

	obj := storage.NewEncodedObject()
	r, err := objfile.NewReader(f)
	if err != nil {
		return nil, err
	}

	defer ioutil.CheckClose(r, &err)

	t, size, err := r.Header()
	if err != nil {
		return nil, err
	}

	obj.SetType(t)
	obj.SetSize(size)
	w, err := obj.Writer()
	if err != nil {
		return nil, err
	}

	_, err = io.Copy(w, r)

	o, err = object.DecodeObject(storage, obj)
	return
}

type repoObjectDecoder struct {
	repo     string
	packfile plumbing.Hash
	decoder  *packfile.Decoder
	storage  storer.EncodedObjectStorer
}

func newRepoObjectDecoder(
	repo repository,
	hash plumbing.Hash,
) (*repoObjectDecoder, error) {
	fs, err := repo.FS()
	if err != nil {
		return nil, err
	}

	fs, err = findDotGit(fs)
	if err != nil {
		return nil, err
	}

	dot := dotgit.New(fs)
	packf, err := dot.ObjectPack(hash)
	if err != nil {
		return nil, err
	}

	storage, err := filesystem.NewStorage(fs)
	if err != nil {
		_ = packf.Close()
		return nil, err
	}

	decoder, err := packfile.NewDecoder(packfile.NewScanner(packf), storage)
	if err != nil {
		_ = packf.Close()
		return nil, err
	}

	return &repoObjectDecoder{
		repo:     repo.ID(),
		packfile: hash,
		decoder:  decoder,
		storage:  storage,
	}, nil
}

func (d *repoObjectDecoder) equals(repo string, packfile plumbing.Hash) bool {
	return d.repo == repo && d.packfile == packfile
}

func (d *repoObjectDecoder) get(offset int64) (object.Object, error) {
	encodedObj, err := d.decoder.DecodeObjectAt(offset)
	if err != nil {
		return nil, err
	}

	return object.DecodeObject(d.storage, encodedObj)
}

func (d *repoObjectDecoder) Close() error {
	return d.decoder.Close()
}

type objectDecoder struct {
	pool    *RepositoryPool
	decoder *repoObjectDecoder
}

func newObjectDecoder(pool *RepositoryPool) *objectDecoder {
	return &objectDecoder{pool: pool}
}

func (d *objectDecoder) decode(
	repository string,
	packfile plumbing.Hash,
	offset int64,
	hash plumbing.Hash,
) (object.Object, error) {
	if offset >= 0 {
		if d.decoder == nil || !d.decoder.equals(repository, packfile) {
			if d.decoder != nil {
				if err := d.decoder.Close(); err != nil {
					return nil, err
				}
			}

			var err error
			d.decoder, err = newRepoObjectDecoder(d.pool.repositories[repository], packfile)
			if err != nil {
				return nil, err
			}
		}

		return d.decoder.get(offset)
	}

	return getUnpackedObject(d.pool.repositories[repository], hash)
}

func (d *objectDecoder) Close() error {
	if d.decoder != nil {
		return d.decoder.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>// Package page helps fetching a HTML page and its referenced CSS files.
package page

import (
	"fmt"
	"io/ioutil"
	"log"

	"net/http"
	"net/url"
	"strings"

	"github.com/nochso/colourl/cache"
	"golang.org/x/net/html"
)

// Default limits for fetching a Page.
var (
	DefaultMaxPageSize  int64 = 1024 * 1024 * 10
	DefaultMaxFileCount int   = 15
	DefaultMaxFileSize  int64 = 1024 * 1024 * 5
)

// Limits for fetching a Page.
var (
	MaxPageSize  = DefaultMaxPageSize
	MaxFileCount = DefaultMaxFileCount
	MaxFileSize  = DefaultMaxFileSize
)

// Page contains HTML and linked CSS files for a specific URL.
type Page struct {
	HTML *File
	CSS  []*File
}

// File consists of the content and URL of a single file.
type File struct {
	Body string
	URL  *url.URL
}

// Count returns the amount of files.
func (p *Page) Count() int {
	return 1 + len(p.CSS)
}

// Size returns the length of files.
func (p *Page) Size() int64 {
	var s int64
	s = int64(len(p.HTML.Body))
	for _, c := range p.CSS {
		s += int64(len(c.Body))
	}
	return s
}

// New Page from a URL.
// Any linked CSS stylesheets will be downloaded.
func New(u string) (*Page, error) {
	p := &Page{}
	html, err := NewFile(u) // Get HTML body
	if err != nil {
		return nil, err
	}
	p.HTML = html
	for _, c := range p.cssURLs() { // Iterate over links to CSS files
		if p.Size() > 1024*1024*10 { // Stop once we've got more than 10MB of HTML & CSS
			break
		}
		css, err := NewFile(c.String())
		if err != nil { // Log and continue on error
			log.Printf("Warning: Could not get CSS mentioned in '%s': %s", p.HTML.URL, err)
		} else {
			p.CSS = append(p.CSS, css)
		}
	}
	return p, nil
}

// NewFile creates a new File by GETting it from url.
func NewFile(url string) (*File, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	// Remember the original URL. It might change afterwards because of redirects.
	f := &File{URL: req.URL}

	v, err := cache.Page.Get(url)
	if err == nil {
		f.Body = v.(string)
		return f, nil
	}
	r, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	b, err := ioutil.ReadAll(r.Body)
	if r.StatusCode != http.StatusOK { // Handle anything but 200/OK as an error
		return f, fmt.Errorf("HTTP GET '%s': %s", url, r.Status)
	}
	if err != nil {
		return f, err
	}
	f.Body = string(b)
	cache.Page.Set(url, f.Body)
	return f, nil
}

// cssURLs extracts URLs to CSS files embedded in a Page's HTML body.
func (p *Page) cssURLs() []*url.URL {
	tokenizer := html.NewTokenizer(strings.NewReader(p.HTML.Body))
	urls := make([]*url.URL, 0)
	var tt html.TokenType
	var t html.Token
	for {
		tt = tokenizer.Next()
		if tt == html.ErrorToken { // End of document
			break
		}
		// Look for <link> elements
		if tt != html.StartTagToken && tt != html.SelfClosingTagToken {
			continue
		}
		t = tokenizer.Token()
		if t.Data != "link" {
			continue
		}
		isStyleSheet := false
		var link string
		for _, attr := range t.Attr {
			if attr.Key == "rel" && attr.Val == "stylesheet" {
				isStyleSheet = true
			} else if attr.Key == "href" {
				link = attr.Val
			}
		}

		// If it links to a stylesheet, resolve the URL based on the URL referencing it
		if isStyleSheet {
			uri, err := url.Parse(link)
			if err != nil {
				log.Printf("Warning: Could not parse CSS link '%s': %s", link, err)
				continue
			}
			urls = append(urls, p.HTML.URL.ResolveReference(uri))
		}
	}
	return urls
}
<commit_msg>Use defined MaxPageSize<commit_after>// Package page helps fetching a HTML page and its referenced CSS files.
package page

import (
	"fmt"
	"io/ioutil"
	"log"

	"net/http"
	"net/url"
	"strings"

	"github.com/nochso/colourl/cache"
	"golang.org/x/net/html"
)

// Default limits for fetching a Page.
var (
	DefaultMaxPageSize  int64 = 1024 * 1024 * 10
	DefaultMaxFileCount int   = 15
	DefaultMaxFileSize  int64 = 1024 * 1024 * 5
)

// Limits for fetching a Page.
var (
	MaxPageSize  = DefaultMaxPageSize
	MaxFileCount = DefaultMaxFileCount
	MaxFileSize  = DefaultMaxFileSize
)

// Page contains HTML and linked CSS files for a specific URL.
type Page struct {
	HTML *File
	CSS  []*File
}

// File consists of the content and URL of a single file.
type File struct {
	Body string
	URL  *url.URL
}

// Count returns the amount of files.
func (p *Page) Count() int {
	return 1 + len(p.CSS)
}

// Size returns the length of files.
func (p *Page) Size() int64 {
	var s int64
	s = int64(len(p.HTML.Body))
	for _, c := range p.CSS {
		s += int64(len(c.Body))
	}
	return s
}

// New Page from a URL.
// Any linked CSS stylesheets will be downloaded.
func New(u string) (*Page, error) {
	p := &Page{}
	html, err := NewFile(u) // Get HTML body
	if err != nil {
		return nil, err
	}
	p.HTML = html
	for _, c := range p.cssURLs() { // Iterate over links to CSS files
		if p.Size() >= MaxPageSize { // Stop if the complete page grows too large
			break
		}
		css, err := NewFile(c.String())
		if err != nil { // Log and continue on error
			log.Printf("Warning: Could not get CSS mentioned in '%s': %s", p.HTML.URL, err)
		} else {
			p.CSS = append(p.CSS, css)
		}
	}
	return p, nil
}

// NewFile creates a new File by GETting it from url.
func NewFile(url string) (*File, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	// Remember the original URL. It might change afterwards because of redirects.
	f := &File{URL: req.URL}

	v, err := cache.Page.Get(url)
	if err == nil {
		f.Body = v.(string)
		return f, nil
	}
	r, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	b, err := ioutil.ReadAll(r.Body)
	if r.StatusCode != http.StatusOK { // Handle anything but 200/OK as an error
		return f, fmt.Errorf("HTTP GET '%s': %s", url, r.Status)
	}
	if err != nil {
		return f, err
	}
	f.Body = string(b)
	cache.Page.Set(url, f.Body)
	return f, nil
}

// cssURLs extracts URLs to CSS files embedded in a Page's HTML body.
func (p *Page) cssURLs() []*url.URL {
	tokenizer := html.NewTokenizer(strings.NewReader(p.HTML.Body))
	urls := make([]*url.URL, 0)
	var tt html.TokenType
	var t html.Token
	for {
		tt = tokenizer.Next()
		if tt == html.ErrorToken { // End of document
			break
		}
		// Look for <link> elements
		if tt != html.StartTagToken && tt != html.SelfClosingTagToken {
			continue
		}
		t = tokenizer.Token()
		if t.Data != "link" {
			continue
		}
		isStyleSheet := false
		var link string
		for _, attr := range t.Attr {
			if attr.Key == "rel" && attr.Val == "stylesheet" {
				isStyleSheet = true
			} else if attr.Key == "href" {
				link = attr.Val
			}
		}

		// If it links to a stylesheet, resolve the URL based on the URL referencing it
		if isStyleSheet {
			uri, err := url.Parse(link)
			if err != nil {
				log.Printf("Warning: Could not parse CSS link '%s': %s", link, err)
				continue
			}
			urls = append(urls, p.HTML.URL.ResolveReference(uri))
		}
	}
	return urls
}
<|endoftext|>


<commit_before>// Package pagerduty is a client to the Pager Duty Integration API
// described here: http://developer.pagerduty.com/documentation/integration/events
package pagerduty

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
)

// PagerDuty's API endpoint
var Endpoint = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"

// See http://developer.pagerduty.com/documentation/integration/events/trigger
type Event struct {
	ServiceKey  string                 `json:"service_key"`
	EventType   string                 `json:"event_type"`
	Description string                 `json:"description"`
	IncidentKey string                 `json:"incident_key"`
	Details     map[string]interface{} `json:"details"`
}

// See http://developer.pagerduty.com/documentation/integration/events
type Response struct {
	Status      string
	Message     string
	IncidentKey string `json:"incident_key"`
	Errors      []string
}

func newEvent(serviceKey, eventType, description string) *Event {
	return &Event{ServiceKey: serviceKey, EventType: eventType, Description: description, Details: make(map[string]interface{})}
}

// Instantiates a new "trigger" Event struct
// see: http://developer.pagerduty.com/documentation/integration/events/trigger
func NewTriggerEvent(serviceKey, description string) *Event {
	return newEvent(serviceKey, "trigger", description)
}

// Instantiates a new "acknowledge" Event struct
// see: http://developer.pagerduty.com/documentation/integration/events/acknowledge
func NewAcknowledgeEvent(serviceKey, description string) *Event {
	return newEvent(serviceKey, "acknowledge", description)
}

// Instantiates a new "acknowledge" Event struct
// http://developer.pagerduty.com/documentation/integration/events/resolve
func NewResolveEvent(serviceKey, description string) *Event {
	return newEvent(serviceKey, "resolve", description)
}

// Prepare a http.Request object that can be used by the http.Client of your choice.
func PrepareRequest(event *Event) (*http.Request, error) {
	marshalled, err := json.Marshal(event)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", Endpoint, bytes.NewBuffer(marshalled))
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")

	return req, nil
}

// Creates a Response struct from a HTTP Response of the PagerDuty's API
func NewResponse(event *Event, resp *http.Response) (*Response, error) {
	buffer, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	response := &Response{}
	if err := json.Unmarshal(buffer, response); err != nil {
		return nil, err
	}

	if event.IncidentKey == "" {
		event.IncidentKey = response.IncidentKey
	}

	return response, nil
}

// Equivalent to a call to PrepareRequest, http.DefaultClient.Do() and NewResponse
func Submit(event *Event) (response *Response, statusCode int, err error) {
	req, err := PrepareRequest(event)
	if err != nil {
		return
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	statusCode = resp.StatusCode
	response, err = NewResponse(event, resp)
	if err != nil {
		return
	}

	return
}
<commit_msg>Add example in package documentation<commit_after>/*
Package pagerduty is a client to the Pager Duty Integration API.
The API is described here: http://developer.pagerduty.com/documentation/integration/events

    // Create a new "trigger" event
    event := pagerduty.NewTriggerEvent(myKey, "test API")

    // Customize the incident key. If not done, pager duty will assign one to you.
    event.IncidentKey = "My Incident Key"

    // Submit the event to pager duty's API
    response, statusCode, err := pagerduty.Submit(event)
*/
package pagerduty

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
)

// PagerDuty's API endpoint
var Endpoint = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"

// See http://developer.pagerduty.com/documentation/integration/events/trigger
type Event struct {
	ServiceKey  string                 `json:"service_key"`
	EventType   string                 `json:"event_type"`
	Description string                 `json:"description"`
	IncidentKey string                 `json:"incident_key"`
	Details     map[string]interface{} `json:"details"`
}

// See http://developer.pagerduty.com/documentation/integration/events
type Response struct {
	Status      string
	Message     string
	IncidentKey string `json:"incident_key"`
	Errors      []string
}

func newEvent(serviceKey, eventType, description string) *Event {
	return &Event{ServiceKey: serviceKey, EventType: eventType, Description: description, Details: make(map[string]interface{})}
}

// Instantiates a new "trigger" Event struct
// see: http://developer.pagerduty.com/documentation/integration/events/trigger
func NewTriggerEvent(serviceKey, description string) *Event {
	return newEvent(serviceKey, "trigger", description)
}

// Instantiates a new "acknowledge" Event struct
// see: http://developer.pagerduty.com/documentation/integration/events/acknowledge
func NewAcknowledgeEvent(serviceKey, description string) *Event {
	return newEvent(serviceKey, "acknowledge", description)
}

// Instantiates a new "acknowledge" Event struct
// http://developer.pagerduty.com/documentation/integration/events/resolve
func NewResolveEvent(serviceKey, description string) *Event {
	return newEvent(serviceKey, "resolve", description)
}

// Prepare a http.Request object that can be used by the http.Client of your choice.
func PrepareRequest(event *Event) (*http.Request, error) {
	marshalled, err := json.Marshal(event)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", Endpoint, bytes.NewBuffer(marshalled))
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")

	return req, nil
}

// Creates a Response struct from a HTTP Response of the PagerDuty's API
func NewResponse(event *Event, resp *http.Response) (*Response, error) {
	buffer, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	response := &Response{}
	if err := json.Unmarshal(buffer, response); err != nil {
		return nil, err
	}

	if event.IncidentKey == "" {
		event.IncidentKey = response.IncidentKey
	}

	return response, nil
}

// Equivalent to a call to PrepareRequest, http.DefaultClient.Do() and NewResponse
func Submit(event *Event) (response *Response, statusCode int, err error) {
	req, err := PrepareRequest(event)
	if err != nil {
		return
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}
	statusCode = resp.StatusCode
	response, err = NewResponse(event, resp)
	if err != nil {
		return
	}

	return
}
<|endoftext|>


<commit_before>// Copyright ©2012 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bgzf

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"io"
)

// Reader implements BGZF blocked gzip decompression.
type Reader struct {
	gzip.Header
	r io.Reader

	// lastChunk is the virtual file offset
	// interval of the last successful read
	// or seek operation.
	lastChunk Chunk

	// nextBase is the file offset of the
	// block following the current block.
	nextBase int64

	active *decompressor

	// Cache is the Reader block cache. If Cache is not nil,
	// the cache is queried for blocks before an attempt to
	// read from the underlying io.Reader.
	Cache Cache

	err error
}

type decompressor struct {
	owner *Reader

	gz gzip.Reader

	// Underlying Reader.
	r flate.Reader

	// Positions within underlying data stream
	offset int64 // Current offset in stream - possibly virtual.
	mark   int64 // Offset at start of useUnderlying.

	// Buffered compressed data from read ahead.
	i   int // Current position in buffered data.
	n   int // Total size of buffered data.
	buf [MaxBlockSize]byte

	// Decompressed data.
	decompressed Block

	err error
}

func makeReader(r io.Reader) flate.Reader {
	switch r := r.(type) {
	case *decompressor:
		panic("bgzf: illegal use of internal type")
	case flate.Reader:
		return r
	default:
		return bufio.NewReader(r)
	}
}

func newDecompressor() *decompressor { return &decompressor{} }

func (d *decompressor) init(r io.Reader) (*decompressor, error) {
	d.r = makeReader(r)
	d.err = d.gz.Reset(d)
	if d.err != nil {
		return d, d.err
	}
	if expectedBlockSize(d.gz.Header) < 0 {
		d.err = ErrNoBlockSize
	}
	return d, d.err
}

// lazyBlock conditionally creates a ready to use Block and returns whether
// the Block subsequently held by the decompressor needs to be reset before
// being filled.
func (d *decompressor) lazyBlock() bool {
	if d.decompressed == nil {
		if w, ok := d.owner.Cache.(Wrapper); ok {
			d.decompressed = w.Wrap(&block{owner: d.owner})
		} else {
			d.decompressed = &block{owner: d.owner}
		}
		return false
	}
	if !d.decompressed.ownedBy(d.owner) {
		d.decompressed.setOwner(d.owner)
	}
	return true
}

func (d *decompressor) header() gzip.Header {
	return d.gz.Header
}

func (d *decompressor) isBuffered() bool { return d.n != 0 }

// Read provides the Read method for the decompressor's gzip.Reader.
func (d *decompressor) Read(p []byte) (int, error) {
	var (
		n   int
		err error
	)
	if d.isBuffered() {
		if d.i >= d.n {
			return 0, io.EOF
		}
		if n := d.n - d.i; len(p) > n {
			p = p[:n]
		}
		n = copy(p, d.buf[d.i:])
		d.i += n
	} else {
		n, err = d.r.Read(p)
	}
	d.offset += int64(n)
	return n, err
}

// ReadByte provides the ReadByte method for the decompressor's gzip.Reader.
func (d *decompressor) ReadByte() (byte, error) {
	var (
		b   byte
		err error
	)
	if d.isBuffered() {
		if d.i == d.n {
			return 0, io.EOF
		}
		b = d.buf[d.i]
		d.i++
	} else {
		b, err = d.r.ReadByte()
	}
	if err == nil {
		d.offset++
	}
	return b, err
}

func (d *decompressor) reset() {
	needReset := d.lazyBlock()

	if d.gotBlockFor(d.owner.nextBase) {
		return
	}

	if needReset && d.offset != d.owner.nextBase {
		// It should not be possible for the expected next block base
		// to be out of register with the count reader unless Seek
		// has been called, so we know the base reader must be an
		// io.ReadSeeker.
		d.err = d.seek(d.owner.r.(io.ReadSeeker), d.owner.nextBase)
		if d.err != nil {
			return
		}
	}

	d.err = d.fill(needReset)
}

func (d *decompressor) seekRead(r io.ReadSeeker, off int64) {
	d.lazyBlock()

	if off == d.decompressed.Base() && d.decompressed.hasData() {
		return
	}

	if d.gotBlockFor(off) {
		return
	}

	d.err = d.seek(r, off)
	if d.err != nil {
		return
	}

	d.err = d.fill(true)
}

func (d *decompressor) seek(r io.ReadSeeker, off int64) error {
	_, err := r.Seek(off, 0)
	if err != nil {
		return err
	}

	type reseter interface {
		Reset(io.Reader)
	}
	switch cr := d.r.(type) {
	case reseter:
		cr.Reset(r)
	default:
		d.r = makeReader(r)
	}
	d.offset = off

	return nil
}

// gotBlockFor returns true if the decompressor has access to a cache
// and that cache holds the block with given base and the correct
// owner, otherwise it returns false.
// gotBlockFor has side effects of recovering the block and putting
// the currently active block into the cache. If the cache returns
// a block owned by another reader, it is discarded.
func (d *decompressor) gotBlockFor(base int64) bool {
	if d.owner.Cache != nil {
		dec := d.decompressed
		if blk := d.owner.Cache.Get(base); blk != nil && blk.ownedBy(d.owner) {
			if dec != nil && dec.hasData() {
				// TODO(kortschak): Under some conditions, e.g. FIFO
				// cache we will be discarding a non-nil evicted Block.
				// Consider retaining these in a sync.Pool.
				d.owner.Cache.Put(dec)
			}
			if d.err = blk.seek(0); d.err == nil {
				d.decompressed = blk
				d.owner.nextBase = blk.nextBase()
				return true
			}
		}
		if dec != nil && dec.hasData() {
			dec, retained := d.owner.Cache.Put(dec)
			if retained {
				d.decompressed = dec
				d.lazyBlock()
			}
		}
	}

	return false
}

func (d *decompressor) useUnderlying() { d.n = 0; d.mark = d.offset }

func (d *decompressor) readAhead(n int) error {
	d.i, d.n = 0, n
	var err error
	lr := io.LimitedReader{R: d.r, N: int64(n)}
	for i, _n := 0, 0; i < n && err == nil; i += _n {
		_n, err = lr.Read(d.buf[i:])
		if err != nil {
			break
		}
	}
	return err
}

func (d *decompressor) deltaOffset() int64 { return d.offset - d.mark }

func (d *decompressor) fill(reset bool) error {
	dec := d.decompressed

	if reset {
		dec.setBase(d.offset)

		d.useUnderlying()
		err := d.gz.Reset(d)
		bs := expectedBlockSize(d.gz.Header)
		if err == nil && bs < 0 {
			err = ErrNoBlockSize
		}
		if err != nil {
			return err
		}
		err = d.readAhead(bs - int(d.deltaOffset()))
		if err != nil {
			return err
		}
	}

	dec.setHeader(d.gz.Header)
	d.gz.Multistream(false)
	return dec.readFrom(&d.gz)
}

func expectedBlockSize(h gzip.Header) int {
	i := bytes.Index(h.Extra, bgzfExtraPrefix)
	if i < 0 || i+5 >= len(h.Extra) {
		return -1
	}
	return (int(h.Extra[i+4]) | int(h.Extra[i+5])<<8) + 1
}

// NewReader returns a new BGZF reader.
//
// The number of concurrent read decompressors is specified by
// rd (currently ignored).
func NewReader(r io.Reader, rd int) (*Reader, error) {
	d, err := newDecompressor().init(r)
	if err != nil {
		return nil, err
	}
	bg := &Reader{
		Header: d.header(),
		r:      r,
		active: d,
	}
	d.owner = bg
	return bg, nil
}

// Offset is a BGZF virtual offset.
type Offset struct {
	File  int64
	Block uint16
}

// Chunk is a region of a BGZF file.
type Chunk struct {
	Begin Offset
	End   Offset
}

// Seek performs a seek operation to the given virtual offset.
func (bg *Reader) Seek(off Offset) error {
	rs, ok := bg.r.(io.ReadSeeker)
	if !ok {
		return ErrNotASeeker
	}

	bg.active.seekRead(rs, off.File)
	bg.err = bg.active.err
	if bg.err != nil {
		return bg.err
	}
	bg.Header = bg.active.header()
	bg.nextBase = bg.active.decompressed.nextBase()

	bg.err = bg.active.decompressed.seek(int64(off.Block))
	if bg.err == nil {
		bg.lastChunk = Chunk{Begin: off, End: off}
	}

	return bg.err
}

// LastChunk returns the region of the BGZF file read by the last read
// operation or the resulting virtual offset of the last successful
// seek operation.
func (bg *Reader) LastChunk() Chunk { return bg.lastChunk }

// Close closes the reader and releases resources.
func (bg *Reader) Close() error {
	bg.Cache = nil
	return bg.active.gz.Close()
}

// Read implements the io.Reader interface.
func (bg *Reader) Read(p []byte) (int, error) {
	if bg.err != nil {
		return 0, bg.err
	}

	dec := bg.active.decompressed
	if dec != nil {
		dec.beginTx()
	} else {
		bs := expectedBlockSize(bg.Header)
		bg.err = bg.active.readAhead(bs - int(bg.active.deltaOffset()))
		if bg.err != nil {
			return 0, bg.err
		}
	}

	if dec == nil || dec.len() == 0 {
		dec, bg.err = bg.resetDecompressor()
		if bg.err != nil {
			return 0, bg.err
		}
	}

	var n int
	for n < len(p) && bg.err == nil {
		var _n int
		_n, bg.err = dec.Read(p[n:])
		if _n > 0 {
			bg.lastChunk = dec.endTx()
		}
		n += _n
		if bg.err == io.EOF {
			if n == len(p) {
				bg.err = nil
				break
			}

			dec, bg.err = bg.resetDecompressor()
			if bg.err != nil {
				break
			}
		}
	}

	return n, bg.err
}

func (bg *Reader) resetDecompressor() (Block, error) {
	bg.active.reset()
	if bg.active.err != nil {
		return nil, bg.active.err
	}
	bg.Header = bg.active.header()
	bg.nextBase = bg.active.decompressed.nextBase()
	return bg.active.decompressed, nil
}
<commit_msg>Make use of init code in fill<commit_after>// Copyright ©2012 The bíogo.bam Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bgzf

import (
	"bufio"
	"bytes"
	"compress/flate"
	"compress/gzip"
	"io"
)

// Reader implements BGZF blocked gzip decompression.
type Reader struct {
	gzip.Header
	r io.Reader

	// lastChunk is the virtual file offset
	// interval of the last successful read
	// or seek operation.
	lastChunk Chunk

	// nextBase is the file offset of the
	// block following the current block.
	nextBase int64

	active *decompressor

	// Cache is the Reader block cache. If Cache is not nil,
	// the cache is queried for blocks before an attempt to
	// read from the underlying io.Reader.
	Cache Cache

	err error
}

type decompressor struct {
	owner *Reader

	gz gzip.Reader

	// Underlying Reader.
	r flate.Reader

	// Current block size.
	blockSize int

	// Positions within underlying data stream
	offset int64 // Current offset in stream - possibly virtual.
	mark   int64 // Offset at start of useUnderlying.

	// Buffered compressed data from read ahead.
	i   int // Current position in buffered data.
	n   int // Total size of buffered data.
	buf [MaxBlockSize]byte

	// Decompressed data.
	decompressed Block

	err error
}

func makeReader(r io.Reader) flate.Reader {
	switch r := r.(type) {
	case *decompressor:
		panic("bgzf: illegal use of internal type")
	case flate.Reader:
		return r
	default:
		return bufio.NewReader(r)
	}
}

func newDecompressor() *decompressor { return &decompressor{} }

func (d *decompressor) init(r flate.Reader) (*decompressor, error) {
	d.r = r
	d.useUnderlying()
	err := d.gz.Reset(d)
	if err != nil {
		d.blockSize = -1
		return d, err
	}
	d.blockSize = expectedBlockSize(d.gz.Header)
	if d.blockSize < 0 {
		err = ErrNoBlockSize
	}
	return d, err
}

// lazyBlock conditionally creates a ready to use Block and returns whether
// the Block subsequently held by the decompressor needs to be reset before
// being filled.
func (d *decompressor) lazyBlock() bool {
	if d.decompressed == nil {
		if w, ok := d.owner.Cache.(Wrapper); ok {
			d.decompressed = w.Wrap(&block{owner: d.owner})
		} else {
			d.decompressed = &block{owner: d.owner}
		}
		return false
	}
	if !d.decompressed.ownedBy(d.owner) {
		d.decompressed.setOwner(d.owner)
	}
	return true
}

func (d *decompressor) header() gzip.Header {
	return d.gz.Header
}

func (d *decompressor) isBuffered() bool { return d.n != 0 }

// Read provides the Read method for the decompressor's gzip.Reader.
func (d *decompressor) Read(p []byte) (int, error) {
	var (
		n   int
		err error
	)
	if d.isBuffered() {
		if d.i >= d.n {
			return 0, io.EOF
		}
		if n := d.n - d.i; len(p) > n {
			p = p[:n]
		}
		n = copy(p, d.buf[d.i:])
		d.i += n
	} else {
		n, err = d.r.Read(p)
	}
	d.offset += int64(n)
	return n, err
}

// ReadByte provides the ReadByte method for the decompressor's gzip.Reader.
func (d *decompressor) ReadByte() (byte, error) {
	var (
		b   byte
		err error
	)
	if d.isBuffered() {
		if d.i == d.n {
			return 0, io.EOF
		}
		b = d.buf[d.i]
		d.i++
	} else {
		b, err = d.r.ReadByte()
	}
	if err == nil {
		d.offset++
	}
	return b, err
}

func (d *decompressor) reset() {
	needReset := d.lazyBlock()

	if d.gotBlockFor(d.owner.nextBase) {
		return
	}

	if needReset && d.offset != d.owner.nextBase {
		// It should not be possible for the expected next block base
		// to be out of register with the count reader unless Seek
		// has been called, so we know the base reader must be an
		// io.ReadSeeker.
		d.err = d.seek(d.owner.r.(io.ReadSeeker), d.owner.nextBase)
		if d.err != nil {
			return
		}
	}

	d.err = d.fill(needReset)
}

func (d *decompressor) seekRead(r io.ReadSeeker, off int64) {
	d.lazyBlock()

	if off == d.decompressed.Base() && d.decompressed.hasData() {
		return
	}

	if d.gotBlockFor(off) {
		return
	}

	d.err = d.seek(r, off)
	if d.err != nil {
		return
	}

	d.err = d.fill(true)
}

func (d *decompressor) seek(r io.ReadSeeker, off int64) error {
	_, err := r.Seek(off, 0)
	if err != nil {
		return err
	}

	type reseter interface {
		Reset(io.Reader)
	}
	switch cr := d.r.(type) {
	case reseter:
		cr.Reset(r)
	default:
		d.r = makeReader(r)
	}
	d.offset = off

	return nil
}

// gotBlockFor returns true if the decompressor has access to a cache
// and that cache holds the block with given base and the correct
// owner, otherwise it returns false.
// gotBlockFor has side effects of recovering the block and putting
// the currently active block into the cache. If the cache returns
// a block owned by another reader, it is discarded.
func (d *decompressor) gotBlockFor(base int64) bool {
	if d.owner.Cache != nil {
		dec := d.decompressed
		if blk := d.owner.Cache.Get(base); blk != nil && blk.ownedBy(d.owner) {
			if dec != nil && dec.hasData() {
				// TODO(kortschak): Under some conditions, e.g. FIFO
				// cache we will be discarding a non-nil evicted Block.
				// Consider retaining these in a sync.Pool.
				d.owner.Cache.Put(dec)
			}
			if d.err = blk.seek(0); d.err == nil {
				d.decompressed = blk
				d.owner.nextBase = blk.nextBase()
				return true
			}
		}
		if dec != nil && dec.hasData() {
			dec, retained := d.owner.Cache.Put(dec)
			if retained {
				d.decompressed = dec
				d.lazyBlock()
			}
		}
	}

	return false
}

func (d *decompressor) useUnderlying() { d.n = 0; d.mark = d.offset }

func (d *decompressor) readAhead() error {
	n := d.blockSize - d.deltaOffset()

	d.i, d.n = 0, n
	var err error
	lr := io.LimitedReader{R: d.r, N: int64(n)}
	for i, _n := 0, 0; i < n && err == nil; i += _n {
		_n, err = lr.Read(d.buf[i:])
		if err != nil {
			break
		}
	}
	return err
}

func (d *decompressor) deltaOffset() int { return int(d.offset - d.mark) }

func (d *decompressor) fill(reset bool) error {
	dec := d.decompressed

	if reset {
		dec.setBase(d.offset)

		_, err := d.init(d.r)
		if err != nil {
			return err
		}

		err = d.readAhead()
		if err != nil {
			return err
		}
	}

	dec.setHeader(d.gz.Header)
	d.gz.Multistream(false)
	return dec.readFrom(&d.gz)
}

func expectedBlockSize(h gzip.Header) int {
	i := bytes.Index(h.Extra, bgzfExtraPrefix)
	if i < 0 || i+5 >= len(h.Extra) {
		return -1
	}
	return (int(h.Extra[i+4]) | int(h.Extra[i+5])<<8) + 1
}

// NewReader returns a new BGZF reader.
//
// The number of concurrent read decompressors is specified by
// rd (currently ignored).
func NewReader(r io.Reader, rd int) (*Reader, error) {
	d, err := newDecompressor().init(makeReader(r))
	if err != nil {
		return nil, err
	}
	bg := &Reader{
		Header: d.header(),
		r:      r,
		active: d,
	}
	d.owner = bg
	return bg, nil
}

// Offset is a BGZF virtual offset.
type Offset struct {
	File  int64
	Block uint16
}

// Chunk is a region of a BGZF file.
type Chunk struct {
	Begin Offset
	End   Offset
}

// Seek performs a seek operation to the given virtual offset.
func (bg *Reader) Seek(off Offset) error {
	rs, ok := bg.r.(io.ReadSeeker)
	if !ok {
		return ErrNotASeeker
	}

	bg.active.seekRead(rs, off.File)
	bg.err = bg.active.err
	if bg.err != nil {
		return bg.err
	}
	bg.Header = bg.active.header()
	bg.nextBase = bg.active.decompressed.nextBase()

	bg.err = bg.active.decompressed.seek(int64(off.Block))
	if bg.err == nil {
		bg.lastChunk = Chunk{Begin: off, End: off}
	}

	return bg.err
}

// LastChunk returns the region of the BGZF file read by the last read
// operation or the resulting virtual offset of the last successful
// seek operation.
func (bg *Reader) LastChunk() Chunk { return bg.lastChunk }

// Close closes the reader and releases resources.
func (bg *Reader) Close() error {
	bg.Cache = nil
	return bg.active.gz.Close()
}

// Read implements the io.Reader interface.
func (bg *Reader) Read(p []byte) (int, error) {
	if bg.err != nil {
		return 0, bg.err
	}

	dec := bg.active.decompressed
	if dec != nil {
		dec.beginTx()
	} else {
		bg.err = bg.active.readAhead()
		if bg.err != nil {
			return 0, bg.err
		}
	}

	if dec == nil || dec.len() == 0 {
		dec, bg.err = bg.resetDecompressor()
		if bg.err != nil {
			return 0, bg.err
		}
	}

	var n int
	for n < len(p) && bg.err == nil {
		var _n int
		_n, bg.err = dec.Read(p[n:])
		if _n > 0 {
			bg.lastChunk = dec.endTx()
		}
		n += _n
		if bg.err == io.EOF {
			if n == len(p) {
				bg.err = nil
				break
			}

			dec, bg.err = bg.resetDecompressor()
			if bg.err != nil {
				break
			}
		}
	}

	return n, bg.err
}

func (bg *Reader) resetDecompressor() (Block, error) {
	bg.active.reset()
	if bg.active.err != nil {
		return nil, bg.active.err
	}
	bg.Header = bg.active.header()
	bg.nextBase = bg.active.decompressed.nextBase()
	return bg.active.decompressed, nil
}
<|endoftext|>