<commit_before>// Copyright 2017-18 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pbf

import (
	"bytes"
	"compress/zlib"
	"context"
	"errors"
	"fmt"
	"io"
	"reflect"
	"runtime"
	"time"

	"github.com/golang/protobuf/proto"
	"m4o.io/pbf/protobuf"
)

const (
	// DefaultBufferSize is the default buffer size for protobuf un-marshaling
	DefaultBufferSize           = 1024 * 1024

	// DefaultInputChannelLength is the default channel length of raw blobs
	DefaultInputChannelLength   = 16

	// DefaultOutputChannelLength is the default channel length of decoded arrays of element
	DefaultOutputChannelLength  = 8

	// DefaultDecodedChannelLength is the default channel length of decoded elements coalesced from output channels
	DefaultDecodedChannelLength = 8000
)

// DefaultNCpu provides the default number of CPUs.
func DefaultNCpu() uint16 {
	return uint16(runtime.GOMAXPROCS(-1))
}

type encoded struct {
	header *protobuf.BlobHeader
	blob   *protobuf.Blob
	err    error
}

type decoded struct {
	elements []interface{}
	err      error
}

type pair struct {
	element interface{}
	err     error
}

// Decoder reads and decodes OpenStreetMap PBF data from an input stream.
type Decoder struct {
	Header Header
	pairs  chan pair
	cancel context.CancelFunc
}

// decoderOptions provides optional configuration parameters for Decoder construction.
type decoderOptions struct {
	protoBufferSize      int    // buffer size for protobuf un-marshaling
	inputChannelLength   int    // channel length of raw blobs
	outputChannelLength  int    // channel length of decoded arrays of element
	decodedChannelLength int    // channel length of decoded elements coalesced from output channels
	nCpu                 uint16 // the number of CPUs to use for background processing
}

// DecoderOption configures how we set up the decoder.
type DecoderOption func(*decoderOptions)

// WithProtoBufferSize lets you set the buffer size for protobuf un-marshaling.
func WithProtoBufferSize(s int) DecoderOption {
	return func(o *decoderOptions) {
		o.protoBufferSize = s
	}
}

// WithInputChannelLength lets you set the channel length of raw blobs.
func WithInputChannelLength(l int) DecoderOption {
	return func(o *decoderOptions) {
		o.inputChannelLength = l
	}
}

// WithOutputChannelLength lets you set the channel length of decoded arrays of element.
func WithOutputChannelLength(l int) DecoderOption {
	return func(o *decoderOptions) {
		o.outputChannelLength = l
	}
}

// WithDecodedChannelLength lets you set the channel length of decoded elements coalesced from output channels.
func WithDecodedChannelLength(l int) DecoderOption {
	return func(o *decoderOptions) {
		o.decodedChannelLength = l
	}
}

// WithNCpus lets you set the number of CPUs to use for background processing.
func WithNCpus(n uint16) DecoderOption {
	return func(o *decoderOptions) {
		o.nCpu = n
	}
}

// defaultDecoderConfig provides a default configuration for decoders.
var defaultDecoderConfig = decoderOptions{
	protoBufferSize:      DefaultBufferSize,
	inputChannelLength:   DefaultInputChannelLength,
	outputChannelLength:  DefaultOutputChannelLength,
	decodedChannelLength: DefaultDecodedChannelLength,
	nCpu:                 DefaultNCpu(),
}

// NewDecoder returns a new decoder, configured with cfg, that reads from
// reader.  The decoder is initialized with the OSM header.
func NewDecoder(ctx context.Context, reader io.Reader, opts ...DecoderOption) (*Decoder, error) {

	d := &Decoder{}
	c := defaultDecoderConfig

	for _, opt := range opts {
		opt(&c)
	}

	ctx, d.cancel = context.WithCancel(ctx)

	r := newBlobReader(reader)
	buf := bytes.NewBuffer(make([]byte, 0, c.protoBufferSize))

	h, err := r.readBlobHeader(buf)
	if err != nil {
		return nil, err
	}
	b, err := r.readBlob(buf, h)
	if err != nil {
		return nil, err
	}
	e, err := elements(h, b, bytes.NewBuffer(make([]byte, 0, 1024)))
	if err != nil {
		return nil, err
	}

	if e[0].(*Header) == nil {
		err = fmt.Errorf("expected header data but got %v", reflect.TypeOf(e[0]))
		return nil, err
	}
	d.Header = *e[0].(*Header)

	// create decoding pipelines
	var outputs []chan decoded
	for _, input := range read(ctx, r, c) {
		outputs = append(outputs, decode(input, c))
	}
	d.pairs = coalesce(c, outputs...)

	return d, nil
}

// Decode reads the next OSM object and returns either a pointer to Node, Way
// or Relation struct representing the underlying OpenStreetMap PBF data, or
// error encountered. The end of the input stream is reported by an io.EOF
// error.
func (d *Decoder) Decode() (interface{}, error) {
	decoded, more := <-d.pairs
	if !more {
		return nil, io.EOF
	}
	return decoded.element, decoded.err
}

// Close will cancel the background decoding pipeline.
func (d *Decoder) Close() {
	d.cancel()
}

// read obtains OSM blobs and sends them down, in a round-robin manner, a list
// of channels to be decoded.
func read(ctx context.Context, b blobReader, cfg decoderOptions) (inputs []chan encoded) {

	n := cfg.nCpu
	for i := uint16(0); i < n; i++ {
		inputs = append(inputs, make(chan encoded, cfg.inputChannelLength))
	}

	go func() {
		defer func() {
			for _, input := range inputs {
				close(input)
			}
		}()

		buffer := bytes.NewBuffer(make([]byte, 0, cfg.protoBufferSize))
		var i uint16
		for {
			input := inputs[i]
			i = (i + 1) % n

			h, err := b.readBlobHeader(buffer)
			if err == io.EOF {
				return
			} else if err != nil {
				input <- encoded{err: err}
				return
			}

			b, err := b.readBlob(buffer, h)
			if err != nil {
				input <- encoded{err: err}
				return
			}

			select {
			case <-ctx.Done():
				return
			case input <- encoded{header: h, blob: b}:
			}
		}
	}()

	return
}

// decode decodes blob/header pairs into an array of OSM elements.  These
// arrays are placed onto an output channel where they will be coalesced into
// their correct order.
func decode(input <-chan encoded, cfg decoderOptions) (output chan decoded) {

	output = make(chan decoded, cfg.outputChannelLength)

	buf := bytes.NewBuffer(make([]byte, 0, cfg.protoBufferSize))

	go func() {
		defer close(output)

		for {
			raw, more := <-input
			if !more {
				return
			}

			if raw.err != nil {
				output <- decoded{nil, raw.err}
				return
			}

			elements, err := elements(raw.header, raw.blob, buf)

			output <- decoded{elements, err}
		}
	}()

	return
}

// coalesce merges the list of channels in a round-robin manner and sends the
// elements in pairs down a channel of pairs.
func coalesce(cfg decoderOptions, outputs ...chan decoded) (pairs chan pair) {

	pairs = make(chan pair, cfg.decodedChannelLength)

	go func() {
		defer close(pairs)

		n := len(outputs)
		var i int
		for {
			output := outputs[i]
			i = (i + 1) % n

			decoded, more := <-output
			if !more {
				// Since the channels are inspected round-robin, when one channel
				// is done, all subsequent channels are done.
				return
			}

			if decoded.err != nil {
				pairs <- pair{nil, decoded.err}
				return
			}

			for _, e := range decoded.elements {
				pairs <- pair{e, nil}
			}
		}
	}()

	return
}

// elements unmarshals an array of OSM elements from an array of protobuf encoded
// bytes.  The bytes could possibly be compressed; zlibBuf is used to facilitate
// decompression.
func elements(header *protobuf.BlobHeader, blob *protobuf.Blob, zlibBuf *bytes.Buffer) ([]interface{}, error) {

	var buf []byte

	switch {
	case blob.Raw != nil:
		buf = blob.GetRaw()

	case blob.ZlibData != nil:
		r, err := zlib.NewReader(bytes.NewReader(blob.GetZlibData()))
		if err != nil {
			return nil, err
		}
		zlibBuf.Reset()
		rawBufferSize := int(blob.GetRawSize() + bytes.MinRead)
		if rawBufferSize > zlibBuf.Cap() {
			zlibBuf.Grow(rawBufferSize)
		}
		_, err = zlibBuf.ReadFrom(r)
		if err != nil {
			return nil, err
		}
		if zlibBuf.Len() != int(blob.GetRawSize()) {
			err = fmt.Errorf("raw blob data size %d but expected %d", zlibBuf.Len(), blob.GetRawSize())
			return nil, err
		}
		buf = zlibBuf.Bytes()

	default:
		return nil, errors.New("unknown blob data type")
	}

	ht := *header.Type
	if ht == "OSMHeader" {
		h, err := parseOSMHeader(buf)
		if err != nil {
			return nil, err
		}
		return []interface{}{h}, nil
	} else if ht == "OSMData" {
		return parsePrimitiveBlock(buf)
	} else {
		return nil, fmt.Errorf("unknown header type %s", ht)
	}
}

// parseOSMHeader unmarshals the OSM header from an array of protobuf encoded bytes.
func parseOSMHeader(buffer []byte) (*Header, error) {
	hb := &protobuf.HeaderBlock{}
	if err := proto.Unmarshal(buffer, hb); err != nil {
		return nil, err
	}

	header := &Header{
		RequiredFeatures: hb.GetRequiredFeatures(),
		OptionalFeatures: hb.GetOptionalFeatures(),
		WritingProgram:   hb.GetWritingprogram(),
		Source:           hb.GetSource(),
		OsmosisReplicationBaseURL:        hb.GetOsmosisReplicationBaseUrl(),
		OsmosisReplicationSequenceNumber: hb.GetOsmosisReplicationSequenceNumber(),
	}

	if hb.Bbox != nil {
		header.BoundingBox = BoundingBox{
			Left:   toDegrees(0, 1, hb.Bbox.GetLeft()),
			Right:  toDegrees(0, 1, hb.Bbox.GetRight()),
			Top:    toDegrees(0, 1, hb.Bbox.GetTop()),
			Bottom: toDegrees(0, 1, hb.Bbox.GetBottom()),
		}
	}

	if hb.OsmosisReplicationTimestamp != nil {
		header.OsmosisReplicationTimestamp = time.Unix(*hb.OsmosisReplicationTimestamp, 0)
	}

	return header, nil
}

// toDegrees converts a coordinate into Degrees, given the offset and
// granularity of the coordinate.
func toDegrees(offset int64, granularity int32, coordinate int64) Degrees {
	return 1e-9 * Degrees(offset+(int64(granularity)*coordinate))
}
<commit_msg>Rename nCpu to nCPU<commit_after>// Copyright 2017-18 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pbf

import (
	"bytes"
	"compress/zlib"
	"context"
	"errors"
	"fmt"
	"io"
	"reflect"
	"runtime"
	"time"

	"github.com/golang/protobuf/proto"
	"m4o.io/pbf/protobuf"
)

const (
	// DefaultBufferSize is the default buffer size for protobuf un-marshaling
	DefaultBufferSize           = 1024 * 1024

	// DefaultInputChannelLength is the default channel length of raw blobs
	DefaultInputChannelLength   = 16

	// DefaultOutputChannelLength is the default channel length of decoded arrays of element
	DefaultOutputChannelLength  = 8

	// DefaultDecodedChannelLength is the default channel length of decoded elements coalesced from output channels
	DefaultDecodedChannelLength = 8000
)

// DefaultNCpu provides the default number of CPUs.
func DefaultNCpu() uint16 {
	return uint16(runtime.GOMAXPROCS(-1))
}

type encoded struct {
	header *protobuf.BlobHeader
	blob   *protobuf.Blob
	err    error
}

type decoded struct {
	elements []interface{}
	err      error
}

type pair struct {
	element interface{}
	err     error
}

// Decoder reads and decodes OpenStreetMap PBF data from an input stream.
type Decoder struct {
	Header Header
	pairs  chan pair
	cancel context.CancelFunc
}

// decoderOptions provides optional configuration parameters for Decoder construction.
type decoderOptions struct {
	protoBufferSize      int    // buffer size for protobuf un-marshaling
	inputChannelLength   int    // channel length of raw blobs
	outputChannelLength  int    // channel length of decoded arrays of element
	decodedChannelLength int    // channel length of decoded elements coalesced from output channels
	nCPU                 uint16 // the number of CPUs to use for background processing
}

// DecoderOption configures how we set up the decoder.
type DecoderOption func(*decoderOptions)

// WithProtoBufferSize lets you set the buffer size for protobuf un-marshaling.
func WithProtoBufferSize(s int) DecoderOption {
	return func(o *decoderOptions) {
		o.protoBufferSize = s
	}
}

// WithInputChannelLength lets you set the channel length of raw blobs.
func WithInputChannelLength(l int) DecoderOption {
	return func(o *decoderOptions) {
		o.inputChannelLength = l
	}
}

// WithOutputChannelLength lets you set the channel length of decoded arrays of element.
func WithOutputChannelLength(l int) DecoderOption {
	return func(o *decoderOptions) {
		o.outputChannelLength = l
	}
}

// WithDecodedChannelLength lets you set the channel length of decoded elements coalesced from output channels.
func WithDecodedChannelLength(l int) DecoderOption {
	return func(o *decoderOptions) {
		o.decodedChannelLength = l
	}
}

// WithNCpus lets you set the number of CPUs to use for background processing.
func WithNCpus(n uint16) DecoderOption {
	return func(o *decoderOptions) {
		o.nCPU = n
	}
}

// defaultDecoderConfig provides a default configuration for decoders.
var defaultDecoderConfig = decoderOptions{
	protoBufferSize:      DefaultBufferSize,
	inputChannelLength:   DefaultInputChannelLength,
	outputChannelLength:  DefaultOutputChannelLength,
	decodedChannelLength: DefaultDecodedChannelLength,
	nCPU:                 DefaultNCpu(),
}

// NewDecoder returns a new decoder, configured with cfg, that reads from
// reader.  The decoder is initialized with the OSM header.
func NewDecoder(ctx context.Context, reader io.Reader, opts ...DecoderOption) (*Decoder, error) {

	d := &Decoder{}
	c := defaultDecoderConfig

	for _, opt := range opts {
		opt(&c)
	}

	ctx, d.cancel = context.WithCancel(ctx)

	r := newBlobReader(reader)
	buf := bytes.NewBuffer(make([]byte, 0, c.protoBufferSize))

	h, err := r.readBlobHeader(buf)
	if err != nil {
		return nil, err
	}
	b, err := r.readBlob(buf, h)
	if err != nil {
		return nil, err
	}
	e, err := elements(h, b, bytes.NewBuffer(make([]byte, 0, 1024)))
	if err != nil {
		return nil, err
	}

	if e[0].(*Header) == nil {
		err = fmt.Errorf("expected header data but got %v", reflect.TypeOf(e[0]))
		return nil, err
	}
	d.Header = *e[0].(*Header)

	// create decoding pipelines
	var outputs []chan decoded
	for _, input := range read(ctx, r, c) {
		outputs = append(outputs, decode(input, c))
	}
	d.pairs = coalesce(c, outputs...)

	return d, nil
}

// Decode reads the next OSM object and returns either a pointer to Node, Way
// or Relation struct representing the underlying OpenStreetMap PBF data, or
// error encountered. The end of the input stream is reported by an io.EOF
// error.
func (d *Decoder) Decode() (interface{}, error) {
	decoded, more := <-d.pairs
	if !more {
		return nil, io.EOF
	}
	return decoded.element, decoded.err
}

// Close will cancel the background decoding pipeline.
func (d *Decoder) Close() {
	d.cancel()
}

// read obtains OSM blobs and sends them down, in a round-robin manner, a list
// of channels to be decoded.
func read(ctx context.Context, b blobReader, cfg decoderOptions) (inputs []chan encoded) {

	n := cfg.nCPU
	for i := uint16(0); i < n; i++ {
		inputs = append(inputs, make(chan encoded, cfg.inputChannelLength))
	}

	go func() {
		defer func() {
			for _, input := range inputs {
				close(input)
			}
		}()

		buffer := bytes.NewBuffer(make([]byte, 0, cfg.protoBufferSize))
		var i uint16
		for {
			input := inputs[i]
			i = (i + 1) % n

			h, err := b.readBlobHeader(buffer)
			if err == io.EOF {
				return
			} else if err != nil {
				input <- encoded{err: err}
				return
			}

			b, err := b.readBlob(buffer, h)
			if err != nil {
				input <- encoded{err: err}
				return
			}

			select {
			case <-ctx.Done():
				return
			case input <- encoded{header: h, blob: b}:
			}
		}
	}()

	return
}

// decode decodes blob/header pairs into an array of OSM elements.  These
// arrays are placed onto an output channel where they will be coalesced into
// their correct order.
func decode(input <-chan encoded, cfg decoderOptions) (output chan decoded) {

	output = make(chan decoded, cfg.outputChannelLength)

	buf := bytes.NewBuffer(make([]byte, 0, cfg.protoBufferSize))

	go func() {
		defer close(output)

		for {
			raw, more := <-input
			if !more {
				return
			}

			if raw.err != nil {
				output <- decoded{nil, raw.err}
				return
			}

			elements, err := elements(raw.header, raw.blob, buf)

			output <- decoded{elements, err}
		}
	}()

	return
}

// coalesce merges the list of channels in a round-robin manner and sends the
// elements in pairs down a channel of pairs.
func coalesce(cfg decoderOptions, outputs ...chan decoded) (pairs chan pair) {

	pairs = make(chan pair, cfg.decodedChannelLength)

	go func() {
		defer close(pairs)

		n := len(outputs)
		var i int
		for {
			output := outputs[i]
			i = (i + 1) % n

			decoded, more := <-output
			if !more {
				// Since the channels are inspected round-robin, when one channel
				// is done, all subsequent channels are done.
				return
			}

			if decoded.err != nil {
				pairs <- pair{nil, decoded.err}
				return
			}

			for _, e := range decoded.elements {
				pairs <- pair{e, nil}
			}
		}
	}()

	return
}

// elements unmarshals an array of OSM elements from an array of protobuf encoded
// bytes.  The bytes could possibly be compressed; zlibBuf is used to facilitate
// decompression.
func elements(header *protobuf.BlobHeader, blob *protobuf.Blob, zlibBuf *bytes.Buffer) ([]interface{}, error) {

	var buf []byte

	switch {
	case blob.Raw != nil:
		buf = blob.GetRaw()

	case blob.ZlibData != nil:
		r, err := zlib.NewReader(bytes.NewReader(blob.GetZlibData()))
		if err != nil {
			return nil, err
		}
		zlibBuf.Reset()
		rawBufferSize := int(blob.GetRawSize() + bytes.MinRead)
		if rawBufferSize > zlibBuf.Cap() {
			zlibBuf.Grow(rawBufferSize)
		}
		_, err = zlibBuf.ReadFrom(r)
		if err != nil {
			return nil, err
		}
		if zlibBuf.Len() != int(blob.GetRawSize()) {
			err = fmt.Errorf("raw blob data size %d but expected %d", zlibBuf.Len(), blob.GetRawSize())
			return nil, err
		}
		buf = zlibBuf.Bytes()

	default:
		return nil, errors.New("unknown blob data type")
	}

	ht := *header.Type
	if ht == "OSMHeader" {
		h, err := parseOSMHeader(buf)
		if err != nil {
			return nil, err
		}
		return []interface{}{h}, nil
	} else if ht == "OSMData" {
		return parsePrimitiveBlock(buf)
	} else {
		return nil, fmt.Errorf("unknown header type %s", ht)
	}
}

// parseOSMHeader unmarshals the OSM header from an array of protobuf encoded bytes.
func parseOSMHeader(buffer []byte) (*Header, error) {
	hb := &protobuf.HeaderBlock{}
	if err := proto.Unmarshal(buffer, hb); err != nil {
		return nil, err
	}

	header := &Header{
		RequiredFeatures: hb.GetRequiredFeatures(),
		OptionalFeatures: hb.GetOptionalFeatures(),
		WritingProgram:   hb.GetWritingprogram(),
		Source:           hb.GetSource(),
		OsmosisReplicationBaseURL:        hb.GetOsmosisReplicationBaseUrl(),
		OsmosisReplicationSequenceNumber: hb.GetOsmosisReplicationSequenceNumber(),
	}

	if hb.Bbox != nil {
		header.BoundingBox = BoundingBox{
			Left:   toDegrees(0, 1, hb.Bbox.GetLeft()),
			Right:  toDegrees(0, 1, hb.Bbox.GetRight()),
			Top:    toDegrees(0, 1, hb.Bbox.GetTop()),
			Bottom: toDegrees(0, 1, hb.Bbox.GetBottom()),
		}
	}

	if hb.OsmosisReplicationTimestamp != nil {
		header.OsmosisReplicationTimestamp = time.Unix(*hb.OsmosisReplicationTimestamp, 0)
	}

	return header, nil
}

// toDegrees converts a coordinate into Degrees, given the offset and
// granularity of the coordinate.
func toDegrees(offset int64, granularity int32, coordinate int64) Degrees {
	return 1e-9 * Degrees(offset+(int64(granularity)*coordinate))
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fixed implements fixed-point integer types.
package fixed // import "golang.org/x/image/math/fixed"

import (
	"fmt"
)

// TODO: implement fmt.Formatter for %f and %g.

// Int26_6 is a signed 26.6 fixed-point number.
//
// The integer part ranges from -33554432 to 33554431, inclusive. The
// fractional part has 6 bits of precision.
//
// For example, the number one-and-a-quarter is Int26_6(1<<6 + 1<<4).
type Int26_6 int32

// String returns a human-readable representation of a 26.6 fixed-point number.
//
// For example, the number one-and-a-quarter becomes "1:16".
func (x Int26_6) String() string {
	const shift, mask = 6, 1<<6 - 1
	if x >= 0 {
		return fmt.Sprintf("%d:%02d", int32(x>>shift), int32(x&mask))
	}
	x = -x
	if x >= 0 {
		return fmt.Sprintf("-%d:%02d", int32(x>>shift), int32(x&mask))
	}
	return "-33554432:00" // The minimum value is -(1<<25).
}

// Int52_12 is a signed 52.12 fixed-point number.
//
// The integer part ranges from -2251799813685248 to 2251799813685247,
// inclusive. The fractional part has 12 bits of precision.
//
// For example, the number one-and-a-quarter is Int52_12(1<<12 + 1<<10).
type Int52_12 int64

// String returns a human-readable representation of a 52.12 fixed-point
// number.
//
// For example, the number one-and-a-quarter becomes "1:1024".
func (x Int52_12) String() string {
	const shift, mask = 12, 1<<12 - 1
	if x >= 0 {
		return fmt.Sprintf("%d:%04d", int64(x>>shift), int64(x&mask))
	}
	x = -x
	if x >= 0 {
		return fmt.Sprintf("-%d:%04d", int64(x>>shift), int64(x&mask))
	}
	return "-2251799813685248:0000" // The minimum value is -(1<<51).
}
<commit_msg>math/fixed: add point types.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fixed implements fixed-point integer types.
package fixed // import "golang.org/x/image/math/fixed"

import (
	"fmt"
)

// TODO: implement fmt.Formatter for %f and %g.

// Int26_6 is a signed 26.6 fixed-point number.
//
// The integer part ranges from -33554432 to 33554431, inclusive. The
// fractional part has 6 bits of precision.
//
// For example, the number one-and-a-quarter is Int26_6(1<<6 + 1<<4).
type Int26_6 int32

// String returns a human-readable representation of a 26.6 fixed-point number.
//
// For example, the number one-and-a-quarter becomes "1:16".
func (x Int26_6) String() string {
	const shift, mask = 6, 1<<6 - 1
	if x >= 0 {
		return fmt.Sprintf("%d:%02d", int32(x>>shift), int32(x&mask))
	}
	x = -x
	if x >= 0 {
		return fmt.Sprintf("-%d:%02d", int32(x>>shift), int32(x&mask))
	}
	return "-33554432:00" // The minimum value is -(1<<25).
}

// Int52_12 is a signed 52.12 fixed-point number.
//
// The integer part ranges from -2251799813685248 to 2251799813685247,
// inclusive. The fractional part has 12 bits of precision.
//
// For example, the number one-and-a-quarter is Int52_12(1<<12 + 1<<10).
type Int52_12 int64

// String returns a human-readable representation of a 52.12 fixed-point
// number.
//
// For example, the number one-and-a-quarter becomes "1:1024".
func (x Int52_12) String() string {
	const shift, mask = 12, 1<<12 - 1
	if x >= 0 {
		return fmt.Sprintf("%d:%04d", int64(x>>shift), int64(x&mask))
	}
	x = -x
	if x >= 0 {
		return fmt.Sprintf("-%d:%04d", int64(x>>shift), int64(x&mask))
	}
	return "-2251799813685248:0000" // The minimum value is -(1<<51).
}

// Point26_6 is a 26.6 fixed-point coordinate pair.
type Point26_6 struct {
	X, Y Int26_6
}

// Point52_12 is a 52.12 fixed-point coordinate pair.
type Point52_12 struct {
	X, Y Int52_12
}
<|endoftext|>


<commit_before>/*
	Repeatr is focused on telling a story about formulas: when you put the same
	things in, you should get the same things out.

	Formulas describe a piece of computation, its inputs, and how to collect
	its outputs.  After that, repeatr can help you make sure your formula
	produces the same thing time and time again.

	We call the parts of the formula that should be deterministic the "conjecture".
	We'll use this word consistently throughout the documentation.
	Anything that is part of the conjecture is hashed when processing the formula,
	and any output marked as part of the conjecture is hashed after the formula's task
	is executed.  (You can choose which of the outputs are part of your conjecture!
	But everything about your inputs must be part of your conjecture, because
	if the inputs change, output consistency is impossible -- except stuff like
	the network locations of data is skipped from the conjecture, since
	that can change without changing the meaning of your formula.)

	### Mathwise:

	Given a Formula j, and the []Output v, and some hash h:

	h(j.Inputs||j.Action||filter(j.Outputs, where Conjecture=true)) -> h(v)

	should be an onto relationship.

	In other words, a Formula should define a "pure" function.  And we'll let you know if it doesn't.

	### Misc docs:

	- The root filesystem of your execution engine is just another `Input` with the rest, with MountPath="/".
	Exactly one input with the root location is required at runtime.

	- Formula.SchedulingInfo, since it's *not* included in the 'conjecture',
	is expected not to have a major impact on your execution correctness.
*/
package def

import (
	"io"
	"time"

	"github.com/spacemonkeygo/errors"
	"polydawn.net/repeatr/lib/streamer"
)

/*
	Formula describes `(inputs, computation) -> (outputs)`.

	Values may be mutated during final validation if missing,
	i.e. the special `Output` that describes stdout and stderr is required
	and will be supplied for you if not already specifically configured.
*/
type Formula struct {
	Inputs  []Input  `json:"inputs"`  // total set of inputs.  sorted order.  included in the conjecture.
	Action  Action   `json:"action"`  // description of the computation to be performed.  included in the conjecture.
	Outputs []Output `json:"outputs"` // set of expected outputs.  sorted order.  conditionally included in the conjecture (configurable per output).
	//SchedulingInfo interface{} // configures what execution framework is used and impl-specific additional parameters to that (minimum node memory, etc).  not considered part of the conjecture.
}

/*
	Input specifies a data source to feed into the beginning of a computation.

	Inputs can be one of many different `Type`s of data source.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	Inputs must specify both a `Hash` and a `URI`.
	`Input.Hash` is the content identity descriptor and will always be verified for all types of data source.
	`Input.Hash` is both identifies the data and provides integrity (in other words,
	all repeatr's input types will use a cryptographically strong hash here,
	so given a hash even an untrusted datastore is safe to use).
	Repeatr requires this to be accurate because if the inputs change, output
	consistency is impossible -- so even for plain http downloads, this is enforced.

	`Input.URI` is a secondary content lookup descriptor, like where on
	the filesystem or network information can be fetched from.
	`Input.URI` might contain extra description to answer questions like
	"which git remote url should i fetch from" or
	"which ipfs swarm do i coordinate with".

	The `URI` is *not* included in the conjecture, because repeatr understands
	that your data will be mobile -- that's why we have the `Input.Hash` take the leading role
	and why the `Input.Hash` should be sufficient to identify the information.
	Changes in the `Input.URI` field may make or break whether the data is accessible,
	but should never actually change the content of the data -- it's just supposed to talk about
	transport details; and content itself is still checked by `Input.Hash`.
*/
type Input struct {
	Name       string   // the map key.  by default will also be used for MountPath.
	Type       string   `json:"type"`  // implementation name (repeatr-internal).  included in the conjecture.
	Hash       string   `json:"hash"`  // identifying hash of input data.  included in the conjecture.
	Warehouses []string `json:"silo"`  // secondary content lookup descriptor.  not considered part of the conjecture.
	MountPath  string   `json:"mount"` // filepath where this input should be mounted in the execution context.  included in the conjecture.
}

type InputsByName []Input

func (a InputsByName) Len() int           { return len(a) }
func (a InputsByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a InputsByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

/*
	Action describes the computation to be run once the inputs have been set up.
	All content is part of the conjecture.
*/
type Action struct {
	Entrypoint []string          `json:"command"` // executable to invoke as the task.  included in the conjecture.
	Cwd        string            `json:"cwd"`     // working directory to set when invoking the executable.  if not set, will be defaulted to "/".
	Env        map[string]string `json:"env"`     // environment variables.  included in the conjecture.
}

/*
	Output describes where we intend to pick up data after a task completes.

	Outputs can be one of many different `Type`s of data sink.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	`Output.MountPath` states where we should collect information from the
	task execution environment.
	After the task completes, repeatr will pick up this data, ship it off
	to storage, and also calculate a checksum of the data so we can see
	whether it matches any prior (or future) runs of this `Formula`.

	Outputs must specify a `URI`; repeatr will ship your data to this address.
	`Output.URI` has similar properties to `Input.URI` (and also similarly,
	is not included in the conjecture, because repeatr understands that
	your data can be mobile).

	The `Output.Hash` field will be filled in with a value computed
	from the data present in `Output.MountPath` after the task has completed.
	As with `Input.Hash`, the `Output.Hash` in repeatr will always be a
	cryptographically strong hash, which means it precisely describes your
	data, and makes it virtually impossible to accidentally get the same
	`Hash` as other data -- any changes to your output will always result
	in a very different `Hash` value.

	(In a content-addressable data store, repeatr may just lift the data store's
	address to use as `Output.Hash`, which is super efficient for everyone involved.
	For other more legacy-oriented systems, this may be a hash of the
	of the working filesystem right before before export.)

	Whether or not to include an `Output` in the overall `Formula`'s conjecture
	is up to you!  Many things in the world are not deterministic; repeatr
	is here to help you with the ones that should be, and stay out of the way
	for the ones that aren't.  Just set the `Output.Conjecture` boolean.

	Some examples of using `Conjecture` conditionally: if you have a job
	which does a bunch of calculations and should spit out a consistent result,
	but also does a lot of progress logging, gather those in two separate outputs.
	Mark the output of your computation in one output and set that to be
	included in the conjecture so repeatr can help you check your algorithm's
	correctness.  Now, since you may want to keep your logs for later, mark
	those as another output, and since these probably contain timestamps and
	other info that isn't *supposed* to be repeated exactly on another run,
	just set `Conjecture=false` on this one so repeatr knows not to check.

	`Output.Filters` may also be used to do a clean up pass on output files
	before committing them to storage or doing repeatr's consistency checks.
	(One typical example, which is engaged by default for you when an output
	is configured to be included in the conjecture, is setting all the file
	modification times to a standard value.)
*/
type Output struct {
	Name       string   // the map key.  by default will also be used for MountPath.
	Type       string   `json:"type"`           // implementation name (repeatr-internal).  included in the conjecture (iff the whole output is).
	Hash       string   `json:"hash"`           // identifying hash of output data.  generated by the output handling implementation during data export when a task is complete.  included in the conjecture (iff the whole output is).
	Warehouses []string `json:"silo,omitempty"` // where to ship the output data.  not considered part of the conjecture.
	MountPath  string   `json:"mount"`          // filepath where this output will be yanked from the job when it reaches completion.  included in the conjecture (iff the whole output is).
	Filters    Filters
	Conjecture bool `json:"cnj,omitempty"` // whether or not this output is expected to contain the same result, every time, when given the same set of `Input` items.
}

type OutputsByName []Output

func (a OutputsByName) Len() int           { return len(a) }
func (a OutputsByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a OutputsByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

/*
	Filters are transformations that can be applied to data, either to
	normalize it for storage or to apply attributes to it before feeding
	the data into an action's inputs.

	The following filters are available:

		- uid   -- the posix user ownership number
		- gid   -- the posix group ownership number
		- mtime -- the posix file modification timestamp

	'uid', 'gid', and 'mtime' are all filtered by default on formula outputs --
	most use cases do not need these attributes, and they are a source of nondeterminism.
	If you want to keep them, you may specify	`uid keep`, `gid keep`, `mtime keep`,
	etc; if you want the filters to flatten to different values than the defaults,
	you may specify `uid 12000`, etc.
	(Note that the default mtime filter flattens the time to Jan 1, 2010 --
	*not* epoch.  Some contemporary software has been known to regard zero/epoch
	timestamps as errors or empty values, so we've choosen a different value in
	the interest of practicality.)

	Filters on inputs will be applied after the data is fetched and before your
	job starts.
	Filters on outputs will be applied after your job process exits, but before
	the output hash is computed and the data committed to any warehouses for storage.

	Note that these filters are built-ins (and there are no extensions possible).
	If you need more complex data transformations, incorporate it into your job
	itself!  These filters are built-in because they cover the most common sources
	of nondeterminism, and because they are efficient to implement as special
	cases in the IO engines (and in some cases, e.g. ownership filters, are also
	necessary for security properties an dusing repeatr IO with minimal host
	system priviledges).
*/
type Filters struct {
	UidMode   FilterMode
	Uid       int
	GidMode   FilterMode
	Gid       int
	MtimeMode FilterMode
	Mtime     time.Time
}

type FilterMode int

const (
	FilterUninitialized FilterMode = iota
	FilterUse
	FilterKeep
	FilterHost
)

var (
	FilterDefaultUid   = 1000
	FilterDefaultGid   = 1000
	FilterDefaultMtime = time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC)
)

/*
	Job is an interface for observing actively running tasks.
	It contains progress reporting interfaces, streams that get realtime
	stdout/stderr, wait for finish, return exit codes, etc.

	All of the data available from `Job` should also be accessible as
	some form of `Output`s	after the execution is complete, but `Job` can
	provide them live.
*/
type Job interface {
	// question the first: provide readables, or accept writables for stdout?
	// probably provide.  the downside of course is this often forces a byte copy somewhere.
	// however, we're going to store these streams anyway.  so the most useful thing to do actually turns out to be log outputs immediately, and just reexpose that readers to that stream.

	Id() JobID // the ID of this job

	/*
		Returns a new reader that delivers the combined stdout and
		stderr of a command, from the beginning of execution.

		Shorthand for `Outputs().Reader(1, 2)`.
	*/
	OutputReader() io.Reader

	/*
		Returns a mux of readable streams.  Numbering is as typical
		unix convention (1=stdout, 2=stderr, etc).
	*/
	Outputs() streamer.ROMux

	/*
		Waits for completion if necessary, then returns the job's results
	*/
	Wait() JobResult
}

type JobID string // type def just to make it hard to accidentally get ids crossed.

/*
	Very much a work-in-progress.

	Holds all information you might want from a completed Job.
*/
type JobResult struct {
	ID JobID

	Error *errors.Error // if the executor experienced a problem running this job. REVIEW: type discussion? semantics?

	ExitCode int // The return code of this job

	Outputs []Output //The hashed outputs from this job

	// More?
}
<commit_msg>Begin spec'ing mounts interacting with the host.<commit_after>/*
	Repeatr is focused on telling a story about formulas: when you put the same
	things in, you should get the same things out.

	Formulas describe a piece of computation, its inputs, and how to collect
	its outputs.  After that, repeatr can help you make sure your formula
	produces the same thing time and time again.

	We call the parts of the formula that should be deterministic the "conjecture".
	We'll use this word consistently throughout the documentation.
	Anything that is part of the conjecture is hashed when processing the formula,
	and any output marked as part of the conjecture is hashed after the formula's task
	is executed.  (You can choose which of the outputs are part of your conjecture!
	But everything about your inputs must be part of your conjecture, because
	if the inputs change, output consistency is impossible -- except stuff like
	the network locations of data is skipped from the conjecture, since
	that can change without changing the meaning of your formula.)

	### Mathwise:

	Given a Formula j, and the []Output v, and some hash h:

	h(j.Inputs||j.Action||filter(j.Outputs, where Conjecture=true)) -> h(v)

	should be an onto relationship.

	In other words, a Formula should define a "pure" function.  And we'll let you know if it doesn't.

	### Misc docs:

	- The root filesystem of your execution engine is just another `Input` with the rest, with MountPath="/".
	Exactly one input with the root location is required at runtime.

	- Formula.SchedulingInfo, since it's *not* included in the 'conjecture',
	is expected not to have a major impact on your execution correctness.
*/
package def

import (
	"io"
	"time"

	"github.com/spacemonkeygo/errors"
	"polydawn.net/repeatr/lib/streamer"
)

/*
	Formula describes `(inputs, computation) -> (outputs)`.

	Values may be mutated during final validation if missing,
	i.e. the special `Output` that describes stdout and stderr is required
	and will be supplied for you if not already specifically configured.
*/
type Formula struct {
	Inputs  []Input  `json:"inputs"`  // total set of inputs.  sorted order.  included in the conjecture.
	Action  Action   `json:"action"`  // description of the computation to be performed.  included in the conjecture.
	Outputs []Output `json:"outputs"` // set of expected outputs.  sorted order.  conditionally included in the conjecture (configurable per output).
	//SchedulingInfo interface{} // configures what execution framework is used and impl-specific additional parameters to that (minimum node memory, etc).  not considered part of the conjecture.
}

/*
	Input specifies a data source to feed into the beginning of a computation.

	Inputs can be one of many different `Type`s of data source.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	Inputs must specify both a `Hash` and a `URI`.
	`Input.Hash` is the content identity descriptor and will always be verified for all types of data source.
	`Input.Hash` is both identifies the data and provides integrity (in other words,
	all repeatr's input types will use a cryptographically strong hash here,
	so given a hash even an untrusted datastore is safe to use).
	Repeatr requires this to be accurate because if the inputs change, output
	consistency is impossible -- so even for plain http downloads, this is enforced.

	`Input.URI` is a secondary content lookup descriptor, like where on
	the filesystem or network information can be fetched from.
	`Input.URI` might contain extra description to answer questions like
	"which git remote url should i fetch from" or
	"which ipfs swarm do i coordinate with".

	The `URI` is *not* included in the conjecture, because repeatr understands
	that your data will be mobile -- that's why we have the `Input.Hash` take the leading role
	and why the `Input.Hash` should be sufficient to identify the information.
	Changes in the `Input.URI` field may make or break whether the data is accessible,
	but should never actually change the content of the data -- it's just supposed to talk about
	transport details; and content itself is still checked by `Input.Hash`.
*/
type Input struct {
	Name       string   // the map key.  by default will also be used for MountPath.
	Type       string   `json:"type"`  // implementation name (repeatr-internal).  included in the conjecture.
	Hash       string   `json:"hash"`  // identifying hash of input data.  included in the conjecture.
	Warehouses []string `json:"silo"`  // secondary content lookup descriptor.  not considered part of the conjecture.
	MountPath  string   `json:"mount"` // filepath where this input should be mounted in the execution context.  included in the conjecture.
}

type InputsByName []Input

func (a InputsByName) Len() int           { return len(a) }
func (a InputsByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a InputsByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

/*
	Action describes the computation to be run once the inputs have been set up.
	All content is part of the conjecture.
*/
type Action struct {
	Entrypoint []string          `json:"command"` // executable to invoke as the task.  included in the conjecture.
	Cwd        string            `json:"cwd"`     // working directory to set when invoking the executable.  if not set, will be defaulted to "/".
	Env        map[string]string `json:"env"`     // environment variables.  included in the conjecture.
	Escapes    Escapes
}

/*
	Output describes where we intend to pick up data after a task completes.

	Outputs can be one of many different `Type`s of data sink.
	Examples may include "tar", "git", "hadoop", "ipfs", etc.

	`Output.MountPath` states where we should collect information from the
	task execution environment.
	After the task completes, repeatr will pick up this data, ship it off
	to storage, and also calculate a checksum of the data so we can see
	whether it matches any prior (or future) runs of this `Formula`.

	Outputs must specify a `URI`; repeatr will ship your data to this address.
	`Output.URI` has similar properties to `Input.URI` (and also similarly,
	is not included in the conjecture, because repeatr understands that
	your data can be mobile).

	The `Output.Hash` field will be filled in with a value computed
	from the data present in `Output.MountPath` after the task has completed.
	As with `Input.Hash`, the `Output.Hash` in repeatr will always be a
	cryptographically strong hash, which means it precisely describes your
	data, and makes it virtually impossible to accidentally get the same
	`Hash` as other data -- any changes to your output will always result
	in a very different `Hash` value.

	(In a content-addressable data store, repeatr may just lift the data store's
	address to use as `Output.Hash`, which is super efficient for everyone involved.
	For other more legacy-oriented systems, this may be a hash of the
	of the working filesystem right before before export.)

	Whether or not to include an `Output` in the overall `Formula`'s conjecture
	is up to you!  Many things in the world are not deterministic; repeatr
	is here to help you with the ones that should be, and stay out of the way
	for the ones that aren't.  Just set the `Output.Conjecture` boolean.

	Some examples of using `Conjecture` conditionally: if you have a job
	which does a bunch of calculations and should spit out a consistent result,
	but also does a lot of progress logging, gather those in two separate outputs.
	Mark the output of your computation in one output and set that to be
	included in the conjecture so repeatr can help you check your algorithm's
	correctness.  Now, since you may want to keep your logs for later, mark
	those as another output, and since these probably contain timestamps and
	other info that isn't *supposed* to be repeated exactly on another run,
	just set `Conjecture=false` on this one so repeatr knows not to check.

	`Output.Filters` may also be used to do a clean up pass on output files
	before committing them to storage or doing repeatr's consistency checks.
	(One typical example, which is engaged by default for you when an output
	is configured to be included in the conjecture, is setting all the file
	modification times to a standard value.)
*/
type Output struct {
	Name       string   // the map key.  by default will also be used for MountPath.
	Type       string   `json:"type"`           // implementation name (repeatr-internal).  included in the conjecture (iff the whole output is).
	Hash       string   `json:"hash"`           // identifying hash of output data.  generated by the output handling implementation during data export when a task is complete.  included in the conjecture (iff the whole output is).
	Warehouses []string `json:"silo,omitempty"` // where to ship the output data.  not considered part of the conjecture.
	MountPath  string   `json:"mount"`          // filepath where this output will be yanked from the job when it reaches completion.  included in the conjecture (iff the whole output is).
	Filters    Filters
	Conjecture bool `json:"cnj,omitempty"` // whether or not this output is expected to contain the same result, every time, when given the same set of `Input` items.
}

type OutputsByName []Output

func (a OutputsByName) Len() int           { return len(a) }
func (a OutputsByName) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a OutputsByName) Less(i, j int) bool { return a[i].Name < a[j].Name }

/*
	Filters are transformations that can be applied to data, either to
	normalize it for storage or to apply attributes to it before feeding
	the data into an action's inputs.

	The following filters are available:

		- uid   -- the posix user ownership number
		- gid   -- the posix group ownership number
		- mtime -- the posix file modification timestamp

	'uid', 'gid', and 'mtime' are all filtered by default on formula outputs --
	most use cases do not need these attributes, and they are a source of nondeterminism.
	If you want to keep them, you may specify	`uid keep`, `gid keep`, `mtime keep`,
	etc; if you want the filters to flatten to different values than the defaults,
	you may specify `uid 12000`, etc.
	(Note that the default mtime filter flattens the time to Jan 1, 2010 --
	*not* epoch.  Some contemporary software has been known to regard zero/epoch
	timestamps as errors or empty values, so we've choosen a different value in
	the interest of practicality.)

	Filters on inputs will be applied after the data is fetched and before your
	job starts.
	Filters on outputs will be applied after your job process exits, but before
	the output hash is computed and the data committed to any warehouses for storage.

	Note that these filters are built-ins (and there are no extensions possible).
	If you need more complex data transformations, incorporate it into your job
	itself!  These filters are built-in because they cover the most common sources
	of nondeterminism, and because they are efficient to implement as special
	cases in the IO engines (and in some cases, e.g. ownership filters, are also
	necessary for security properties an dusing repeatr IO with minimal host
	system priviledges).
*/
type Filters struct {
	UidMode   FilterMode
	Uid       int
	GidMode   FilterMode
	Gid       int
	MtimeMode FilterMode
	Mtime     time.Time
}

type FilterMode int

const (
	FilterUninitialized FilterMode = iota
	FilterUse
	FilterKeep
	FilterHost
)

var (
	FilterDefaultUid   = 1000
	FilterDefaultGid   = 1000
	FilterDefaultMtime = time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC)
)

/*
	Escapes are features that give up repeatr's promises about repeatability,
	but make it possible to use repeatr's execution engines and data transports
	anyway.

	For example, one "escape" is to make a writable mount of a host
	filesystem.  This instantly breaks all portability guarantees... but is
	incredibly useful if you want to use repeatr inputs to ship an application
	that is then allowed to interact statefully with a host machine.

	If you want to use data from a host machine, but still want trackable
	repeatability guarantees, consider using a pipeline instead of host mounts:
	use `repeatr scan --type=dir` to hash the data (and optionally store copies),
	then pipe the hash reported by the scan into the formula you hand to `repeatr run`.
*/
type Escapes struct {
	Mounts []Mount
}

type Mount struct {
	SourcePath string
	TargetPath string
	Writable   bool
	// CONSIDER: not sure what should be default for writable.  ro should usually be a scan; but not required; you might have sockets, be using this as ipc, whatever; all of which are "crazy", relatively speaking, but that's what this is called an escape valve for.
}

/*
	Job is an interface for observing actively running tasks.
	It contains progress reporting interfaces, streams that get realtime
	stdout/stderr, wait for finish, return exit codes, etc.

	All of the data available from `Job` should also be accessible as
	some form of `Output`s	after the execution is complete, but `Job` can
	provide them live.
*/
type Job interface {
	// question the first: provide readables, or accept writables for stdout?
	// probably provide.  the downside of course is this often forces a byte copy somewhere.
	// however, we're going to store these streams anyway.  so the most useful thing to do actually turns out to be log outputs immediately, and just reexpose that readers to that stream.

	Id() JobID // the ID of this job

	/*
		Returns a new reader that delivers the combined stdout and
		stderr of a command, from the beginning of execution.

		Shorthand for `Outputs().Reader(1, 2)`.
	*/
	OutputReader() io.Reader

	/*
		Returns a mux of readable streams.  Numbering is as typical
		unix convention (1=stdout, 2=stderr, etc).
	*/
	Outputs() streamer.ROMux

	/*
		Waits for completion if necessary, then returns the job's results
	*/
	Wait() JobResult
}

type JobID string // type def just to make it hard to accidentally get ids crossed.

/*
	Very much a work-in-progress.

	Holds all information you might want from a completed Job.
*/
type JobResult struct {
	ID JobID

	Error *errors.Error // if the executor experienced a problem running this job. REVIEW: type discussion? semantics?

	ExitCode int // The return code of this job

	Outputs []Output //The hashed outputs from this job

	// More?
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// CacheControl sets the default cache policy on static asset responses
func CacheControl(next http.Handler) http.HandlerFunc {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		etag := ConfigCache("etag")
		policy := fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 60*60*24*30)
		res.Header().Add("Etag", etag)
		res.Header().Add("Cache-Control", policy)

		if match := req.Header.Get("If-None-Match"); match != "" {
			if strings.Contains(match, etag) {
				fmt.Println("matched etag")
				res.WriteHeader(http.StatusNotModified)
				return
			}

			fmt.Println("checked, no match")
		}

		next.ServeHTTP(res, req)
	})
}

// NewEtag generates a new Etag for response caching
func NewEtag() string {
	now := fmt.Sprintf("%d", time.Now().Unix())
	etag := base64.StdEncoding.EncodeToString([]byte(now))

	return etag
}

// InvalidateCache sets a new Etag for http responses
func InvalidateCache() error {
	kv := make(map[string]interface{})

	c, err := ConfigAll()
	if err != nil {
		return err
	}

	err = json.Unmarshal(c, &kv)
	if err != nil {
		return err
	}

	kv["etag"] = NewEtag()

	data := make(url.Values)
	for k, v := range kv {
		switch v.(type) {
		case string:
			data.Set(k, v.(string))
		case []string:
			vv := v.([]string)
			for i := range vv {
				if i == 0 {
					data.Set(k, vv[i])
				} else {
					data.Add(k, vv[i])
				}
			}
		}
	}

	err = SetConfig(data)
	if err != nil {

	}

	return nil
}
<commit_msg>removing print debug for cache check<commit_after>package db

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// CacheControl sets the default cache policy on static asset responses
func CacheControl(next http.Handler) http.HandlerFunc {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		etag := ConfigCache("etag")
		policy := fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 60*60*24*30)
		res.Header().Add("Etag", etag)
		res.Header().Add("Cache-Control", policy)

		if match := req.Header.Get("If-None-Match"); match != "" {
			if strings.Contains(match, etag) {
				res.WriteHeader(http.StatusNotModified)
				return
			}
		}

		next.ServeHTTP(res, req)
	})
}

// NewEtag generates a new Etag for response caching
func NewEtag() string {
	now := fmt.Sprintf("%d", time.Now().Unix())
	etag := base64.StdEncoding.EncodeToString([]byte(now))

	return etag
}

// InvalidateCache sets a new Etag for http responses
func InvalidateCache() error {
	kv := make(map[string]interface{})

	c, err := ConfigAll()
	if err != nil {
		return err
	}

	err = json.Unmarshal(c, &kv)
	if err != nil {
		return err
	}

	kv["etag"] = NewEtag()

	data := make(url.Values)
	for k, v := range kv {
		switch v.(type) {
		case string:
			data.Set(k, v.(string))
		case []string:
			vv := v.([]string)
			for i := range vv {
				if i == 0 {
					data.Set(k, vv[i])
				} else {
					data.Add(k, vv[i])
				}
			}
		}
	}

	err = SetConfig(data)
	if err != nil {

	}

	return nil
}
<|endoftext|>


<commit_before>package sftp

import (
	"encoding"
	"sort"
	"sync"
)

// The goal of the packetManager is to keep the outgoing packets in the same
// order as the incoming as is requires by section 7 of the RFC.

type packetManager struct {
	requests    chan orderedPacket
	responses   chan orderedPacket
	fini        chan struct{}
	incoming    orderedPackets
	outgoing    orderedPackets
	sender      packetSender // connection object
	working     *sync.WaitGroup
	packetCount uint32
}

type packetSender interface {
	sendPacket(encoding.BinaryMarshaler) error
}

func newPktMgr(sender packetSender) *packetManager {
	s := &packetManager{
		requests:  make(chan orderedPacket, SftpServerWorkerCount),
		responses: make(chan orderedPacket, SftpServerWorkerCount),
		fini:      make(chan struct{}),
		incoming:  make([]orderedPacket, 0, SftpServerWorkerCount),
		outgoing:  make([]orderedPacket, 0, SftpServerWorkerCount),
		sender:    sender,
		working:   &sync.WaitGroup{},
	}
	go s.controller()
	return s
}

//// packet ordering
func (s *packetManager) newOrderId() uint32 {
	s.packetCount++
	return s.packetCount
}

type orderedRequest struct {
	requestPacket
	orderid uint32
}

func (s *packetManager) newOrderedRequest(p requestPacket) orderedRequest {
	return orderedRequest{requestPacket: p, orderid: s.newOrderId()}
}
func (p orderedRequest) orderId() uint32       { return p.orderid }
func (p orderedRequest) setOrderId(oid uint32) { p.orderid = oid }

type orderedResponse struct {
	responsePacket
	orderid uint32
}

func (s *packetManager) newOrderedResponse(p responsePacket, id uint32,
) orderedResponse {
	return orderedResponse{responsePacket: p, orderid: id}
}
func (p orderedResponse) orderId() uint32       { return p.orderid }
func (p orderedResponse) setOrderId(oid uint32) { p.orderid = oid }

type orderedPacket interface {
	id() uint32
	orderId() uint32
}
type orderedPackets []orderedPacket

func (o orderedPackets) Sort() {
	sort.Slice(o, func(i, j int) bool {
		return o[i].orderId() < o[j].orderId()
	})
}

//// packet registry
// register incoming packets to be handled
func (s *packetManager) incomingPacket(pkt orderedRequest) {
	s.working.Add(1)
	s.requests <- pkt
}

// register outgoing packets as being ready
func (s *packetManager) readyPacket(pkt orderedResponse) {
	s.responses <- pkt
	s.working.Done()
}

// shut down packetManager controller
func (s *packetManager) close() {
	// pause until current packets are processed
	s.working.Wait()
	close(s.fini)
}

// Passed a worker function, returns a channel for incoming packets.
// Keep process packet responses in the order they are received while
// maximizing throughput of file transfers.
func (s *packetManager) workerChan(runWorker func(chan orderedRequest),
) chan orderedRequest {

	// multiple workers for faster read/writes
	rwChan := make(chan orderedRequest, SftpServerWorkerCount)
	for i := 0; i < SftpServerWorkerCount; i++ {
		runWorker(rwChan)
	}

	// single worker to enforce sequential processing of everything else
	cmdChan := make(chan orderedRequest)
	runWorker(cmdChan)

	pktChan := make(chan orderedRequest, SftpServerWorkerCount)
	go func() {
		for pkt := range pktChan {
			switch pkt.requestPacket.(type) {
			case *sshFxpReadPacket, *sshFxpWritePacket:
				s.incomingPacket(pkt)
				rwChan <- pkt
				continue
			case *sshFxpClosePacket:
				// wait for reads/writes to finish when file is closed
				// incomingPacket() call must occur after this
				s.working.Wait()
			}
			s.incomingPacket(pkt)
			// all non-RW use sequential cmdChan
			cmdChan <- pkt
		}
		close(rwChan)
		close(cmdChan)
		s.close()
	}()

	return pktChan
}

// process packets
func (s *packetManager) controller() {
	for {
		select {
		case pkt := <-s.requests:
			debug("incoming id (oid): %v (%v)", pkt.id(), pkt.orderId())
			s.incoming = append(s.incoming, pkt)
			s.incoming.Sort()
		case pkt := <-s.responses:
			debug("outgoing id (oid): %v (%v)", pkt.id(), pkt.orderId())
			s.outgoing = append(s.outgoing, pkt)
			s.outgoing.Sort()
		case <-s.fini:
			return
		}
		s.maybeSendPackets()
	}
}

// send as many packets as are ready
func (s *packetManager) maybeSendPackets() {
	for {
		if len(s.outgoing) == 0 || len(s.incoming) == 0 {
			debug("break! -- outgoing: %v; incoming: %v",
				len(s.outgoing), len(s.incoming))
			break
		}
		out := s.outgoing[0]
		in := s.incoming[0]
		// debug("incoming: %v", ids(s.incoming))
		// debug("outgoing: %v", ids(s.outgoing))
		if in.orderId() == out.orderId() {
			debug("Sending packet: %v", out.id())
			s.sender.sendPacket(out.(encoding.BinaryMarshaler))
			// pop off heads
			copy(s.incoming, s.incoming[1:])            // shift left
			s.incoming = s.incoming[:len(s.incoming)-1] // remove last
			copy(s.outgoing, s.outgoing[1:])            // shift left
			s.outgoing = s.outgoing[:len(s.outgoing)-1] // remove last
		} else {
			break
		}
	}
}

// func oids(o []orderedPacket) []uint32 {
// 	res := make([]uint32, 0, len(o))
// 	for _, v := range o {
// 		res = append(res, v.orderId())
// 	}
// 	return res
// }
// func ids(o []orderedPacket) []uint32 {
// 	res := make([]uint32, 0, len(o))
// 	for _, v := range o {
// 		res = append(res, v.id())
// 	}
// 	return res
// }
<commit_msg>prevent memory leaks in slice buffer<commit_after>package sftp

import (
	"encoding"
	"sort"
	"sync"
)

// The goal of the packetManager is to keep the outgoing packets in the same
// order as the incoming as is requires by section 7 of the RFC.

type packetManager struct {
	requests    chan orderedPacket
	responses   chan orderedPacket
	fini        chan struct{}
	incoming    orderedPackets
	outgoing    orderedPackets
	sender      packetSender // connection object
	working     *sync.WaitGroup
	packetCount uint32
}

type packetSender interface {
	sendPacket(encoding.BinaryMarshaler) error
}

func newPktMgr(sender packetSender) *packetManager {
	s := &packetManager{
		requests:  make(chan orderedPacket, SftpServerWorkerCount),
		responses: make(chan orderedPacket, SftpServerWorkerCount),
		fini:      make(chan struct{}),
		incoming:  make([]orderedPacket, 0, SftpServerWorkerCount),
		outgoing:  make([]orderedPacket, 0, SftpServerWorkerCount),
		sender:    sender,
		working:   &sync.WaitGroup{},
	}
	go s.controller()
	return s
}

//// packet ordering
func (s *packetManager) newOrderId() uint32 {
	s.packetCount++
	return s.packetCount
}

type orderedRequest struct {
	requestPacket
	orderid uint32
}

func (s *packetManager) newOrderedRequest(p requestPacket) orderedRequest {
	return orderedRequest{requestPacket: p, orderid: s.newOrderId()}
}
func (p orderedRequest) orderId() uint32       { return p.orderid }
func (p orderedRequest) setOrderId(oid uint32) { p.orderid = oid }

type orderedResponse struct {
	responsePacket
	orderid uint32
}

func (s *packetManager) newOrderedResponse(p responsePacket, id uint32,
) orderedResponse {
	return orderedResponse{responsePacket: p, orderid: id}
}
func (p orderedResponse) orderId() uint32       { return p.orderid }
func (p orderedResponse) setOrderId(oid uint32) { p.orderid = oid }

type orderedPacket interface {
	id() uint32
	orderId() uint32
}
type orderedPackets []orderedPacket

func (o orderedPackets) Sort() {
	sort.Slice(o, func(i, j int) bool {
		return o[i].orderId() < o[j].orderId()
	})
}

//// packet registry
// register incoming packets to be handled
func (s *packetManager) incomingPacket(pkt orderedRequest) {
	s.working.Add(1)
	s.requests <- pkt
}

// register outgoing packets as being ready
func (s *packetManager) readyPacket(pkt orderedResponse) {
	s.responses <- pkt
	s.working.Done()
}

// shut down packetManager controller
func (s *packetManager) close() {
	// pause until current packets are processed
	s.working.Wait()
	close(s.fini)
}

// Passed a worker function, returns a channel for incoming packets.
// Keep process packet responses in the order they are received while
// maximizing throughput of file transfers.
func (s *packetManager) workerChan(runWorker func(chan orderedRequest),
) chan orderedRequest {

	// multiple workers for faster read/writes
	rwChan := make(chan orderedRequest, SftpServerWorkerCount)
	for i := 0; i < SftpServerWorkerCount; i++ {
		runWorker(rwChan)
	}

	// single worker to enforce sequential processing of everything else
	cmdChan := make(chan orderedRequest)
	runWorker(cmdChan)

	pktChan := make(chan orderedRequest, SftpServerWorkerCount)
	go func() {
		for pkt := range pktChan {
			switch pkt.requestPacket.(type) {
			case *sshFxpReadPacket, *sshFxpWritePacket:
				s.incomingPacket(pkt)
				rwChan <- pkt
				continue
			case *sshFxpClosePacket:
				// wait for reads/writes to finish when file is closed
				// incomingPacket() call must occur after this
				s.working.Wait()
			}
			s.incomingPacket(pkt)
			// all non-RW use sequential cmdChan
			cmdChan <- pkt
		}
		close(rwChan)
		close(cmdChan)
		s.close()
	}()

	return pktChan
}

// process packets
func (s *packetManager) controller() {
	for {
		select {
		case pkt := <-s.requests:
			debug("incoming id (oid): %v (%v)", pkt.id(), pkt.orderId())
			s.incoming = append(s.incoming, pkt)
			s.incoming.Sort()
		case pkt := <-s.responses:
			debug("outgoing id (oid): %v (%v)", pkt.id(), pkt.orderId())
			s.outgoing = append(s.outgoing, pkt)
			s.outgoing.Sort()
		case <-s.fini:
			return
		}
		s.maybeSendPackets()
	}
}

// send as many packets as are ready
func (s *packetManager) maybeSendPackets() {
	for {
		if len(s.outgoing) == 0 || len(s.incoming) == 0 {
			debug("break! -- outgoing: %v; incoming: %v",
				len(s.outgoing), len(s.incoming))
			break
		}
		out := s.outgoing[0]
		in := s.incoming[0]
		// debug("incoming: %v", ids(s.incoming))
		// debug("outgoing: %v", ids(s.outgoing))
		if in.orderId() == out.orderId() {
			debug("Sending packet: %v", out.id())
			s.sender.sendPacket(out.(encoding.BinaryMarshaler))
			// pop off heads
			copy(s.incoming, s.incoming[1:])            // shift left
			s.incoming[len(s.incoming)-1] = nil         // clear last
			s.incoming = s.incoming[:len(s.incoming)-1] // remove last
			copy(s.outgoing, s.outgoing[1:])            // shift left
			s.outgoing[len(s.outgoing)-1] = nil         // clear last
			s.outgoing = s.outgoing[:len(s.outgoing)-1] // remove last
		} else {
			break
		}
	}
}

// func oids(o []orderedPacket) []uint32 {
// 	res := make([]uint32, 0, len(o))
// 	for _, v := range o {
// 		res = append(res, v.orderId())
// 	}
// 	return res
// }
// func ids(o []orderedPacket) []uint32 {
// 	res := make([]uint32, 0, len(o))
// 	for _, v := range o {
// 		res = append(res, v.id())
// 	}
// 	return res
// }
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2016 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
	"github.com/minio/minio/pkg/madmin"
)

var adminServiceRestartCmd = cli.Command{
	Name:         "restart",
	Usage:        "restart all MinIO servers",
	Action:       mainAdminServiceRestart,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        globalFlags,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Restart MinIO server represented by its alias 'play'.
     {{.Prompt}} {{.HelpName}} play/
`,
}

// serviceRestartCommand is container for service restart command success and failure messages.
type serviceRestartCommand struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
}

// String colorized service restart command message.
func (s serviceRestartCommand) String() string {
	return console.Colorize("ServiceRestart", "Restart command successfully sent to `"+s.ServerURL+"`. Type Ctrl-C or wait to see the status of the restart.")
}

// JSON jsonified service restart command message.
func (s serviceRestartCommand) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// serviceRestartMessage is container for service restart success and failure messages.
type serviceRestartMessage struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
	Err       error  `json:"error,omitempty"`
}

// String colorized service restart message.
func (s serviceRestartMessage) String() string {
	if s.Err == nil {
		return console.Colorize("ServiceRestart", "\nRestarted `"+s.ServerURL+"` successfully.")
	}
	return console.Colorize("FailedServiceRestart", "Failed to restart `"+s.ServerURL+"`. error: "+s.Err.Error())
}

// JSON jsonified service restart message.
func (s serviceRestartMessage) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// checkAdminServiceRestartSyntax - validate all the passed arguments
func checkAdminServiceRestartSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 2 {
		cli.ShowCommandHelpAndExit(ctx, "restart", 1) // last argument is exit code
	}
}

func mainAdminServiceRestart(ctx *cli.Context) error {

	// Validate serivce restart syntax.
	checkAdminServiceRestartSyntax(ctx)

	// Set color.
	console.SetColor("ServiceOffline", color.New(color.FgRed, color.Bold))
	console.SetColor("ServiceInitializing", color.New(color.FgYellow, color.Bold))
	console.SetColor("ServiceRestart", color.New(color.FgGreen, color.Bold))
	console.SetColor("FailedServiceRestart", color.New(color.FgRed, color.Bold))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	// Restart the specified MinIO server
	fatalIf(probe.NewError(client.ServiceRestart(globalContext)), "Unable to restart the server.")

	// Success..
	printMsg(serviceRestartCommand{Status: "success", ServerURL: aliasedURL})

	coloring := color.New(color.FgRed)
	mark := "."

	// Print restart progress
	printProgress := func() {
		if !globalQuiet && !globalJSON {
			coloring.Printf(mark)
		}
	}

	for {
		select {
		case <-globalContext.Done():
			return globalContext.Err()
		case <-time.NewTimer(3 * time.Second).C:
			ctx, cancel := context.WithTimeout(globalContext, 1*time.Second)
			// Fetch the service status of the specified MinIO server
			info, e := client.ServerInfo(ctx)
			cancel()
			switch {
			case e == nil && info.Mode == madmin.ObjectLayerOnline:
				printMsg(serviceRestartMessage{Status: "success", ServerURL: aliasedURL})
				return nil
			case err == nil && info.Mode == madmin.ObjectLayerInitializing:
				coloring = color.New(color.FgYellow)
				mark = "!"
				fallthrough
			default:
				printProgress()
			}
		}
	}
}
<commit_msg>fix: use madmin constants<commit_after>/*
 * MinIO Client (C) 2016 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"time"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/console"
	"github.com/minio/minio/pkg/madmin"
)

var adminServiceRestartCmd = cli.Command{
	Name:         "restart",
	Usage:        "restart all MinIO servers",
	Action:       mainAdminServiceRestart,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        globalFlags,
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Restart MinIO server represented by its alias 'play'.
     {{.Prompt}} {{.HelpName}} play/
`,
}

// serviceRestartCommand is container for service restart command success and failure messages.
type serviceRestartCommand struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
}

// String colorized service restart command message.
func (s serviceRestartCommand) String() string {
	return console.Colorize("ServiceRestart", "Restart command successfully sent to `"+s.ServerURL+"`. Type Ctrl-C or wait to see the status of the restart.")
}

// JSON jsonified service restart command message.
func (s serviceRestartCommand) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// serviceRestartMessage is container for service restart success and failure messages.
type serviceRestartMessage struct {
	Status    string `json:"status"`
	ServerURL string `json:"serverURL"`
	Err       error  `json:"error,omitempty"`
}

// String colorized service restart message.
func (s serviceRestartMessage) String() string {
	if s.Err == nil {
		return console.Colorize("ServiceRestart", "\nRestarted `"+s.ServerURL+"` successfully.")
	}
	return console.Colorize("FailedServiceRestart", "Failed to restart `"+s.ServerURL+"`. error: "+s.Err.Error())
}

// JSON jsonified service restart message.
func (s serviceRestartMessage) JSON() string {
	serviceRestartJSONBytes, e := json.MarshalIndent(s, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(serviceRestartJSONBytes)
}

// checkAdminServiceRestartSyntax - validate all the passed arguments
func checkAdminServiceRestartSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 2 {
		cli.ShowCommandHelpAndExit(ctx, "restart", 1) // last argument is exit code
	}
}

func mainAdminServiceRestart(ctx *cli.Context) error {

	// Validate serivce restart syntax.
	checkAdminServiceRestartSyntax(ctx)

	// Set color.
	console.SetColor("ServiceOffline", color.New(color.FgRed, color.Bold))
	console.SetColor("ServiceInitializing", color.New(color.FgYellow, color.Bold))
	console.SetColor("ServiceRestart", color.New(color.FgGreen, color.Bold))
	console.SetColor("FailedServiceRestart", color.New(color.FgRed, color.Bold))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Unable to initialize admin connection.")

	// Restart the specified MinIO server
	fatalIf(probe.NewError(client.ServiceRestart(globalContext)), "Unable to restart the server.")

	// Success..
	printMsg(serviceRestartCommand{Status: "success", ServerURL: aliasedURL})

	coloring := color.New(color.FgRed)
	mark := "."

	// Print restart progress
	printProgress := func() {
		if !globalQuiet && !globalJSON {
			coloring.Printf(mark)
		}
	}

	for {
		select {
		case <-globalContext.Done():
			return globalContext.Err()
		case <-time.NewTimer(3 * time.Second).C:
			ctx, cancel := context.WithTimeout(globalContext, 1*time.Second)
			// Fetch the service status of the specified MinIO server
			info, e := client.ServerInfo(ctx)
			cancel()
			switch {
			case e == nil && info.Mode == string(madmin.ItemOnline):
				printMsg(serviceRestartMessage{Status: "success", ServerURL: aliasedURL})
				return nil
			case err == nil && info.Mode == string(madmin.ItemInitializing):
				coloring = color.New(color.FgYellow)
				mark = "!"
				fallthrough
			default:
				printProgress()
			}
		}
	}
}
<|endoftext|>


<commit_before>package cockroach

import "time"

// BuildJobLogs represents the logs for a build job
type BuildJobLogs struct {
	RepoFullName string
	RepoBranch   string
	RepoRevision string
	RepoSource   string
	LogText      string
}

// BuildJobLogRow represents the logs for a build job as stored in the database
type BuildJobLogRow struct {
	ID           int
	RepoFullName string
	RepoBranch   string
	RepoRevision string
	RepoSource   string
	LogText      string
	InsertedAt   time.Time
}

// BuildVersionDetail represents a specific build, including version number, repo, branch, revision and manifest
type BuildVersionDetail struct {
	ID           int
	BuildVersion string
	RepoSource   string
	RepoFullName string
	RepoBranch   string
	RepoRevision string
	Manifest     string
	InsertedAt   time.Time
}

// Build represents a specific build, including version number, repo, branch, revision, labels and manifest
type Build struct {
	ID           int       `jsonapi:"primary,pipeline"`
	RepoSource   string    `jsonapi:"attr,repo-source"`
	RepoOwner    string    `jsonapi:"attr,repo-owner"`
	RepoName     string    `jsonapi:"attr,repo-name"`
	RepoBranch   string    `jsonapi:"attr,repo-branch"`
	RepoRevision string    `jsonapi:"attr,repo-revision"`
	BuildVersion string    `jsonapi:"attr,build-version"`
	BuildStatus  string    `jsonapi:"attr,build-status"`
	Labels       string    `jsonapi:"attr,labels"`
	Manifest     string    `jsonapi:"attr,manifest"`
	InsertedAt   time.Time `jsonapi:"attr,inserted-at"`
	UpdatedAt    time.Time `jsonapi:"attr,updated-at"`
}
<commit_msg>use plural for type<commit_after>package cockroach

import "time"

// BuildJobLogs represents the logs for a build job
type BuildJobLogs struct {
	RepoFullName string
	RepoBranch   string
	RepoRevision string
	RepoSource   string
	LogText      string
}

// BuildJobLogRow represents the logs for a build job as stored in the database
type BuildJobLogRow struct {
	ID           int
	RepoFullName string
	RepoBranch   string
	RepoRevision string
	RepoSource   string
	LogText      string
	InsertedAt   time.Time
}

// BuildVersionDetail represents a specific build, including version number, repo, branch, revision and manifest
type BuildVersionDetail struct {
	ID           int
	BuildVersion string
	RepoSource   string
	RepoFullName string
	RepoBranch   string
	RepoRevision string
	Manifest     string
	InsertedAt   time.Time
}

// Build represents a specific build, including version number, repo, branch, revision, labels and manifest
type Build struct {
	ID           int       `jsonapi:"primary,pipelines"`
	RepoSource   string    `jsonapi:"attr,repo-source"`
	RepoOwner    string    `jsonapi:"attr,repo-owner"`
	RepoName     string    `jsonapi:"attr,repo-name"`
	RepoBranch   string    `jsonapi:"attr,repo-branch"`
	RepoRevision string    `jsonapi:"attr,repo-revision"`
	BuildVersion string    `jsonapi:"attr,build-version"`
	BuildStatus  string    `jsonapi:"attr,build-status"`
	Labels       string    `jsonapi:"attr,labels"`
	Manifest     string    `jsonapi:"attr,manifest"`
	InsertedAt   time.Time `jsonapi:"attr,inserted-at"`
	UpdatedAt    time.Time `jsonapi:"attr,updated-at"`
}
<|endoftext|>


<commit_before>package digioceanloadbalancer

import (
  "testing"
  digioceanAuth "github.com/cloudlibz/gocloud/digioceanauth"
)

func init() {
	digioceanAuth.LoadConfig()
}

func TestCreateloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  forwardingrules := []map[string]interface{}{
      {
        "EntryProtocol":  "https",
        "EntryPort":  444,
        "TargetProtocol": "https",
        "TargetPort": 443,
        "TLSPassthrough": true,
      },
  }

  healthcheck := map[string]interface{}{
    "Protocol": "http",
    "Port": 80,
    "Path": "/",
    "CheckIntervalSeconds": 10,
    "ResponseTimeoutSeconds": 5,
    "HealthyThreshold": 5,
    "UnhealthyThreshold": 3,
	}

  stickysessions := map[string]interface{}{
    "Type": "none",
  }

	create := map[string]interface{}{
    "Name": "example-01",
    "Algorithm":  "round_robin",
    "Region": "nyc3",
    "ForwardingRules":  forwardingrules,
    "HealthCheck":  healthcheck,
    "StickySessions": stickysessions,
    "DropletIDs":  []int{3164444, 3164445},
    "Tag":  nil,
    "RedirectHTTPToHTTPS": false,
	}

	_, err := digioceancloud.Createloadbalancer(create)

	if err != nil {
		t.Errorf("Test to create DigitalOcean LoadBalancer failed.")
	}
}

func TestDeleteloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  delete := map[string]string{
    "ID": "86407564",
   }

	_, err := digioceancloud.Deleteloadbalancer(delete)

	if err != nil {
		t.Errorf("Test to delete DigitalOcean LoadBalancer failed")
	}
}

func TestListloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

	_, err := digioceancloud.Listloadbalancer(nil)

	if err != nil {
		t.Errorf("Test to list DigitalOcean LoadBalancer failed")
	}
}

func TestAttachnodewithloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  attachnodewithloadbalancer := map[string]interface{}{
    "LoadBalancerID":   "my-load-balancer",
    "DropletIDs":        []int{31331, 31313},
	}

  _, err := digioceancloud.Attachnodewithloadbalancer(attachnodewithloadbalancer)

	if err != nil {
		t.Errorf("Test to attach Droplets to DigitalOcean LoadBalancer failed")
	}
}

func TestDetachnodewithloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  detachnodewithloadbalancer := map[string]interface{}{
    "LoadBalancerID":   "my-load-balancer",
    "DropletIDs":        []int{31331, 31313},
	}

  _, err := digioceancloud.Detachnodewithloadbalancer(detachnodewithloadbalancer)

	if err != nil {
		t.Errorf("Test to detach Droplets to DigitalOcean LoadBalancer failed")
	}
}
<commit_msg>Corrected grammatical mistake<commit_after>package digioceanloadbalancer

import (
  "testing"
  digioceanAuth "github.com/cloudlibz/gocloud/digioceanauth"
)

func init() {
	digioceanAuth.LoadConfig()
}

func TestCreateloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  forwardingrules := []map[string]interface{}{
      {
        "EntryProtocol":  "https",
        "EntryPort":  444,
        "TargetProtocol": "https",
        "TargetPort": 443,
        "TLSPassthrough": true,
      },
  }

  healthcheck := map[string]interface{}{
    "Protocol": "http",
    "Port": 80,
    "Path": "/",
    "CheckIntervalSeconds": 10,
    "ResponseTimeoutSeconds": 5,
    "HealthyThreshold": 5,
    "UnhealthyThreshold": 3,
	}

  stickysessions := map[string]interface{}{
    "Type": "none",
  }

	create := map[string]interface{}{
    "Name": "example-01",
    "Algorithm":  "round_robin",
    "Region": "nyc3",
    "ForwardingRules":  forwardingrules,
    "HealthCheck":  healthcheck,
    "StickySessions": stickysessions,
    "DropletIDs":  []int{3164444, 3164445},
    "Tag":  nil,
    "RedirectHTTPToHTTPS": false,
	}

	_, err := digioceancloud.Createloadbalancer(create)

	if err != nil {
		t.Errorf("Test to create DigitalOcean LoadBalancer failed.")
	}
}

func TestDeleteloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  delete := map[string]string{
    "ID": "86407564",
   }

	_, err := digioceancloud.Deleteloadbalancer(delete)

	if err != nil {
		t.Errorf("Test to delete DigitalOcean LoadBalancer failed")
	}
}

func TestListloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

	_, err := digioceancloud.Listloadbalancer(nil)

	if err != nil {
		t.Errorf("Test to list DigitalOcean LoadBalancer failed")
	}
}

func TestAttachnodewithloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  attachnodewithloadbalancer := map[string]interface{}{
    "LoadBalancerID":   "my-load-balancer",
    "DropletIDs":        []int{31331, 31313},
	}

  _, err := digioceancloud.Attachnodewithloadbalancer(attachnodewithloadbalancer)

	if err != nil {
		t.Errorf("Test to attach Droplets to DigitalOcean LoadBalancer failed")
	}
}

func TestDetachnodewithloadbalancer(t *testing.T) {

  var digioceancloud LoadBalancer

  detachnodewithloadbalancer := map[string]interface{}{
    "LoadBalancerID":   "my-load-balancer",
    "DropletIDs":        []int{31331, 31313},
	}

  _, err := digioceancloud.Detachnodewithloadbalancer(detachnodewithloadbalancer)

	if err != nil {
		t.Errorf("Test to detach Droplets from DigitalOcean LoadBalancer failed")
	}
}
<|endoftext|>