<commit_before>// Copyright 2017 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pbf

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"reflect"
	"runtime"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/maguro/pbf/protobuf"
)

const (
	initialBufferSize = 1024 * 1024
)

type encoded struct {
	header *protobuf.BlobHeader
	blob   *protobuf.Blob
	err    error
}

type decoded struct {
	elements []interface{}
	err      error
}

type pair struct {
	element interface{}
	err     error
}

// Decoder reads and decodes OpenStreetMap PBF data from an input stream.
type Decoder struct {
	InputBufferSize      int
	ZlibBufferSize       int
	OutputChannelLength  int
	DecodedChannelLength int
	inputs               []chan<- encoded
	outputs              []<-chan decoded
	decoded              chan pair
	done                 chan struct{}
	start                sync.Once

	reader io.Reader
	buffer *bytes.Buffer

	Header *Header
}

// NewDecoder returns a new decoder that reads from r.
func NewDecoder(r io.Reader) (*Decoder, error) {
	decoder := &Decoder{
		InputBufferSize:      16,
		ZlibBufferSize:       initialBufferSize,
		OutputChannelLength:  8,
		DecodedChannelLength: 8000,
		reader:               r,
		buffer:               bytes.NewBuffer(make([]byte, 0, initialBufferSize)),
	}

	bh, err := decoder.readBlobHeader()
	if err != nil {
		return nil, err
	}
	b, err := decoder.readBlob(bh)
	if err != nil {
		return nil, err
	}
	elements, err := decode(bh, b, bytes.NewBuffer(make([]byte, 0, 1024)))
	if err != nil {
		return nil, err
	}
	decoder.Header = elements[0].(*Header)
	if decoder.Header == nil {
		err = fmt.Errorf("expected header data but got %v", reflect.TypeOf(elements[0]))
		return nil, err
	}

	return decoder, nil
}

// SetBufferSize sets initial size of decoding buffer. Any value will produce
// valid results; buffer will grow automatically if required.
func (d *Decoder) SetBufferSize(n int) {
	d.buffer = bytes.NewBuffer(make([]byte, 0, n))
}

// Start begins parsing in the background using n goroutines.  The background
// processing can be canceled by calling Stop.
func (d *Decoder) Start(n int) {
	if n < 1 {
		n = 1
	}

	d.start.Do(func() {
		d.inputs = make([]chan<- encoded, n)
		d.outputs = make([]<-chan decoded, n)
		d.decoded = make(chan pair, d.DecodedChannelLength)
		d.done = make(chan struct{})

		// start data decoders
		for i := range d.inputs {
			input := make(chan encoded, d.InputBufferSize)
			output := make(chan decoded, d.OutputChannelLength)

			go func() {
				defer close(output)

				zlibBuffer := bytes.NewBuffer(make([]byte, 0, d.ZlibBufferSize))

				for {
					select {
					case <-d.done:
						return
					case raw, more := <-input:
						if !more {
							return
						}

						if raw.err != nil {
							output <- decoded{nil, raw.err}
							return
						}

						elements, err := decode(raw.header, raw.blob, zlibBuffer)

						select {
						case <-d.done:
							return
						case output <- decoded{elements, err}:
						}
					}
				}
			}()

			d.inputs[i] = input
			d.outputs[i] = output
		}

		// read raw blobs and distribute amongst inputs
		go func() {
			defer func() {
				for _, input := range d.inputs {
					close(input)
				}
			}()

			var i int
			for {
				input := d.inputs[i]
				i = (i + 1) % n

				h, err := d.readBlobHeader()
				if err == io.EOF {
					return
				} else if err != nil {
					input <- encoded{err: err}
					return
				}

				b, err := d.readBlob(h)
				if err != nil {
					input <- encoded{err: err}
					return
				}

				select {
				case <-d.done:
					return
				case input <- encoded{header: h, blob: b}:
				}
			}
		}()

		// coalesce decoded elements
		go func() {
			defer close(d.decoded)

			var i int
			for {
				output := d.outputs[i]
				i = (i + 1) % n

				select {
				case <-d.done:
					return
				case decoded, more := <-output:
					if !more {
						return
					}

					if decoded.err != nil {
						d.decoded <- pair{nil, decoded.err}
						return
					}

					for _, e := range decoded.elements {
						d.decoded <- pair{e, nil}
					}
				}
			}
		}()
	})
}

// Stop will cancel the background decoding pipeline.
func (d *Decoder) Stop() {
	close(d.done)
}

// Decode reads the next OSM object and returns either a pointer to Node, Way
// or Relation struct representing the underlying OpenStreetMap PBF data, or
// error encountered. The end of the input stream is reported by an io.EOF
// error.
//
// If background parsing was not begun by calling Start, Start is called with
// the maximum number of CPUs returned by GOMAXPROCS.
func (d *Decoder) Decode() (interface{}, error) {

	d.Start(runtime.GOMAXPROCS(-1))

	decoded, more := <-d.decoded
	if !more {
		return nil, io.EOF
	}
	return decoded.element, decoded.err
}

func (d *Decoder) readBlobHeader() (header *protobuf.BlobHeader, err error) {
	var size uint32
	err = binary.Read(d.reader, binary.BigEndian, &size)
	if err != nil {
		return nil, err
	}

	d.buffer.Reset()
	if _, err := io.CopyN(d.buffer, d.reader, int64(size)); err != nil {
		return nil, err
	}

	header = &protobuf.BlobHeader{}
	if err := proto.Unmarshal(d.buffer.Bytes(), header); err != nil {
		return nil, err
	}

	return header, nil
}

func (d *Decoder) readBlob(header *protobuf.BlobHeader) (*protobuf.Blob, error) {
	size := header.GetDatasize()

	d.buffer.Reset()
	if _, err := io.CopyN(d.buffer, d.reader, int64(size)); err != nil {
		return nil, err
	}

	blob := &protobuf.Blob{}
	if err := proto.Unmarshal(d.buffer.Bytes(), blob); err != nil {
		return nil, err
	}

	return blob, nil
}

func decode(header *protobuf.BlobHeader, blob *protobuf.Blob, zlibBuf *bytes.Buffer) ([]interface{}, error) {
	var buffer []byte
	switch {
	case blob.Raw != nil:
		buffer = blob.GetRaw()

	case blob.ZlibData != nil:
		r, err := zlib.NewReader(bytes.NewReader(blob.GetZlibData()))
		if err != nil {
			return nil, err
		}
		zlibBuf.Reset()
		rawBufferSize := int(blob.GetRawSize() + bytes.MinRead)
		if rawBufferSize > zlibBuf.Cap() {
			zlibBuf.Grow(rawBufferSize)
		}
		_, err = zlibBuf.ReadFrom(r)
		if err != nil {
			return nil, err
		}
		if zlibBuf.Len() != int(blob.GetRawSize()) {
			err = fmt.Errorf("raw blob data size %d but expected %d", zlibBuf.Len(), blob.GetRawSize())
			return nil, err
		}
		buffer = zlibBuf.Bytes()

	default:
		return nil, errors.New("unknown blob data type")
	}

	ht := *header.Type
	if ht == "OSMHeader" {
		h, err := parseOSMHeader(buffer)
		if err != nil {
			return nil, err
		}

		return []interface{}{h}, nil
	} else if ht == "OSMData" {
		return parsePrimitiveBlock(buffer)
	} else {
		return nil, fmt.Errorf("unknown header type %s", ht)
	}
}

func parseOSMHeader(buffer []byte) (*Header, error) {
	hb := &protobuf.HeaderBlock{}
	if err := proto.Unmarshal(buffer, hb); err != nil {
		return nil, err
	}

	header := &Header{
		RequiredFeatures: hb.GetRequiredFeatures(),
		OptionalFeatures: hb.GetOptionalFeatures(),
		WritingProgram:   hb.GetWritingprogram(),
		Source:           hb.GetSource(),
		OsmosisReplicationBaseURL:        hb.GetOsmosisReplicationBaseUrl(),
		OsmosisReplicationSequenceNumber: hb.GetOsmosisReplicationSequenceNumber(),
	}

	if hb.Bbox != nil {
		header.BoundingBox = &BoundingBox{
			Left:   toDecimalDegrees(0, 1, hb.Bbox.GetLeft()),
			Right:  toDecimalDegrees(0, 1, hb.Bbox.GetRight()),
			Top:    toDecimalDegrees(0, 1, hb.Bbox.GetTop()),
			Bottom: toDecimalDegrees(0, 1, hb.Bbox.GetBottom()),
		}
	}

	if hb.OsmosisReplicationTimestamp != nil {
		header.OsmosisReplicationTimestamp = time.Unix(*hb.OsmosisReplicationTimestamp, 0)
	}

	return header, nil
}

// toDecimalDegrees converts a coordinate into DecimalDegrees, given the offset and
// granularity of the coordinate.
func toDecimalDegrees(offset int64, granularity int32, coordinate int64) DecimalDegrees {
	return NanoDecimalDegrees * DecimalDegrees(offset+(int64(granularity)*coordinate))
}
<commit_msg>Change Stop() to not close done it not initialized<commit_after>// Copyright 2017 the original author or authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pbf

import (
	"bytes"
	"compress/zlib"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"reflect"
	"runtime"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/maguro/pbf/protobuf"
)

const (
	initialBufferSize = 1024 * 1024
)

type encoded struct {
	header *protobuf.BlobHeader
	blob   *protobuf.Blob
	err    error
}

type decoded struct {
	elements []interface{}
	err      error
}

type pair struct {
	element interface{}
	err     error
}

// Decoder reads and decodes OpenStreetMap PBF data from an input stream.
type Decoder struct {
	InputBufferSize      int
	ZlibBufferSize       int
	OutputChannelLength  int
	DecodedChannelLength int
	inputs               []chan<- encoded
	outputs              []<-chan decoded
	decoded              chan pair
	done                 chan struct{}
	start                sync.Once

	reader io.Reader
	buffer *bytes.Buffer

	Header *Header
}

// NewDecoder returns a new decoder that reads from r.
func NewDecoder(r io.Reader) (*Decoder, error) {
	decoder := &Decoder{
		InputBufferSize:      16,
		ZlibBufferSize:       initialBufferSize,
		OutputChannelLength:  8,
		DecodedChannelLength: 8000,
		reader:               r,
		buffer:               bytes.NewBuffer(make([]byte, 0, initialBufferSize)),
	}

	bh, err := decoder.readBlobHeader()
	if err != nil {
		return nil, err
	}
	b, err := decoder.readBlob(bh)
	if err != nil {
		return nil, err
	}
	elements, err := decode(bh, b, bytes.NewBuffer(make([]byte, 0, 1024)))
	if err != nil {
		return nil, err
	}
	decoder.Header = elements[0].(*Header)
	if decoder.Header == nil {
		err = fmt.Errorf("expected header data but got %v", reflect.TypeOf(elements[0]))
		return nil, err
	}

	return decoder, nil
}

// SetBufferSize sets initial size of decoding buffer. Any value will produce
// valid results; buffer will grow automatically if required.
func (d *Decoder) SetBufferSize(n int) {
	d.buffer = bytes.NewBuffer(make([]byte, 0, n))
}

// Start begins parsing in the background using n goroutines.  The background
// processing can be canceled by calling Stop.
func (d *Decoder) Start(n int) {
	if n < 1 {
		n = 1
	}

	d.start.Do(func() {
		d.inputs = make([]chan<- encoded, n)
		d.outputs = make([]<-chan decoded, n)
		d.decoded = make(chan pair, d.DecodedChannelLength)
		d.done = make(chan struct{})

		// start data decoders
		for i := range d.inputs {
			input := make(chan encoded, d.InputBufferSize)
			output := make(chan decoded, d.OutputChannelLength)

			go func() {
				defer close(output)

				zlibBuffer := bytes.NewBuffer(make([]byte, 0, d.ZlibBufferSize))

				for {
					select {
					case <-d.done:
						return
					case raw, more := <-input:
						if !more {
							return
						}

						if raw.err != nil {
							output <- decoded{nil, raw.err}
							return
						}

						elements, err := decode(raw.header, raw.blob, zlibBuffer)

						select {
						case <-d.done:
							return
						case output <- decoded{elements, err}:
						}
					}
				}
			}()

			d.inputs[i] = input
			d.outputs[i] = output
		}

		// read raw blobs and distribute amongst inputs
		go func() {
			defer func() {
				for _, input := range d.inputs {
					close(input)
				}
			}()

			var i int
			for {
				input := d.inputs[i]
				i = (i + 1) % n

				h, err := d.readBlobHeader()
				if err == io.EOF {
					return
				} else if err != nil {
					input <- encoded{err: err}
					return
				}

				b, err := d.readBlob(h)
				if err != nil {
					input <- encoded{err: err}
					return
				}

				select {
				case <-d.done:
					return
				case input <- encoded{header: h, blob: b}:
				}
			}
		}()

		// coalesce decoded elements
		go func() {
			defer close(d.decoded)

			var i int
			for {
				output := d.outputs[i]
				i = (i + 1) % n

				select {
				case <-d.done:
					return
				case decoded, more := <-output:
					if !more {
						return
					}

					if decoded.err != nil {
						d.decoded <- pair{nil, decoded.err}
						return
					}

					for _, e := range decoded.elements {
						d.decoded <- pair{e, nil}
					}
				}
			}
		}()
	})
}

// Stop will cancel the background decoding pipeline.
func (d *Decoder) Stop() {
	if d.done != nil {
		close(d.done)
	}
}

// Decode reads the next OSM object and returns either a pointer to Node, Way
// or Relation struct representing the underlying OpenStreetMap PBF data, or
// error encountered. The end of the input stream is reported by an io.EOF
// error.
//
// If background parsing was not begun by calling Start, Start is called with
// the maximum number of CPUs returned by GOMAXPROCS.
func (d *Decoder) Decode() (interface{}, error) {

	d.Start(runtime.GOMAXPROCS(-1))

	decoded, more := <-d.decoded
	if !more {
		return nil, io.EOF
	}
	return decoded.element, decoded.err
}

func (d *Decoder) readBlobHeader() (header *protobuf.BlobHeader, err error) {
	var size uint32
	err = binary.Read(d.reader, binary.BigEndian, &size)
	if err != nil {
		return nil, err
	}

	d.buffer.Reset()
	if _, err := io.CopyN(d.buffer, d.reader, int64(size)); err != nil {
		return nil, err
	}

	header = &protobuf.BlobHeader{}
	if err := proto.Unmarshal(d.buffer.Bytes(), header); err != nil {
		return nil, err
	}

	return header, nil
}

func (d *Decoder) readBlob(header *protobuf.BlobHeader) (*protobuf.Blob, error) {
	size := header.GetDatasize()

	d.buffer.Reset()
	if _, err := io.CopyN(d.buffer, d.reader, int64(size)); err != nil {
		return nil, err
	}

	blob := &protobuf.Blob{}
	if err := proto.Unmarshal(d.buffer.Bytes(), blob); err != nil {
		return nil, err
	}

	return blob, nil
}

func decode(header *protobuf.BlobHeader, blob *protobuf.Blob, zlibBuf *bytes.Buffer) ([]interface{}, error) {
	var buffer []byte
	switch {
	case blob.Raw != nil:
		buffer = blob.GetRaw()

	case blob.ZlibData != nil:
		r, err := zlib.NewReader(bytes.NewReader(blob.GetZlibData()))
		if err != nil {
			return nil, err
		}
		zlibBuf.Reset()
		rawBufferSize := int(blob.GetRawSize() + bytes.MinRead)
		if rawBufferSize > zlibBuf.Cap() {
			zlibBuf.Grow(rawBufferSize)
		}
		_, err = zlibBuf.ReadFrom(r)
		if err != nil {
			return nil, err
		}
		if zlibBuf.Len() != int(blob.GetRawSize()) {
			err = fmt.Errorf("raw blob data size %d but expected %d", zlibBuf.Len(), blob.GetRawSize())
			return nil, err
		}
		buffer = zlibBuf.Bytes()

	default:
		return nil, errors.New("unknown blob data type")
	}

	ht := *header.Type
	if ht == "OSMHeader" {
		h, err := parseOSMHeader(buffer)
		if err != nil {
			return nil, err
		}

		return []interface{}{h}, nil
	} else if ht == "OSMData" {
		return parsePrimitiveBlock(buffer)
	} else {
		return nil, fmt.Errorf("unknown header type %s", ht)
	}
}

func parseOSMHeader(buffer []byte) (*Header, error) {
	hb := &protobuf.HeaderBlock{}
	if err := proto.Unmarshal(buffer, hb); err != nil {
		return nil, err
	}

	header := &Header{
		RequiredFeatures: hb.GetRequiredFeatures(),
		OptionalFeatures: hb.GetOptionalFeatures(),
		WritingProgram:   hb.GetWritingprogram(),
		Source:           hb.GetSource(),
		OsmosisReplicationBaseURL:        hb.GetOsmosisReplicationBaseUrl(),
		OsmosisReplicationSequenceNumber: hb.GetOsmosisReplicationSequenceNumber(),
	}

	if hb.Bbox != nil {
		header.BoundingBox = &BoundingBox{
			Left:   toDecimalDegrees(0, 1, hb.Bbox.GetLeft()),
			Right:  toDecimalDegrees(0, 1, hb.Bbox.GetRight()),
			Top:    toDecimalDegrees(0, 1, hb.Bbox.GetTop()),
			Bottom: toDecimalDegrees(0, 1, hb.Bbox.GetBottom()),
		}
	}

	if hb.OsmosisReplicationTimestamp != nil {
		header.OsmosisReplicationTimestamp = time.Unix(*hb.OsmosisReplicationTimestamp, 0)
	}

	return header, nil
}

// toDecimalDegrees converts a coordinate into DecimalDegrees, given the offset and
// granularity of the coordinate.
func toDecimalDegrees(offset int64, granularity int32, coordinate int64) DecimalDegrees {
	return NanoDecimalDegrees * DecimalDegrees(offset+(int64(granularity)*coordinate))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"

	"github.com/deis/deisctl/client"
	"github.com/deis/deisctl/cmd"

	docopt "github.com/docopt/docopt-go"
)

const (
	// Version of deisctl client
	Version string = "0.12.0-dev"
)

func exit(err error, code int) {
	fmt.Printf("Error: %v\n", err)
	os.Exit(code)
}

func setGlobalFlags(args map[string]interface{}) {
	client.Flags.Version = args["--version"].(bool)
	client.Flags.Endpoint = args["--endpoint"].(string)
	client.Flags.EtcdKeyPrefix = args["--etcd-key-prefix"].(string)
	client.Flags.EtcdKeyFile = args["--etcd-keyfile"].(string)
	client.Flags.EtcdCertFile = args["--etcd-certfile"].(string)
	client.Flags.EtcdCAFile = args["--etcd-cafile"].(string)
	//client.Flags.UseAPI = args["--experimental-api"].(bool)
	client.Flags.KnownHostsFile = args["--known-hosts-file"].(string)
	client.Flags.StrictHostKeyChecking = args["--strict-host-key-checking"].(bool)
	timeout, _ := strconv.ParseFloat(args["--request-timeout"].(string), 64)
	client.Flags.RequestTimeout = timeout
	tunnel := args["--tunnel"].(string)
	if tunnel != "" {
		client.Flags.Tunnel = tunnel
	} else {
		client.Flags.Tunnel = os.Getenv("DEISCTL_TUNNEL")
	}
}

func main() {
	usage := `Deis Control Utility

Usage:
  deisctl <command> [<target>...] [options]

Example Commands:

  deisctl install platform
  deisctl uninstall builder@1
  deisctl list
  deisctl scale router=2
  deisctl start router@2
  deisctl stop router builder
  deisctl status controller
  deisctl journal controller

Options:
  --version                   print version and exit
  --endpoint=<url>            etcd endpoint for fleet [default: http://127.0.0.1:4001]
  --etcd-key-prefix=<path>    keyspace for fleet data in etcd [default: /_coreos.com/fleet/]
  --etcd-keyfile=<path>       etcd key file authentication [default: ]
  --etcd-certfile=<path>      etcd cert file authentication [default: ]
  --etcd-cafile=<path>        etcd CA file authentication [default: ]
  --known-hosts-file=<path>   file used to store remote machine fingerprints [default: ~/.ssh/known_hosts]
  --strict-host-key-checking  verify SSH host keys [default: true]
  --tunnel=<host>             establish an SSH tunnel for communication with fleet and etcd [default: ]
  --request-timeout=<secs>    amount of time to allow a single request before considering it failed. [default: 3.0]
`
	// special handling for version
	if len(os.Args) == 2 && os.Args[1] == "--version" {
		fmt.Println(Version)
		os.Exit(0)
	}
	// parse command-line arguments
	args, err := docopt.Parse(usage, nil, true, "", true)
	if err != nil {
		exit(err, 2)
	}
	command := args["<command>"]
	targets := args["<target>"].([]string)
	setGlobalFlags(args)
	// construct a client
	c, err := client.NewClient()
	if err != nil {
		exit(err, 1)
	}
	// dispatch the command
	switch command {
	case "list":
		err = cmd.ListUnits(c)
	case "list-units":
		err = cmd.ListUnits(c)
	case "list-unit-files":
		err = cmd.ListUnitFiles(c)
	case "scale":
		err = cmd.Scale(c, targets)
	case "start":
		err = cmd.Start(c, targets)
	case "restart":
		err = cmd.Restart(c, targets)
	case "stop":
		err = cmd.Stop(c, targets)
	case "status":
		err = cmd.Status(c, targets)
	case "journal":
		err = cmd.Journal(c, targets)
	case "install":
		err = cmd.Install(c, targets)
	case "uninstall":
		err = cmd.Uninstall(c, targets)
	case "config":
		err = cmd.Config()
	case "update":
		err = cmd.Update()
	case "refresh-units":
		err = cmd.RefreshUnits()
	default:
		fmt.Printf(usage)
		os.Exit(2)
	}
	if err != nil {
		exit(err, 1)
	}
}
<commit_msg>fix(deisctl): use docopt's native version parser<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"

	"github.com/deis/deisctl/client"
	"github.com/deis/deisctl/cmd"

	docopt "github.com/docopt/docopt-go"
)

const (
	// Version of deisctl client
	Version string = "0.12.0-dev"
)

func exit(err error, code int) {
	fmt.Printf("Error: %v\n", err)
	os.Exit(code)
}

func setGlobalFlags(args map[string]interface{}) {
	client.Flags.Version = args["--version"].(bool)
	client.Flags.Endpoint = args["--endpoint"].(string)
	client.Flags.EtcdKeyPrefix = args["--etcd-key-prefix"].(string)
	client.Flags.EtcdKeyFile = args["--etcd-keyfile"].(string)
	client.Flags.EtcdCertFile = args["--etcd-certfile"].(string)
	client.Flags.EtcdCAFile = args["--etcd-cafile"].(string)
	//client.Flags.UseAPI = args["--experimental-api"].(bool)
	client.Flags.KnownHostsFile = args["--known-hosts-file"].(string)
	client.Flags.StrictHostKeyChecking = args["--strict-host-key-checking"].(bool)
	timeout, _ := strconv.ParseFloat(args["--request-timeout"].(string), 64)
	client.Flags.RequestTimeout = timeout
	tunnel := args["--tunnel"].(string)
	if tunnel != "" {
		client.Flags.Tunnel = tunnel
	} else {
		client.Flags.Tunnel = os.Getenv("DEISCTL_TUNNEL")
	}
}

func main() {
	usage := `Deis Control Utility

Usage:
  deisctl <command> [<target>...] [options]

Example Commands:

  deisctl install platform
  deisctl uninstall builder@1
  deisctl list
  deisctl scale router=2
  deisctl start router@2
  deisctl stop router builder
  deisctl status controller
  deisctl journal controller

Options:
  --version                   print version and exit
  --endpoint=<url>            etcd endpoint for fleet [default: http://127.0.0.1:4001]
  --etcd-key-prefix=<path>    keyspace for fleet data in etcd [default: /_coreos.com/fleet/]
  --etcd-keyfile=<path>       etcd key file authentication [default: ]
  --etcd-certfile=<path>      etcd cert file authentication [default: ]
  --etcd-cafile=<path>        etcd CA file authentication [default: ]
  --known-hosts-file=<path>   file used to store remote machine fingerprints [default: ~/.ssh/known_hosts]
  --strict-host-key-checking  verify SSH host keys [default: true]
  --tunnel=<host>             establish an SSH tunnel for communication with fleet and etcd [default: ]
  --request-timeout=<secs>    amount of time to allow a single request before considering it failed. [default: 3.0]
`
	// parse command-line arguments
	args, err := docopt.Parse(usage, nil, true, Version, true)
	if err != nil {
		exit(err, 2)
	}
	command := args["<command>"]
	targets := args["<target>"].([]string)
	setGlobalFlags(args)
	// construct a client
	c, err := client.NewClient()
	if err != nil {
		exit(err, 1)
	}
	// dispatch the command
	switch command {
	case "list":
		err = cmd.ListUnits(c)
	case "list-units":
		err = cmd.ListUnits(c)
	case "list-unit-files":
		err = cmd.ListUnitFiles(c)
	case "scale":
		err = cmd.Scale(c, targets)
	case "start":
		err = cmd.Start(c, targets)
	case "restart":
		err = cmd.Restart(c, targets)
	case "stop":
		err = cmd.Stop(c, targets)
	case "status":
		err = cmd.Status(c, targets)
	case "journal":
		err = cmd.Journal(c, targets)
	case "install":
		err = cmd.Install(c, targets)
	case "uninstall":
		err = cmd.Uninstall(c, targets)
	case "config":
		err = cmd.Config()
	case "update":
		err = cmd.Update()
	case "refresh-units":
		err = cmd.RefreshUnits()
	default:
		fmt.Printf(usage)
		os.Exit(2)
	}
	if err != nil {
		exit(err, 1)
	}
}
<|endoftext|>


<commit_before>package wordnet

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

var synsets *Synsets
var nonAlphanumeric *regexp.Regexp
var stopwords *regexp.Regexp

// Synsets contains maps from string to byte offset, and byte offset to tokens
type Synsets struct {
	Index map[string][]uint64
	Data  map[uint64][]string
}

// TODO Download WordNet database if it doesn't exist where expected

// Initialize pulls WordNet's synset database into memory
func init() {
	nonAlphanumeric = regexp.MustCompile("\\W+")
	stopwords = regexp.MustCompile("\\s+(I|a|an|as|at|by|he|she|his|hers|it|its|me|or|thou|us|who)\\s")

	currentDirectory, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	downloadDatabase(currentDirectory)

	synsets = &Synsets{Index: make(map[string][]uint64), Data: make(map[uint64][]string)}

	for _, filename := range []string{"index.adj", "index.adv", "index.noun", "index.verb", "data.adj", "data.adv", "data.noun", "data.verb"} {
		isIndexFile := strings.HasPrefix(filename, "index")
		isDataFile := !isIndexFile
		f, err := os.Open(os.ExpandEnv(path.Join(currentDirectory, "dict", filename)))
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, "  ") {
				continue
			}

			fields := strings.Fields(line)
			switch {
			case isIndexFile: // load index files containing word->synset byte offset
				word := fields[0]
				synsetCnt, err := strconv.ParseUint(fields[2], 10, 32)
				ptrCnt, err := strconv.ParseUint(fields[3], 10, 32)
				if err != nil {
					log.Fatal(err)
				}
				synsetIndices := make([]uint64, synsetCnt)
				for i, synsetOffset := range fields[6+ptrCnt:] {
					wideIndex, err := strconv.ParseUint(synsetOffset, 10, 32)
					if err != nil {
						log.Fatal(err)
					}
					synsetIndices[i] = wideIndex
				}
				synsets.Index[word] = synsetIndices

			case isDataFile: // load data files containing the synset byte offset -> synonyms
				synsetOffset, err := strconv.ParseUint(fields[0], 10, 32)
				if err != nil {
					log.Fatal(err)
				}
				wordCnt, err := strconv.ParseUint(fields[3], 16, 32)
				if err != nil {
					log.Fatal(err)
				}
				// TODO finish parsing out the data file <---
				words := make([]string, wordCnt)
				for i, word := range fields[4 : 3+(2*wordCnt)] {
					if i%2 == 0 {
						words[i/2] = word
					}
				}
				synsets.Data[synsetOffset] = words
			}
		}
	}
}

// Tokenize is a relatively naive sentence tokenizer
// TODO further normalize
func Tokenize(sentence string) []string {
	alphanumericOnly := nonAlphanumeric.ReplaceAllString(sentence, " ")
	withoutStopwords := stopwords.ReplaceAllString(" "+alphanumericOnly+" ", " ")
	fields := strings.Fields(withoutStopwords)
	for i, field := range fields {
		fields[i] = strings.ToLower(field)
	}
	return fields
}

// SynonymsForWord returns the synonyms of a given word.
func SynonymsForWord(word string) []string {
	sliceOfOffsets := synsets.Index[word]
	var allOffsets []string
	for _, offset := range sliceOfOffsets {
		allOffsets = append(allOffsets, synsets.Data[offset]...)
	}
	return deDup(allOffsets)
}

func deDup(s []string) []string {
	result := []string{}
	seen := map[string]bool{}
	for _, val := range s {
		if _, ok := seen[val]; !ok {
			result = append(result, val)
			seen[val] = true
		}
	}
	return result
}

// downloadDatabase downloads the WordNet 3.1 database if necessary
func downloadDatabase(intoDirectory string) {
	// Don't already have it? Download it.
	if _, err := os.Stat(path.Join(intoDirectory, "dict")); os.IsNotExist(err) {
		log.Println("Downloading WordNet 3.1 database files...")
		out, err := os.Create(path.Join(intoDirectory, "dict.tar.gz"))
		if err != nil {
			log.Println("Couldn't create file dict.tar.gz in " + intoDirectory)
			log.Fatal(err)
		}
		resp, err := http.Get("http://wordnetcode.princeton.edu/wn3.1.dict.tar.gz")
		if err != nil {
			log.Println("Download failed")
			log.Fatal(err)
		}
		defer resp.Body.Close()
		_, err = io.Copy(out, resp.Body)
		if err != nil {
			log.Println("Couldn't write file after download")
			log.Fatal(err)
		}

		// extract it
		if err := ungzip(path.Join(intoDirectory, "dict.tar.gz"), intoDirectory); err != nil {
			log.Println("Couldn't ungzip file")
			log.Fatal(err)
		}
		if err := untar(path.Join(intoDirectory, "dict.tar"), intoDirectory); err != nil {
			log.Println("Couldn't untar file")
			log.Fatal(err)
		}
		// Cleanup
		os.Remove(path.Join(intoDirectory, "dict.tar.gz"))
		os.Remove(path.Join(intoDirectory, "dict.tar"))
	}
}

// ungzip originally from http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/

func ungzip(source, target string) error {
	log.Println("source: " + source)
	log.Println("target: " + target)
	reader, err := os.Open(source)
	if err != nil {
		return err
	}
	defer reader.Close()

	archive, err := gzip.NewReader(reader)
	if err != nil {
		return err
	}
	defer archive.Close()

	ext := regexp.MustCompile("\\.gz$")
	target = ext.ReplaceAllString(filepath.Join(target, path.Base(source)), "")
	writer, err := os.Create(target)
	if err != nil {
		return err
	}
	defer writer.Close()

	_, err = io.Copy(writer, archive)
	return err
}

// untar originally from http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/
func untar(tarball, target string) error {
	reader, err := os.Open(tarball)
	if err != nil {
		return err
	}
	defer reader.Close()
	tarReader := tar.NewReader(reader)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		path := filepath.Join(target, header.Name)
		info := header.FileInfo()
		if info.IsDir() {
			if err = os.MkdirAll(path, info.Mode()); err != nil {
				return err
			}
			continue
		}

		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(file, tarReader)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>using temporary directory location for WordNet database<commit_after>package wordnet

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

var synsets *Synsets
var nonAlphanumeric *regexp.Regexp
var stopwords *regexp.Regexp

// Synsets contains maps from string to byte offset, and byte offset to tokens
type Synsets struct {
	Index map[string][]uint64
	Data  map[uint64][]string
}

// TODO Download WordNet database if it doesn't exist where expected

// Initialize pulls WordNet's synset database into memory
func init() {
	nonAlphanumeric = regexp.MustCompile("\\W+")
	stopwords = regexp.MustCompile("\\s+(I|a|an|as|at|by|he|she|his|hers|it|its|me|or|thou|us|who)\\s")

	currentDirectory := path.Join(os.TempDir(), "synonyms-service-wordnet-db")
	os.Mkdir(currentDirectory, 755)

	downloadDatabase(currentDirectory)

	synsets = &Synsets{Index: make(map[string][]uint64), Data: make(map[uint64][]string)}

	for _, filename := range []string{"index.adj", "index.adv", "index.noun", "index.verb", "data.adj", "data.adv", "data.noun", "data.verb"} {
		isIndexFile := strings.HasPrefix(filename, "index")
		isDataFile := !isIndexFile
		f, err := os.Open(os.ExpandEnv(path.Join(currentDirectory, "dict", filename)))
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()

		scanner := bufio.NewScanner(f)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, "  ") {
				continue
			}

			fields := strings.Fields(line)
			switch {
			case isIndexFile: // load index files containing word->synset byte offset
				word := fields[0]
				synsetCnt, err := strconv.ParseUint(fields[2], 10, 32)
				ptrCnt, err := strconv.ParseUint(fields[3], 10, 32)
				if err != nil {
					log.Fatal(err)
				}
				synsetIndices := make([]uint64, synsetCnt)
				for i, synsetOffset := range fields[6+ptrCnt:] {
					wideIndex, err := strconv.ParseUint(synsetOffset, 10, 32)
					if err != nil {
						log.Fatal(err)
					}
					synsetIndices[i] = wideIndex
				}
				synsets.Index[word] = synsetIndices

			case isDataFile: // load data files containing the synset byte offset -> synonyms
				synsetOffset, err := strconv.ParseUint(fields[0], 10, 32)
				if err != nil {
					log.Fatal(err)
				}
				wordCnt, err := strconv.ParseUint(fields[3], 16, 32)
				if err != nil {
					log.Fatal(err)
				}
				// TODO finish parsing out the data file <---
				words := make([]string, wordCnt)
				for i, word := range fields[4 : 3+(2*wordCnt)] {
					if i%2 == 0 {
						words[i/2] = word
					}
				}
				synsets.Data[synsetOffset] = words
			}
		}
	}
}

// Tokenize is a relatively naive sentence tokenizer
// TODO further normalize
func Tokenize(sentence string) []string {
	alphanumericOnly := nonAlphanumeric.ReplaceAllString(sentence, " ")
	withoutStopwords := stopwords.ReplaceAllString(" "+alphanumericOnly+" ", " ")
	fields := strings.Fields(withoutStopwords)
	for i, field := range fields {
		fields[i] = strings.ToLower(field)
	}
	return fields
}

// SynonymsForWord returns the synonyms of a given word.
func SynonymsForWord(word string) []string {
	sliceOfOffsets := synsets.Index[word]
	var allOffsets []string
	for _, offset := range sliceOfOffsets {
		allOffsets = append(allOffsets, synsets.Data[offset]...)
	}
	return deDup(allOffsets)
}

func deDup(s []string) []string {
	result := []string{}
	seen := map[string]bool{}
	for _, val := range s {
		if _, ok := seen[val]; !ok {
			result = append(result, val)
			seen[val] = true
		}
	}
	return result
}

// downloadDatabase downloads the WordNet 3.1 database if necessary
func downloadDatabase(intoDirectory string) {
	// Don't already have it? Download it.
	if _, err := os.Stat(path.Join(intoDirectory, "dict")); os.IsNotExist(err) {
		log.Println("Downloading WordNet 3.1 database files...")
		out, err := os.Create(path.Join(intoDirectory, "dict.tar.gz"))
		if err != nil {
			log.Println("Couldn't create file dict.tar.gz in " + intoDirectory)
			log.Fatal(err)
		}
		resp, err := http.Get("http://wordnetcode.princeton.edu/wn3.1.dict.tar.gz")
		if err != nil {
			log.Println("Download failed")
			log.Fatal(err)
		}
		defer resp.Body.Close()
		_, err = io.Copy(out, resp.Body)
		if err != nil {
			log.Println("Couldn't write file after download")
			log.Fatal(err)
		}

		// extract it
		if err := ungzip(path.Join(intoDirectory, "dict.tar.gz"), intoDirectory); err != nil {
			log.Println("Couldn't ungzip file")
			log.Fatal(err)
		}
		if err := untar(path.Join(intoDirectory, "dict.tar"), intoDirectory); err != nil {
			log.Println("Couldn't untar file")
			log.Fatal(err)
		}
		// Cleanup
		os.Remove(path.Join(intoDirectory, "dict.tar.gz"))
		os.Remove(path.Join(intoDirectory, "dict.tar"))
	}
}

// ungzip originally from http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/

func ungzip(source, target string) error {
	reader, err := os.Open(source)
	if err != nil {
		return err
	}
	defer reader.Close()

	archive, err := gzip.NewReader(reader)
	if err != nil {
		return err
	}
	defer archive.Close()

	ext := regexp.MustCompile("\\.gz$")
	target = ext.ReplaceAllString(filepath.Join(target, path.Base(source)), "")
	writer, err := os.Create(target)
	if err != nil {
		return err
	}
	defer writer.Close()

	_, err = io.Copy(writer, archive)
	return err
}

// untar originally from http://blog.ralch.com/tutorial/golang-working-with-tar-and-gzip/
func untar(tarball, target string) error {
	reader, err := os.Open(tarball)
	if err != nil {
		return err
	}
	defer reader.Close()
	tarReader := tar.NewReader(reader)

	for {
		header, err := tarReader.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		}

		path := filepath.Join(target, header.Name)
		info := header.FileInfo()
		if info.IsDir() {
			if err = os.MkdirAll(path, info.Mode()); err != nil {
				return err
			}
			continue
		}

		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(file, tarReader)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"go/format"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type Unit struct {
	Name          string
	Receiver      string
	Offset        int    // From normal (for example, mass base unit is kg, not kg)
	PrintString   string // print string for the unit (kg for mass)
	ExtraConstant []Constant
	Suffix        string
	Singular      string
	TypeComment   string // Text to comment the type
	Dimensions    []Dimension
	ErForm        string //For Xxxer interface
}

type Dimension struct {
	Name  string
	Power int
}

const (
	TimeName   string = "TimeDim"
	LengthName string = "LengthDim"
	MassName   string = "MassDim"
)

type Constant struct {
	Name  string
	Value string
}

type Prefix struct {
	Name  string
	Power int
}

var Prefixes = []Prefix{
	{
		Name:  "Yotta",
		Power: 24,
	},
	{
		Name:  "Zetta",
		Power: 21,
	},
	{
		Name:  "Exa",
		Power: 18,
	},
	{
		Name:  "Peta",
		Power: 15,
	},
	{
		Name:  "Tera",
		Power: 12,
	},
	{
		Name:  "Giga",
		Power: 9,
	},
	{
		Name:  "Mega",
		Power: 6,
	},
	{
		Name:  "Kilo",
		Power: 3,
	},
	{
		Name:  "Hecto",
		Power: 2,
	},
	{
		Name:  "Deca",
		Power: 1,
	},
	{
		Name:  "",
		Power: 0,
	},
	{
		Name:  "Deci",
		Power: -1,
	},
	{
		Name:  "Centi",
		Power: -2,
	},
	{
		Name:  "Milli",
		Power: -3,
	},
	{
		Name:  "Micro",
		Power: -6,
	},
	{
		Name:  "Nano",
		Power: -9,
	},
	{
		Name:  "Pico",
		Power: -12,
	},
	{
		Name:  "Femto",
		Power: -15,
	},
	{
		Name:  "Atto",
		Power: -18,
	},
	{
		Name:  "Zepto",
		Power: -21,
	},
	{
		Name:  "Yocto",
		Power: -24,
	},
}

var Units = []Unit{
	{
		Name:        "Mass",
		Receiver:    "m",
		Offset:      -3,
		PrintString: "kg",
		Suffix:      "gram",
		Singular:    "Gram",
		TypeComment: "Mass represents a mass in kilograms",
		Dimensions: []Dimension{
			{
				Name:  MassName,
				Power: 1,
			},
		},
	},
	{
		Name:        "Length",
		Receiver:    "l",
		PrintString: "m",
		Suffix:      "meter",
		Singular:    "Meter",
		TypeComment: "Length represents a length in meters",
		Dimensions: []Dimension{
			{
				Name:  LengthName,
				Power: 1,
			},
		},
	},
	{
		Name:        "Time",
		Receiver:    "t",
		PrintString: "s",
		Suffix:      "second",
		Singular:    "Second",
		TypeComment: "Time represents a time in seconds",
		ExtraConstant: []Constant{
			{
				Name:  "Hour",
				Value: "3600",
			},
			{
				Name:  "Minute",
				Value: "60",
			},
		},
		Dimensions: []Dimension{
			{
				Name:  TimeName,
				Power: 1,
			},
		},
		ErForm: "Timer",
	},
}

var gopath string
var unitPkgPath string

func init() {
	gopath = os.Getenv("GOPATH")
	if gopath == "" {
		log.Fatal("no gopath")
	}

	unitPkgPath = filepath.Join(gopath, "src", "github.com", "gonum", "unit")
}

// Generate generates a file for each of the units
func main() {
	for _, unit := range Units {
		generate(unit)
	}
}

var headerTemplate = `// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package unit

// This file is autogenerated by github.com/gonum/unit/autogen
// Changes should be made to the autogenerated template rather than this one

import (
	"errors"
	"fmt"
	"math"
)

// {{.TypeComment}}
type {{.Name}} float64
`

var constTemplate = `
const(
	{{$unit := .Unit}}
	{{range $unit.ExtraConstant}} {{.Name}} {{$unit.Name}} = {{.Value}}
	{{end}}
	{{$prefixes := .Prefixes}}
	{{range $prefixes}} {{if .Name}} {{.Name}}{{$unit.Suffix}} {{else}} {{$unit.Singular}} {{end}} {{$unit.Name}} = {{if .Power}} 1e{{.Power}} {{else}} 1.0 {{end}}
	{{end}}
)
`

var methodTemplate = `
// Unit converts the {{.Name}} to a *Unit
func ({{.Receiver}} {{.Name}}) Unit() *Unit{
	return New(float64({{.Receiver}}), Dimensions{
		{{range .Dimensions}} {{.Name}}: {{.Power}},
		{{end}}
		})
}

// {{.Name}} allows {{.Name}} to implement a {{if .ErForm}}{{.ErForm}}{{else}}{{.Name}}er{{end}} interface
func ({{.Receiver}} {{.Name}}) {{.Name}}() {{.Name}} {
	return {{.Receiver}}
}

// From converts the unit into the receiver. From returns an
// error if there is a mismatch in dimension
func ({{.Receiver}} *{{.Name}}) From(u Uniter) error{
	if !DimensionsMatch(u, {{.Singular}}){
		(*{{.Receiver}}) = {{.Name}}(math.NaN())
		return errors.New("Dimension mismatch")
	}
	(*{{.Receiver}}) = {{.Name}}(u.Unit().Value())
	return nil
}
`

var formatTemplate = `
func ({{.Receiver}} {{.Name}}) Format(fs fmt.State, c rune){
	switch c {
	case 'v':
		if fs.Flag('#') {
			fmt.Fprintf(fs, "%T(%v)", {{.Receiver}}, float64({{.Receiver}}))
			return
		}
		fallthrough
	case 'e', 'E', 'f', 'F', 'g', 'G':
		p, pOk := fs.Precision()
		if !pOk {
			p = -1
		}
		w, wOk := fs.Width()
		if !wOk {
			w = -1
		}
		fmt.Fprintf(fs, "%*.*"+string(c), w, p, float64({{.Receiver}}))
		fmt.Fprint(fs, " {{.PrintString}}")
	default:
		fmt.Fprintf(fs, "%%!%c(%T=%g {{.PrintString}})", c, {{.Receiver}}, float64({{.Receiver}}))
	return
}
}
`

func generate(unit Unit) {
	lowerName := strings.ToLower(unit.Name)
	filename := filepath.Join(unitPkgPath, lowerName+".go")
	f, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	// Need to define new prefixes because text/template can't do math.
	// Need to do math because kilogram = 1 not 10^3

	prefixes := make([]Prefix, len(Prefixes))
	for i, p := range Prefixes {
		prefixes[i].Name = p.Name
		prefixes[i].Power = p.Power + unit.Offset
	}

	data := struct {
		Prefixes []Prefix
		Unit     Unit
	}{
		prefixes,
		unit,
	}

	buf := bytes.NewBuffer(make([]byte, 0))

	header := template.Must(template.New("header").Parse(headerTemplate))
	err = header.Execute(buf, unit)
	if err != nil {
		log.Fatal(err)
	}

	prefix := template.Must(template.New("prefix").Parse(constTemplate))
	err = prefix.Execute(buf, data)
	if err != nil {
		log.Fatal(err)
	}

	methods := template.Must(template.New("methods").Parse(methodTemplate))
	err = methods.Execute(buf, unit)
	if err != nil {
		log.Fatal(err)
	}

	form := template.Must(template.New("format").Parse(formatTemplate))
	err = form.Execute(buf, unit)
	if err != nil {
		log.Fatal(err)
	}

	b, err := format.Source(buf.Bytes())
	if err != nil {
		f.Write(buf.Bytes()) // This is here to debug bad format
		log.Fatalf("error formatting: %s", err)
	}

	f.Write(b)
}
<commit_msg>Removed parenthesis around pointer dereference<commit_after>package main

import (
	"bytes"
	"go/format"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type Unit struct {
	Name          string
	Receiver      string
	Offset        int    // From normal (for example, mass base unit is kg, not kg)
	PrintString   string // print string for the unit (kg for mass)
	ExtraConstant []Constant
	Suffix        string
	Singular      string
	TypeComment   string // Text to comment the type
	Dimensions    []Dimension
	ErForm        string //For Xxxer interface
}

type Dimension struct {
	Name  string
	Power int
}

const (
	TimeName   string = "TimeDim"
	LengthName string = "LengthDim"
	MassName   string = "MassDim"
)

type Constant struct {
	Name  string
	Value string
}

type Prefix struct {
	Name  string
	Power int
}

var Prefixes = []Prefix{
	{
		Name:  "Yotta",
		Power: 24,
	},
	{
		Name:  "Zetta",
		Power: 21,
	},
	{
		Name:  "Exa",
		Power: 18,
	},
	{
		Name:  "Peta",
		Power: 15,
	},
	{
		Name:  "Tera",
		Power: 12,
	},
	{
		Name:  "Giga",
		Power: 9,
	},
	{
		Name:  "Mega",
		Power: 6,
	},
	{
		Name:  "Kilo",
		Power: 3,
	},
	{
		Name:  "Hecto",
		Power: 2,
	},
	{
		Name:  "Deca",
		Power: 1,
	},
	{
		Name:  "",
		Power: 0,
	},
	{
		Name:  "Deci",
		Power: -1,
	},
	{
		Name:  "Centi",
		Power: -2,
	},
	{
		Name:  "Milli",
		Power: -3,
	},
	{
		Name:  "Micro",
		Power: -6,
	},
	{
		Name:  "Nano",
		Power: -9,
	},
	{
		Name:  "Pico",
		Power: -12,
	},
	{
		Name:  "Femto",
		Power: -15,
	},
	{
		Name:  "Atto",
		Power: -18,
	},
	{
		Name:  "Zepto",
		Power: -21,
	},
	{
		Name:  "Yocto",
		Power: -24,
	},
}

var Units = []Unit{
	{
		Name:        "Mass",
		Receiver:    "m",
		Offset:      -3,
		PrintString: "kg",
		Suffix:      "gram",
		Singular:    "Gram",
		TypeComment: "Mass represents a mass in kilograms",
		Dimensions: []Dimension{
			{
				Name:  MassName,
				Power: 1,
			},
		},
	},
	{
		Name:        "Length",
		Receiver:    "l",
		PrintString: "m",
		Suffix:      "meter",
		Singular:    "Meter",
		TypeComment: "Length represents a length in meters",
		Dimensions: []Dimension{
			{
				Name:  LengthName,
				Power: 1,
			},
		},
	},
	{
		Name:        "Time",
		Receiver:    "t",
		PrintString: "s",
		Suffix:      "second",
		Singular:    "Second",
		TypeComment: "Time represents a time in seconds",
		ExtraConstant: []Constant{
			{
				Name:  "Hour",
				Value: "3600",
			},
			{
				Name:  "Minute",
				Value: "60",
			},
		},
		Dimensions: []Dimension{
			{
				Name:  TimeName,
				Power: 1,
			},
		},
		ErForm: "Timer",
	},
}

var gopath string
var unitPkgPath string

func init() {
	gopath = os.Getenv("GOPATH")
	if gopath == "" {
		log.Fatal("no gopath")
	}

	unitPkgPath = filepath.Join(gopath, "src", "github.com", "gonum", "unit")
}

// Generate generates a file for each of the units
func main() {
	for _, unit := range Units {
		generate(unit)
	}
}

var headerTemplate = `// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package unit

// This file is autogenerated by github.com/gonum/unit/autogen
// Changes should be made to the autogenerated template rather than this one

import (
	"errors"
	"fmt"
	"math"
)

// {{.TypeComment}}
type {{.Name}} float64
`

var constTemplate = `
const(
	{{$unit := .Unit}}
	{{range $unit.ExtraConstant}} {{.Name}} {{$unit.Name}} = {{.Value}}
	{{end}}
	{{$prefixes := .Prefixes}}
	{{range $prefixes}} {{if .Name}} {{.Name}}{{$unit.Suffix}} {{else}} {{$unit.Singular}} {{end}} {{$unit.Name}} = {{if .Power}} 1e{{.Power}} {{else}} 1.0 {{end}}
	{{end}}
)
`

var methodTemplate = `
// Unit converts the {{.Name}} to a *Unit
func ({{.Receiver}} {{.Name}}) Unit() *Unit{
	return New(float64({{.Receiver}}), Dimensions{
		{{range .Dimensions}} {{.Name}}: {{.Power}},
		{{end}}
		})
}

// {{.Name}} allows {{.Name}} to implement a {{if .ErForm}}{{.ErForm}}{{else}}{{.Name}}er{{end}} interface
func ({{.Receiver}} {{.Name}}) {{.Name}}() {{.Name}} {
	return {{.Receiver}}
}

// From converts the unit into the receiver. From returns an
// error if there is a mismatch in dimension
func ({{.Receiver}} *{{.Name}}) From(u Uniter) error{
	if !DimensionsMatch(u, {{.Singular}}){
		*{{.Receiver}} = {{.Name}}(math.NaN())
		return errors.New("Dimension mismatch")
	}
	*{{.Receiver}} = {{.Name}}(u.Unit().Value())
	return nil
}
`

var formatTemplate = `
func ({{.Receiver}} {{.Name}}) Format(fs fmt.State, c rune){
	switch c {
	case 'v':
		if fs.Flag('#') {
			fmt.Fprintf(fs, "%T(%v)", {{.Receiver}}, float64({{.Receiver}}))
			return
		}
		fallthrough
	case 'e', 'E', 'f', 'F', 'g', 'G':
		p, pOk := fs.Precision()
		if !pOk {
			p = -1
		}
		w, wOk := fs.Width()
		if !wOk {
			w = -1
		}
		fmt.Fprintf(fs, "%*.*"+string(c), w, p, float64({{.Receiver}}))
		fmt.Fprint(fs, " {{.PrintString}}")
	default:
		fmt.Fprintf(fs, "%%!%c(%T=%g {{.PrintString}})", c, {{.Receiver}}, float64({{.Receiver}}))
	return
}
}
`

func generate(unit Unit) {
	lowerName := strings.ToLower(unit.Name)
	filename := filepath.Join(unitPkgPath, lowerName+".go")
	f, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	// Need to define new prefixes because text/template can't do math.
	// Need to do math because kilogram = 1 not 10^3

	prefixes := make([]Prefix, len(Prefixes))
	for i, p := range Prefixes {
		prefixes[i].Name = p.Name
		prefixes[i].Power = p.Power + unit.Offset
	}

	data := struct {
		Prefixes []Prefix
		Unit     Unit
	}{
		prefixes,
		unit,
	}

	buf := bytes.NewBuffer(make([]byte, 0))

	header := template.Must(template.New("header").Parse(headerTemplate))
	err = header.Execute(buf, unit)
	if err != nil {
		log.Fatal(err)
	}

	prefix := template.Must(template.New("prefix").Parse(constTemplate))
	err = prefix.Execute(buf, data)
	if err != nil {
		log.Fatal(err)
	}

	methods := template.Must(template.New("methods").Parse(methodTemplate))
	err = methods.Execute(buf, unit)
	if err != nil {
		log.Fatal(err)
	}

	form := template.Must(template.New("format").Parse(formatTemplate))
	err = form.Execute(buf, unit)
	if err != nil {
		log.Fatal(err)
	}

	b, err := format.Source(buf.Bytes())
	if err != nil {
		f.Write(buf.Bytes()) // This is here to debug bad format
		log.Fatalf("error formatting: %s", err)
	}

	f.Write(b)
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"
	"net/http"

	"fmt"

	piazza "github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
)

type Client struct {
	url    string
	logger *pzsyslog.Logger
	h      *piazza.Http
}

func NewClient(url string, apiKey string, logger *pzsyslog.Logger) (*Client, error) {
	var err error

	err = piazza.WaitForService(piazza.PzUuidgen, url)
	if err != nil {
		return nil, err
	}

	h := &piazza.Http{
		BaseUrl: url,
		ApiKey:  apiKey,
	}

	client := &Client{
		url:    url,
		logger: logger,
		h:      h,
	}

	err = client.logger.Info("Client started")
	if err != nil {
		return nil, err
	}

	return client, nil
}

//------------------------------------------------------------------------------

func (c *Client) getObjectCount(endpoint string) (int, error) {

	resp := c.h.PzGet(endpoint)
	if resp.IsError() {
		return 0, resp.ToError()
	}
	if resp.StatusCode != http.StatusOK {
		return 0, resp.ToError()
	}

	if resp.Pagination == nil {
		return 0, nil
	}

	raw, err := json.Marshal(resp.Pagination.Count)
	if err != nil {
		return 0, err
	}

	var count int
	err = json.Unmarshal(raw, &count)

	return count, err
}

func (c *Client) getObject(endpoint string, out interface{}) error {

	resp := c.h.PzGet(endpoint)
	if resp.IsError() {
		return resp.ToError()
	}

	if resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	err := resp.ExtractData(out)
	return err
}

func (c *Client) postObject(obj interface{}, endpoint string, out interface{}) error {
	resp := c.h.PzPost(endpoint, obj)
	if resp.IsError() {
		return resp.ToError()
	}

	if resp.StatusCode != http.StatusCreated &&
		resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	err := resp.ExtractData(out)
	return err
}

func (c *Client) putObject(obj interface{}, endpoint string, out interface{}) error {
	resp := c.h.PzPut(endpoint, obj)
	if resp.IsError() {
		return resp.ToError()
	}

	if resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	err := resp.ExtractData(out)
	return err
}

func (c *Client) deleteObject(endpoint string) error {
	resp := c.h.PzDelete(endpoint)
	if resp.IsError() {
		return resp.ToError()
	}
	if resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	return nil
}

//------------------------------------------------------------------------------

func (c *Client) GetVersion() (*piazza.Version, error) {
	out := &piazza.Version{}
	err := c.getObject("/version", out)
	return out, err
}

//------------------------------------------------------------------------------

func (c *Client) GetEventType(id piazza.Ident) (*EventType, error) {
	out := &EventType{}
	err := c.getObject("/eventType/"+id.String(), out)
	return out, err
}

func (c *Client) GetEventTypeByName(name string) (*EventType, error) {
	out := &EventType{}
	err := c.getObject("/eventType?name="+name, out)
	return out, err
}

func (c *Client) GetNumEventTypes() (int, error) {
	path := fmt.Sprintf("/eventType")
	return c.getObjectCount(path)
}

func (c *Client) GetAllEventTypes(perPage, page int) (*[]EventType, error) {
	out := &[]EventType{}
	path := fmt.Sprintf("/eventType?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, out)
	return out, err
}

func (c *Client) PostEventType(eventType *EventType) (*EventType, error) {
	out := &EventType{}
	err := c.postObject(eventType, "/eventType", out)
	return out, err
}

func (c *Client) QueryEventTypes(query map[string]interface{}) (*[]EventType, error) {
	out := &[]EventType{}
	err := c.postObject(query, "/eventType/query", out)
	return out, err
}

func (c *Client) PutEventType(eventType *EventType) (*EventType, error) {
	out := &EventType{}
	err := c.putObject(eventType, "/eventType", out)
	return out, err
}

func (c *Client) DeleteEventType(id piazza.Ident) error {
	err := c.deleteObject("/eventType/" + id.String())
	return err
}

//------------------------------------------------------------------------------

func (c *Client) GetEvent(id piazza.Ident) (*Event, error) {
	out := &Event{}
	err := c.getObject("/event/"+id.String(), out)
	return out, err
}

func (c *Client) GetAllEvents(perPage, page int) (*[]Event, error) {
	out := &[]Event{}
	path := fmt.Sprintf("/event?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, out)
	return out, err
}

func (c *Client) GetNumEvents() (int, error) {
	return c.getObjectCount("/event")
}

func (c *Client) GetAllEventsByEventType(eventTypeID piazza.Ident) (*[]Event, error) {
	out := &[]Event{}
	err := c.getObject("/event?eventTypeId="+eventTypeID.String(), out)
	return out, err
}

func (c *Client) PostEvent(event *Event) (*Event, error) {
	out := &Event{}
	err := c.postObject(event, "/event", out)
	return out, err
}

func (c *Client) QueryEvents(query map[string]interface{}) (*[]Event, error) {
	out := &[]Event{}
	err := c.postObject(query, "/event/query", out)
	return out, err
}

func (c *Client) PutEvent(event *Event) (*Event, error) {
	out := &Event{}
	err := c.putObject(event, "/event", out)
	return out, err
}

func (c *Client) DeleteEvent(id piazza.Ident) error {
	err := c.deleteObject("/event/" + id.String())
	return err
}

//------------------------------------------------------------------------------

func (c *Client) GetTrigger(id piazza.Ident) (*Trigger, error) {
	out := &Trigger{}
	err := c.getObject("/trigger/"+id.String(), out)
	return out, err
}

func (c *Client) GetNumTriggers() (int, error) {
	path := fmt.Sprintf("/trigger")
	return c.getObjectCount(path)
}

func (c *Client) GetAllTriggers(perPage int, page int) (*[]Trigger, error) {
	out := &[]Trigger{}
	path := fmt.Sprintf("/trigger?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, &out)
	return out, err
}

func (c *Client) PostTrigger(trigger *Trigger) (*Trigger, error) {
	out := &Trigger{}
	err := c.postObject(trigger, "/trigger", out)
	return out, err
}

func (c *Client) QueryTriggers(query map[string]interface{}) (*[]Trigger, error) {
	out := &[]Trigger{}
	err := c.postObject(query, "/trigger/query", out)
	return out, err
}

func (c *Client) PutTrigger(id piazza.Ident, triggerUpdate *TriggerUpdate) error {
	out := &TriggerUpdate{}
	err := c.putObject(triggerUpdate, "/trigger/"+id.String(), out)
	return err
}

func (c *Client) DeleteTrigger(id piazza.Ident) error {
	err := c.deleteObject("/trigger/" + id.String())
	return err
}

//------------------------------------------------------------------------------

func (c *Client) GetAlert(id piazza.Ident) (*Alert, error) {
	out := &Alert{}
	err := c.getObject("/alert/"+id.String(), out)
	return out, err
}

func (c *Client) GetAlertByTrigger(id piazza.Ident) (*[]Alert, error) {
	out := &[]Alert{}
	err := c.getObject("/alert?triggerId="+id.String(), out)
	return out, err
}

func (c *Client) GetNumAlerts() (int, error) {
	path := fmt.Sprintf("/alert")
	return c.getObjectCount(path)
}

func (c *Client) GetAllAlerts(perPage int, page int) (*[]Alert, error) {
	out := &[]Alert{}
	path := fmt.Sprintf("/alert?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, out)
	return out, err
}

func (c *Client) PostAlert(alert *Alert) (*Alert, error) {
	out := &Alert{}
	err := c.postObject(alert, "/alert", out)
	return out, err
}

func (c *Client) QueryAlerts(query map[string]interface{}) (*[]Alert, error) {
	out := &[]Alert{}
	err := c.postObject(query, "/alert/query", out)
	return out, err
}

func (c *Client) PutAlert(alert *Alert) (*Alert, error) {
	out := &Alert{}
	err := c.putObject(alert, "/alert", out)
	return out, err
}

func (c *Client) DeleteAlert(id piazza.Ident) error {
	return c.deleteObject("/alert/" + id.String())
}

//------------------------------------------------------------------------------

func (c *Client) TestElasticsearchGetVersion() (*string, error) {
	ss := ""
	s := &ss
	err := c.getObject("/_test/elasticsearch/version", s)
	return s, err
}

func (c *Client) TestElasticsearchGetOne(id piazza.Ident) (*TestElasticsearchBody, error) {
	out := &TestElasticsearchBody{}
	err := c.getObject("/_test/elasticsearch/data/"+id.String(), out)
	return out, err
}

func (c *Client) TestElasticsearchPost(body *TestElasticsearchBody) (*TestElasticsearchBody, error) {
	out := &TestElasticsearchBody{}
	err := c.postObject(body, "/_test/elasticsearch/data", out)
	return out, err
}

//------------------------------------------------------------------------------

func (c *Client) GetStats() (*Stats, error) {
	out := &Stats{}
	err := c.getObject("/admin/stats", out)
	return out, err

}
<commit_msg>16663 Remove UUIDGen reference<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"encoding/json"
	"net/http"

	"fmt"

	piazza "github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
)

type Client struct {
	url    string
	logger *pzsyslog.Logger
	h      *piazza.Http
}

func NewClient(url string, apiKey string, logger *pzsyslog.Logger) (*Client, error) {
	var err error

	err = piazza.WaitForService(piazza.PzWorkflow, url)
	if err != nil {
		return nil, err
	}

	h := &piazza.Http{
		BaseUrl: url,
		ApiKey:  apiKey,
	}

	client := &Client{
		url:    url,
		logger: logger,
		h:      h,
	}

	err = client.logger.Info("Client started")
	if err != nil {
		return nil, err
	}

	return client, nil
}

//------------------------------------------------------------------------------

func (c *Client) getObjectCount(endpoint string) (int, error) {

	resp := c.h.PzGet(endpoint)
	if resp.IsError() {
		return 0, resp.ToError()
	}
	if resp.StatusCode != http.StatusOK {
		return 0, resp.ToError()
	}

	if resp.Pagination == nil {
		return 0, nil
	}

	raw, err := json.Marshal(resp.Pagination.Count)
	if err != nil {
		return 0, err
	}

	var count int
	err = json.Unmarshal(raw, &count)

	return count, err
}

func (c *Client) getObject(endpoint string, out interface{}) error {

	resp := c.h.PzGet(endpoint)
	if resp.IsError() {
		return resp.ToError()
	}

	if resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	err := resp.ExtractData(out)
	return err
}

func (c *Client) postObject(obj interface{}, endpoint string, out interface{}) error {
	resp := c.h.PzPost(endpoint, obj)
	if resp.IsError() {
		return resp.ToError()
	}

	if resp.StatusCode != http.StatusCreated &&
		resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	err := resp.ExtractData(out)
	return err
}

func (c *Client) putObject(obj interface{}, endpoint string, out interface{}) error {
	resp := c.h.PzPut(endpoint, obj)
	if resp.IsError() {
		return resp.ToError()
	}

	if resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	err := resp.ExtractData(out)
	return err
}

func (c *Client) deleteObject(endpoint string) error {
	resp := c.h.PzDelete(endpoint)
	if resp.IsError() {
		return resp.ToError()
	}
	if resp.StatusCode != http.StatusOK {
		return resp.ToError()
	}

	return nil
}

//------------------------------------------------------------------------------

func (c *Client) GetVersion() (*piazza.Version, error) {
	out := &piazza.Version{}
	err := c.getObject("/version", out)
	return out, err
}

//------------------------------------------------------------------------------

func (c *Client) GetEventType(id piazza.Ident) (*EventType, error) {
	out := &EventType{}
	err := c.getObject("/eventType/"+id.String(), out)
	return out, err
}

func (c *Client) GetEventTypeByName(name string) (*EventType, error) {
	out := &EventType{}
	err := c.getObject("/eventType?name="+name, out)
	return out, err
}

func (c *Client) GetNumEventTypes() (int, error) {
	path := fmt.Sprintf("/eventType")
	return c.getObjectCount(path)
}

func (c *Client) GetAllEventTypes(perPage, page int) (*[]EventType, error) {
	out := &[]EventType{}
	path := fmt.Sprintf("/eventType?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, out)
	return out, err
}

func (c *Client) PostEventType(eventType *EventType) (*EventType, error) {
	out := &EventType{}
	err := c.postObject(eventType, "/eventType", out)
	return out, err
}

func (c *Client) QueryEventTypes(query map[string]interface{}) (*[]EventType, error) {
	out := &[]EventType{}
	err := c.postObject(query, "/eventType/query", out)
	return out, err
}

func (c *Client) PutEventType(eventType *EventType) (*EventType, error) {
	out := &EventType{}
	err := c.putObject(eventType, "/eventType", out)
	return out, err
}

func (c *Client) DeleteEventType(id piazza.Ident) error {
	err := c.deleteObject("/eventType/" + id.String())
	return err
}

//------------------------------------------------------------------------------

func (c *Client) GetEvent(id piazza.Ident) (*Event, error) {
	out := &Event{}
	err := c.getObject("/event/"+id.String(), out)
	return out, err
}

func (c *Client) GetAllEvents(perPage, page int) (*[]Event, error) {
	out := &[]Event{}
	path := fmt.Sprintf("/event?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, out)
	return out, err
}

func (c *Client) GetNumEvents() (int, error) {
	return c.getObjectCount("/event")
}

func (c *Client) GetAllEventsByEventType(eventTypeID piazza.Ident) (*[]Event, error) {
	out := &[]Event{}
	err := c.getObject("/event?eventTypeId="+eventTypeID.String(), out)
	return out, err
}

func (c *Client) PostEvent(event *Event) (*Event, error) {
	out := &Event{}
	err := c.postObject(event, "/event", out)
	return out, err
}

func (c *Client) QueryEvents(query map[string]interface{}) (*[]Event, error) {
	out := &[]Event{}
	err := c.postObject(query, "/event/query", out)
	return out, err
}

func (c *Client) PutEvent(event *Event) (*Event, error) {
	out := &Event{}
	err := c.putObject(event, "/event", out)
	return out, err
}

func (c *Client) DeleteEvent(id piazza.Ident) error {
	err := c.deleteObject("/event/" + id.String())
	return err
}

//------------------------------------------------------------------------------

func (c *Client) GetTrigger(id piazza.Ident) (*Trigger, error) {
	out := &Trigger{}
	err := c.getObject("/trigger/"+id.String(), out)
	return out, err
}

func (c *Client) GetNumTriggers() (int, error) {
	path := fmt.Sprintf("/trigger")
	return c.getObjectCount(path)
}

func (c *Client) GetAllTriggers(perPage int, page int) (*[]Trigger, error) {
	out := &[]Trigger{}
	path := fmt.Sprintf("/trigger?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, &out)
	return out, err
}

func (c *Client) PostTrigger(trigger *Trigger) (*Trigger, error) {
	out := &Trigger{}
	err := c.postObject(trigger, "/trigger", out)
	return out, err
}

func (c *Client) QueryTriggers(query map[string]interface{}) (*[]Trigger, error) {
	out := &[]Trigger{}
	err := c.postObject(query, "/trigger/query", out)
	return out, err
}

func (c *Client) PutTrigger(id piazza.Ident, triggerUpdate *TriggerUpdate) error {
	out := &TriggerUpdate{}
	err := c.putObject(triggerUpdate, "/trigger/"+id.String(), out)
	return err
}

func (c *Client) DeleteTrigger(id piazza.Ident) error {
	err := c.deleteObject("/trigger/" + id.String())
	return err
}

//------------------------------------------------------------------------------

func (c *Client) GetAlert(id piazza.Ident) (*Alert, error) {
	out := &Alert{}
	err := c.getObject("/alert/"+id.String(), out)
	return out, err
}

func (c *Client) GetAlertByTrigger(id piazza.Ident) (*[]Alert, error) {
	out := &[]Alert{}
	err := c.getObject("/alert?triggerId="+id.String(), out)
	return out, err
}

func (c *Client) GetNumAlerts() (int, error) {
	path := fmt.Sprintf("/alert")
	return c.getObjectCount(path)
}

func (c *Client) GetAllAlerts(perPage int, page int) (*[]Alert, error) {
	out := &[]Alert{}
	path := fmt.Sprintf("/alert?perPage=%d&page=%d", perPage, page)
	err := c.getObject(path, out)
	return out, err
}

func (c *Client) PostAlert(alert *Alert) (*Alert, error) {
	out := &Alert{}
	err := c.postObject(alert, "/alert", out)
	return out, err
}

func (c *Client) QueryAlerts(query map[string]interface{}) (*[]Alert, error) {
	out := &[]Alert{}
	err := c.postObject(query, "/alert/query", out)
	return out, err
}

func (c *Client) PutAlert(alert *Alert) (*Alert, error) {
	out := &Alert{}
	err := c.putObject(alert, "/alert", out)
	return out, err
}

func (c *Client) DeleteAlert(id piazza.Ident) error {
	return c.deleteObject("/alert/" + id.String())
}

//------------------------------------------------------------------------------

func (c *Client) TestElasticsearchGetVersion() (*string, error) {
	ss := ""
	s := &ss
	err := c.getObject("/_test/elasticsearch/version", s)
	return s, err
}

func (c *Client) TestElasticsearchGetOne(id piazza.Ident) (*TestElasticsearchBody, error) {
	out := &TestElasticsearchBody{}
	err := c.getObject("/_test/elasticsearch/data/"+id.String(), out)
	return out, err
}

func (c *Client) TestElasticsearchPost(body *TestElasticsearchBody) (*TestElasticsearchBody, error) {
	out := &TestElasticsearchBody{}
	err := c.postObject(body, "/_test/elasticsearch/data", out)
	return out, err
}

//------------------------------------------------------------------------------

func (c *Client) GetStats() (*Stats, error) {
	out := &Stats{}
	err := c.getObject("/admin/stats", out)
	return out, err

}
<|endoftext|>


<commit_before>package bidi

import (
	"context"
	"encoding/binary"
	"io"
	"net/http"

	"github.com/gorilla/websocket"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/status"
	"google.golang.org/grpc/transport"

	"github.com/johanbrandhorst/protobuf/internal"
)

// Logger is the interface used by the Proxy to log events
type Logger interface {
	Debug(...interface{})
	Warn(...interface{})
}

// Proxy wraps a handler with a websocket to perform
// bidirectional messaging between a gRPC backend and a web frontend.
type Proxy struct {
	h      http.Handler
	logger Logger
	creds  credentials.TransportCredentials
}

// WrapServer warps the input handler with a Websocket-to-Bidi-streaming proxy.
func WrapServer(h http.Handler, logger Logger, creds credentials.TransportCredentials) http.Handler {
	return &Proxy{
		h:      h,
		logger: logger,
		creds:  creds,
	}
}

// TODO: allow modification of upgrader settings?
var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		// TODO: Enforce only local origins
		return true
	},
}

func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !websocket.IsWebSocketUpgrade(r) {
		p.h.ServeHTTP(w, r)
		return
	}

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		p.logger.Warn("Failed to upgrade Websocket: ", err)
		return
	}

	ctx, cancelFn := context.WithCancel(r.Context())
	defer cancelFn()

	p.logger.Debug("Creating new transport with addr: ", r.Host)
	t, err := transport.NewClientTransport(ctx,
		transport.TargetInfo{Addr: r.Host},
		transport.ConnectOptions{
			TransportCredentials: p.creds,
		})
	if err != nil {
		p.logger.Warn("Failed to create transport: ", err)
		return
	}
	defer func() {
		err = t.GracefulClose()
		if err != nil {
			p.logger.Warn("Failed to close transport: ", err)
		}
	}()

	p.logger.Debug("Creating new stream with host: ", r.RemoteAddr, "and method: ", r.RequestURI)
	s, err := t.NewStream(ctx, &transport.CallHdr{
		Host:   r.RemoteAddr,
		Method: r.RequestURI,
	})
	if err != nil {
		p.logger.Warn("Failed to create stream: ", err)
		return
	}

	// Read loop - reads from websocket and puts it on the stream
	go func() {
		for {
			select {
			case <-s.Context().Done():
				p.logger.Debug("[READ] Context canceled, returning")
				return
			default:
			}
			p.logger.Debug("[READ] Reading from Websocket")
			_, payload, err := conn.ReadMessage()
			if err != nil {
				p.logger.Warn("[READ] Failed to read Websocket message: ", err)
				return
			}
			p.logger.Debug("[READ] Read payload: ", payload)
			if internal.IsCloseMessage(payload) {
				err = t.Write(s, nil, &transport.Options{Last: true})
				if err == io.EOF || err == nil {
					return
				}
			} else {
				// Append header
				payload = append(make([]byte, 5), payload...)
				// Skip first byte to indicate no compression
				// TODO: Add compression?
				// Encode size of payload to byte 1-5
				binary.BigEndian.PutUint32(payload[1:5], uint32(len(payload)-5))
				err = t.Write(s, payload, &transport.Options{Last: false})
			}

			if err != nil {
				p.logger.Warn("[READ] Failed to write message to transport", err)
				if _, ok := err.(transport.ConnectionError); !ok {
					t.CloseStream(s, err)
				}
				return
			}
		}
	}()

	// Write loop -- take messages from stream and write to websocket
	var header [5]byte
	var msg []byte
	for {
		// Read header
		_, err := s.Read(header[:])
		if err != nil {
			if err == io.EOF {
				p.logger.Debug("[WRITE] Stream closed")
			} else {
				p.logger.Warn("[WRITE] Failed to read header: ", err)
			}

			// Wait for status to be received
			<-s.Done()
			p.sendStatus(conn, s.Status())
			return
		}

		// TODO: Add compression?
		isCompressed := uint8(header[0]) != 0
		if isCompressed {
			// If payload is compressed, bail out
			p.logger.Warn("[WRITE] Reply was compressed, bailing")
			p.sendStatus(conn, status.New(codes.FailedPrecondition, "Server sent compressed data"))
			return
		}
		len := int(binary.BigEndian.Uint32(header[1:]))

		// TODO: Reuse buffer and resize as necessary instead
		msg = make([]byte, int(len))
		if n, err := s.Read(msg); err != nil || n != len {
			p.logger.Warn("[WRITE] Failed to read message: ", err)
			// Wait for status to be received
			<-s.Done()
			p.sendStatus(conn, s.Status())
			return
		}

		if err = conn.WriteMessage(websocket.BinaryMessage, msg); err != nil {
			p.logger.Warn("[WRITE] Failed to write message: ", err)
			return
		}
		p.logger.Debug("[WRITE] Sent", msg)
	}

}

func (p *Proxy) sendStatus(conn *websocket.Conn, st *status.Status) {
	p.logger.Debug(`Sending status: Msg: "`, st.Message(), `", Code: `, st.Code().String())

	closeMsg := websocket.FormatCloseMessage(internal.FormatErrorCode(st.Code()), st.Message())
	err := conn.WriteMessage(websocket.CloseMessage, closeMsg)
	if err != nil {
		p.logger.Warn("error writing websocket trailer: ", err)
	}

	p.logger.Debug("Sent close message")
	err = conn.Close()
	if err != nil {
		p.logger.Warn("Failed to close connection: ", err)
		return
	}
	p.logger.Debug("Closed connection")
	return
}
<commit_msg>Make log messages more consistent<commit_after>package wsproxy

import (
	"context"
	"encoding/binary"
	"io"
	"net/http"

	"github.com/gorilla/websocket"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/status"
	"google.golang.org/grpc/transport"

	"github.com/johanbrandhorst/protobuf/internal"
)

// Logger is the interface used by the Proxy to log events
type Logger interface {
	Debug(...interface{})
	Warn(...interface{})
}

// Proxy wraps a handler with a websocket to perform
// bidirectional messaging between a gRPC backend and a web frontend.
type Proxy struct {
	h      http.Handler
	logger Logger
	creds  credentials.TransportCredentials
}

// WrapServer wraps the input handler with a Websocket-to-Bidi-Streaming proxy.
func WrapServer(h http.Handler, logger Logger, creds credentials.TransportCredentials) http.Handler {
	return &Proxy{
		h:      h,
		logger: logger,
		creds:  creds,
	}
}

// TODO: allow modification of upgrader settings?
var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		// TODO: Enforce only local origins
		return true
	},
}

func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if !websocket.IsWebSocketUpgrade(r) {
		p.h.ServeHTTP(w, r)
		return
	}

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		p.logger.Warn("Failed to upgrade Websocket: ", err)
		return
	}

	ctx, cancelFn := context.WithCancel(r.Context())
	defer cancelFn()

	p.logger.Debug("Creating new transport with addr: ", r.Host)
	t, err := transport.NewClientTransport(ctx,
		transport.TargetInfo{Addr: r.Host},
		transport.ConnectOptions{
			TransportCredentials: p.creds,
		})
	if err != nil {
		p.logger.Warn("Failed to create transport: ", err)
		return
	}
	defer func() {
		err = t.GracefulClose()
		if err != nil {
			p.logger.Warn("Failed to close transport: ", err)
		}
	}()

	p.logger.Debug("Creating new stream with host: ", r.RemoteAddr, " and method: ", r.RequestURI)
	s, err := t.NewStream(ctx, &transport.CallHdr{
		Host:   r.RemoteAddr,
		Method: r.RequestURI,
	})
	if err != nil {
		p.logger.Warn("Failed to create stream: ", err)
		return
	}

	// Read loop - reads from websocket and puts it on the stream
	go func() {
		for {
			select {
			case <-s.Context().Done():
				p.logger.Debug("[READ] Context canceled, returning")
				return
			default:
			}
			p.logger.Debug("[READ] Reading from Websocket")
			_, payload, err := conn.ReadMessage()
			if err != nil {
				p.logger.Warn("[READ] Failed to read Websocket message: ", err)
				return
			}
			p.logger.Debug("[READ] Read payload: ", payload)
			if internal.IsCloseMessage(payload) {
				err = t.Write(s, nil, &transport.Options{Last: true})
				if err == io.EOF || err == nil {
					return
				}
			} else {
				// Append header
				payload = append(make([]byte, 5), payload...)
				// Skip first byte to indicate no compression
				// TODO: Add compression?
				// Encode size of payload to byte 1-5
				binary.BigEndian.PutUint32(payload[1:5], uint32(len(payload)-5))
				err = t.Write(s, payload, &transport.Options{Last: false})
			}

			if err != nil {
				p.logger.Warn("[READ] Failed to write message to transport", err)
				if _, ok := err.(transport.ConnectionError); !ok {
					t.CloseStream(s, err)
				}
				return
			}
		}
	}()

	// Write loop -- take messages from stream and write to websocket
	var header [5]byte
	var msg []byte
	for {
		// Read header
		_, err := s.Read(header[:])
		if err != nil {
			if err == io.EOF {
				p.logger.Debug("[WRITE] Stream closed")
			} else {
				p.logger.Warn("[WRITE] Failed to read header: ", err)
			}

			// Wait for status to be received
			<-s.Done()
			p.sendStatus(conn, s.Status())
			return
		}

		// TODO: Add compression?
		isCompressed := uint8(header[0]) != 0
		if isCompressed {
			// If payload is compressed, bail out
			p.logger.Warn("[WRITE] Reply was compressed, bailing")
			p.sendStatus(conn, status.New(codes.FailedPrecondition, "Server sent compressed data"))
			return
		}
		len := int(binary.BigEndian.Uint32(header[1:]))

		// TODO: Reuse buffer and resize as necessary instead
		msg = make([]byte, int(len))
		if n, err := s.Read(msg); err != nil || n != len {
			p.logger.Warn("[WRITE] Failed to read message: ", err)
			// Wait for status to be received
			<-s.Done()
			p.sendStatus(conn, s.Status())
			return
		}

		if err = conn.WriteMessage(websocket.BinaryMessage, msg); err != nil {
			p.logger.Warn("[WRITE] Failed to write message: ", err)
			return
		}
		p.logger.Debug("[WRITE] Sent payload:", msg)
	}

}

func (p *Proxy) sendStatus(conn *websocket.Conn, st *status.Status) {
	p.logger.Debug(`[WRITE] Sending status: Msg: "`, st.Message(), `", Code: `, st.Code().String())

	closeMsg := websocket.FormatCloseMessage(internal.FormatErrorCode(st.Code()), st.Message())
	err := conn.WriteMessage(websocket.CloseMessage, closeMsg)
	if err != nil {
		p.logger.Warn("[WRITE] Failed to write Websocket trailer: ", err)
	}

	p.logger.Debug("[WRITE] Sent close message")
	err = conn.Close()
	if err != nil {
		p.logger.Warn("[WRITE] Failed to close connection: ", err)
		return
	}
	p.logger.Debug("[WRITE] Closed connection")
	return
}
<|endoftext|>


<commit_before>package guard

import (
	"math/rand"
	"sync/atomic"
	"time"
)

type BackoffStrategy interface {
	NextInterval() time.Duration
	Reset() BackoffStrategy
}

func NewConstantBackoff(d time.Duration) BackoffStrategy {
	return &ConstantBackoff{d}
}

type ConstantBackoff struct {
	Interval time.Duration
}

func (c *ConstantBackoff) NextInterval() time.Duration {
	return c.Interval
}

func (c *ConstantBackoff) Reset() BackoffStrategy {
	return c
}

func NewNoBackoff() BackoffStrategy {
	return NoBackoff{}
}

type NoBackoff struct{}

func (n NoBackoff) NextInterval() time.Duration {
	return 0
}

func (n NoBackoff) Reset() BackoffStrategy {
	return n
}

func NewExponentialBackoff(options ...ExponentialBackoffOption) BackoffStrategy {
	e := &ExponentialBackoff{
		initialInterval:     float64(time.Second),
		maxInterval:         float64(time.Minute),
		multiplier:          2,
		randomizationFactor: 0.5,
		retryCount:          0,
	}

	for _, o := range options {
		o(e)
	}

	if e.randomizer == nil {
		e.randomizer = rand.New(rand.NewSource(time.Now().Unix()))
	}

	return e
}

type ExponentialBackoff struct {
	initialInterval     float64
	maxInterval         float64
	multiplier          float64
	randomizationFactor float64
	randomizer          Randomizer
	retryCount          int64
}

func (e *ExponentialBackoff) NextInterval() time.Duration {
	n := e.retryCount

	interval := e.initialInterval
	for i := int64(0); i < n; i++ {
		interval *= e.multiplier
	}

	if interval > e.maxInterval {
		interval = e.maxInterval
	} else {
		atomic.CompareAndSwapInt64(&e.retryCount, n, n+1)
	}

	rnd := (1 - e.randomizationFactor) + (2 * e.randomizationFactor * e.randomizer.Float64())
	nextBackoff := time.Duration(interval * rnd)

	return nextBackoff
}

func (e *ExponentialBackoff) Reset() BackoffStrategy {
	clone := *e
	clone.retryCount = 0
	return &clone
}

type ExponentialBackoffOption func(*ExponentialBackoff)

func WithInitialInterval(d time.Duration) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *ExponentialBackoff) {
		e.initialInterval = float64(d)
	})
}

func WithMaxInterval(d time.Duration) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *ExponentialBackoff) {
		e.maxInterval = float64(d)
	})
}

func WithMultiplier(f float64) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *ExponentialBackoff) {
		e.multiplier = f
	})
}

func WithRandomizationFactor(f float64) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *ExponentialBackoff) {
		e.randomizationFactor = f
	})
}

func WithRandomizer(r Randomizer) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *ExponentialBackoff) {
		e.randomizer = r
	})
}
<commit_msg>Unexport backoff strategies<commit_after>package guard

import (
	"math/rand"
	"sync/atomic"
	"time"
)

type BackoffStrategy interface {
	NextInterval() time.Duration
	Reset() BackoffStrategy
}

func ConstantBackoff(d time.Duration) BackoffStrategy {
	return &constantBackoff{d}
}

type constantBackoff struct {
	Interval time.Duration
}

func (c *constantBackoff) NextInterval() time.Duration {
	return c.Interval
}

func (c *constantBackoff) Reset() BackoffStrategy {
	return c
}

func NoBackoff() BackoffStrategy {
	return noBackoff{}
}

type noBackoff struct{}

func (n noBackoff) NextInterval() time.Duration {
	return 0
}

func (n noBackoff) Reset() BackoffStrategy {
	return n
}

func ExponentialBackoff(options ...ExponentialBackoffOption) BackoffStrategy {
	e := &exponentialBackoff{
		initialInterval:     float64(time.Second),
		maxInterval:         float64(time.Minute),
		multiplier:          2,
		randomizationFactor: 0.5,
		retryCount:          0,
	}

	for _, o := range options {
		o(e)
	}

	if e.randomizer == nil {
		e.randomizer = rand.New(rand.NewSource(time.Now().Unix()))
	}

	return e
}

type exponentialBackoff struct {
	initialInterval     float64
	maxInterval         float64
	multiplier          float64
	randomizationFactor float64
	randomizer          Randomizer
	retryCount          int64
}

func (e *exponentialBackoff) NextInterval() time.Duration {
	n := e.retryCount

	interval := e.initialInterval
	for i := int64(0); i < n; i++ {
		interval *= e.multiplier
	}

	if interval > e.maxInterval {
		interval = e.maxInterval
	} else {
		atomic.CompareAndSwapInt64(&e.retryCount, n, n+1)
	}

	rnd := (1 - e.randomizationFactor) + (2 * e.randomizationFactor * e.randomizer.Float64())
	nextBackoff := time.Duration(interval * rnd)

	return nextBackoff
}

func (e *exponentialBackoff) Reset() BackoffStrategy {
	clone := *e
	clone.retryCount = 0
	return &clone
}

type ExponentialBackoffOption func(*exponentialBackoff)

func WithInitialInterval(d time.Duration) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *exponentialBackoff) {
		e.initialInterval = float64(d)
	})
}

func WithMaxInterval(d time.Duration) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *exponentialBackoff) {
		e.maxInterval = float64(d)
	})
}

func WithMultiplier(f float64) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *exponentialBackoff) {
		e.multiplier = f
	})
}

func WithRandomizationFactor(f float64) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *exponentialBackoff) {
		e.randomizationFactor = f
	})
}

func WithRandomizer(r Randomizer) ExponentialBackoffOption {
	return ExponentialBackoffOption(func(e *exponentialBackoff) {
		e.randomizer = r
	})
}
<|endoftext|>


<commit_before>package endly

import "github.com/viant/toolbox/url"

//ProcessStartRequest represents a start request
type ProcessStartRequest struct {
	Name          string
	Target        *url.Resource
	Options       *ExecutionOptions
	SystemService bool
	Directory     string
	Command       string
	Arguments     []string
}

//ProcessStartResponse represents a start response
type ProcessStartResponse struct {
	Command string
	Info    []*ProcessInfo
}
<commit_msg>udpated<commit_after>package endly

import "github.com/viant/toolbox/url"

//ProcessStartRequest represents a start request
type ProcessStartRequest struct {
	Name          string
	Target        *url.Resource
	Options       *ExecutionOptions
	Directory     string
	Arguments     []string
	ImmuneToHangups bool //start process as nohup
}

//ProcessStartResponse represents a start response
type ProcessStartResponse struct {
	Command string
	Info    []*ProcessInfo
}
<|endoftext|>


<commit_before>package worker

import (
	"encoding/json"
	"net"
	"strconv"
	"time"

	"github.com/kr/beanstalk"
)

const (
	BeanstalkHost = "localhost" // Beanstalk default host.
	BeanstalkPort = "11300"     // Beanstalk default port.
	BeanstalkTube = "default"   // Beanstalk default queue.
	BeanstalkPrio = 100         // Beanstalk default job priority.

	beanstalkReadyKey  = "current-jobs-ready"
	beanstalkFailedKey = "current-jobs-buried"
)

var (
	BeanstalkTimeout time.Duration = 1 * time.Second // Beanstalk reserve timeout.
	BeanstalkTTR     time.Duration = 2 * DefaultTTR  // Beanstalk default TTR (time to run).
)

// beanstalkMessage represents data returned by Reserve.
type beanstalkMessage struct {
	ID        uint64 // Message ID.
	*Envelope        // Holds parsed json.
}

// newBeanstalkMessage returns an instance of beanstalkMessage.
func newBeanstalkMessage(id uint64, payload []byte) (*beanstalkMessage, error) {
	base, err := NewEnvelope(payload)
	if err != nil {
		return nil, err
	}

	env := &beanstalkMessage{
		ID:       id,
		Envelope: base,
	}

	return env, nil
}

// BeanstalkQueue represents a Beanstalk queue.
type BeanstalkQueue struct {
	Host string        // Beanstalk host.
	Port string        // Beanstalk port.
	Name string        // Beanstalk tube name.
	Prio uint32        // Beanstalk priority.
	TTR  time.Duration // Beanstalk time to run.

	conn *beanstalk.Conn
	tube *beanstalk.Tube
	tset *beanstalk.TubeSet
}

// NewBeanstalkQueue returns a queue instance using custom options.
func NewBeanstalkQueue(opts ...func(*BeanstalkQueue)) (Queue, error) {
	q := &BeanstalkQueue{
		Host: BeanstalkHost,
		Port: BeanstalkPort,
		Name: BeanstalkTube,
		Prio: BeanstalkPrio,
		TTR:  BeanstalkTTR,
	}

	addr := net.JoinHostPort(q.Host, q.Port)

	conn, err := beanstalk.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}
	q.conn = conn

	tube := &beanstalk.Tube{
		Conn: q.conn,
		Name: q.Name,
	}
	q.tube = tube
	q.tset = beanstalk.NewTubeSet(conn, q.Name)

	return q, nil
}

// Put puts the job in the queue.
func (q *BeanstalkQueue) Put(j Job) error {
	typ, err := StructType(j)
	if err != nil {
		return err
	}

	job := &Payload{
		Type: typ,
		Args: j,
	}

	body, err := json.Marshal(job)
	if err != nil {
		return err
	}

	_, err = q.tube.Put(body, q.Prio, 0, q.TTR)
	if err != nil {
		return err
	}

	return nil
}

// Get peeks a job from the queue.
func (q *BeanstalkQueue) Get() (Message, error) {
	id, payload, err := q.tset.Reserve(BeanstalkTimeout)
	if err != nil {
		if cerr, ok := err.(beanstalk.ConnError); ok && cerr.Err == beanstalk.ErrTimeout {
			return nil, &Error{Err: "timeout", IsTimeout: true}
		}
		return nil, err
	}

	msg, err := newBeanstalkMessage(id, payload)
	if err != nil {
		return nil, err
	}

	return msg, nil
}

// Delete deletes a job from the queue.
func (q *BeanstalkQueue) Delete(m Message) error {
	if env, ok := m.(*beanstalkMessage); ok {
		return q.conn.Delete(env.ID)
	}

	return NewErrorFmt("bad envelope: %v", m)
}

// Reject rejects the job marking it as failed.
func (q *BeanstalkQueue) Reject(m Message) error {
	if env, ok := m.(*beanstalkMessage); ok {
		return q.conn.Bury(env.ID, q.Prio+1)
	}

	return NewErrorFmt("bad envelope: %v", m)
}

// Size returns the queue size, only ready jobs are returned.
func (q *BeanstalkQueue) Size() (uint64, uint64, error) {
	parse := func(key string, dict map[string]string) (uint64, error) {
		if v, ok := dict[key]; !ok {
			return 0, NewErrorFmt("bad dict %v", v)
		} else {
			n, err := strconv.ParseUint(v, 10, 64)
			if err != nil {
				return 0, err
			}
			return n, nil
		}
	}

	dict, err := q.tube.Stats()
	if err != nil {
		return 0, 0, err
	}

	ready, err := parse(beanstalkReadyKey, dict)
	if err != nil {
		return 0, 0, err
	}

	failed, err := parse(beanstalkFailedKey, dict)
	if err != nil {
		return 0, 0, err
	}

	return ready, failed, nil
}
<commit_msg>Apply user options to beanstalk queue<commit_after>package worker

import (
	"encoding/json"
	"net"
	"strconv"
	"time"

	"github.com/kr/beanstalk"
)

const (
	BeanstalkHost = "localhost" // Beanstalk default host.
	BeanstalkPort = "11300"     // Beanstalk default port.
	BeanstalkTube = "default"   // Beanstalk default queue.
	BeanstalkPrio = 100         // Beanstalk default job priority.

	beanstalkReadyKey  = "current-jobs-ready"
	beanstalkFailedKey = "current-jobs-buried"
)

var (
	BeanstalkTimeout time.Duration = 1 * time.Second // Beanstalk reserve timeout.
	BeanstalkTTR     time.Duration = 2 * DefaultTTR  // Beanstalk default TTR (time to run).
)

// beanstalkMessage represents data returned by Reserve.
type beanstalkMessage struct {
	ID        uint64 // Message ID.
	*Envelope        // Holds parsed json.
}

// newBeanstalkMessage returns an instance of beanstalkMessage.
func newBeanstalkMessage(id uint64, payload []byte) (*beanstalkMessage, error) {
	base, err := NewEnvelope(payload)
	if err != nil {
		return nil, err
	}

	env := &beanstalkMessage{
		ID:       id,
		Envelope: base,
	}

	return env, nil
}

// BeanstalkQueue represents a Beanstalk queue.
type BeanstalkQueue struct {
	Host string        // Beanstalk host.
	Port string        // Beanstalk port.
	Name string        // Beanstalk tube name.
	Prio uint32        // Beanstalk priority.
	TTR  time.Duration // Beanstalk time to run.

	conn *beanstalk.Conn
	tube *beanstalk.Tube
	tset *beanstalk.TubeSet
}

// NewBeanstalkQueue returns a queue instance using custom options.
func NewBeanstalkQueue(opts ...func(*BeanstalkQueue)) (Queue, error) {
	q := &BeanstalkQueue{
		Host: BeanstalkHost,
		Port: BeanstalkPort,
		Name: BeanstalkTube,
		Prio: BeanstalkPrio,
		TTR:  BeanstalkTTR,
	}

	// Apply options.
	for _, opt := range opts {
		opt(q)
	}

	addr := net.JoinHostPort(q.Host, q.Port)

	conn, err := beanstalk.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}
	q.conn = conn

	tube := &beanstalk.Tube{
		Conn: q.conn,
		Name: q.Name,
	}
	q.tube = tube
	q.tset = beanstalk.NewTubeSet(conn, q.Name)

	return q, nil
}

// Put puts the job in the queue.
func (q *BeanstalkQueue) Put(j Job) error {
	typ, err := StructType(j)
	if err != nil {
		return err
	}

	job := &Payload{
		Type: typ,
		Args: j,
	}

	body, err := json.Marshal(job)
	if err != nil {
		return err
	}

	_, err = q.tube.Put(body, q.Prio, 0, q.TTR)
	if err != nil {
		return err
	}

	return nil
}

// Get peeks a job from the queue.
func (q *BeanstalkQueue) Get() (Message, error) {
	id, payload, err := q.tset.Reserve(BeanstalkTimeout)
	if err != nil {
		if cerr, ok := err.(beanstalk.ConnError); ok && cerr.Err == beanstalk.ErrTimeout {
			return nil, &Error{Err: "timeout", IsTimeout: true}
		}
		return nil, err
	}

	msg, err := newBeanstalkMessage(id, payload)
	if err != nil {
		return nil, err
	}

	return msg, nil
}

// Delete deletes a job from the queue.
func (q *BeanstalkQueue) Delete(m Message) error {
	if env, ok := m.(*beanstalkMessage); ok {
		return q.conn.Delete(env.ID)
	}

	return NewErrorFmt("bad envelope: %v", m)
}

// Reject rejects the job marking it as failed.
func (q *BeanstalkQueue) Reject(m Message) error {
	if env, ok := m.(*beanstalkMessage); ok {
		return q.conn.Bury(env.ID, q.Prio+1)
	}

	return NewErrorFmt("bad envelope: %v", m)
}

// Size returns the queue size, only ready jobs are returned.
func (q *BeanstalkQueue) Size() (uint64, uint64, error) {
	parse := func(key string, dict map[string]string) (uint64, error) {
		if v, ok := dict[key]; !ok {
			return 0, NewErrorFmt("bad dict %v", v)
		} else {
			n, err := strconv.ParseUint(v, 10, 64)
			if err != nil {
				return 0, err
			}
			return n, nil
		}
	}

	dict, err := q.tube.Stats()
	if err != nil {
		return 0, 0, err
	}

	ready, err := parse(beanstalkReadyKey, dict)
	if err != nil {
		return 0, 0, err
	}

	failed, err := parse(beanstalkFailedKey, dict)
	if err != nil {
		return 0, 0, err
	}

	return ready, failed, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/logger"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/model"
)

func CreateItemIndex(logger logger.Logger) *ItemIndex {
	return &ItemIndex{
		logger: logger,
		items:  make(map[route.Route]*model.Item),
	}
}

type ItemIndex struct {
	logger logger.Logger
	items  map[route.Route]*model.Item
}

func (index *ItemIndex) IsMatch(route route.Route) (item *model.Item, isMatch bool) {
	item, isMatch = index.items[route]
	return
}

func (index *ItemIndex) IsFileMatch(route route.Route) (*model.File, bool) {

	fmt.Printf("Checking if %q is a file match\n", route)

	// skip all virtual parents
	parent := index.GetParent(&route)
	for parent != nil && parent.IsVirtual() {
		fmt.Printf("Parent is %q (IsVirtual: %s)\n", parent.Route(), parent.IsVirtual())
		parent = index.GetParent(parent.Route())
	}

	// abort if there is no non-virtual parent
	if parent == nil {
		return nil, false
	}

	// check if the parent has a file with the supplied route
	if file := parent.GetFile(route); file != nil {
		return file, true
	}

	// file not found
	return nil, false
}

func (index *ItemIndex) GetParent(childRoute *route.Route) *model.Item {

	// already at the root
	if childRoute.Level() == 0 {
		return nil
	}

	// locate the parent item
	for parentRoute, parentItem := range index.items {
		if !parentRoute.IsParentOf(childRoute) {
			continue
		}

		// return the parent
		fmt.Printf("%q is a parent of %q\n", parentRoute, childRoute)
		return parentItem
	}

	// check if there is a parent
	parentRoute := childRoute.Parent()
	if parentRoute == nil {
		return nil // we are already at the root
	}

	// no parent item found - create a virtual parent
	virtualParent, err := newVirtualItem(parentRoute)
	if err != nil {

		// error while creating a virtual parent
		index.logger.Warn("Unable to create a virtual parent for the route %q. Error: %s", parentRoute, err)
		return nil

	}

	// return the virtual parent
	return virtualParent
}

func newVirtualItem(route *route.Route) (*model.Item, error) {

	// create a virtual item
	item, err := model.NewVirtualItem(route)
	if err != nil {
		return nil, err
	}

	// set the item title
	item.Title = route.FolderName()

	return item, nil
}

func (index *ItemIndex) GetChilds(route *route.Route) []*model.Item {

	// locate first and second level childs
	childs := make([]*model.Item, 0)

	for itemRoute, item := range index.items {

		// skip all items which are not a child
		if !itemRoute.IsChildOf(route) {
			continue
		}

		childs = append(childs, item)
	}

	// sort the items by date and folder name
	model.SortItemBy(dateAndName).Sort(childs)

	return childs
}

func (index *ItemIndex) Routes() []route.Route {
	routes := make([]route.Route, 0)
	for route, _ := range index.items {
		routes = append(routes, route)
	}
	return routes
}

func (index *ItemIndex) Add(item *model.Item) {
	index.logger.Debug("Adding item %q to index", item)
	index.items[*item.Route()] = item
}

// sort the items by date and name
func dateAndName(item1, item2 *model.Item) bool {

	if item1.MetaData == nil || item2.MetaData == nil || item1.MetaData.CreationDate.Equal(item2.MetaData.CreationDate) {
		// ascending by route
		return item1.Route().Value() < item2.Route().Value()

	}

	// descending by date
	return item1.MetaData.CreationDate.After(item2.MetaData.CreationDate)
}
<commit_msg>Only sort by route, because the date is often not available<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package index

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/logger"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/model"
)

func CreateItemIndex(logger logger.Logger) *ItemIndex {
	return &ItemIndex{
		logger: logger,
		items:  make(map[route.Route]*model.Item),
	}
}

type ItemIndex struct {
	logger logger.Logger
	items  map[route.Route]*model.Item
}

func (index *ItemIndex) IsMatch(route route.Route) (item *model.Item, isMatch bool) {
	item, isMatch = index.items[route]
	return
}

func (index *ItemIndex) IsFileMatch(route route.Route) (*model.File, bool) {

	fmt.Printf("Checking if %q is a file match\n", route)

	// skip all virtual parents
	parent := index.GetParent(&route)
	for parent != nil && parent.IsVirtual() {
		fmt.Printf("Parent is %q (IsVirtual: %s)\n", parent.Route(), parent.IsVirtual())
		parent = index.GetParent(parent.Route())
	}

	// abort if there is no non-virtual parent
	if parent == nil {
		return nil, false
	}

	// check if the parent has a file with the supplied route
	if file := parent.GetFile(route); file != nil {
		return file, true
	}

	// file not found
	return nil, false
}

func (index *ItemIndex) GetParent(childRoute *route.Route) *model.Item {

	// already at the root
	if childRoute.Level() == 0 {
		return nil
	}

	// locate the parent item
	for parentRoute, parentItem := range index.items {
		if !parentRoute.IsParentOf(childRoute) {
			continue
		}

		// return the parent
		fmt.Printf("%q is a parent of %q\n", parentRoute, childRoute)
		return parentItem
	}

	// check if there is a parent
	parentRoute := childRoute.Parent()
	if parentRoute == nil {
		return nil // we are already at the root
	}

	// no parent item found - create a virtual parent
	virtualParent, err := newVirtualItem(parentRoute)
	if err != nil {

		// error while creating a virtual parent
		index.logger.Warn("Unable to create a virtual parent for the route %q. Error: %s", parentRoute, err)
		return nil

	}

	// return the virtual parent
	return virtualParent
}

func newVirtualItem(route *route.Route) (*model.Item, error) {

	// create a virtual item
	item, err := model.NewVirtualItem(route)
	if err != nil {
		return nil, err
	}

	// set the item title
	item.Title = route.FolderName()

	return item, nil
}

func (index *ItemIndex) GetChilds(route *route.Route) []*model.Item {

	// locate first and second level childs
	childs := make([]*model.Item, 0)

	for itemRoute, item := range index.items {

		// skip all items which are not a child
		if !itemRoute.IsChildOf(route) {
			continue
		}

		childs = append(childs, item)
	}

	// sort the items by ascending by route
	model.SortItemBy(sortItemsByRoute).Sort(childs)

	return childs
}

func (index *ItemIndex) Routes() []route.Route {
	routes := make([]route.Route, 0)
	for route, _ := range index.items {
		routes = append(routes, route)
	}
	return routes
}

func (index *ItemIndex) Add(item *model.Item) {
	index.logger.Debug("Adding item %q to index", item)
	index.items[*item.Route()] = item
}

// sort the items by date and name
func sortItemsByRoute(item1, item2 *model.Item) bool {

	// ascending by route
	return item1.Route().Value() < item2.Route().Value()

}
<|endoftext|>