<commit_before>package main

import (
	"context"
	_ "encoding/json"
	"flag"
	"fmt"
	_ "io/ioutil"
	"os"
	_ "path/filepath"
	"strings"
	"time"

	"a4.io/blobstash/pkg/client/blobstore"
	"a4.io/blobstash/pkg/client/kvstore"
	"a4.io/blobstash/pkg/filetree/filetreeutil/node"
	"a4.io/blobstash/pkg/filetree/reader"
	"a4.io/blobstash/pkg/filetree/writer"

	"github.com/dustin/go-humanize"
	"github.com/google/subcommands"
	_ "github.com/mitchellh/go-homedir"
	_ "gopkg.in/yaml.v2"
)

func rerr(msg string, a ...interface{}) subcommands.ExitStatus {
	fmt.Printf(msg, a...)
	return subcommands.ExitFailure
}

func rsuccess(msg string, a ...interface{}) subcommands.ExitStatus {
	fmt.Printf(msg, a...)
	return subcommands.ExitSuccess
}

type filetreeLsCmd struct {
	bs      *blobstore.BlobStore
	kvs     *kvstore.KvStore
	showRef bool
}

func (*filetreeLsCmd) Name() string     { return "filetree-ls" }
func (*filetreeLsCmd) Synopsis() string { return "Display recent blobs" }
func (*filetreeLsCmd) Usage() string {
	return `recent :
	Display recent blobs.
`
}

func (l *filetreeLsCmd) SetFlags(f *flag.FlagSet) {
	f.BoolVar(&l.showRef, "show-ref", false, "Output references")
}

type Node struct {
	Name     string  `json:"name"`
	Type     string  `json:"type"`
	Size     int     `json:"size"`
	Mode     uint32  `json:"mode"`
	ModTime  string  `json:"mtime"`
	Hash     string  `json:"ref"`
	Children []*Node `json:"children,omitempty"`

	Meta *node.RawNode `json:"meta"`

	Data   map[string]interface{} `json:"data,omitempty"`
	XAttrs map[string]string      `json:"xattrs,omitempty"`
}

func displayNode(c *Node, showRef bool) {
	var ref string
	if showRef {
		ref = fmt.Sprintf("%s\t", c.Hash)
	}
	if c.Type == "file" {
		ref = fmt.Sprintf("%s%s\t", ref, humanize.Bytes(uint64(c.Size)))
	} else {
		ref = fmt.Sprintf("%s0 B\t", ref)
	}

	t, err := time.Parse(time.RFC3339, c.ModTime)
	if err != nil {
		panic(err)
	}
	name := c.Name
	if c.Type == "dir" {
		name = name + "/"
	}
	fmt.Printf("%s\t%s%s\n", t.Format("2006-01-02  15:04"), ref, name)
}

func (l *filetreeLsCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	if f.NArg() == 0 {
		nodes := []*Node{}
		if err := l.kvs.Client().GetJSON(ctx, "/api/filetree/fs/root?prefix=_filetree:root", nil, &nodes); err != nil {
			return rerr("failed to fetch root: %v", err)
		}
		for _, node := range nodes {
			displayNode(node, l.showRef)
		}
	}
	if f.NArg() == 1 {
		data := strings.Split(f.Arg(0), "/")
		path := strings.Join(data[1:], "/")
		// FIXME(tsileo): not store the type in the key anymore
		key, err := l.kvs.Get(ctx, fmt.Sprintf("_filetree:root:dir:%s", data[0]), -1)
		n := &Node{}
		if err != nil {
			return rerr("failed to fetch root key \"%s\": %v", data[0], err)
		}
		if err := l.kvs.Client().GetJSON(ctx, fmt.Sprintf("/api/filetree/fs/ref/%s/%s", key.Hash, path), nil, n); err != nil {
			return rerr("failed to fetch node: %v", err)
		}
		for _, c := range n.Children {
			displayNode(c, l.showRef)
		}
	}
	// TODO(tsileo): support filetree-ls rootname/subdir using fs/path API
	return subcommands.ExitSuccess
}

type filetreeDownloadCmd struct {
	bs  *blobstore.BlobStore
	kvs *kvstore.KvStore
}

func (*filetreeDownloadCmd) Name() string     { return "filetree-get" }
func (*filetreeDownloadCmd) Synopsis() string { return "Display recent blobs" }
func (*filetreeDownloadCmd) Usage() string {
	return `recent :
	Display recent blobs.
`
}

func (*filetreeDownloadCmd) SetFlags(_ *flag.FlagSet) {}

func (r *filetreeDownloadCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	data := strings.Split(f.Arg(0), "/")
	path := strings.Join(data[1:], "/")
	// FIXME(tsileo): not store the type in the key anymore
	// FIXME(tsileo): move this into a filetree client
	key, err := r.kvs.Get(ctx, fmt.Sprintf("_filetree:root:dir:%s", data[0]), -1)
	n := &Node{}
	if err != nil {
		return rerr("failed to fetch root key \"%s\": %v", data[0], err)
	}
	if err := r.kvs.Client().GetJSON(ctx, fmt.Sprintf("/api/filetree/fs/ref/%s/%s", key.Hash, path), nil, n); err != nil {
		return rerr("failed to fetch node: %v", err)
	}

	// since the `Meta` type is used internally, and the `Hash` fields is the blake2b hash of the JSON encoded struct,
	// the Hash is omitted when converted to JSON
	if n.Meta != nil {
		n.Meta.Hash = n.Hash
	}

	downloader := reader.NewDownloader(r.bs)
	fmt.Printf("%+v\n", n)
	fmt.Printf("%+v\n", n.Meta)

	// If no target path is provided, use the filename
	tpath := f.Arg(1)
	if tpath == "" {
		tpath = n.Name
	}

	if err := downloader.Download(context.TODO(), n.Meta, tpath); err != nil {
		return rerr("failed to download: %v", err)
	}

	return subcommands.ExitSuccess
}

type filetreePutCmd struct {
	bs  *blobstore.BlobStore
	kvs *kvstore.KvStore
}

func (*filetreePutCmd) Name() string     { return "filetree-put" }
func (*filetreePutCmd) Synopsis() string { return "Display recent blobs" }
func (*filetreePutCmd) Usage() string {
	return `recent :
	Display recent blobs.
`
}

func (*filetreePutCmd) SetFlags(_ *flag.FlagSet) {}

func (r *filetreePutCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
	finfo, err := os.Stat(f.Arg(0))
	switch {
	case os.IsNotExist(err):
		return rsuccess("path \"%s\" does not exist", f.Arg(0))
	case err == nil:
	default:
		return rerr("failed to stat file: %v", err)
	}
	var m *node.RawNode
	up := writer.NewUploader(r.bs)
	// It's a dir
	if finfo.IsDir() {
		m, err = up.PutDir(f.Arg(0))
		if err != nil {
			return rerr("failed to upload: %v", err)
		}
		// FIXME(tsileo): store a FiletreMeta{Hostname, OS} to display (os icon) (hostname) in the web ui
		// hostname, err := os.Hostname()
		// if err != nil {
		// 	return rerr("failed to get hostname: %v", err)
		// }
		// FIXME(tsileo): a way to set the hostname?
	} else {
		// It's a file
		m, err = up.PutFile(f.Arg(0))
		if err != nil {
			return rerr("failed to upload: %v", err)
		}
	}
	fmt.Printf("meta=%+v", m)
	if _, err := r.kvs.Put(ctx, fmt.Sprintf("_filetree:root:%s:%s", m.Type, m.Name), m.Hash, []byte("TODO meta data"), -1); err != nil {
		return rerr("faile to set kv entry: %v", err)
	}

	return subcommands.ExitSuccess
}

func main() {
	// TODO(tsileo) config file with server address and collection name
	opts := blobstore.DefaultOpts().SetHost(os.Getenv("BLOBSTASH_API_HOST"), os.Getenv("BLOBSTASH_API_KEY"))
	opts.SnappyCompression = false
	bs := blobstore.New(opts)
	// col := ds.Col("notes23")
	kvopts := kvstore.DefaultOpts().SetHost(os.Getenv("BLOBSTASH_API_HOST"), os.Getenv("BLOBSTASH_API_KEY"))

	// FIXME(tsileo): have GetJSON support snappy?
	kvopts.SnappyCompression = false

	kvs := kvstore.New(kvopts)

	subcommands.Register(subcommands.HelpCommand(), "")
	subcommands.Register(subcommands.FlagsCommand(), "")
	subcommands.Register(subcommands.CommandsCommand(), "")
	subcommands.Register(&filetreePutCmd{bs: bs, kvs: kvs}, "")
	subcommands.Register(&filetreeDownloadCmd{bs: bs, kvs: kvs}, "")
	subcommands.Register(&filetreeLsCmd{bs: bs, kvs: kvs}, "")

	flag.Parse()
	ctx := context.Background()
	os.Exit(int(subcommands.Execute(ctx)))
}
<commit_msg>cmd: delete old CLI<commit_after><|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// client-gen makes the individual typed clients using go2idl.
package main

import (
	"fmt"
	"path/filepath"

	"k8s.io/kubernetes/cmd/libs/go2idl/args"
	clientgenargs "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/args"
	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/generators"
	"k8s.io/kubernetes/pkg/api/unversioned"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"
)

var (
	test          = flag.BoolP("test", "t", false, "set this flag to generate the client code for the testdata")
	inputVersions = flag.StringSlice("input", []string{"api/", "extensions/"}, "group/versions that client-gen will generate clients for. At most one version per group is allowed. Specified in the format \"group1/version1,group2/version2...\". Default to \"api/,extensions\"")
	clientsetName = flag.StringP("clientset-name", "n", "internalclientset", "the name of the generated clientset package.")
	clientsetPath = flag.String("clientset-path", "k8s.io/kubernetes/pkg/client/clientset_generated/", "the generated clientset will be output to <clientset-path>/<clientset-name>. Default to \"k8s.io/kubernetes/pkg/client/clientset_generated/\"")
	clientsetOnly = flag.Bool("clientset-only", false, "when set, client-gen only generates the clientset shell, without generating the individual typed clients")
	fakeClient    = flag.Bool("fake-clientset", true, "when set, client-gen will generate the fake clientset that can be used in tests")
)

func versionToPath(group string, version string) (path string) {
	const base = "k8s.io/kubernetes/pkg"
	// special case for the core group
	if group == "api" {
		path = filepath.Join(base, "api", version)
	} else {
		path = filepath.Join(base, "apis", group, version)
	}
	return
}

func parseInputVersions() (paths []string, groupVersions []unversioned.GroupVersion, gvToPath map[unversioned.GroupVersion]string, err error) {
	var visitedGroups = make(map[string]struct{})
	gvToPath = make(map[unversioned.GroupVersion]string)
	for _, gvString := range *inputVersions {
		gv, err := unversioned.ParseGroupVersion(gvString)
		if err != nil {
			return nil, nil, nil, err
		}

		if _, found := visitedGroups[gv.Group]; found {
			return nil, nil, nil, fmt.Errorf("group %q appeared more than once in the input. At most one version is allowed for each group.", gv.Group)
		}
		visitedGroups[gv.Group] = struct{}{}
		groupVersions = append(groupVersions, gv)
		path := versionToPath(gv.Group, gv.Version)
		paths = append(paths, path)
		gvToPath[gv] = path
	}
	return paths, groupVersions, gvToPath, nil
}

func main() {
	arguments := args.Default()
	flag.Parse()
	var cmdArgs string
	flag.VisitAll(func(f *flag.Flag) {
		if !f.Changed {
			return
		}
		cmdArgs = cmdArgs + fmt.Sprintf("--%s=%s ", f.Name, f.Value)
	})

	dependencies := []string{
		"k8s.io/kubernetes/pkg/fields",
		"k8s.io/kubernetes/pkg/labels",
		"k8s.io/kubernetes/pkg/watch",
		"k8s.io/kubernetes/pkg/client/unversioned",
		"k8s.io/kubernetes/pkg/apimachinery/registered",
	}

	if *test {
		arguments.InputDirs = append(dependencies, []string{
			"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup",
		}...)
		// We may change the output path later.
		arguments.OutputPackagePath = "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testoutput"
		arguments.CustomArgs = clientgenargs.Args{
			[]unversioned.GroupVersion{{"testgroup", ""}},
			map[unversioned.GroupVersion]string{
				unversioned.GroupVersion{"testgroup", ""}: "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup",
			},
			"test_internalclientset",
			"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testoutput/clientset_generated/",
			false,
			false,
			cmdArgs,
		}
	} else {
		inputPath, groupVersions, gvToPath, err := parseInputVersions()
		if err != nil {
			glog.Fatalf("Error: %v", err)
		}
		glog.Infof("going to generate clientset from these input paths: %v", inputPath)
		arguments.InputDirs = append(inputPath, dependencies...)
		// TODO: we need to make OutPackagePath a map[string]string. For example,
		// we need clientset and the individual typed clients be output to different
		// output path.

		// We may change the output path later.
		arguments.OutputPackagePath = "k8s.io/kubernetes/pkg/client/typed/generated"

		arguments.CustomArgs = clientgenargs.Args{
			groupVersions,
			gvToPath,
			*clientsetName,
			*clientsetPath,
			*clientsetOnly,
			*fakeClient,
			cmdArgs,
		}
	}

	if err := arguments.Execute(
		generators.NameSystems(),
		generators.DefaultNameSystem(),
		generators.Packages,
	); err != nil {
		glog.Fatalf("Error: %v", err)
	}
}
<commit_msg>ignore verify-only flag<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// client-gen makes the individual typed clients using go2idl.
package main

import (
	"fmt"
	"path/filepath"

	"k8s.io/kubernetes/cmd/libs/go2idl/args"
	clientgenargs "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/args"
	"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/generators"
	"k8s.io/kubernetes/pkg/api/unversioned"

	"github.com/golang/glog"
	flag "github.com/spf13/pflag"
)

var (
	test          = flag.BoolP("test", "t", false, "set this flag to generate the client code for the testdata")
	inputVersions = flag.StringSlice("input", []string{"api/", "extensions/"}, "group/versions that client-gen will generate clients for. At most one version per group is allowed. Specified in the format \"group1/version1,group2/version2...\". Default to \"api/,extensions\"")
	clientsetName = flag.StringP("clientset-name", "n", "internalclientset", "the name of the generated clientset package.")
	clientsetPath = flag.String("clientset-path", "k8s.io/kubernetes/pkg/client/clientset_generated/", "the generated clientset will be output to <clientset-path>/<clientset-name>. Default to \"k8s.io/kubernetes/pkg/client/clientset_generated/\"")
	clientsetOnly = flag.Bool("clientset-only", false, "when set, client-gen only generates the clientset shell, without generating the individual typed clients")
	fakeClient    = flag.Bool("fake-clientset", true, "when set, client-gen will generate the fake clientset that can be used in tests")
)

func versionToPath(group string, version string) (path string) {
	const base = "k8s.io/kubernetes/pkg"
	// special case for the core group
	if group == "api" {
		path = filepath.Join(base, "api", version)
	} else {
		path = filepath.Join(base, "apis", group, version)
	}
	return
}

func parseInputVersions() (paths []string, groupVersions []unversioned.GroupVersion, gvToPath map[unversioned.GroupVersion]string, err error) {
	var visitedGroups = make(map[string]struct{})
	gvToPath = make(map[unversioned.GroupVersion]string)
	for _, gvString := range *inputVersions {
		gv, err := unversioned.ParseGroupVersion(gvString)
		if err != nil {
			return nil, nil, nil, err
		}

		if _, found := visitedGroups[gv.Group]; found {
			return nil, nil, nil, fmt.Errorf("group %q appeared more than once in the input. At most one version is allowed for each group.", gv.Group)
		}
		visitedGroups[gv.Group] = struct{}{}
		groupVersions = append(groupVersions, gv)
		path := versionToPath(gv.Group, gv.Version)
		paths = append(paths, path)
		gvToPath[gv] = path
	}
	return paths, groupVersions, gvToPath, nil
}

func main() {
	arguments := args.Default()
	flag.Parse()
	var cmdArgs string
	flag.VisitAll(func(f *flag.Flag) {
		if !f.Changed || f.Name == "verify-only" {
			return
		}
		cmdArgs = cmdArgs + fmt.Sprintf("--%s=%s ", f.Name, f.Value)
	})

	dependencies := []string{
		"k8s.io/kubernetes/pkg/fields",
		"k8s.io/kubernetes/pkg/labels",
		"k8s.io/kubernetes/pkg/watch",
		"k8s.io/kubernetes/pkg/client/unversioned",
		"k8s.io/kubernetes/pkg/apimachinery/registered",
	}

	if *test {
		arguments.InputDirs = append(dependencies, []string{
			"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup",
		}...)
		// We may change the output path later.
		arguments.OutputPackagePath = "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testoutput"
		arguments.CustomArgs = clientgenargs.Args{
			[]unversioned.GroupVersion{{"testgroup", ""}},
			map[unversioned.GroupVersion]string{
				unversioned.GroupVersion{"testgroup", ""}: "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testdata/apis/testgroup",
			},
			"test_internalclientset",
			"k8s.io/kubernetes/cmd/libs/go2idl/client-gen/testoutput/clientset_generated/",
			false,
			false,
			cmdArgs,
		}
	} else {
		inputPath, groupVersions, gvToPath, err := parseInputVersions()
		if err != nil {
			glog.Fatalf("Error: %v", err)
		}
		glog.Infof("going to generate clientset from these input paths: %v", inputPath)
		arguments.InputDirs = append(inputPath, dependencies...)
		// TODO: we need to make OutPackagePath a map[string]string. For example,
		// we need clientset and the individual typed clients be output to different
		// output path.

		// We may change the output path later.
		arguments.OutputPackagePath = "k8s.io/kubernetes/pkg/client/typed/generated"

		arguments.CustomArgs = clientgenargs.Args{
			groupVersions,
			gvToPath,
			*clientsetName,
			*clientsetPath,
			*clientsetOnly,
			*fakeClient,
			cmdArgs,
		}
	}

	if err := arguments.Execute(
		generators.NameSystems(),
		generators.DefaultNameSystem(),
		generators.Packages,
	); err != nil {
		glog.Fatalf("Error: %v", err)
	}
}
<|endoftext|>


<commit_before>package flake

import (
	"encoding/binary"
	"os"
	"sync"
	"time"
)

//  ---------------------------------------------------------------------------
//  Layout - Big Endian
//  ---------------------------------------------------------------------------
//  [0:6]   48 bits | Upper 48 bits of timestamp (milliseconds since the epoch)
//  [6:8]   16 bits | a per-interval sequence # (interval == 1 millisecond)
//  [9:14]  48 bits | a hardware id
//  [14:16] 16 bits | process ID
//  ---------------------------------------------------------------------------
//  | 0 | 1 | 2 | 3 | 4 | 5 | 6 |  7  |  8  | 9 | A | B | C | D |  E  |  F  |
//  ---------------------------------------------------------------------------
//  |           48 bits         |  16 bits  |     48 bits       |  16 bits  |
//  ---------------------------------------------------------------------------
//  |          timestamp        |  interval |    HardwareID     | ProcessID |
//  ---------------------------------------------------------------------------
//  Notes
//  ---------------------------------------------------------------------------
//  The time bits are the most signficant bits because they have the primary
//  impact on the sort order of ids. The interval/seq # is next most significant
//  as it is the tie-breaker when the time portions are equivalent.
//
//  Note that the lower 64 bits are basically random and not specifically
//  useful for ordering, although they play their party when the upper 64-bits
//  are equivalent between two ideas. Again, the ordering outcome in this
//  situation is somewhat random, but generally somewhat repeatable (hardware
//  id should be consistent and stable a vast majority of the time).
//  ---------------------------------------------------------------------------

var sequenceBits uint64 = 16
var sequenceMask = uint64(int64(-1) ^ (int64(-1) << sequenceBits))
var maxSequenceNumber = uint64(^sequenceMask)

type generator struct {
	epoch      int64
	hardwareID HardwareID
	processID  int
	lastTime   int64
	sequence   uint64
	mutex      sync.Mutex

	machineID uint64
}

// NewGenerator creates an instance of generator which implements Generator
func NewGenerator(epoch int64, hardwareID HardwareID, processID int) Generator {
	// binary.BigEndian.Uint64 won't work on a []byte < len(8) so we need to
	// copy our 6-byte hardwareID into the most-signficant bits
	tempBytes := make([]byte, 8)
	copy(tempBytes[0:6], hardwareID[0:6])

	return &generator{
		epoch:      epoch,
		hardwareID: hardwareID,
		processID:  processID & 0xFFFF,
		machineID:  binary.BigEndian.Uint64(tempBytes) | uint64(processID&0xFFFF),
	}
}

// NewOvertoneEpochGenerator creates an instance of generator using the Overtone Epoch
func NewOvertoneEpochGenerator(hardwareID HardwareID) Generator {
	return NewGenerator(OvertoneEpochMs, hardwareID, os.Getpid())
}

func (gen *generator) Epoch() int64 {
	return gen.epoch
}

func (gen *generator) HardwareID() HardwareID {
	return gen.hardwareID
}

func (gen *generator) ProcessID() int {
	return gen.processID
}

func (gen *generator) GenerateAsStream(count int, buffer []byte, callback func(int, []byte) error) (totalAllocated int, err error) {
	if len(buffer) < OvertFlakeIDLength {
		return 0, ErrBufferTooSmall
	}

	// while we still have ids to allocate/generate
	for count > 0 {
		var allocated uint64
		var interval int64
		var index int

		// allocate as many ids as available up to count
		allocated, interval, err = gen.allocate(count)
		if err != nil {
			return totalAllocated, err
		}

		totalAllocated += int(allocated)

		// calculate the delta between the interval (Unix Epoch in milliseconds)
		// and the epoch being used for id generation
		delta := uint64((interval - gen.epoch) << 16)

		// for each ID that was allocated, write the bytes for the ID to
		// the results array
		for j := uint64(0); j < allocated; j++ {
			var upper = delta | (j & sequenceMask)
			binary.BigEndian.PutUint64(buffer[index:index+8], upper)
			binary.BigEndian.PutUint64(buffer[index+8:index+16], gen.machineID)
			index += 16

			// buffer is full
			if index >= len(buffer) {
				err = callback(index/16, buffer)
				if err != nil {
					return
				}
				index = 0
			}
		}

		// partial buffer fill
		if index > 0 {
			callback(index/16, buffer)
			if err != nil {
				return
			}
			index = 0
		}

		count -= int(allocated)
	}

	return
}

// Generate uses allocate to allocate as many ids as required, and writes
// each id into a contiguous []byte
func (gen *generator) Generate(count int) (results []byte, err error) {
	// allocate a buffer that will hold count IDs
	results = make([]byte, OvertFlakeIDLength*count)

	var allocated int

	// use the stream API but because our buffer can hold all allocated ids we dont need
	// to react in the callback
	allocated, err = gen.GenerateAsStream(count, results, func(allocated int, ids []byte) error {
		return nil
	})

	// we do not want to return a partial result
	if (allocated != count) || (err != nil) {
		results = nil
	}

	return
}

// allocate does all the magic of time and sequence management. It does not
// perfomm the generation of the ids, but provides the data required to do so
func (gen *generator) allocate(count int) (uint64, int64, error) {
	if uint64(count) > maxSequenceNumber {
		return 0, 0, ErrTooManyRequested
	}

	// We need to take the lock so we can manipulate the generator state
	gen.mutex.Lock()
	defer gen.mutex.Unlock()

	// current time since Unix Epoch in milliseconds
	current := timestamp()

	// Is time going backwards? Thats a problem
	if current < gen.lastTime {
		return 0, 0, ErrTimeIsMovingBackwards
	}

	if gen.lastTime != current {
		gen.lastTime = current
		gen.sequence = 0
	} else {
		// When all the ids have been allocated for this interval then we end up
		// here and we need to spin for the next cycle
		if gen.sequence == 0 {
			for current <= gen.lastTime {
				current = timestamp()
			}
		}
	}

	// allocated the request # of items, or whatever is remaining for this cycle
	var allocated uint64
	if uint64(count) > gen.sequence-sequenceMask {
		allocated = gen.sequence - sequenceMask
	} else {
		allocated = uint64(count)
	}

	// advance the sequence for the # of items allocated
	gen.sequence = (gen.sequence + allocated) & sequenceMask

	return allocated, current, nil
}

// timestamp returns the # of milliseconds that have passed since
// the unix epoch
func timestamp() int64 {
	return time.Now().UnixNano() / 1e6
}
<commit_msg>added generator.lastAllocatedTime implemented Generator.LastAllocatedTime changed how totalAllocated is incremented in GenerateAsStream save lastAllocatedTime after an allocation occurs<commit_after>package flake

import (
	"encoding/binary"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

//  ---------------------------------------------------------------------------
//  Layout - Big Endian
//  ---------------------------------------------------------------------------
//  [0:6]   48 bits | Upper 48 bits of timestamp (milliseconds since the epoch)
//  [6:8]   16 bits | a per-interval sequence # (interval == 1 millisecond)
//  [9:14]  48 bits | a hardware id
//  [14:16] 16 bits | process ID
//  ---------------------------------------------------------------------------
//  | 0 | 1 | 2 | 3 | 4 | 5 | 6 |  7  |  8  | 9 | A | B | C | D |  E  |  F  |
//  ---------------------------------------------------------------------------
//  |           48 bits         |  16 bits  |     48 bits       |  16 bits  |
//  ---------------------------------------------------------------------------
//  |          timestamp        |  interval |    HardwareID     | ProcessID |
//  ---------------------------------------------------------------------------
//  Notes
//  ---------------------------------------------------------------------------
//  The time bits are the most signficant bits because they have the primary
//  impact on the sort order of ids. The interval/seq # is next most significant
//  as it is the tie-breaker when the time portions are equivalent.
//
//  Note that the lower 64 bits are basically random and not specifically
//  useful for ordering, although they play their party when the upper 64-bits
//  are equivalent between two ideas. Again, the ordering outcome in this
//  situation is somewhat random, but generally somewhat repeatable (hardware
//  id should be consistent and stable a vast majority of the time).
//  ---------------------------------------------------------------------------

var sequenceBits uint64 = 16
var sequenceMask = uint64(int64(-1) ^ (int64(-1) << sequenceBits))
var maxSequenceNumber = uint64(^sequenceMask)

// generator is an implementtion of Generator
type generator struct {
	epoch      int64
	hardwareID HardwareID
	processID  int
	machineID  uint64

	lastTime          int64
	lastAllocatedTime int64
	sequence          uint64

	mutex sync.Mutex
}

// NewGenerator creates an instance of generator which implements Generator
func NewGenerator(epoch int64, hardwareID HardwareID, processID int) Generator {
	// binary.BigEndian.Uint64 won't work on a []byte < len(8) so we need to
	// copy our 6-byte hardwareID into the most-signficant bits
	tempBytes := make([]byte, 8)
	copy(tempBytes[0:6], hardwareID[0:6])

	return &generator{
		epoch:      epoch,
		hardwareID: hardwareID,
		processID:  processID & 0xFFFF,
		machineID:  binary.BigEndian.Uint64(tempBytes) | uint64(processID&0xFFFF),
	}
}

// NewOvertoneEpochGenerator creates an instance of generator using the Overtone Epoch
func NewOvertoneEpochGenerator(hardwareID HardwareID) Generator {
	return NewGenerator(OvertoneEpochMs, hardwareID, os.Getpid())
}

func (gen *generator) Epoch() int64 {
	return gen.epoch
}

func (gen *generator) HardwareID() HardwareID {
	return gen.hardwareID
}

func (gen *generator) ProcessID() int {
	return gen.processID
}

func (gen *generator) LastAllocatedTime() int64 {
	// use the atomic api for both reads and writes of this value so that we do not
	// need to incur the overhead of the mutex or create additional contention
	return atomic.LoadInt64(&gen.lastAllocatedTime)
}

func (gen *generator) GenerateAsStream(count int, buffer []byte, callback func(int, []byte) error) (totalAllocated int, err error) {
	if len(buffer) < OvertFlakeIDLength {
		return 0, ErrBufferTooSmall
	}

	// while we still have ids to allocate/generate
	for count > 0 {
		var allocated uint64
		var interval int64
		var index int

		// allocate as many ids as available up to count
		allocated, interval, err = gen.allocate(count)
		if err != nil {
			return totalAllocated, err
		}

		// calculate the delta between the interval (Unix Epoch in milliseconds)
		// and the epoch being used for id generation
		delta := uint64((interval - gen.epoch) << 16)

		// for each ID that was allocated, write the bytes for the ID to
		// the results array
		for j := uint64(0); j < allocated; j++ {
			var upper = delta | (j & sequenceMask)
			binary.BigEndian.PutUint64(buffer[index:index+8], upper)
			binary.BigEndian.PutUint64(buffer[index+8:index+16], gen.machineID)
			index += 16

			// buffer is full
			if index >= len(buffer) {
				err = callback(index/16, buffer)
				if err != nil {
					return
				}

				// more were delivered so update our return value
				totalAllocated += int(index / 16)

				// back to beginning of the buffer
				index = 0
			}
		}

		// partial buffer fill
		if index > 0 {
			callback(index/16, buffer)
			if err != nil {
				return
			}

			// more were delivered so update our return value
			totalAllocated += int(index / 16)

			// back to beginning of the buffer
			index = 0
		}

		count -= int(allocated)
	}

	return
}

// Generate uses allocate to allocate as many ids as required, and writes
// each id into a contiguous []byte
func (gen *generator) Generate(count int) (results []byte, err error) {
	// allocate a buffer that will hold count IDs
	results = make([]byte, OvertFlakeIDLength*count)

	var allocated int

	// use the stream API but because our buffer can hold all allocated ids we dont need
	// to react in the callback
	allocated, err = gen.GenerateAsStream(count, results, func(allocated int, ids []byte) error {
		return nil
	})

	// we do not want to return a partial result
	if (allocated != count) || (err != nil) {
		results = nil
	}

	return
}

// allocate does all the magic of time and sequence management. It does not
// perfomm the generation of the ids, but provides the data required to do so
func (gen *generator) allocate(count int) (uint64, int64, error) {
	if uint64(count) > maxSequenceNumber {
		return 0, 0, ErrTooManyRequested
	}

	// We need to take the lock so we can manipulate the generator state
	gen.mutex.Lock()
	defer gen.mutex.Unlock()

	// current time since Unix Epoch in milliseconds
	current := timestamp()

	// Is time going backwards? Thats a problem
	if current < gen.lastTime {
		return 0, 0, ErrTimeIsMovingBackwards
	}

	if gen.lastTime != current {
		gen.lastTime = current
		gen.sequence = 0
	} else {
		// When all the ids have been allocated for this interval then we end up
		// here and we need to spin for the next cycle
		if gen.sequence == 0 {
			for current <= gen.lastTime {
				current = timestamp()
			}
		}
	}

	// allocated the request # of items, or whatever is remaining for this cycle
	var allocated uint64
	if uint64(count) > gen.sequence-sequenceMask {
		allocated = gen.sequence - sequenceMask
	} else {
		allocated = uint64(count)
	}

	// advance the sequence for the # of items allocated
	gen.sequence = (gen.sequence + allocated) & sequenceMask

	// remember the last time interval where we allocated one or more ids.
	//
	// Note that although we own the mutex, the reader uses atomic so
	// we (the writer) do the same
	atomic.StoreInt64(&gen.lastAllocatedTime, gen.lastTime)

	return allocated, current, nil
}

// timestamp returns the # of milliseconds that have passed since
// the unix epoch
func timestamp() int64 {
	return time.Now().UnixNano() / 1e6
}
<|endoftext|>


<commit_before>package main_test

import (
	"os"
	"testing"

	"github.com/fraenkel/candiedyaml"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/switchboard/config"
)

func TestSwitchboard(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Switchboard Executable Suite")
}

var switchboardBinPath string
var dummyBackendBinPath string
var dummyHealthcheckBinPath string
var switchboardPort uint
var backendPort uint
var backendPort2 uint
var dummyHealthcheckPort uint
var dummyHealthcheckPort2 uint
var proxyConfigFile string
var proxyConfig config.Proxy

var _ = BeforeSuite(func() {
	var err error
	switchboardBinPath, err = gexec.Build("github.com/pivotal-cf-experimental/switchboard/cmd/switchboard", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	dummyBackendBinPath, err = gexec.Build("github.com/pivotal-cf-experimental/switchboard/cmd/switchboard/internal/dummy_backend", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	dummyHealthcheckBinPath, err = gexec.Build("github.com/pivotal-cf-experimental/switchboard/cmd/switchboard/internal/dummy_healthcheck", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	switchboardPort = uint(39900 + GinkgoParallelNode())
	healthcheckTimeoutInMS := uint(500)

	backendPort = uint(45000 + GinkgoParallelNode())
	backendPort2 = uint(46000 + GinkgoParallelNode())
	dummyHealthcheckPort = uint(45500 + GinkgoParallelNode())
	dummyHealthcheckPort2 = uint(46500 + GinkgoParallelNode())

	backend1 := config.Backend{
		BackendIP:       "localhost",
		BackendPort:     backendPort,
		HealthcheckPort: dummyHealthcheckPort,
	}

	backend2 := config.Backend{
		BackendIP:       "localhost",
		BackendPort:     backendPort2,
		HealthcheckPort: dummyHealthcheckPort2,
	}

	backends := []config.Backend{backend1, backend2}

	proxyConfig = config.Proxy{
		Pidfile:                "/tmp/switchboard.pid",
		Backends:               backends,
		HealthcheckTimeoutInMS: healthcheckTimeoutInMS,
		Port: switchboardPort,
	}

	proxyConfigFile = "/tmp/proxyConfig.yml"

	fileToWrite, err := os.Create(proxyConfigFile)
	if err != nil {
		println("Failed to open file for writing:", err.Error())
		os.Exit(1)
	}

	encoder := candiedyaml.NewEncoder(fileToWrite)
	err = encoder.Encode(proxyConfig)

	if err != nil {
		println("Failed to encode document:", err.Error())
		os.Exit(1)
	}

})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Avoid hardcoding /tmp for better test isolation<commit_after>package main_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/fraenkel/candiedyaml"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/switchboard/config"
)

func TestSwitchboard(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Switchboard Executable Suite")
}

var switchboardBinPath string
var dummyBackendBinPath string
var dummyHealthcheckBinPath string
var switchboardPort uint
var backendPort uint
var backendPort2 uint
var dummyHealthcheckPort uint
var dummyHealthcheckPort2 uint
var proxyConfigFile string
var proxyConfig config.Proxy

var _ = BeforeSuite(func() {
	var err error
	switchboardBinPath, err = gexec.Build("github.com/pivotal-cf-experimental/switchboard/cmd/switchboard", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	dummyBackendBinPath, err = gexec.Build("github.com/pivotal-cf-experimental/switchboard/cmd/switchboard/internal/dummy_backend", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	dummyHealthcheckBinPath, err = gexec.Build("github.com/pivotal-cf-experimental/switchboard/cmd/switchboard/internal/dummy_healthcheck", "-race")
	Ω(err).ShouldNot(HaveOccurred())

	switchboardPort = uint(39900 + GinkgoParallelNode())
	healthcheckTimeoutInMS := uint(500)

	backendPort = uint(45000 + GinkgoParallelNode())
	backendPort2 = uint(46000 + GinkgoParallelNode())
	dummyHealthcheckPort = uint(45500 + GinkgoParallelNode())
	dummyHealthcheckPort2 = uint(46500 + GinkgoParallelNode())

	backend1 := config.Backend{
		BackendIP:       "localhost",
		BackendPort:     backendPort,
		HealthcheckPort: dummyHealthcheckPort,
	}

	backend2 := config.Backend{
		BackendIP:       "localhost",
		BackendPort:     backendPort2,
		HealthcheckPort: dummyHealthcheckPort2,
	}

	backends := []config.Backend{backend1, backend2}

	tempDir, err := ioutil.TempDir(os.TempDir(), "switchboard")
	Expect(err).NotTo(HaveOccurred())

	proxyConfig = config.Proxy{
		Pidfile:                filepath.Join(tempDir, "switchboard.pid"),
		Backends:               backends,
		HealthcheckTimeoutInMS: healthcheckTimeoutInMS,
		Port: switchboardPort,
	}

	proxyConfigFile = filepath.Join(tempDir, "proxyConfig.yml")
	fileToWrite, err := os.Create(proxyConfigFile)
	if err != nil {
		println("Failed to open file for writing:", err.Error())
		os.Exit(1)
	}

	encoder := candiedyaml.NewEncoder(fileToWrite)
	err = encoder.Encode(proxyConfig)

	if err != nil {
		println("Failed to encode document:", err.Error())
		os.Exit(1)
	}

})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>package junos

// To View the entire configuration, use the keyword "full" for the first
// argument. If anything else outside of "full" is specified, it will return
// the configuration of the specified top-level stanza only. So "security"
// would return everything under the "security" stanza.
func ExampleJunos_viewConfiguration() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// Output format can be "text" or "xml".
	config, err := jnpr.GetConfig("full", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(config)
}

// Comparing and working with rollback configurations.
func ExampleJunos_rollbackConfigurations() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// If you want to view the difference between the current configuration and a rollback
	// one, then you can use the ConfigDiff() function to specify a previous config:
	diff, err := jnpr.ConfigDiff(3)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(diff)

	// You can rollback to a previous state, or the rescue configuration by using
	// the RollbackConfig() function:
	err := jnpr.RollbackConfig(3)
	if err != nil {
		fmt.Println(err)
	}

	// Create a rescue config from the active configuration.
	jnpr.Rescue("save")

	// You can also delete a rescue config.
	jnpr.Rescue("delete")

	// Rollback to the "rescue" configuration.
	err := jnpr.RollbackConfig("rescue")
	if err != nil {
		fmt.Println(err)
	}
}

// Configuring devices.
func ExampleJunos_configuringDevices() {
	// Use the LoadConfig() function to load the configuration from a file.

	// When configuring a device, it is good practice to lock the configuration database,
	// load the config, commit the configuration, and then unlock the configuration database.
	// You can do this with the following functions: Lock(), Commit(), Unlock().

	// Multiple ways to commit a configuration.

	// Commit the configuration as normal.
	Commit()

	// Check the configuration for any syntax errors (NOTE: you must still issue a
	// Commit() afterwards).
	CommitCheck()

	// Commit at a later time, i.e. 4:30 PM.
	CommitAt("16:30:00")

	// Rollback configuration if a Commit() is not issued within the given <minutes>.
	CommitConfirm(15)

	// You can configure the Junos device by uploading a local file, or pulling from an
	// FTP/HTTP server. The LoadConfig() function takes three arguments:

	// filename or URL, format, and a boolean (true/false) "commit-on-load".

	// If you specify a URL, it must be in the following format:

	// ftp://<username>:<password>@hostname/pathname/file-name
	// http://<username>:<password>@hostname/pathname/file-name

	// Note: The default value for the FTP path variable is the user’s home directory. Thus,
	// by default the file path to the configuration file is relative to the user directory.
	// To specify an absolute path when using FTP, start the path with the characters %2F;
	// for example: ftp://username:password@hostname/%2Fpath/filename.

	// The format of the commands within the file must be one of the following types:

	// set
	// system name-server 1.1.1.1

	// text
	// system {
	//     name-server 1.1.1.1;
	// }

	// xml
	// <system>
	//     <name-server>
	//         <name>1.1.1.1</name>
	//     </name-server>
	// </system>

	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// If the third option is "true" then after the configuration is loaded, a commit
	// will be issued. If set to "false," you will have to commit the configuration
	// using one of the Commit() functions.
	jnpr.Lock()
	err := jnpr.LoadConfig("path-to-file.txt", "set", true)
	if err != nil {
		fmt.Println(err)
	}
	jnpr.Unlock()
}

// Running operational mode commands on a device.
func ExampleJunos_runningCommands() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// You can run operational mode commands such as "show" and "request" by using the
	// Command() function. Output formats can be "text" or "xml".

	// Results returned in text format.
	txtOutput, err := jnpr.Command("show chassis hardware", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(txtOutput)

	// Results returned in XML format.
	xmlOutput, err := jnpr.Command("show chassis hardware", "xml")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(xmlOutput)
}

// Viewing basic information about the device.
func ExampleJunos_deviceInformation() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// When you call the PrintFacts() function, it just prints out the platform
	// and software information to the console.
	jnpr.PrintFacts()

	// You can also loop over the struct field that contains this information yourself:
	fmt.Printf("Hostname: %s", jnpr.Hostname)
	for _, data := range jnpr.Platform {
		fmt.Printf("Model: %s, Version: %s", data.Model, data.Version)
	}
	// Output: Model: SRX240H2, Version: 12.1X47-D10.4
}

// Establishing a connection to Junos Space and working with devices.
func ExampleJunosSpace_devices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Get the list of devices.
	devices, err := space.Devices()
	if err != nil {
		fmt.Println(err)
	}

	// Iterate over our device list and display some information about them.
	for _, device := range devices.Devices {
		fmt.Printf("Name: %s, IP Address: %s, Platform: %s\n", device.Name, device.IP, device.Platform)
	}

	// Add a device to Junos Space.
	jobID, err = space.AddDevice("sdubs-fw", "admin", "juniper123")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(jobID)
	// Output: 1345283

	// Remove a device from Junos Space.
	err = space.RemoveDevice("sdubs-fw")
	if err != nil {
		fmt.Println(err)
	}

	// Resynchronize a device. A good option if you do a lot of configuration to a device
	// outside of Junos Space.
	job, err := space.Resync("firewall-A")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}

// Software upgrades using Junos Space.
func ExampleJunosSpace_softwareUpgrade() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Staging software on a device. The last parameter is whether or not to remove any
	// existing images from the device; boolean.
	//
	// This will not upgrade the device, but only place the image there to be used at a later
	// time.
	jobID, err := space.StageSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", false)
	if err != nil {
		fmt.Println(err)
	}

	// If you want to issue a software upgrade to the device, here's how:

	// Configure our options, such as whether or not to reboot the device, etc.
	options := &junos.SoftwareUpgrade{
		UseDownloaded: true,
		Validate:      false,
		Reboot:        false,
		RebootAfter:   0,
		Cleanup:       false,
		RemoveAfter:   false,
	}

	jobID, err := space.DeploySoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", options)
	if err != nil {
		fmt.Println(err)
	}

	// Remove a staged image from the device.
	jobID, err := space.RemoveStagedSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz")
	if err != nil {
		fmt.Println(err)
	}
}

// Viewing information about Security Director devices (SRX, J-series, etc.).
func ExampleJunosSpace_securityDirectorDevices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security devices:
	devices, err := space.SecurityDevices()
	if err != nil {
		fmt.Println(err)
	}

	for _, device := range devices.Devices {
		fmt.Printf("%+v\n", device)
	}
}

// Working with address and service objects.
func ExampleJunosSpace_addressObjects() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// To view the address and service objects, you use the Addresses() and Services() functions. Both of them
	// take a "filter" parameter, which lets you search for objects matching your filter.

	//If you leave the parameter blank (e.g. ""), or specify "all", then every object is returned.

	// Address objects
	addresses, err := space.Addresses("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, address := range addresses.Addresses {
		fmt.Printf("%+v\n", address)
	}

	// Service objects
	services, err := space.Services("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, service := range services.Services {
		fmt.Printf("%+v\n", service)
	}

	// Add an address group. "true" as the first parameter means that we assume the
	// group is going to be an address group.
	space.AddGroup(true, "Blacklist-IPs", "Blacklisted IP addresses")

	// Add a service group. We do this by specifying "false" as the first parameter.
	space.AddGroup(false, "Web-Protocols", "All web-based protocols and ports")

	// Add an address object
	space.AddAddress("my-laptop", "2.2.2.2", "My personal laptop")

	// Add a network
	space.AddAddress("corporate-users", "192.168.1.0/24", "People on campus")

	// Add a service object with an 1800 second inactivity timeout (using "0" disables this feature)
	space.AddService("udp", "udp-5000", 5000, 5000, "UDP port 5000", 1800)

	// Add a service object with a port range
	space.AddService("tcp", "high-port-range", 40000, 65000, "TCP high ports", 0)

	// If you want to modify an existing object group, you do this with the ModifyObject() function. The
	// first parameter is whether the object is an address group (true) or a service group (false).

	// Add a service to a group
	space.ModifyObject(false, "add", "service-group", "service-name")

	// Remove an address object from a group
	space.ModifyObject(true, "remove", "Whitelisted-Addresses", "bad-ip")

	// Rename an object
	space.ModifyObject(false, "rename", "Web-Services", "Web-Ports")

	// Delete an object
	space.ModifyObject(true, "delete", "my-laptop")
}

// Working with polymorphic (variable) objects.
func ExampleJunosSpace_variables() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Add a variable
	// The parameters are as follows: variable-name, description, default-value
	space.AddVariable("test-variable", "Our test variable", "default-object")

	// Create our session state for modifying variables
	v, err := space.ModifyVariable()
	if err != nil {
		log.Fatal(err)
	}

	// Adding objects to the variable
	v.Add("test-variable", "srx-1", "user-pc")
	v.Add("test-variable", "corp-firewall", "db-server")

	// Delete a variable
	space.DeleteVariable("test-variable")
}

// Working with policies.
func ExampleJunosSpace_policies() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security policies Junos Space manages:
	policies, err := space.Policies()
	if err != nil {
		fmt.Println(err)
	}

	for _, policy := range policies.Policies {
		fmt.Printf("%s\n", policy.Name)
	}

	// For example, say we have been adding and removing objects in a group, and that group
	// is referenced in a firewall policy. Here's how to update the policy:

	// Update the policy. If "false" is specified, then the policy is only published, and the
	// device is not updated.
	job, err := space.PublishPolicy("Internet-Firewall-Policy", true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)

	// Let's update a device knowing that we have some previously published services.
	job, err := space.UpdateDevice("firewall-1.company.com")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}
<commit_msg>Updated Resync() example<commit_after>package junos

// To View the entire configuration, use the keyword "full" for the first
// argument. If anything else outside of "full" is specified, it will return
// the configuration of the specified top-level stanza only. So "security"
// would return everything under the "security" stanza.
func ExampleJunos_viewConfiguration() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// Output format can be "text" or "xml".
	config, err := jnpr.GetConfig("full", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(config)
}

// Comparing and working with rollback configurations.
func ExampleJunos_rollbackConfigurations() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// If you want to view the difference between the current configuration and a rollback
	// one, then you can use the ConfigDiff() function to specify a previous config:
	diff, err := jnpr.ConfigDiff(3)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(diff)

	// You can rollback to a previous state, or the rescue configuration by using
	// the RollbackConfig() function:
	err := jnpr.RollbackConfig(3)
	if err != nil {
		fmt.Println(err)
	}

	// Create a rescue config from the active configuration.
	jnpr.Rescue("save")

	// You can also delete a rescue config.
	jnpr.Rescue("delete")

	// Rollback to the "rescue" configuration.
	err := jnpr.RollbackConfig("rescue")
	if err != nil {
		fmt.Println(err)
	}
}

// Configuring devices.
func ExampleJunos_configuringDevices() {
	// Use the LoadConfig() function to load the configuration from a file.

	// When configuring a device, it is good practice to lock the configuration database,
	// load the config, commit the configuration, and then unlock the configuration database.
	// You can do this with the following functions: Lock(), Commit(), Unlock().

	// Multiple ways to commit a configuration.

	// Commit the configuration as normal.
	Commit()

	// Check the configuration for any syntax errors (NOTE: you must still issue a
	// Commit() afterwards).
	CommitCheck()

	// Commit at a later time, i.e. 4:30 PM.
	CommitAt("16:30:00")

	// Rollback configuration if a Commit() is not issued within the given <minutes>.
	CommitConfirm(15)

	// You can configure the Junos device by uploading a local file, or pulling from an
	// FTP/HTTP server. The LoadConfig() function takes three arguments:

	// filename or URL, format, and a boolean (true/false) "commit-on-load".

	// If you specify a URL, it must be in the following format:

	// ftp://<username>:<password>@hostname/pathname/file-name
	// http://<username>:<password>@hostname/pathname/file-name

	// Note: The default value for the FTP path variable is the user’s home directory. Thus,
	// by default the file path to the configuration file is relative to the user directory.
	// To specify an absolute path when using FTP, start the path with the characters %2F;
	// for example: ftp://username:password@hostname/%2Fpath/filename.

	// The format of the commands within the file must be one of the following types:

	// set
	// system name-server 1.1.1.1

	// text
	// system {
	//     name-server 1.1.1.1;
	// }

	// xml
	// <system>
	//     <name-server>
	//         <name>1.1.1.1</name>
	//     </name-server>
	// </system>

	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// If the third option is "true" then after the configuration is loaded, a commit
	// will be issued. If set to "false," you will have to commit the configuration
	// using one of the Commit() functions.
	jnpr.Lock()
	err := jnpr.LoadConfig("path-to-file.txt", "set", true)
	if err != nil {
		fmt.Println(err)
	}
	jnpr.Unlock()
}

// Running operational mode commands on a device.
func ExampleJunos_runningCommands() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// You can run operational mode commands such as "show" and "request" by using the
	// Command() function. Output formats can be "text" or "xml".

	// Results returned in text format.
	txtOutput, err := jnpr.Command("show chassis hardware", "text")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(txtOutput)

	// Results returned in XML format.
	xmlOutput, err := jnpr.Command("show chassis hardware", "xml")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(xmlOutput)
}

// Viewing basic information about the device.
func ExampleJunos_deviceInformation() {
	// Establish our session first.
	jnpr, err := junos.NewSession(host, user, password)
	if err != nil {
		log.Fatal(err)
	}
	defer jnpr.Close()

	// When you call the PrintFacts() function, it just prints out the platform
	// and software information to the console.
	jnpr.PrintFacts()

	// You can also loop over the struct field that contains this information yourself:
	fmt.Printf("Hostname: %s", jnpr.Hostname)
	for _, data := range jnpr.Platform {
		fmt.Printf("Model: %s, Version: %s", data.Model, data.Version)
	}
	// Output: Model: SRX240H2, Version: 12.1X47-D10.4
}

// Establishing a connection to Junos Space and working with devices.
func ExampleJunosSpace_devices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Get the list of devices.
	devices, err := space.Devices()
	if err != nil {
		fmt.Println(err)
	}

	// Iterate over our device list and display some information about them.
	for _, device := range devices.Devices {
		fmt.Printf("Name: %s, IP Address: %s, Platform: %s\n", device.Name, device.IP, device.Platform)
	}

	// Add a device to Junos Space.
	jobID, err = space.AddDevice("sdubs-fw", "admin", "juniper123")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(jobID)
	// Output: 1345283

	// Remove a device from Junos Space.
	err = space.RemoveDevice("sdubs-fw")
	if err != nil {
		fmt.Println(err)
	}

	// Resynchronize a device. A good option if you do a lot of configuration to a device
	// outside of Junos Space.
	job, err := space.Resync("firewall-A")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(job)
	// Output: 1542348
}

// Software upgrades using Junos Space.
func ExampleJunosSpace_softwareUpgrade() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Staging software on a device. The last parameter is whether or not to remove any
	// existing images from the device; boolean.
	//
	// This will not upgrade the device, but only place the image there to be used at a later
	// time.
	jobID, err := space.StageSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", false)
	if err != nil {
		fmt.Println(err)
	}

	// If you want to issue a software upgrade to the device, here's how:

	// Configure our options, such as whether or not to reboot the device, etc.
	options := &junos.SoftwareUpgrade{
		UseDownloaded: true,
		Validate:      false,
		Reboot:        false,
		RebootAfter:   0,
		Cleanup:       false,
		RemoveAfter:   false,
	}

	jobID, err := space.DeploySoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz", options)
	if err != nil {
		fmt.Println(err)
	}

	// Remove a staged image from the device.
	jobID, err := space.RemoveStagedSoftware("sdubs-fw", "junos-srxsme-12.1X46-D30.2-domestic.tgz")
	if err != nil {
		fmt.Println(err)
	}
}

// Viewing information about Security Director devices (SRX, J-series, etc.).
func ExampleJunosSpace_securityDirectorDevices() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security devices:
	devices, err := space.SecurityDevices()
	if err != nil {
		fmt.Println(err)
	}

	for _, device := range devices.Devices {
		fmt.Printf("%+v\n", device)
	}
}

// Working with address and service objects.
func ExampleJunosSpace_addressObjects() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// To view the address and service objects, you use the Addresses() and Services() functions. Both of them
	// take a "filter" parameter, which lets you search for objects matching your filter.

	//If you leave the parameter blank (e.g. ""), or specify "all", then every object is returned.

	// Address objects
	addresses, err := space.Addresses("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, address := range addresses.Addresses {
		fmt.Printf("%+v\n", address)
	}

	// Service objects
	services, err := space.Services("all")
	if err != nil {
		fmt.Println(err)
	}

	for _, service := range services.Services {
		fmt.Printf("%+v\n", service)
	}

	// Add an address group. "true" as the first parameter means that we assume the
	// group is going to be an address group.
	space.AddGroup(true, "Blacklist-IPs", "Blacklisted IP addresses")

	// Add a service group. We do this by specifying "false" as the first parameter.
	space.AddGroup(false, "Web-Protocols", "All web-based protocols and ports")

	// Add an address object
	space.AddAddress("my-laptop", "2.2.2.2", "My personal laptop")

	// Add a network
	space.AddAddress("corporate-users", "192.168.1.0/24", "People on campus")

	// Add a service object with an 1800 second inactivity timeout (using "0" disables this feature)
	space.AddService("udp", "udp-5000", 5000, 5000, "UDP port 5000", 1800)

	// Add a service object with a port range
	space.AddService("tcp", "high-port-range", 40000, 65000, "TCP high ports", 0)

	// If you want to modify an existing object group, you do this with the ModifyObject() function. The
	// first parameter is whether the object is an address group (true) or a service group (false).

	// Add a service to a group
	space.ModifyObject(false, "add", "service-group", "service-name")

	// Remove an address object from a group
	space.ModifyObject(true, "remove", "Whitelisted-Addresses", "bad-ip")

	// Rename an object
	space.ModifyObject(false, "rename", "Web-Services", "Web-Ports")

	// Delete an object
	space.ModifyObject(true, "delete", "my-laptop")
}

// Working with polymorphic (variable) objects.
func ExampleJunosSpace_variables() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// Add a variable
	// The parameters are as follows: variable-name, description, default-value
	space.AddVariable("test-variable", "Our test variable", "default-object")

	// Create our session state for modifying variables
	v, err := space.ModifyVariable()
	if err != nil {
		log.Fatal(err)
	}

	// Adding objects to the variable
	v.Add("test-variable", "srx-1", "user-pc")
	v.Add("test-variable", "corp-firewall", "db-server")

	// Delete a variable
	space.DeleteVariable("test-variable")
}

// Working with policies.
func ExampleJunosSpace_policies() {
	// Establish a connection to a Junos Space server.
	space := junos.NewServer("space.company.com", "admin", "juniper123")

	// List all security policies Junos Space manages:
	policies, err := space.Policies()
	if err != nil {
		fmt.Println(err)
	}

	for _, policy := range policies.Policies {
		fmt.Printf("%s\n", policy.Name)
	}

	// For example, say we have been adding and removing objects in a group, and that group
	// is referenced in a firewall policy. Here's how to update the policy:

	// Update the policy. If "false" is specified, then the policy is only published, and the
	// device is not updated.
	job, err := space.PublishPolicy("Internet-Firewall-Policy", true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)

	// Let's update a device knowing that we have some previously published services.
	job, err := space.UpdateDevice("firewall-1.company.com")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Printf("Job ID: %d\n", job)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"reflect"
	"testing"
)

func testIconsFindHelper(terms []string) icons {
	return newIcons().find(terms)
}

func TestIcons_iconsYamlPath_TestEnv(t *testing.T) {
	actual := iconsYamlPath()
	expected := "workflow/icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsYamlPath_ProductionEnv(t *testing.T) {
	resetEnv := setTestEnvHelper("FAW_ICONS_YAML_PATH", "")
	defer resetEnv()

	actual := iconsYamlPath()
	expected := "icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsReadYaml(t *testing.T) {
	path := "workflow/icons.yml"
	actual, _ := iconsReadYaml(path)

	expected, _ := ioutil.ReadFile(path)
	if !reflect.DeepEqual(actual, expected) {
		t.Error("failed to read file")
	}
}

func TestIcons_iconsReadYaml_Error(t *testing.T) {
	path := ""
	_, err := iconsReadYaml(path)

	if err == nil {
		t.Error("expected error, but nil")
	}
}

func TestIcons_iconsUnmarshalYaml(t *testing.T) {
	b := []byte(`
icons:
  - name:       Glass
    id:         glass
    unicode:    f000
    created:    1.0
    filter:
      - martini
`)
	actual, _ := iconsUnmarshalYaml(b)

	icon := icon{
		Name:    "Glass",
		ID:      "glass",
		Unicode: "f000",
		Created: "1.0",
		Filter:  []string{"martini"},
	}
	expected := iconsYaml{icons{icon}}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_AllIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := len(fi)
	expected := 585
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_ZeroIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_OneIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TwoIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_FirstIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[0].ID
	expected := "500px"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_LastIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "youtube-square"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TaxiIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Aliases[0]
	expected = "cab"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Filter[0]
	expected = "vehicle"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Categories[0]
	expected = "Web Application Icons"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

// for https://github.com/ruedap/alfred2-font-awesome-workflow/issues/74
func TestIcons_find_PlusIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"plus"})[4]

	actual := fi.Name
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi.ID
	expected = "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi.Unicode
	expected = "f067"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi.Created
	expected = "1.0"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_Aliases(t *testing.T) {
	fi := testIconsFindHelper([]string{"navicon"})

	actual := fi[0].ID
	expected := "bars"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}

func TestIcons_findByUnicode(t *testing.T) {
	fi := newIcons().findByUnicode("f067")[0]

	actual := fi.ID
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}
<commit_msg>Fix tests<commit_after>package main

import (
	"io/ioutil"
	"reflect"
	"testing"
)

func testIconsFindHelper(terms []string) icons {
	return newIcons().find(terms)
}

func TestIcons_iconsYamlPath_TestEnv(t *testing.T) {
	actual := iconsYamlPath()
	expected := "workflow/icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsYamlPath_ProductionEnv(t *testing.T) {
	resetEnv := setTestEnvHelper("FAW_ICONS_YAML_PATH", "")
	defer resetEnv()

	actual := iconsYamlPath()
	expected := "icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsReadYaml(t *testing.T) {
	path := "workflow/icons.yml"
	actual, _ := iconsReadYaml(path)

	expected, _ := ioutil.ReadFile(path)
	if !reflect.DeepEqual(actual, expected) {
		t.Error("failed to read file")
	}
}

func TestIcons_iconsReadYaml_Error(t *testing.T) {
	path := ""
	_, err := iconsReadYaml(path)

	if err == nil {
		t.Error("expected error, but nil")
	}
}

func TestIcons_iconsUnmarshalYaml(t *testing.T) {
	b := []byte(`
icons:
  - name:       Glass
    id:         glass
    unicode:    f000
    created:    1.0
    filter:
      - martini
`)
	actual, _ := iconsUnmarshalYaml(b)

	icon := icon{
		Name:    "Glass",
		ID:      "glass",
		Unicode: "f000",
		Created: "1.0",
		Filter:  []string{"martini"},
	}
	expected := iconsYaml{icons{icon}}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_AllIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := len(fi)
	expected := 585
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_ZeroIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_OneIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TwoIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_FirstIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[0].ID
	expected := "500px"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_LastIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "youtube-square"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TaxiIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Aliases[0]
	expected = "cab"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Filter[0]
	expected = "vehicle"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Categories[0]
	expected = "Web Application Icons"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_Aliases(t *testing.T) {
	fi := testIconsFindHelper([]string{"navicon"})

	actual := fi[0].ID
	expected := "bars"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}

func TestIcons_findByUnicode(t *testing.T) {
	fi := newIcons().findByUnicode("f067")

	actual := fi[0].ID
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	uuidpkg "github.com/pborman/uuid"
	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

//-TRIGGER----------------------------------------------------------------------

// TriggerDBMapping is the name of the Elasticsearch type to which Triggers are added
const TriggerDBMapping string = "Trigger"

// TriggerIndexSettings is the mapping for the "trigger" index in Elasticsearch
const TriggerIndexSettings = `
{
	"mappings": {
		"Trigger": {
			"properties": {
				"triggerId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"title": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"enabled": {
					"type": "boolean",
					"index": "not_analyzed"
				},
				"condition": {
					"properties": {
						"eventTypeIds": {
							"type": "string",
							"index": "not_analyzed"
						},
						"query": {
							"dynamic": true,
							"properties": {}
						}
					}
				},
				"job": {
					"properties": {
						"createdBy": {
							"type": "string",
							"index": "not_analyzed"
						},
						"jobType": {
							"dynamic": true,
							"properties": {}
						}
					}
				},
				"percolationId": {
					"type": "string",
					"index": "not_analyzed"
				}
			}
		}
	}
}
`

// Condition expresses the idea of "this ES query returns an event"
// Query is specific to the event type
type Condition struct {
	EventTypeIds []piazza.Ident         `json:"eventTypeIds" binding:"required"`
	Query        map[string]interface{} `json:"query" binding:"required"`
}

type JobRequest struct {
	CreatedBy string  `json:"createdBy"`
	JobType   JobType `json:"jobType" binding:"required"`
}

type JobType struct {
	Data map[string]interface{} `json:"data" binding:"required"`
	Type string                 `json:"type" binding:"required"`
}

// Trigger does something when the and'ed set of Conditions all are true
// Events are the results of the Conditions queries
// Job is the JobMessage to submit back to Pz
type Trigger struct {
	TriggerId     piazza.Ident `json:"triggerId"`
	Name          string       `json:"name" binding:"required"`
	Condition     Condition    `json:"condition" binding:"required"`
	Job           JobRequest   `json:"job" binding:"required"`
	PercolationId piazza.Ident `json:"percolationId"`
	CreatedBy     string       `json:"createdBy"`
	CreatedOn     time.Time    `json:"createdOn"`
	Enabled       bool         `json:"enabled"`
}

// TriggerList is a list of triggers
type TriggerList []Trigger

//-EVENT------------------------------------------------------------------------

// EventIndexSettings is the mapping for the "events" index in Elasticsearch
const EventIndexSettings = `
{
	"settings": {
		"index.mapping.coerce": false
	},
	"mappings": {
		"_default_": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"eventId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"data": {
					"properties": {}
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"cronSchedule": {
					"type": "string",
					"index": "not_analyzed"
				}
			}
		}
	}
}
`

// An Event is posted by some source (service, user, etc) to indicate Something Happened
// Data is specific to the event type
type Event struct {
	EventId      piazza.Ident           `json:"eventId"`
	EventTypeId  piazza.Ident           `json:"eventTypeId" binding:"required"`
	Data         map[string]interface{} `json:"data"`
	CreatedBy    string                 `json:"createdBy"`
	CreatedOn    time.Time              `json:"createdOn"`
	CronSchedule string                 `json:"cronSchedule"`
}

// EventList is a list of events
type EventList []Event

//-EVENTTYPE--------------------------------------------------------------------

// EventTypeDBMapping is the name of the Elasticsearch type to which Events are added
const EventTypeDBMapping string = "EventType"

// EventTypeIndexSettings is the mapping for the "eventtypes" index in Elasticsearch
const EventTypeIndexSettings = `
{
	"mappings": {
		"EventType": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"name": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"mapping": {
					"dynamic": true,
					"properties": {}
				}
			}
		}
	}
}
`

// EventType describes an Event that is to be sent to workflow by a client or service
type EventType struct {
	EventTypeId piazza.Ident                                    `json:"eventTypeId"`
	Name        string                                          `json:"name" binding:"required"`
	Mapping     map[string]elasticsearch.MappingElementTypeName `json:"mapping" binding:"required"`
	CreatedBy   string                                          `json:"createdBy"`
	CreatedOn   time.Time                                       `json:"createdOn"`
}

// EventTypeList is a list of EventTypes
type EventTypeList []EventType

//-ALERT------------------------------------------------------------------------

// AlertDBMapping is the name of the Elasticsearch type to which Alerts are added
const AlertDBMapping string = "Alert"

// AlertIndexSettings are the default settings for our Elasticsearch alerts index
// Explanation:
//   "index": "not_analyzed"
//     This means that these properties are not analyzed by Elasticsearch.
//     Previously, these ids were analyzed by ES and thus broken up into chunks;
//     in the case of a UUID this would happen via break-up by the "-" character.
//     For example, the UUID "ab3142cd-1a8e-44f8-6a01-5ce8a9328fb2" would be broken
//     into "ab3142cd", "1a8e", "44f8", "6a01" and "5ce8a9328fb2", and queries would
//     match on all of these separate strings, which was undesired behavior.
const AlertIndexSettings = `
{
	"mappings": {
		"Alert": {
			"properties": {
				"alertId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"triggerId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"jobId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"eventId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				}
			}
		}
	}
}
`

// Alert is a notification, automatically created when a Trigger happens
type Alert struct {
	AlertId   piazza.Ident `json:"alertId"`
	TriggerId piazza.Ident `json:"triggerId"`
	EventId   piazza.Ident `json:"eventId"`
	JobId     piazza.Ident `json:"jobId"`
	CreatedBy string       `json:"createdBy"`
	CreatedOn time.Time    `json:"createdOn"`
}

//-CRON-------------------------------------------------------------------------

const CronIndexSettings = EventIndexSettings

const cronDBMapping = "Cron"

//-- Stats ------------------------------------------------------------

type workflowStats struct {
	sync.Mutex
	CreatedOn        time.Time `json:"createdOn"`
	NumEventTypes    int       `json:"numEventTypes"`
	NumEvents        int       `json:"numEvents"`
	NumTriggers      int       `json:"numTriggers"`
	NumAlerts        int       `json:"numAlerts"`
	NumTriggeredJobs int       `json:"numTriggeredJobs"`
}

func (stats *workflowStats) incrCounter(counter *int) {
	stats.Lock()
	*counter++
	stats.Unlock()
}

func (stats *workflowStats) IncrEventTypes() {
	stats.incrCounter(&stats.NumEventTypes)
}

func (stats *workflowStats) IncrEvents() {
	stats.incrCounter(&stats.NumEvents)
}

func (stats *workflowStats) IncrTriggers() {
	stats.incrCounter(&stats.NumTriggers)
}

func (stats *workflowStats) IncrAlerts() {
	stats.incrCounter(&stats.NumAlerts)
}

func (stats *workflowStats) IncrTriggerJobs() {
	stats.incrCounter(&stats.NumTriggeredJobs)
}

//-UTILITY----------------------------------------------------------------------

// LoggedError logs the error's message and creates an error
func LoggedError(mssg string, args ...interface{}) error {
	str := fmt.Sprintf(mssg, args)
	log.Printf(str)
	return errors.New(str)
}

// isUUID checks to see if the UUID is valid
func isUUID(uuid string) bool {
	return uuidpkg.Parse(uuid) != nil
}

//-INIT-------------------------------------------------------------------------

func init() {
	piazza.JsonResponseDataTypes["*workflow.EventType"] = "eventtype"
	piazza.JsonResponseDataTypes["[]workflow.EventType"] = "eventtype-list"
	piazza.JsonResponseDataTypes["*workflow.Event"] = "event"
	piazza.JsonResponseDataTypes["[]workflow.Event"] = "event-list"
	piazza.JsonResponseDataTypes["*workflow.Trigger"] = "trigger"
	piazza.JsonResponseDataTypes["[]workflow.Trigger"] = "trigger-list"
	piazza.JsonResponseDataTypes["*workflow.Alert"] = "alert"
	piazza.JsonResponseDataTypes["[]workflow.Alert"] = "alert-list"
	piazza.JsonResponseDataTypes["workflow.workflowStats"] = "workflowstats"
}
<commit_msg>Update CronIndexSettings<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package workflow

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	uuidpkg "github.com/pborman/uuid"
	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
)

//-TRIGGER----------------------------------------------------------------------

// TriggerDBMapping is the name of the Elasticsearch type to which Triggers are added
const TriggerDBMapping string = "Trigger"

// TriggerIndexSettings is the mapping for the "trigger" index in Elasticsearch
const TriggerIndexSettings = `
{
	"mappings": {
		"Trigger": {
			"properties": {
				"triggerId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"title": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"enabled": {
					"type": "boolean",
					"index": "not_analyzed"
				},
				"condition": {
					"properties": {
						"eventTypeIds": {
							"type": "string",
							"index": "not_analyzed"
						},
						"query": {
							"dynamic": true,
							"properties": {}
						}
					}
				},
				"job": {
					"properties": {
						"createdBy": {
							"type": "string",
							"index": "not_analyzed"
						},
						"jobType": {
							"dynamic": true,
							"properties": {}
						}
					}
				},
				"percolationId": {
					"type": "string",
					"index": "not_analyzed"
				}
			}
		}
	}
}
`

// Condition expresses the idea of "this ES query returns an event"
// Query is specific to the event type
type Condition struct {
	EventTypeIds []piazza.Ident         `json:"eventTypeIds" binding:"required"`
	Query        map[string]interface{} `json:"query" binding:"required"`
}

type JobRequest struct {
	CreatedBy string  `json:"createdBy"`
	JobType   JobType `json:"jobType" binding:"required"`
}

type JobType struct {
	Data map[string]interface{} `json:"data" binding:"required"`
	Type string                 `json:"type" binding:"required"`
}

// Trigger does something when the and'ed set of Conditions all are true
// Events are the results of the Conditions queries
// Job is the JobMessage to submit back to Pz
type Trigger struct {
	TriggerId     piazza.Ident `json:"triggerId"`
	Name          string       `json:"name" binding:"required"`
	Condition     Condition    `json:"condition" binding:"required"`
	Job           JobRequest   `json:"job" binding:"required"`
	PercolationId piazza.Ident `json:"percolationId"`
	CreatedBy     string       `json:"createdBy"`
	CreatedOn     time.Time    `json:"createdOn"`
	Enabled       bool         `json:"enabled"`
}

// TriggerList is a list of triggers
type TriggerList []Trigger

//-EVENT------------------------------------------------------------------------

// EventIndexSettings is the mapping for the "events" index in Elasticsearch
const EventIndexSettings = `
{
	"settings": {
		"index.mapping.coerce": false
	},
	"mappings": {
		"_default_": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"eventId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"data": {
					"properties": {}
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"cronSchedule": {
					"type": "string",
					"index": "not_analyzed"
				}
			}
		}
	}
}
`

// An Event is posted by some source (service, user, etc) to indicate Something Happened
// Data is specific to the event type
type Event struct {
	EventId      piazza.Ident           `json:"eventId"`
	EventTypeId  piazza.Ident           `json:"eventTypeId" binding:"required"`
	Data         map[string]interface{} `json:"data"`
	CreatedBy    string                 `json:"createdBy"`
	CreatedOn    time.Time              `json:"createdOn"`
	CronSchedule string                 `json:"cronSchedule"`
}

// EventList is a list of events
type EventList []Event

//-EVENTTYPE--------------------------------------------------------------------

// EventTypeDBMapping is the name of the Elasticsearch type to which Events are added
const EventTypeDBMapping string = "EventType"

// EventTypeIndexSettings is the mapping for the "eventtypes" index in Elasticsearch
const EventTypeIndexSettings = `
{
	"mappings": {
		"EventType": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"name": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"mapping": {
					"dynamic": true,
					"properties": {}
				}
			}
		}
	}
}
`

// EventType describes an Event that is to be sent to workflow by a client or service
type EventType struct {
	EventTypeId piazza.Ident                                    `json:"eventTypeId"`
	Name        string                                          `json:"name" binding:"required"`
	Mapping     map[string]elasticsearch.MappingElementTypeName `json:"mapping" binding:"required"`
	CreatedBy   string                                          `json:"createdBy"`
	CreatedOn   time.Time                                       `json:"createdOn"`
}

// EventTypeList is a list of EventTypes
type EventTypeList []EventType

//-ALERT------------------------------------------------------------------------

// AlertDBMapping is the name of the Elasticsearch type to which Alerts are added
const AlertDBMapping string = "Alert"

// AlertIndexSettings are the default settings for our Elasticsearch alerts index
// Explanation:
//   "index": "not_analyzed"
//     This means that these properties are not analyzed by Elasticsearch.
//     Previously, these ids were analyzed by ES and thus broken up into chunks;
//     in the case of a UUID this would happen via break-up by the "-" character.
//     For example, the UUID "ab3142cd-1a8e-44f8-6a01-5ce8a9328fb2" would be broken
//     into "ab3142cd", "1a8e", "44f8", "6a01" and "5ce8a9328fb2", and queries would
//     match on all of these separate strings, which was undesired behavior.
const AlertIndexSettings = `
{
	"mappings": {
		"Alert": {
			"properties": {
				"alertId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"triggerId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"jobId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"eventId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				}
			}
		}
	}
}
`

// Alert is a notification, automatically created when a Trigger happens
type Alert struct {
	AlertId   piazza.Ident `json:"alertId"`
	TriggerId piazza.Ident `json:"triggerId"`
	EventId   piazza.Ident `json:"eventId"`
	JobId     piazza.Ident `json:"jobId"`
	CreatedBy string       `json:"createdBy"`
	CreatedOn time.Time    `json:"createdOn"`
}

//-CRON-------------------------------------------------------------------------

const CronIndexSettings = `
{
	"settings": {
		"index.mapping.coerce": false
	},
	"mappings": {
		"Cron": {
			"properties": {
				"eventTypeId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"eventId": {
					"type": "string",
					"index": "not_analyzed"
				},
				"data": {
					"properties": {}
				},
				"createdBy": {
					"type": "string",
					"index": "not_analyzed"
				},
				"createdOn": {
					"type": "date"
				},
				"cronSchedule": {
					"type": "string",
					"index": "not_analyzed"
				}
			}
		}
	}
}
`

const cronDBMapping = "Cron"

//-- Stats ------------------------------------------------------------

type workflowStats struct {
	sync.Mutex
	CreatedOn        time.Time `json:"createdOn"`
	NumEventTypes    int       `json:"numEventTypes"`
	NumEvents        int       `json:"numEvents"`
	NumTriggers      int       `json:"numTriggers"`
	NumAlerts        int       `json:"numAlerts"`
	NumTriggeredJobs int       `json:"numTriggeredJobs"`
}

func (stats *workflowStats) incrCounter(counter *int) {
	stats.Lock()
	*counter++
	stats.Unlock()
}

func (stats *workflowStats) IncrEventTypes() {
	stats.incrCounter(&stats.NumEventTypes)
}

func (stats *workflowStats) IncrEvents() {
	stats.incrCounter(&stats.NumEvents)
}

func (stats *workflowStats) IncrTriggers() {
	stats.incrCounter(&stats.NumTriggers)
}

func (stats *workflowStats) IncrAlerts() {
	stats.incrCounter(&stats.NumAlerts)
}

func (stats *workflowStats) IncrTriggerJobs() {
	stats.incrCounter(&stats.NumTriggeredJobs)
}

//-UTILITY----------------------------------------------------------------------

// LoggedError logs the error's message and creates an error
func LoggedError(mssg string, args ...interface{}) error {
	str := fmt.Sprintf(mssg, args)
	log.Printf(str)
	return errors.New(str)
}

// isUUID checks to see if the UUID is valid
func isUUID(uuid string) bool {
	return uuidpkg.Parse(uuid) != nil
}

//-INIT-------------------------------------------------------------------------

func init() {
	piazza.JsonResponseDataTypes["*workflow.EventType"] = "eventtype"
	piazza.JsonResponseDataTypes["[]workflow.EventType"] = "eventtype-list"
	piazza.JsonResponseDataTypes["*workflow.Event"] = "event"
	piazza.JsonResponseDataTypes["[]workflow.Event"] = "event-list"
	piazza.JsonResponseDataTypes["*workflow.Trigger"] = "trigger"
	piazza.JsonResponseDataTypes["[]workflow.Trigger"] = "trigger-list"
	piazza.JsonResponseDataTypes["*workflow.Alert"] = "alert"
	piazza.JsonResponseDataTypes["[]workflow.Alert"] = "alert-list"
	piazza.JsonResponseDataTypes["workflow.workflowStats"] = "workflowstats"
}
<|endoftext|>