<commit_before>package archive

import (
	"fmt"
	"io"
	"path"
	"path/filepath"
	"time"

	"crypto"
	"crypto/sha512"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/clearsign"
	"golang.org/x/crypto/openpgp/packet"

	"pault.ag/go/blobstore"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/transput"
)

// Archive {{{

// Core Archive abstrcation. This contains helpers to write out package files,
// as well as handles creating underlying abstractions (such as Suites).
type Archive struct {
	Store      blobstore.Store
	signingKey *openpgp.Entity
	path       string
	Pool       Pool
}

// Create a new Archive at the given `root` on the filesystem, with the
// openpgp.Entity `signer` (an Entity which contains an OpenPGP Private
// Key).
//
// This interface is intended to *write* Archives, not *read* them. Extra
// steps must be taken to load an Archive over the network, and attention
// must be paid when handling the Cryptographic chain of trust.
func New(path string, signer *openpgp.Entity) (*Archive, error) {
	var err error
	path, err = filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	store, err := blobstore.Load(path)
	if err != nil {
		return nil, err
	}

	return &Archive{
		Store:      *store,
		signingKey: signer,
		path:       path,
		Pool:       Pool{Store: *store},
	}, nil
}

func (a Archive) Path() string {
	return a.path
}

// Use the default backend to remove any unlinked files from the Blob store.
//
// If files you care about are not linked onto the stage, they will be removed
// by the garbage collector. Decruft only when you're sure the stage has been
// set.
func (a Archive) Decruft() error {
	return a.Store.GC(blobstore.DumbGarbageCollector{})
}

// Given a list of objects, link them to the keyed paths.
func (a Archive) Link(blobs ArchiveState) error {
	for path, obj := range blobs {
		if err := a.Store.Link(obj, path); err != nil {
			return err
		}
	}
	return nil
}

// Create a new Release object from a Suite, passing off the Name, Description
// and constructing the rest of the goodies.
//
// This will be an entirely empty object, without anything read off disk.
func newRelease(suite Suite) (*Release, error) {
	when := time.Now()

	var validUntil string = ""
	if suite.features.Duration != "" {
		duration, err := time.ParseDuration(suite.features.Duration)
		if err != nil {
			return nil, err
		}
		validUntil = when.Add(duration).Format(time.RFC1123Z)
	}

	release := Release{
		Suite:       suite.Name,
		Description: suite.Description,
		ValidUntil:  validUntil,
		Origin:      suite.Origin,
		Label:       suite.Label,
		Version:     suite.Version,
	}
	release.Date = when.Format(time.RFC1123Z)
	release.Architectures = []dependency.Arch{}
	release.Components = []string{}
	release.SHA256 = []control.SHA256FileHash{}
	release.SHA1 = []control.SHA1FileHash{}
	release.SHA512 = []control.SHA512FileHash{}
	release.MD5Sum = []control.MD5FileHash{}
	return &release, nil
}

// This is a set of file changes ready to be passed to `Link` to link in.
// Basically, this maps file paths to blobstore objects, which will be
// swapped in all at once. This allows errors to avoid mutating state in
// the archive.
type ArchiveState map[string]blobstore.Object

// Engross a Suite for signing and final commit into the blobstore. This
// will return handle(s) to the signed and ready Objects, fit for passage
// to Link.
//
// This will contain all the related Packages and Release files.
func (a Archive) Engross(suite Suite) (ArchiveState, error) {
	release, err := newRelease(suite)
	if err != nil {
		return nil, err
	}

	files := ArchiveState{}
	arches := map[dependency.Arch]bool{}

	for name, component := range suite.components {
		release.Components = append(release.Components, name)
		for arch, writer := range component.packageWriters {
			arches[arch] = true

			// For each Binary entry, do the same as above (todo: someone
			// DRY this out a bit. I'm too lazy.

			suitePath := path.Join(name, fmt.Sprintf("binary-%s", arch),
				"Packages")

			obj, err := a.Store.Commit(*writer.handle)
			if err != nil {
				return nil, err
			}

			for _, hasher := range writer.hashers {
				fileHash := control.FileHashFromHasher(suitePath, *hasher)
				release.AddHash(fileHash)
			}

			filePath := path.Join("dists", suite.Name, suitePath)
			files[filePath] = *obj
		}
	}

	for arch, _ := range arches {
		release.Architectures = append(release.Architectures, arch)
	}

	/* Now, let's do some magic */

	// Now, let's write out the Release file (and sign it normally)
	obj, sig, err := suite.archive.encodeSigned(release)
	if err != nil {
		return nil, err
	}

	filePath := path.Join("dists", suite.Name, "Release")
	files[filePath] = *obj
	files[fmt.Sprintf("%s.gpg", filePath)] = *sig

	// Ditto with the clearsigned version (Should we merge the two above?)
	obj, err = suite.archive.encodeClearsigned(release)
	if err != nil {
		return nil, err
	}

	files[path.Join("dists", suite.Name, "InRelease")] = *obj

	return files, nil
}

// Given a control.Marshal'able object, encode it to the blobstore, while
// also clearsigning the data.
func (a Archive) encodeClearsigned(data interface{}) (*blobstore.Object, error) {

	if a.signingKey == nil {
		return nil, fmt.Errorf("No signing key loaded")
	}

	fd, err := a.Store.Create()
	if err != nil {
		return nil, err
	}

	defer fd.Close()
	wc, err := clearsign.Encode(fd, a.signingKey.PrivateKey, nil)
	if err != nil {
		return nil, err
	}

	encoder, err := control.NewEncoder(wc)
	if err != nil {
		return nil, err
	}

	if err := encoder.Encode(data); err != nil {
		return nil, err
	}

	if err := wc.Close(); err != nil {
		return nil, err
	}

	return a.Store.Commit(*fd)
}

// Given a control.Marshal'able object, encode it to the blobstore, while
// also doing a detached OpenPGP signature. The objects returned (in order)
// are data, commited to the blobstore, the signature for that object, commited
// to the blobstore, and any error(s), finally.
func (a Archive) encodeSigned(data interface{}) (*blobstore.Object, *blobstore.Object, error) {
	/* Right, so, the trick here is that we secretly call out to encode,
	 * but tap it with a pipe into the signing code */

	if a.signingKey == nil {
		return nil, nil, fmt.Errorf("No signing key loaded")
	}

	signature, err := a.Store.Create()
	if err != nil {
		return nil, nil, err
	}
	defer signature.Close()

	hash := sha512.New()

	obj, err := a.encode(data, hash)
	if err != nil {
		return nil, nil, err
	}

	sig := new(packet.Signature)
	sig.SigType = packet.SigTypeBinary
	sig.PubKeyAlgo = a.signingKey.PrivateKey.PubKeyAlgo

	sig.Hash = crypto.SHA512

	sig.CreationTime = new(packet.Config).Now()
	sig.IssuerKeyId = &(a.signingKey.PrivateKey.KeyId)

	err = sig.Sign(hash, a.signingKey.PrivateKey, nil)
	if err != nil {
		return nil, nil, err
	}

	if err := sig.Serialize(signature); err != nil {
		return nil, nil, err
	}

	sigObj, err := a.Store.Commit(*signature)
	if err != nil {
		return nil, nil, err
	}

	return obj, sigObj, nil

}

// Encode a given control.Marshal'able object into the Blobstore, and return
// a handle to its object.
//
// The optinal argument `tap` will be written to as the object gets sent into
// the blobstore. This may be useful if you wish to have a copy of the data
// going into the store.
func (a Archive) encode(data interface{}, tap io.Writer) (*blobstore.Object, error) {
	fd, err := a.Store.Create()
	if err != nil {
		return nil, err
	}

	var writer io.Writer = fd
	if tap != nil {
		writer = io.MultiWriter(fd, tap)
	}

	encoder, err := control.NewEncoder(writer)
	if err != nil {
		return nil, err
	}

	if err := encoder.Encode(data); err != nil {
		return nil, err
	}

	return a.Store.Commit(*fd)
}

// }}}

// Suite {{{

// Abstraction to handle writing data into a Suite. This is a write-only
// target, and is not intended to read a Release file.
//
// This contains no state read off disk, and is purely for writing to.
type Suite struct {
	control.Paragraph

	archive *Archive

	Name        string `control:"Suite"`
	Description string
	Origin      string
	Label       string
	Version     string

	components map[string]*Component `control:"-"`

	features struct {
		Hashes   []string
		Duration string
	} `control:"-"`
}

// Get a handle to write a given Suite from an Archive.
// The suite will be entirely blank, and attributes will not be
// read from the existing files, if any.
func (a Archive) Suite(name string) (*Suite, error) {
	suite := Suite{
		Name:       name,
		archive:    &a,
		components: map[string]*Component{},
	}

	suite.features.Hashes = []string{"sha256", "sha1", "sha512"}
	suite.features.Duration = "240h"

	return &suite, nil
}

// Get or create a Component for a given Suite. If no such Component
// has been created so far, this will create a new object, otherwise
// it will return the existing entry.
//
// This contains no state read off disk, and is purely for writing to.
func (s Suite) Component(name string) (*Component, error) {
	if _, ok := s.components[name]; !ok {
		comp, err := newComponent(&s)
		if err != nil {
			return nil, err
		}
		s.components[name] = comp
		return comp, nil
	}
	el := s.components[name]
	return el, nil
}

// }}}

// Component {{{

// Small wrapper to represent a Component of a Suite, which, at its core
// is simply a set of Indexes to be written to.
//
// This contains no state read off disk, and is purely for writing to.
type Component struct {
	suite          *Suite
	packageWriters map[dependency.Arch]*IndexWriter
}

// Create a new Component, configured for use.
func newComponent(suite *Suite) (*Component, error) {
	return &Component{
		suite:          suite,
		packageWriters: map[dependency.Arch]*IndexWriter{},
	}, nil
}

// Get a given IndexWriter for an arch, or create one if none exists.
func (c *Component) getWriter(arch dependency.Arch) (*IndexWriter, error) {
	if _, ok := c.packageWriters[arch]; !ok {
		writer, err := newIndexWriter(c.suite)
		if err != nil {
			return nil, err
		}
		c.packageWriters[arch] = writer
	}
	return c.packageWriters[arch], nil
}

// Add a given Package to a Package List. Under the hood, this will
// get or create a IndexWriter, and invoke the .Add method on the
// Package Writer.
func (c *Component) AddPackage(pkg Package) error {
	writer, err := c.getWriter(pkg.Architecture)
	if err != nil {
		return err
	}
	return writer.Add(pkg)
}

// }}}

// IndexWriter {{{

// This writer represents a Package list - which is to say, a list of
// binary .deb files, for a particular Architecture, in a particular Component
// in a particular Suite, in a particular Archive.
//
// This is not an encapsulation to store the entire Index in memory, rather,
// it's a wrapper to help write Package entries into the Index.
type IndexWriter struct {
	archive *Archive

	handle  *blobstore.Writer
	closer  func() error
	encoder *control.Encoder

	hashers []*transput.Hasher
}

func getHashers(suite *Suite) (io.Writer, []*transput.Hasher, error) {
	ret := []*transput.Hasher{}
	writers := []io.Writer{}

	for _, algo := range suite.features.Hashes {
		hasher, err := transput.NewHasher(algo)
		if err != nil {
			return nil, nil, err
		}
		writers = append(writers, hasher)
		ret = append(ret, hasher)
	}

	return io.MultiWriter(writers...), ret, nil
}

// given a Suite, create a new Package Writer, configured with
// the appropriate Hashing, and targeting a new file blob in the
// underlying blobstore.
func newIndexWriter(suite *Suite) (*IndexWriter, error) {
	handle, err := suite.archive.Store.Create()
	if err != nil {
		return nil, err
	}

	writer, hashers, err := getHashers(suite)
	if err != nil {
		return nil, err
	}

	encoder, err := control.NewEncoder(io.MultiWriter(writer, handle))
	if err != nil {
		handle.Close()
		return nil, err
	}

	return &IndexWriter{
		archive: suite.archive,
		closer:  handle.Close,
		encoder: encoder,
		handle:  handle,
		hashers: hashers,
	}, nil
}

// Write a Package entry into the Packages index.
func (p IndexWriter) Add(data interface{}) error {
	return p.encoder.Encode(data)
}

// }}}

// vim: foldmethod=marker
<commit_msg>Ensure we're always in UTC<commit_after>package archive

import (
	"fmt"
	"io"
	"path"
	"path/filepath"
	"time"

	"crypto"
	"crypto/sha512"

	"golang.org/x/crypto/openpgp"
	"golang.org/x/crypto/openpgp/clearsign"
	"golang.org/x/crypto/openpgp/packet"

	"pault.ag/go/blobstore"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/transput"
)

// Archive {{{

// Core Archive abstrcation. This contains helpers to write out package files,
// as well as handles creating underlying abstractions (such as Suites).
type Archive struct {
	Store      blobstore.Store
	signingKey *openpgp.Entity
	path       string
	Pool       Pool
}

// Create a new Archive at the given `root` on the filesystem, with the
// openpgp.Entity `signer` (an Entity which contains an OpenPGP Private
// Key).
//
// This interface is intended to *write* Archives, not *read* them. Extra
// steps must be taken to load an Archive over the network, and attention
// must be paid when handling the Cryptographic chain of trust.
func New(path string, signer *openpgp.Entity) (*Archive, error) {
	var err error
	path, err = filepath.Abs(path)
	if err != nil {
		return nil, err
	}

	store, err := blobstore.Load(path)
	if err != nil {
		return nil, err
	}

	return &Archive{
		Store:      *store,
		signingKey: signer,
		path:       path,
		Pool:       Pool{Store: *store},
	}, nil
}

func (a Archive) Path() string {
	return a.path
}

// Use the default backend to remove any unlinked files from the Blob store.
//
// If files you care about are not linked onto the stage, they will be removed
// by the garbage collector. Decruft only when you're sure the stage has been
// set.
func (a Archive) Decruft() error {
	return a.Store.GC(blobstore.DumbGarbageCollector{})
}

// Given a list of objects, link them to the keyed paths.
func (a Archive) Link(blobs ArchiveState) error {
	for path, obj := range blobs {
		if err := a.Store.Link(obj, path); err != nil {
			return err
		}
	}
	return nil
}

// Create a new Release object from a Suite, passing off the Name, Description
// and constructing the rest of the goodies.
//
// This will be an entirely empty object, without anything read off disk.
func newRelease(suite Suite) (*Release, error) {
	when := time.Now()

	var validUntil string = ""
	if suite.features.Duration != "" {
		duration, err := time.ParseDuration(suite.features.Duration)
		if err != nil {
			return nil, err
		}
		validUntil = when.Add(duration).Format(time.RFC1123Z)
	}

	release := Release{
		Suite:       suite.Name,
		Description: suite.Description,
		ValidUntil:  validUntil,
		Origin:      suite.Origin,
		Label:       suite.Label,
		Version:     suite.Version,
	}
	release.Date = when.In(time.UTC).Format(time.RFC1123Z)
	release.Architectures = []dependency.Arch{}
	release.Components = []string{}
	release.SHA256 = []control.SHA256FileHash{}
	release.SHA1 = []control.SHA1FileHash{}
	release.SHA512 = []control.SHA512FileHash{}
	release.MD5Sum = []control.MD5FileHash{}
	return &release, nil
}

// This is a set of file changes ready to be passed to `Link` to link in.
// Basically, this maps file paths to blobstore objects, which will be
// swapped in all at once. This allows errors to avoid mutating state in
// the archive.
type ArchiveState map[string]blobstore.Object

// Engross a Suite for signing and final commit into the blobstore. This
// will return handle(s) to the signed and ready Objects, fit for passage
// to Link.
//
// This will contain all the related Packages and Release files.
func (a Archive) Engross(suite Suite) (ArchiveState, error) {
	release, err := newRelease(suite)
	if err != nil {
		return nil, err
	}

	files := ArchiveState{}
	arches := map[dependency.Arch]bool{}

	for name, component := range suite.components {
		release.Components = append(release.Components, name)
		for arch, writer := range component.packageWriters {
			arches[arch] = true

			// For each Binary entry, do the same as above (todo: someone
			// DRY this out a bit. I'm too lazy.

			suitePath := path.Join(name, fmt.Sprintf("binary-%s", arch),
				"Packages")

			obj, err := a.Store.Commit(*writer.handle)
			if err != nil {
				return nil, err
			}

			for _, hasher := range writer.hashers {
				fileHash := control.FileHashFromHasher(suitePath, *hasher)
				release.AddHash(fileHash)
			}

			filePath := path.Join("dists", suite.Name, suitePath)
			files[filePath] = *obj
		}
	}

	for arch, _ := range arches {
		release.Architectures = append(release.Architectures, arch)
	}

	/* Now, let's do some magic */

	// Now, let's write out the Release file (and sign it normally)
	obj, sig, err := suite.archive.encodeSigned(release)
	if err != nil {
		return nil, err
	}

	filePath := path.Join("dists", suite.Name, "Release")
	files[filePath] = *obj
	files[fmt.Sprintf("%s.gpg", filePath)] = *sig

	// Ditto with the clearsigned version (Should we merge the two above?)
	obj, err = suite.archive.encodeClearsigned(release)
	if err != nil {
		return nil, err
	}

	files[path.Join("dists", suite.Name, "InRelease")] = *obj

	return files, nil
}

// Given a control.Marshal'able object, encode it to the blobstore, while
// also clearsigning the data.
func (a Archive) encodeClearsigned(data interface{}) (*blobstore.Object, error) {

	if a.signingKey == nil {
		return nil, fmt.Errorf("No signing key loaded")
	}

	fd, err := a.Store.Create()
	if err != nil {
		return nil, err
	}

	defer fd.Close()
	wc, err := clearsign.Encode(fd, a.signingKey.PrivateKey, nil)
	if err != nil {
		return nil, err
	}

	encoder, err := control.NewEncoder(wc)
	if err != nil {
		return nil, err
	}

	if err := encoder.Encode(data); err != nil {
		return nil, err
	}

	if err := wc.Close(); err != nil {
		return nil, err
	}

	return a.Store.Commit(*fd)
}

// Given a control.Marshal'able object, encode it to the blobstore, while
// also doing a detached OpenPGP signature. The objects returned (in order)
// are data, commited to the blobstore, the signature for that object, commited
// to the blobstore, and any error(s), finally.
func (a Archive) encodeSigned(data interface{}) (*blobstore.Object, *blobstore.Object, error) {
	/* Right, so, the trick here is that we secretly call out to encode,
	 * but tap it with a pipe into the signing code */

	if a.signingKey == nil {
		return nil, nil, fmt.Errorf("No signing key loaded")
	}

	signature, err := a.Store.Create()
	if err != nil {
		return nil, nil, err
	}
	defer signature.Close()

	hash := sha512.New()

	obj, err := a.encode(data, hash)
	if err != nil {
		return nil, nil, err
	}

	sig := new(packet.Signature)
	sig.SigType = packet.SigTypeBinary
	sig.PubKeyAlgo = a.signingKey.PrivateKey.PubKeyAlgo

	sig.Hash = crypto.SHA512

	sig.CreationTime = new(packet.Config).Now()
	sig.IssuerKeyId = &(a.signingKey.PrivateKey.KeyId)

	err = sig.Sign(hash, a.signingKey.PrivateKey, nil)
	if err != nil {
		return nil, nil, err
	}

	if err := sig.Serialize(signature); err != nil {
		return nil, nil, err
	}

	sigObj, err := a.Store.Commit(*signature)
	if err != nil {
		return nil, nil, err
	}

	return obj, sigObj, nil

}

// Encode a given control.Marshal'able object into the Blobstore, and return
// a handle to its object.
//
// The optinal argument `tap` will be written to as the object gets sent into
// the blobstore. This may be useful if you wish to have a copy of the data
// going into the store.
func (a Archive) encode(data interface{}, tap io.Writer) (*blobstore.Object, error) {
	fd, err := a.Store.Create()
	if err != nil {
		return nil, err
	}

	var writer io.Writer = fd
	if tap != nil {
		writer = io.MultiWriter(fd, tap)
	}

	encoder, err := control.NewEncoder(writer)
	if err != nil {
		return nil, err
	}

	if err := encoder.Encode(data); err != nil {
		return nil, err
	}

	return a.Store.Commit(*fd)
}

// }}}

// Suite {{{

// Abstraction to handle writing data into a Suite. This is a write-only
// target, and is not intended to read a Release file.
//
// This contains no state read off disk, and is purely for writing to.
type Suite struct {
	control.Paragraph

	archive *Archive

	Name        string `control:"Suite"`
	Description string
	Origin      string
	Label       string
	Version     string

	components map[string]*Component `control:"-"`

	features struct {
		Hashes   []string
		Duration string
	} `control:"-"`
}

// Get a handle to write a given Suite from an Archive.
// The suite will be entirely blank, and attributes will not be
// read from the existing files, if any.
func (a Archive) Suite(name string) (*Suite, error) {
	suite := Suite{
		Name:       name,
		archive:    &a,
		components: map[string]*Component{},
	}

	suite.features.Hashes = []string{"sha256", "sha1", "sha512"}
	suite.features.Duration = "240h"

	return &suite, nil
}

// Get or create a Component for a given Suite. If no such Component
// has been created so far, this will create a new object, otherwise
// it will return the existing entry.
//
// This contains no state read off disk, and is purely for writing to.
func (s Suite) Component(name string) (*Component, error) {
	if _, ok := s.components[name]; !ok {
		comp, err := newComponent(&s)
		if err != nil {
			return nil, err
		}
		s.components[name] = comp
		return comp, nil
	}
	el := s.components[name]
	return el, nil
}

// }}}

// Component {{{

// Small wrapper to represent a Component of a Suite, which, at its core
// is simply a set of Indexes to be written to.
//
// This contains no state read off disk, and is purely for writing to.
type Component struct {
	suite          *Suite
	packageWriters map[dependency.Arch]*IndexWriter
}

// Create a new Component, configured for use.
func newComponent(suite *Suite) (*Component, error) {
	return &Component{
		suite:          suite,
		packageWriters: map[dependency.Arch]*IndexWriter{},
	}, nil
}

// Get a given IndexWriter for an arch, or create one if none exists.
func (c *Component) getWriter(arch dependency.Arch) (*IndexWriter, error) {
	if _, ok := c.packageWriters[arch]; !ok {
		writer, err := newIndexWriter(c.suite)
		if err != nil {
			return nil, err
		}
		c.packageWriters[arch] = writer
	}
	return c.packageWriters[arch], nil
}

// Add a given Package to a Package List. Under the hood, this will
// get or create a IndexWriter, and invoke the .Add method on the
// Package Writer.
func (c *Component) AddPackage(pkg Package) error {
	writer, err := c.getWriter(pkg.Architecture)
	if err != nil {
		return err
	}
	return writer.Add(pkg)
}

// }}}

// IndexWriter {{{

// This writer represents a Package list - which is to say, a list of
// binary .deb files, for a particular Architecture, in a particular Component
// in a particular Suite, in a particular Archive.
//
// This is not an encapsulation to store the entire Index in memory, rather,
// it's a wrapper to help write Package entries into the Index.
type IndexWriter struct {
	archive *Archive

	handle  *blobstore.Writer
	closer  func() error
	encoder *control.Encoder

	hashers []*transput.Hasher
}

func getHashers(suite *Suite) (io.Writer, []*transput.Hasher, error) {
	ret := []*transput.Hasher{}
	writers := []io.Writer{}

	for _, algo := range suite.features.Hashes {
		hasher, err := transput.NewHasher(algo)
		if err != nil {
			return nil, nil, err
		}
		writers = append(writers, hasher)
		ret = append(ret, hasher)
	}

	return io.MultiWriter(writers...), ret, nil
}

// given a Suite, create a new Package Writer, configured with
// the appropriate Hashing, and targeting a new file blob in the
// underlying blobstore.
func newIndexWriter(suite *Suite) (*IndexWriter, error) {
	handle, err := suite.archive.Store.Create()
	if err != nil {
		return nil, err
	}

	writer, hashers, err := getHashers(suite)
	if err != nil {
		return nil, err
	}

	encoder, err := control.NewEncoder(io.MultiWriter(writer, handle))
	if err != nil {
		handle.Close()
		return nil, err
	}

	return &IndexWriter{
		archive: suite.archive,
		closer:  handle.Close,
		encoder: encoder,
		handle:  handle,
		hashers: hashers,
	}, nil
}

// Write a Package entry into the Packages index.
func (p IndexWriter) Add(data interface{}) error {
	return p.encoder.Encode(data)
}

// }}}

// vim: foldmethod=marker
<|endoftext|>


<commit_before>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"strings"
	"text/template"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/nodetemple/nodetemple/version"
)

var (
	commandUsageTemplate *template.Template
	templFuncs           = template.FuncMap{
		"descToLines": func(s string) []string {
			return strings.Split(strings.Trim(s, "\n\t "), "\n")
		},
		"cmdName": func(cmd *cobra.Command, startCmd *cobra.Command) string {
			parts := []string{cmd.Name()}
			for cmd.HasParent() && cmd.Parent().Name() != startCmd.Name() {
				cmd = cmd.Parent()
				parts = append([]string{cmd.Name()}, parts...)
			}
			return strings.Join(parts, " ")
		},
	}
)

func init() {
	commandUsage := `
{{ $cmd := .Cmd }}\
{{ $cmdname := cmdName .Cmd .Cmd.Root }}\
NAME:
{{ if not .Cmd.HasParent }}\
{{printf "\t%s - %s" .Cmd.Name .Cmd.Short}}
{{else}}\
{{printf "\t%s - %s" $cmdname .Cmd.Short}}
{{end}}\

USAGE:
{{printf "\t%s" .Cmd.UseLine}}
{{ if not .Cmd.HasParent }}\

VERSION:
{{printf "\t%s" .Version}}
{{end}}\
{{if .Cmd.HasSubCommands}}\

COMMANDS:
{{range .SubCommands}}\
{{ $cmdname := cmdName . $cmd }}\
{{ if .Runnable }}\
{{printf "\t%s\t%s" $cmdname .Short}}
{{end}}\
{{end}}\
{{end}}\
{{ if .Cmd.Long }}\

DESCRIPTION:
{{range $line := descToLines .Cmd.Long}}{{printf "\t%s" $line}}
{{end}}\
{{end}}\
{{if .Cmd.HasLocalFlags}}\

OPTIONS:
{{.Cmd.LocalFlags.FlagUsages}}\
{{end}}\
{{if .Cmd.HasInheritedFlags}}\

GLOBAL OPTIONS:
{{.Cmd.InheritedFlags.FlagUsages}}

Global options can also be configured via upper-case environment variables prefixed with "{{.EnvFlag}}_".
For example: "--some-flag" => "{{.EnvFlag}}_SOME_FLAG".\
{{end}}
`[1:]

	commandUsageTemplate = template.Must(template.New("command_usage").Funcs(templFuncs).Parse(strings.Replace(commandUsage, "\\\n", "", -1)))
}

func getSubCommands(cmd *cobra.Command) []*cobra.Command {
	subCommands := []*cobra.Command{}
	for _, subCmd := range cmd.Commands() {
		subCommands = append(subCommands, subCmd)
		subCommands = append(subCommands, getSubCommands(subCmd)...)
	}
	return subCommands
}

func usageFunc(cmd *cobra.Command) error {
	subCommands := getSubCommands(cmd)
	commandUsageTemplate.Execute(tabOut, struct {
		Cmd         *cobra.Command
		CmdFlags    *pflag.FlagSet
		SubCommands []*cobra.Command
		EnvFlag     string
		Version     string
	}{
		cmd,
		cmd.Flags(),
		subCommands,
		strings.ToUpper(cliName),
		version.Version,
	})
	tabOut.Flush()
	return nil
}
<commit_msg>Expand help<commit_after>/*
Copyright 2015 Nodetemple <hostmaster@nodetemple.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"strings"
	"text/template"

	"github.com/spf13/cobra"
	"github.com/nodetemple/nodetemple/version"
)

var (
	commandUsageTemplate *template.Template
	templFuncs           = template.FuncMap{
		"descToLines": func(s string) []string {
			return strings.Split(strings.Trim(s, "\n\t "), "\n")
		},
		"cmdName": func(cmd *cobra.Command, startCmd *cobra.Command) string {
			parts := []string{cmd.Name()}
			for cmd.HasParent() && cmd.Parent().Name() != startCmd.Name() {
				cmd = cmd.Parent()
				parts = append([]string{cmd.Name()}, parts...)
			}
			return strings.Join(parts, " ")
		},
	}
)

func init() {
	commandUsage := `
{{ $cmd := .Cmd }}\
{{ $cmdname := cmdName .Cmd .Cmd.Root }}\
NAME:
{{ if not .Cmd.HasParent }}\
{{printf "\t%s - %s" .Cmd.Name .Cmd.Short}}
{{else}}\
{{printf "\t%s - %s" $cmdname .Cmd.Short}}
{{end}}\

USAGE:
{{printf "\t%s" .Cmd.UseLine}}
{{ if not .Cmd.HasParent }}\

VERSION:
{{printf "\t%s" .Version}}
{{end}}\
{{if .Cmd.HasSubCommands}}\

COMMANDS:
{{range .SubCommands}}\
{{ $cmdname := cmdName . $cmd }}\
{{ if .Runnable }}\
{{printf "\t%s\t%s" $cmdname .Short}}
{{end}}\
{{end}}\
{{end}}\
{{ if .Cmd.Long }}\

DESCRIPTION:
{{range $line := descToLines .Cmd.Long}}{{printf "\t%s" $line}}
{{end}}\
{{end}}\
{{if .Cmd.HasLocalFlags}}\

OPTIONS:
{{.Cmd.LocalFlags.FlagUsages}}\
{{end}}\
{{if .Cmd.HasInheritedFlags}}\

GLOBAL OPTIONS:
{{.Cmd.InheritedFlags.FlagUsages}}
Global options can also be configured via upper-case environment variables prefixed with "{{.EnvFlag}}_".
For example: "--some-flag" => "{{.EnvFlag}}_SOME_FLAG".
{{end}}\
{{ if .Cmd.HasSubCommands }}\

Run "{{.Cmd.Name}} help [command]" for more details on a specific command
{{else}}\

For help on global options run "{{.Cmd.Name}} help"
{{end}}`[1:]

	commandUsageTemplate = template.Must(template.New("command_usage").Funcs(templFuncs).Parse(strings.Replace(commandUsage, "\\\n", "", -1)))
}

func getSubCommands(cmd *cobra.Command) []*cobra.Command {
	subCommands := []*cobra.Command{}
	for _, subCmd := range cmd.Commands() {
		subCommands = append(subCommands, subCmd)
		subCommands = append(subCommands, getSubCommands(subCmd)...)
	}
	return subCommands
}

func usageFunc(cmd *cobra.Command) error {
	subCommands := getSubCommands(cmd)
	commandUsageTemplate.Execute(tabOut, struct {
		Cmd         *cobra.Command
		SubCommands []*cobra.Command
		EnvFlag     string
		Version     string
	}{
		cmd,
		subCommands,
		strings.ToUpper(cliName),
		version.Version,
	})
	tabOut.Flush()
	return nil
}
<|endoftext|>


<commit_before>package main

// This file contains the implementation of an arduino interface

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/tarm/serial"
)

var cmd = `#!/bin/bash
for sysdevpath in $(find /sys/bus/usb/devices/usb*/ -name dev); do
(
syspath="${sysdevpath%/dev}"
devname="$(udevadm info -q name -p $syspath)"
[[ "$devname" == "bus/"* ]] && continue
eval "$(udevadm info -q property --export -p $syspath)"
[[ -z "$ID_SERIAL" ]] && continue
echo -e "/dev/$devname - $ID_SERIAL\n"
)
done `

func sendError(timeout time.Duration, err error, errorC chan<- error) {
	select {
	case <-time.After(timeout):
	case errorC <- err:
	}
}

func run(timeout time.Duration, outputC chan string, errorC chan error, command string, args ...string) {

	defer func() {
		errorC <- nil
	}()

	// instantiate new command
	cmd := exec.Command(command, args...)

	// get pipe to standard output
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		sendError(timeout, err, errorC)
		return
	}

	// start process via command
	if err := cmd.Start(); err != nil {
		sendError(timeout, err, errorC)
		return
	}

	// setup a buffer to capture standard output
	var buf bytes.Buffer

	// create a channel to capture any errors from wait
	done := make(chan error)
	go func() {
		if _, err := buf.ReadFrom(stdout); err != nil {
			sendError(timeout, err, errorC)
		}
		done <- cmd.Wait()
	}()

	// block on select, and switch based on actions received
	select {
	case <-time.After(timeout):
		if err := cmd.Process.Kill(); err != nil {
			sendError(timeout, err, errorC)
			return
		}
		sendError(timeout, fmt.Errorf("process timed out"), errorC)
		return
	case err := <-done:
		if err != nil {
			sendError(timeout, err, errorC)
			close(done)

		}
		outputC <- buf.String()
	}
}

// findArduinos locates devices that implement the Arduino
// serial connection.
//
// This function returns a collection of the devices
// that are likely candidates for Arduino.
//
func findArduinos() (devices [][]string, err error) {

	// Create a script to do some device discovery
	tmpfile, err := ioutil.TempFile("", "example")
	if err != nil {
		return nil, err
	}

	defer os.Remove(tmpfile.Name()) // clean up

	if _, err = tmpfile.Write([]byte(cmd)); err != nil {
		return nil, err
	}
	os.Chmod(tmpfile.Name(), 0700)
	if err = tmpfile.Close(); err != nil {
		return nil, err
	}

	devices = [][]string{}

	outputC := make(chan string, 1)
	defer close(outputC)

	errorC := make(chan error, 1)
	defer close(errorC)

	go run(time.Duration(5*time.Second), outputC, errorC, tmpfile.Name())

	for {
		select {
		case dev := <-outputC:
			for _, line := range strings.Split(dev, "\n") {
				if strings.Contains(line, "arduino") {
					details := strings.Split(line, " -")
					serial := strings.Split(line, "_")
					devices = append(devices, []string{details[0], strings.TrimSpace(serial[len(serial)-1])})
					continue
				}
				if strings.Contains(line, "ttyUSB") && strings.Contains(line, "USB_UART") {
					details := strings.Split(line, " -")
					serial := strings.Split(line, " -/")[1]
					devices = append(devices, []string{details[0], strings.TrimSpace(serial)})
					continue
				}
			}
		case err = <-errorC:
			if err == nil {
				return devices, nil
			}
			return nil, err
		}
	}
}

type arduino struct {
	port    *serial.Port
	portal  string // The name of ingress portal that this control device is associated with
	devName string // The tty style device name
	role    string // The type of arduino that is present, core, or resonator cluster
}

// startDevice is used to start an individual arduino USB Serial device
//
func startDevice(portalName string, devName string) (device *arduino, err error) {

	device = &arduino{}

	device.port, err = serial.OpenPort(&serial.Config{Name: devName, Baud: 115200, ReadTimeout: time.Duration(time.Second * 5)})

	if err != nil {
		logW.Error(fmt.Sprintf("unable to open arduino at %s due to %s", devName, err.Error()), "error", err)
		return nil, err
	}

	// Let the device stabilize before continuing
	select {
	case <-time.After(2 * time.Second):
	}

	if device.role, err = device.ping(); err != nil {
		device.close()

		logW.Error(fmt.Sprintf("unable to ping arduino at %s due to %s", devName, err.Error()), "error", err)
		return nil, err
	}

	device.devName = devName
	device.portal = portalName
	return device, nil
}

func findDevices() (devices []string) {
	// Parse the comma seperated device list
	devices = strings.Split(*arduinos, ",")

	// If the user did not specify arduinos to be used add then automatically
	if len(devices) == 1 && len(devices[0]) == 0 {
		deviceCatalog, err := findArduinos()
		if err != nil {
			logW.Error(err.Error())
			os.Exit(-3)
		}
		if len(deviceCatalog) == 0 {
			logW.Warn("No arduinos were specified and none could not be found, software will continue running and looking for devices")
		}

		devices = make([]string, 0, len(deviceCatalog))
		for _, attribs := range deviceCatalog {
			logW.Info(fmt.Sprintf("Found arduino '%s' serial # '%s'", attribs[0], attribs[1]), "arduinoDevice", attribs[0], "audrinoSerial", attribs[1])
			devices = append(devices, attribs[0])
		}
	}
	return devices
}

func (dev *arduino) close() (err error) {
	defer func() {
		dev.port = nil
	}()

	return dev.port.Close()
}

func (dev *arduino) ping() (line string, err error) {

	dev.port.Flush()

	n, err := dev.port.Write([]byte("*\n"))
	if err != nil {
		return line, err
	}
	if n != 2 {
		logW.Warn(fmt.Sprintf("%d bytes written out of %d", n, 2))
	}

	buf := make([]byte, 256)
	reader := bufio.NewReader(dev.port)
	buf, err = reader.ReadBytes('\x0a')

	return strings.TrimSpace(string(buf)), nil
}

func (dev *arduino) sendCmd(cmd []byte) (err error) {

	// TODO Add an incremental write loop for serial devices
	n, err := dev.port.Write(cmd)
	if err != nil {
		return err
	}
	if n != len(cmd) {
		logW.Warn(fmt.Sprintf("%d bytes written out of %d", n, len(cmd)))
	}

	return nil
}
<commit_msg>Added debugging lines<commit_after>package main

// This file contains the implementation of an arduino interface

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/tarm/serial"
)

var cmd = `#!/bin/bash
for sysdevpath in $(find /sys/bus/usb/devices/usb*/ -name dev); do
(
syspath="${sysdevpath%/dev}"
devname="$(udevadm info -q name -p $syspath)"
[[ "$devname" == "bus/"* ]] && continue
eval "$(udevadm info -q property --export -p $syspath)"
[[ -z "$ID_SERIAL" ]] && continue
echo -e "/dev/$devname - $ID_SERIAL\n"
)
done `

func sendError(timeout time.Duration, err error, errorC chan<- error) {
	select {
	case <-time.After(timeout):
	case errorC <- err:
	}
}

func run(timeout time.Duration, outputC chan string, errorC chan error, command string, args ...string) {

	defer func() {
		errorC <- nil
	}()

	// instantiate new command
	cmd := exec.Command(command, args...)

	// get pipe to standard output
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		sendError(timeout, err, errorC)
		return
	}

	// start process via command
	if err := cmd.Start(); err != nil {
		sendError(timeout, err, errorC)
		return
	}

	// setup a buffer to capture standard output
	var buf bytes.Buffer

	// create a channel to capture any errors from wait
	done := make(chan error)
	go func() {
		if _, err := buf.ReadFrom(stdout); err != nil {
			sendError(timeout, err, errorC)
		}
		done <- cmd.Wait()
	}()

	// block on select, and switch based on actions received
	select {
	case <-time.After(timeout):
		if err := cmd.Process.Kill(); err != nil {
			sendError(timeout, err, errorC)
			return
		}
		sendError(timeout, fmt.Errorf("process timed out"), errorC)
		return
	case err := <-done:
		if err != nil {
			sendError(timeout, err, errorC)
			close(done)

		}
		outputC <- buf.String()
	}
}

// findArduinos locates devices that implement the Arduino
// serial connection.
//
// This function returns a collection of the devices
// that are likely candidates for Arduino.
//
func findArduinos() (devices [][]string, err error) {

	// Create a script to do some device discovery
	tmpfile, err := ioutil.TempFile("", "example")
	if err != nil {
		return nil, err
	}

	defer os.Remove(tmpfile.Name()) // clean up

	if _, err = tmpfile.Write([]byte(cmd)); err != nil {
		return nil, err
	}
	os.Chmod(tmpfile.Name(), 0700)
	if err = tmpfile.Close(); err != nil {
		return nil, err
	}

	devices = [][]string{}

	outputC := make(chan string, 1)
	defer close(outputC)

	errorC := make(chan error, 1)
	defer close(errorC)

	go run(time.Duration(5*time.Second), outputC, errorC, tmpfile.Name())

	for {
		select {
		case dev := <-outputC:
			for _, line := range strings.Split(dev, "\n") {
				if strings.Contains(line, "arduino") {
					details := strings.Split(line, " -")
					serial := strings.Split(line, "_")
					devices = append(devices, []string{details[0], strings.TrimSpace(serial[len(serial)-1])})
					continue
				}
				if strings.Contains(line, "ttyUSB") && strings.Contains(line, "USB_UART") {
					details := strings.Split(line, " -")
					lineParts := strings.Split(line, " -/")
					if len(lineParts) < 2 {
						logW.Warn(fmt.Sprintf("Unexpected device line format %s", line))
						continue
					}
					serial := lineParts[1]
					devices = append(devices, []string{details[0], strings.TrimSpace(serial)})
					continue
				}
			}
		case err = <-errorC:
			if err == nil {
				return devices, nil
			}
			return nil, err
		}
	}
}

type arduino struct {
	port    *serial.Port
	portal  string // The name of ingress portal that this control device is associated with
	devName string // The tty style device name
	role    string // The type of arduino that is present, core, or resonator cluster
}

// startDevice is used to start an individual arduino USB Serial device
//
func startDevice(portalName string, devName string) (device *arduino, err error) {

	device = &arduino{}

	device.port, err = serial.OpenPort(&serial.Config{Name: devName, Baud: 115200, ReadTimeout: time.Duration(time.Second * 5)})

	if err != nil {
		logW.Error(fmt.Sprintf("unable to open arduino at %s due to %s", devName, err.Error()), "error", err)
		return nil, err
	}

	// Let the device stabilize before continuing
	select {
	case <-time.After(2 * time.Second):
	}

	if device.role, err = device.ping(); err != nil {
		device.close()

		logW.Error(fmt.Sprintf("unable to ping arduino at %s due to %s", devName, err.Error()), "error", err)
		return nil, err
	}

	device.devName = devName
	device.portal = portalName
	return device, nil
}

func findDevices() (devices []string) {
	// Parse the comma seperated device list
	devices = strings.Split(*arduinos, ",")

	// If the user did not specify arduinos to be used add then automatically
	if len(devices) == 1 && len(devices[0]) == 0 {
		deviceCatalog, err := findArduinos()
		if err != nil {
			logW.Error(err.Error())
			os.Exit(-3)
		}
		if len(deviceCatalog) == 0 {
			logW.Warn("No arduinos were specified and none could not be found, software will continue running and looking for devices")
		}

		devices = make([]string, 0, len(deviceCatalog))
		for _, attribs := range deviceCatalog {
			logW.Info(fmt.Sprintf("Found arduino '%s' serial # '%s'", attribs[0], attribs[1]), "arduinoDevice", attribs[0], "audrinoSerial", attribs[1])
			devices = append(devices, attribs[0])
		}
	}
	return devices
}

func (dev *arduino) close() (err error) {
	defer func() {
		dev.port = nil
	}()

	return dev.port.Close()
}

func (dev *arduino) ping() (line string, err error) {

	dev.port.Flush()

	n, err := dev.port.Write([]byte("*\n"))
	if err != nil {
		return line, err
	}
	if n != 2 {
		logW.Warn(fmt.Sprintf("%d bytes written out of %d", n, 2))
	}

	buf := make([]byte, 256)
	reader := bufio.NewReader(dev.port)
	buf, err = reader.ReadBytes('\x0a')

	return strings.TrimSpace(string(buf)), nil
}

func (dev *arduino) sendCmd(cmd []byte) (err error) {

	// TODO Add an incremental write loop for serial devices
	n, err := dev.port.Write(cmd)
	if err != nil {
		return err
	}
	if n != len(cmd) {
		logW.Warn(fmt.Sprintf("%d bytes written out of %d", n, len(cmd)))
	}

	return nil
}
<|endoftext|>


<commit_before>// +build integration
//
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package integration

import (
	"context"
	"fmt"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"go.uber.org/zap"

	"github.com/m3db/m3/src/dbnode/client"
	"github.com/m3db/m3/src/dbnode/encoding"
	"github.com/m3db/m3/src/dbnode/storage/index"
	"github.com/m3db/m3/src/m3ninx/idx"
	"github.com/m3db/m3/src/query/storage/m3/consolidators"
	"github.com/m3db/m3/src/x/ident"
	xtime "github.com/m3db/m3/src/x/time"
)

// TestIndexWrites holds index writes for testing.
type TestIndexWrites []TestIndexWrite

// TestSeriesIterator is a minimal subset of encoding.SeriesIterator.
type TestSeriesIterator interface {
	encoding.Iterator

	// ID gets the ID of the series.
	ID() ident.ID

	// Tags returns an iterator over the tags associated with the ID.
	Tags() ident.TagIterator
}

// TestSeriesIterators is a an iterator over TestSeriesIterator.
type TestSeriesIterators interface {

	// Next moves to the next item.
	Next() bool

	// Current returns the current value.
	Current() TestSeriesIterator
}

type testSeriesIterators struct {
	encoding.SeriesIterators
	idx int
}

func (t *testSeriesIterators) Next() bool {
	if t.idx >= t.Len() {
		return false
	}
	t.idx++

	return true
}

func (t *testSeriesIterators) Current() TestSeriesIterator {
	return t.Iters()[t.idx-1]
}

// MatchesSeriesIters matches index writes with expected series.
func (w TestIndexWrites) MatchesSeriesIters(
	t *testing.T,
	seriesIters encoding.SeriesIterators,
) {
	actualCount := w.MatchesTestSeriesIters(t, &testSeriesIterators{SeriesIterators: seriesIters})

	uniqueIDs := make(map[string]struct{})
	for _, wi := range w {
		uniqueIDs[wi.ID.String()] = struct{}{}
	}
	require.Equal(t, len(uniqueIDs), actualCount)
}

// MatchesTestSeriesIters matches index writes with expected test series.
func (w TestIndexWrites) MatchesTestSeriesIters(
	t *testing.T,
	seriesIters TestSeriesIterators,
) int {
	writesByID := make(map[string]TestIndexWrites)
	for _, wi := range w {
		writesByID[wi.ID.String()] = append(writesByID[wi.ID.String()], wi)
	}
	var actualCount int
	for seriesIters.Next() {
		iter := seriesIters.Current()
		id := iter.ID().String()
		writes, ok := writesByID[id]
		require.True(t, ok, id)
		writes.matchesSeriesIter(t, iter)
		actualCount++
	}

	return actualCount
}

func (w TestIndexWrites) matchesSeriesIter(t *testing.T, iter TestSeriesIterator) {
	found := make([]bool, len(w))
	count := 0
	for iter.Next() {
		count++
		dp, _, _ := iter.Current()
		for i := 0; i < len(w); i++ {
			if found[i] {
				continue
			}
			wi := w[i]
			if !ident.NewTagIterMatcher(wi.Tags.Duplicate()).Matches(iter.Tags().Duplicate()) {
				require.FailNow(t, "tags don't match provided id", iter.ID().String())
			}
			if dp.TimestampNanos.Equal(wi.Timestamp) && dp.Value == wi.Value {
				found[i] = true
				break
			}
		}
	}
	require.Equal(t, len(w), count, iter.ID().String())
	require.NoError(t, iter.Err())
	for i := 0; i < len(found); i++ {
		require.True(t, found[i], iter.ID().String())
	}
}

// Write test data.
func (w TestIndexWrites) Write(t *testing.T, ns ident.ID, s client.Session) {
	for i := 0; i < len(w); i++ {
		wi := w[i]
		require.NoError(t, s.WriteTagged(ns,
			wi.ID,
			wi.Tags.Duplicate(),
			wi.Timestamp,
			wi.Value,
			xtime.Second,
			nil,
		), "%v", wi)
	}
}

// NumIndexed gets number of indexed series.
func (w TestIndexWrites) NumIndexed(t *testing.T, ns ident.ID, s client.Session) int {
	return w.NumIndexedWithOptions(t, ns, s, NumIndexedOptions{})
}

// NumIndexedOptions is options when performing num indexed check.
type NumIndexedOptions struct {
	Logger *zap.Logger
}

// NumIndexedWithOptions gets number of indexed series with a set of options.
func (w TestIndexWrites) NumIndexedWithOptions(
	t *testing.T,
	ns ident.ID,
	s client.Session,
	opts NumIndexedOptions,
) int {
	numFound := 0
	for i := 0; i < len(w); i++ {
		wi := w[i]
		q := newQuery(t, wi.Tags)
		iter, _, err := s.FetchTaggedIDs(ContextWithDefaultTimeout(), ns,
			index.Query{Query: q},
			index.QueryOptions{
				StartInclusive: wi.Timestamp.Add(-1 * time.Second),
				EndExclusive:   wi.Timestamp.Add(1 * time.Second),
				SeriesLimit:    10,
			})
		if err != nil {
			if l := opts.Logger; l != nil {
				l.Error("fetch tagged IDs error", zap.Error(err))
			}
			continue
		}
		if !iter.Next() {
			if l := opts.Logger; l != nil {
				l.Warn("missing result",
					zap.String("queryID", wi.ID.String()),
					zap.ByteString("queryTags", consolidators.MustIdentTagIteratorToTags(wi.Tags, nil).ID()))
			}
			continue
		}
		cuNs, cuID, cuTag := iter.Current()
		if ns.String() != cuNs.String() {
			if l := opts.Logger; l != nil {
				l.Warn("namespace mismatch",
					zap.String("queryNamespace", ns.String()),
					zap.String("resultNamespace", cuNs.String()))
			}
			continue
		}
		if wi.ID.String() != cuID.String() {
			if l := opts.Logger; l != nil {
				l.Warn("id mismatch",
					zap.String("queryID", wi.ID.String()),
					zap.String("resultID", cuID.String()))
			}
			continue
		}
		if !ident.NewTagIterMatcher(wi.Tags).Matches(cuTag) {
			if l := opts.Logger; l != nil {
				l.Warn("tag mismatch",
					zap.ByteString("queryTags", consolidators.MustIdentTagIteratorToTags(wi.Tags, nil).ID()),
					zap.ByteString("resultTags", consolidators.MustIdentTagIteratorToTags(cuTag, nil).ID()))
			}
			continue
		}
		numFound++
	}
	return numFound
}

type TestIndexWrite struct {
	ID        ident.ID
	Tags      ident.TagIterator
	Timestamp xtime.UnixNano
	Value     float64
}

// GenerateTestIndexWrite generates test index writes.
func GenerateTestIndexWrite(periodID, numWrites, numTags int, startTime, endTime xtime.UnixNano) TestIndexWrites {
	writes := make([]TestIndexWrite, 0, numWrites)
	step := endTime.Sub(startTime) / time.Duration(numWrites+1)
	for i := 0; i < numWrites; i++ {
		id, tags := genIDTags(periodID, i, numTags)
		writes = append(writes, TestIndexWrite{
			ID:        id,
			Tags:      tags,
			Timestamp: startTime.Add(time.Duration(i) * step).Truncate(time.Second),
			Value:     float64(i),
		})
	}
	return writes
}

type genIDTagsOption func(ident.Tags) ident.Tags

func genIDTags(i int, j int, numTags int, opts ...genIDTagsOption) (ident.ID, ident.TagIterator) {
	id := fmt.Sprintf("foo.%d.%d", i, j)
	tags := make([]ident.Tag, 0, numTags)
	for i := 0; i < numTags; i++ {
		tags = append(tags, ident.StringTag(
			fmt.Sprintf("%s.tagname.%d", id, i),
			fmt.Sprintf("%s.tagvalue.%d", id, i),
		))
	}
	tags = append(tags,
		ident.StringTag("common_i", strconv.Itoa(i)),
		ident.StringTag("common_j", strconv.Itoa(j)),
		ident.StringTag("shared", "shared"))

	result := ident.NewTags(tags...)
	for _, fn := range opts {
		result = fn(result)
	}

	return ident.StringID(id), ident.NewTagsIterator(result)
}

func isIndexed(t *testing.T, s client.Session, ns ident.ID, id ident.ID, tags ident.TagIterator) bool {
	result, err := isIndexedChecked(t, s, ns, id, tags)
	if err != nil {
		return false
	}
	return result
}

func isIndexedChecked(t *testing.T, s client.Session, ns ident.ID, id ident.ID, tags ident.TagIterator) (bool, error) {
	q := newQuery(t, tags)
	iter, _, err := s.FetchTaggedIDs(ContextWithDefaultTimeout(), ns,
		index.Query{Query: q},
		index.QueryOptions{
			StartInclusive: xtime.Now(),
			EndExclusive:   xtime.Now(),
			SeriesLimit:    10,
		})
	if err != nil {
		return false, err
	}

	defer iter.Finalize()

	if !iter.Next() {
		return false, nil
	}

	cuNs, cuID, cuTag := iter.Current()
	if err := iter.Err(); err != nil {
		return false, fmt.Errorf("iter err: %v", err)
	}

	if ns.String() != cuNs.String() {
		return false, fmt.Errorf("namespace not matched")
	}
	if id.String() != cuID.String() {
		return false, fmt.Errorf("id not matched")
	}
	if !ident.NewTagIterMatcher(tags).Matches(cuTag) {
		return false, fmt.Errorf("tags did not match")
	}

	return true, nil
}

func newQuery(t *testing.T, tags ident.TagIterator) idx.Query {
	tags = tags.Duplicate()
	filters := make([]idx.Query, 0, tags.Remaining())
	for tags.Next() {
		tag := tags.Current()
		tq := idx.NewTermQuery(tag.Name.Bytes(), tag.Value.Bytes())
		filters = append(filters, tq)
	}
	return idx.NewConjunctionQuery(filters...)
}

// ContextWithDefaultTimeout returns a context with a default timeout
// set of one minute.
func ContextWithDefaultTimeout() context.Context {
	ctx, _ := context.WithTimeout(context.Background(), time.Minute) //nolint
	return ctx
}
<commit_msg>[dbnode] Add WriteAttempt integration test index helper method (#3860)<commit_after>// +build integration
//
// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package integration

import (
	"context"
	"fmt"
	"strconv"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"go.uber.org/zap"

	"github.com/m3db/m3/src/dbnode/client"
	"github.com/m3db/m3/src/dbnode/encoding"
	"github.com/m3db/m3/src/dbnode/storage/index"
	"github.com/m3db/m3/src/m3ninx/idx"
	"github.com/m3db/m3/src/query/storage/m3/consolidators"
	"github.com/m3db/m3/src/x/ident"
	xtime "github.com/m3db/m3/src/x/time"
)

// TestIndexWrites holds index writes for testing.
type TestIndexWrites []TestIndexWrite

// TestSeriesIterator is a minimal subset of encoding.SeriesIterator.
type TestSeriesIterator interface {
	encoding.Iterator

	// ID gets the ID of the series.
	ID() ident.ID

	// Tags returns an iterator over the tags associated with the ID.
	Tags() ident.TagIterator
}

// TestSeriesIterators is a an iterator over TestSeriesIterator.
type TestSeriesIterators interface {

	// Next moves to the next item.
	Next() bool

	// Current returns the current value.
	Current() TestSeriesIterator
}

type testSeriesIterators struct {
	encoding.SeriesIterators
	idx int
}

func (t *testSeriesIterators) Next() bool {
	if t.idx >= t.Len() {
		return false
	}
	t.idx++

	return true
}

func (t *testSeriesIterators) Current() TestSeriesIterator {
	return t.Iters()[t.idx-1]
}

// MatchesSeriesIters matches index writes with expected series.
func (w TestIndexWrites) MatchesSeriesIters(
	t *testing.T,
	seriesIters encoding.SeriesIterators,
) {
	actualCount := w.MatchesTestSeriesIters(t, &testSeriesIterators{SeriesIterators: seriesIters})

	uniqueIDs := make(map[string]struct{})
	for _, wi := range w {
		uniqueIDs[wi.ID.String()] = struct{}{}
	}
	require.Equal(t, len(uniqueIDs), actualCount)
}

// MatchesTestSeriesIters matches index writes with expected test series.
func (w TestIndexWrites) MatchesTestSeriesIters(
	t *testing.T,
	seriesIters TestSeriesIterators,
) int {
	writesByID := make(map[string]TestIndexWrites)
	for _, wi := range w {
		writesByID[wi.ID.String()] = append(writesByID[wi.ID.String()], wi)
	}
	var actualCount int
	for seriesIters.Next() {
		iter := seriesIters.Current()
		id := iter.ID().String()
		writes, ok := writesByID[id]
		require.True(t, ok, id)
		writes.matchesSeriesIter(t, iter)
		actualCount++
	}

	return actualCount
}

func (w TestIndexWrites) matchesSeriesIter(t *testing.T, iter TestSeriesIterator) {
	found := make([]bool, len(w))
	count := 0
	for iter.Next() {
		count++
		dp, _, _ := iter.Current()
		for i := 0; i < len(w); i++ {
			if found[i] {
				continue
			}
			wi := w[i]
			if !ident.NewTagIterMatcher(wi.Tags.Duplicate()).Matches(iter.Tags().Duplicate()) {
				require.FailNow(t, "tags don't match provided id", iter.ID().String())
			}
			if dp.TimestampNanos.Equal(wi.Timestamp) && dp.Value == wi.Value {
				found[i] = true
				break
			}
		}
	}
	require.Equal(t, len(w), count, iter.ID().String())
	require.NoError(t, iter.Err())
	for i := 0; i < len(found); i++ {
		require.True(t, found[i], iter.ID().String())
	}
}

// Write writes test data and asserts the result.
func (w TestIndexWrites) Write(t *testing.T, ns ident.ID, s client.Session) {
	require.NoError(t, w.WriteAttempt(ns, s))
}

// WriteAttempt writes test data and returns an error if encountered.
func (w TestIndexWrites) WriteAttempt(ns ident.ID, s client.Session) error {
	for i := 0; i < len(w); i++ {
		wi := w[i]
		err := s.WriteTagged(ns, wi.ID, wi.Tags.Duplicate(), wi.Timestamp,
			wi.Value, xtime.Second, nil)
		if err != nil {
			return err
		}
	}
	return nil
}

// NumIndexed gets number of indexed series.
func (w TestIndexWrites) NumIndexed(t *testing.T, ns ident.ID, s client.Session) int {
	return w.NumIndexedWithOptions(t, ns, s, NumIndexedOptions{})
}

// NumIndexedOptions is options when performing num indexed check.
type NumIndexedOptions struct {
	Logger *zap.Logger
}

// NumIndexedWithOptions gets number of indexed series with a set of options.
func (w TestIndexWrites) NumIndexedWithOptions(
	t *testing.T,
	ns ident.ID,
	s client.Session,
	opts NumIndexedOptions,
) int {
	numFound := 0
	for i := 0; i < len(w); i++ {
		wi := w[i]
		q := newQuery(t, wi.Tags)
		iter, _, err := s.FetchTaggedIDs(ContextWithDefaultTimeout(), ns,
			index.Query{Query: q},
			index.QueryOptions{
				StartInclusive: wi.Timestamp.Add(-1 * time.Second),
				EndExclusive:   wi.Timestamp.Add(1 * time.Second),
				SeriesLimit:    10,
			})
		if err != nil {
			if l := opts.Logger; l != nil {
				l.Error("fetch tagged IDs error", zap.Error(err))
			}
			continue
		}
		if !iter.Next() {
			if l := opts.Logger; l != nil {
				l.Warn("missing result",
					zap.String("queryID", wi.ID.String()),
					zap.ByteString("queryTags", consolidators.MustIdentTagIteratorToTags(wi.Tags, nil).ID()))
			}
			continue
		}
		cuNs, cuID, cuTag := iter.Current()
		if ns.String() != cuNs.String() {
			if l := opts.Logger; l != nil {
				l.Warn("namespace mismatch",
					zap.String("queryNamespace", ns.String()),
					zap.String("resultNamespace", cuNs.String()))
			}
			continue
		}
		if wi.ID.String() != cuID.String() {
			if l := opts.Logger; l != nil {
				l.Warn("id mismatch",
					zap.String("queryID", wi.ID.String()),
					zap.String("resultID", cuID.String()))
			}
			continue
		}
		if !ident.NewTagIterMatcher(wi.Tags).Matches(cuTag) {
			if l := opts.Logger; l != nil {
				l.Warn("tag mismatch",
					zap.ByteString("queryTags", consolidators.MustIdentTagIteratorToTags(wi.Tags, nil).ID()),
					zap.ByteString("resultTags", consolidators.MustIdentTagIteratorToTags(cuTag, nil).ID()))
			}
			continue
		}
		numFound++
	}
	return numFound
}

type TestIndexWrite struct {
	ID        ident.ID
	Tags      ident.TagIterator
	Timestamp xtime.UnixNano
	Value     float64
}

// GenerateTestIndexWrite generates test index writes.
func GenerateTestIndexWrite(periodID, numWrites, numTags int, startTime, endTime xtime.UnixNano) TestIndexWrites {
	writes := make([]TestIndexWrite, 0, numWrites)
	step := endTime.Sub(startTime) / time.Duration(numWrites+1)
	for i := 0; i < numWrites; i++ {
		id, tags := genIDTags(periodID, i, numTags)
		writes = append(writes, TestIndexWrite{
			ID:        id,
			Tags:      tags,
			Timestamp: startTime.Add(time.Duration(i) * step).Truncate(time.Second),
			Value:     float64(i),
		})
	}
	return writes
}

type genIDTagsOption func(ident.Tags) ident.Tags

func genIDTags(i int, j int, numTags int, opts ...genIDTagsOption) (ident.ID, ident.TagIterator) {
	id := fmt.Sprintf("foo.%d.%d", i, j)
	tags := make([]ident.Tag, 0, numTags)
	for i := 0; i < numTags; i++ {
		tags = append(tags, ident.StringTag(
			fmt.Sprintf("%s.tagname.%d", id, i),
			fmt.Sprintf("%s.tagvalue.%d", id, i),
		))
	}
	tags = append(tags,
		ident.StringTag("common_i", strconv.Itoa(i)),
		ident.StringTag("common_j", strconv.Itoa(j)),
		ident.StringTag("shared", "shared"))

	result := ident.NewTags(tags...)
	for _, fn := range opts {
		result = fn(result)
	}

	return ident.StringID(id), ident.NewTagsIterator(result)
}

func isIndexed(t *testing.T, s client.Session, ns ident.ID, id ident.ID, tags ident.TagIterator) bool {
	result, err := isIndexedChecked(t, s, ns, id, tags)
	if err != nil {
		return false
	}
	return result
}

func isIndexedChecked(t *testing.T, s client.Session, ns ident.ID, id ident.ID, tags ident.TagIterator) (bool, error) {
	q := newQuery(t, tags)
	iter, _, err := s.FetchTaggedIDs(ContextWithDefaultTimeout(), ns,
		index.Query{Query: q},
		index.QueryOptions{
			StartInclusive: xtime.Now(),
			EndExclusive:   xtime.Now(),
			SeriesLimit:    10,
		})
	if err != nil {
		return false, err
	}

	defer iter.Finalize()

	if !iter.Next() {
		return false, nil
	}

	cuNs, cuID, cuTag := iter.Current()
	if err := iter.Err(); err != nil {
		return false, fmt.Errorf("iter err: %v", err)
	}

	if ns.String() != cuNs.String() {
		return false, fmt.Errorf("namespace not matched")
	}
	if id.String() != cuID.String() {
		return false, fmt.Errorf("id not matched")
	}
	if !ident.NewTagIterMatcher(tags).Matches(cuTag) {
		return false, fmt.Errorf("tags did not match")
	}

	return true, nil
}

func newQuery(t *testing.T, tags ident.TagIterator) idx.Query {
	tags = tags.Duplicate()
	filters := make([]idx.Query, 0, tags.Remaining())
	for tags.Next() {
		tag := tags.Current()
		tq := idx.NewTermQuery(tag.Name.Bytes(), tag.Value.Bytes())
		filters = append(filters, tq)
	}
	return idx.NewConjunctionQuery(filters...)
}

// ContextWithDefaultTimeout returns a context with a default timeout
// set of one minute.
func ContextWithDefaultTimeout() context.Context {
	ctx, _ := context.WithTimeout(context.Background(), time.Minute) //nolint
	return ctx
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Citrix Systems, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package netscaler

import (
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"strings"
)

//AddResource adds a resource of supplied type and name
func (c *NitroClient) AddResource(resourceType string, name string, resourceStruct interface{}) (string, error) {

	if c.ResourceExists(resourceType, name) == false {

		nsResource := make(map[string]interface{})
		nsResource[resourceType] = resourceStruct

		resourceJSON, err := json.Marshal(nsResource)

		log.Println("Resourcejson is " + string(resourceJSON))

		body, err := c.createResource(resourceType, resourceJSON)
		if err != nil {
			log.Fatal("Failed to create resource of type %s, name=%s, err=%s", resourceType, name, err)
			return "", err
		}
		_ = body
	}

	return name, nil
}

//UpdateResource updates a resource of supplied type and name
func (c *NitroClient) UpdateResource(resourceType string, name string, resourceStruct interface{}) (string, error) {

	if c.ResourceExists(resourceType, name) == true {
		nsResource := make(map[string]interface{})
		nsResource[resourceType] = resourceStruct
		resourceJSON, err := json.Marshal(nsResource)

		log.Println("Resourcejson is " + string(resourceJSON))

		body, err := c.updateResource(resourceType, name, resourceJSON)
		if err != nil {
			log.Fatal(fmt.Sprintf("Failed to update resource of type %s, name=%s err=%s", resourceType, name, err))
			return "", err
		}
		_ = body
	}

	return name, nil
}

//DeleteResource deletes a resource of supplied type and name
func (c *NitroClient) DeleteResource(resourceType string, resourceName string) error {

	_, err := c.listResource(resourceType, resourceName)
	if err == nil { // resource exists
		log.Printf("Found resource of type %s: %s", resourceType, resourceName)
		_, err = c.deleteResource(resourceType, resourceName)
		if err != nil {
			log.Println(fmt.Sprintf("Failed to delete resourceType %s: %s, err=%s", resourceType, resourceName, err))
			return err
		}
	} else {
		log.Printf("Resource %s already deleted ", resourceName)
	}
	return nil
}

//BindResource binds the 'bindingResourceName' to the 'bindToResourceName'.
func (c *NitroClient) BindResource(bindToResourceType string, bindToResourceName string, bindingResourceType string, bindingResourceName string, bindingStruct interface{}) error {
	if c.ResourceExists(bindToResourceType, bindToResourceName) == false {
		return fmt.Errorf("BindTo Resource %s of type %s does not exist", bindToResourceType, bindToResourceName)
	}

	if c.ResourceExists(bindingResourceType, bindingResourceName) == false {
		return fmt.Errorf("Binding Resource %s of type %s does not exist", bindingResourceType, bindingResourceName)
	}
	bindingName := bindToResourceType + "_" + bindingResourceType + "_binding"
	nsBinding := make(map[string]interface{})
	nsBinding[bindingName] = bindingStruct

	resourceJSON, err := json.Marshal(nsBinding)

	body, err := c.createResource(bindingName, resourceJSON)
	if err != nil {
		log.Fatal("Failed to bind resource %s to resource %s, err=%s", bindToResourceName, bindingResourceName, err)
		return err
	}
	_ = body
	return nil
}

//UnbindResource unbinds 'boundResourceName' from 'boundToResource'
func (c *NitroClient) UnbindResource(boundToResourceType string, boundToResourceName string, boundResourceType string, boundResourceName string, bindingFilterName string) error {

	if c.ResourceExists(boundToResourceType, boundToResourceName) == false {
		log.Println(fmt.Sprintf("Unbind: BoundTo Resource %s of type %s does not exist", boundToResourceType, boundToResourceName))
		return nil
	}

	if c.ResourceExists(boundResourceType, boundResourceName) == false {
		log.Println("Unbind: Bound Resource %s of type %s does not exist", boundResourceType, boundResourceName)
		return nil
	}

	_, err := c.unbindResource(boundToResourceType, boundToResourceName, boundResourceType, boundResourceName, bindingFilterName)
	if err != nil {
		return fmt.Errorf("Failed to unbind  %s:%s from %s:%s, err=%s", boundResourceType, boundResourceName, boundToResourceType, boundToResourceName, err)
	}

	return nil
}

//ResourceExists returns true if supplied resource name and type exists
func (c *NitroClient) ResourceExists(resourceType string, resourceName string) bool {
	_, err := c.listResource(resourceType, resourceName)
	if err != nil {
		log.Printf("No %s %s found", resourceType, resourceName)
		return false
	}
	log.Printf("%s %s is already present", resourceType, resourceName)
	return true
}

//FindResource returns the config of the supplied resource name and type if it exists
func (c *NitroClient) FindResource(resourceType string, resourceName string) (map[string]interface{}, error) {
	var data map[string]interface{}
	result, err := c.listResource(resourceType, resourceName)
	if err != nil {
		log.Printf("No %s %s found", resourceType, resourceName)
		return data, fmt.Errorf("No resource %s of type %s found", resourceName, resourceType)
	}
	if err = json.Unmarshal(result, &data); err != nil {
		log.Println("Failed to unmarshal Netscaler Response!")
		return data, fmt.Errorf("Failed to unmarshal Netscaler Response:resource %s of type %s", resourceName, resourceType)
	}
	if data[resourceType] == nil {
		log.Printf("No %s %s found", resourceType, resourceName)
		return data, fmt.Errorf("No resource %s of type %s found", resourceName, resourceType)
	}
	resource := data[resourceType].([]interface{})[0] //only one resource obviously

	return resource.(map[string]interface{}), nil
}

//ResourceBindingExists returns true if the supplied binding exists
func (c *NitroClient) ResourceBindingExists(resourceType string, resourceName string, boundResourceType string, boundResourceFilterName string, boundResourceFilterValue string) bool {
	result, err := c.listBoundResources(resourceName, resourceType, boundResourceType, boundResourceFilterName, boundResourceFilterValue)
	if err != nil {
		log.Printf("No %s %s to %s %s binding found", resourceType, resourceName, boundResourceType, boundResourceFilterValue)
		return false
	}

	var data map[string]interface{}
	if err := json.Unmarshal(result, &data); err != nil {
		log.Println("Failed to unmarshal Netscaler Response!")
		return false
	}
	if data[fmt.Sprintf("%s_%s_binding", resourceType, boundResourceType)] == nil {
		return false
	}

	log.Printf("%s of type  %s is bound to %s type and name %s", resourceType, resourceName, boundResourceType, boundResourceFilterValue)
	return true
}

//Enable a list of features
func (c *NitroClient) EnableFeatures(featureNames []string) error {
	/* construct this:
	{
	        "nsfeature":
		{
		    "feature": [ "LB", ]
		}
	}
	*/
	featureStruct := make(map[string]map[string][]string)
	featureStruct["nsfeature"] = make(map[string][]string)
	featureStruct["nsfeature"]["feature"] = featureNames

	featureJSON, err := json.Marshal(featureStruct)
	if err != nil {
		log.Println("Failed to marshal features to JSON")
		return fmt.Errorf("Failed to marshal features to JSON")
	}

	_, err = c.enableFeatures(featureJSON)
	if err != nil {
		return fmt.Errorf("Failed to enable feature ", err)
	}
	return nil
}

//Determine list of features
func (c *NitroClient) ListEnabledFeatures() ([]string, error) {

	bytes, err := c.listEnabledFeatures()
	if err != nil {
		return []string{}, fmt.Errorf("Failed to enable feature ", err)
	}
	var data map[string]interface{}
	if err = json.Unmarshal(bytes, &data); err != nil {
		log.Println("Failed to unmarshal Netscaler Response!")
		return []string{}, fmt.Errorf("Failed to unmarshal Netscaler Response to list Features")
	}
	if data["nsfeature"] == nil {
		log.Printf("No features found")
		return []string{}, fmt.Errorf("No features found")
	}
	features := data["nsfeature"].(map[string]interface{})
	log.Println("Features: ", features)
	log.Println("Features[feature]: ", features["feature"], " type ", reflect.TypeOf(features["feature"]))
	//result := make([]string, len(features["feature"]), len(features["feature"]))
	//for i, v := range features["feature"] {
	//result[i] = v.(string)
	//}

	//return result, nil
	result := fmt.Sprintf("%v", features["feature"])
	result = strings.TrimPrefix(result, "[")
	result = strings.TrimSuffix(result, "]")
	flist := strings.Split(result, " ")
	log.Println("result: ", result, "flist: ", flist)
	//return features["feature"].([]string), nil
	return flist, nil
}
<commit_msg>remove / add some comments<commit_after>/*
Copyright 2016 Citrix Systems, Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package netscaler

import (
	"encoding/json"
	"fmt"
	"log"
	"strings"
)

//AddResource adds a resource of supplied type and name
func (c *NitroClient) AddResource(resourceType string, name string, resourceStruct interface{}) (string, error) {

	if c.ResourceExists(resourceType, name) == false {

		nsResource := make(map[string]interface{})
		nsResource[resourceType] = resourceStruct

		resourceJSON, err := json.Marshal(nsResource)

		log.Println("Resourcejson is " + string(resourceJSON))

		body, err := c.createResource(resourceType, resourceJSON)
		if err != nil {
			log.Fatal("Failed to create resource of type %s, name=%s, err=%s", resourceType, name, err)
			return "", err
		}
		_ = body
	}

	return name, nil
}

//UpdateResource updates a resource of supplied type and name
func (c *NitroClient) UpdateResource(resourceType string, name string, resourceStruct interface{}) (string, error) {

	if c.ResourceExists(resourceType, name) == true {
		nsResource := make(map[string]interface{})
		nsResource[resourceType] = resourceStruct
		resourceJSON, err := json.Marshal(nsResource)

		log.Println("Resourcejson is " + string(resourceJSON))

		body, err := c.updateResource(resourceType, name, resourceJSON)
		if err != nil {
			log.Fatal(fmt.Sprintf("Failed to update resource of type %s, name=%s err=%s", resourceType, name, err))
			return "", err
		}
		_ = body
	}

	return name, nil
}

//DeleteResource deletes a resource of supplied type and name
func (c *NitroClient) DeleteResource(resourceType string, resourceName string) error {

	_, err := c.listResource(resourceType, resourceName)
	if err == nil { // resource exists
		log.Printf("Found resource of type %s: %s", resourceType, resourceName)
		_, err = c.deleteResource(resourceType, resourceName)
		if err != nil {
			log.Println(fmt.Sprintf("Failed to delete resourceType %s: %s, err=%s", resourceType, resourceName, err))
			return err
		}
	} else {
		log.Printf("Resource %s already deleted ", resourceName)
	}
	return nil
}

//BindResource binds the 'bindingResourceName' to the 'bindToResourceName'.
func (c *NitroClient) BindResource(bindToResourceType string, bindToResourceName string, bindingResourceType string, bindingResourceName string, bindingStruct interface{}) error {
	if c.ResourceExists(bindToResourceType, bindToResourceName) == false {
		return fmt.Errorf("BindTo Resource %s of type %s does not exist", bindToResourceType, bindToResourceName)
	}

	if c.ResourceExists(bindingResourceType, bindingResourceName) == false {
		return fmt.Errorf("Binding Resource %s of type %s does not exist", bindingResourceType, bindingResourceName)
	}
	bindingName := bindToResourceType + "_" + bindingResourceType + "_binding"
	nsBinding := make(map[string]interface{})
	nsBinding[bindingName] = bindingStruct

	resourceJSON, err := json.Marshal(nsBinding)

	body, err := c.createResource(bindingName, resourceJSON)
	if err != nil {
		log.Fatal("Failed to bind resource %s to resource %s, err=%s", bindToResourceName, bindingResourceName, err)
		return err
	}
	_ = body
	return nil
}

//UnbindResource unbinds 'boundResourceName' from 'boundToResource'
func (c *NitroClient) UnbindResource(boundToResourceType string, boundToResourceName string, boundResourceType string, boundResourceName string, bindingFilterName string) error {

	if c.ResourceExists(boundToResourceType, boundToResourceName) == false {
		log.Println(fmt.Sprintf("Unbind: BoundTo Resource %s of type %s does not exist", boundToResourceType, boundToResourceName))
		return nil
	}

	if c.ResourceExists(boundResourceType, boundResourceName) == false {
		log.Println("Unbind: Bound Resource %s of type %s does not exist", boundResourceType, boundResourceName)
		return nil
	}

	_, err := c.unbindResource(boundToResourceType, boundToResourceName, boundResourceType, boundResourceName, bindingFilterName)
	if err != nil {
		return fmt.Errorf("Failed to unbind  %s:%s from %s:%s, err=%s", boundResourceType, boundResourceName, boundToResourceType, boundToResourceName, err)
	}

	return nil
}

//ResourceExists returns true if supplied resource name and type exists
func (c *NitroClient) ResourceExists(resourceType string, resourceName string) bool {
	_, err := c.listResource(resourceType, resourceName)
	if err != nil {
		log.Printf("No %s %s found", resourceType, resourceName)
		return false
	}
	log.Printf("%s %s is already present", resourceType, resourceName)
	return true
}

//FindResource returns the config of the supplied resource name and type if it exists
func (c *NitroClient) FindResource(resourceType string, resourceName string) (map[string]interface{}, error) {
	var data map[string]interface{}
	result, err := c.listResource(resourceType, resourceName)
	if err != nil {
		log.Printf("No %s %s found", resourceType, resourceName)
		return data, fmt.Errorf("No resource %s of type %s found", resourceName, resourceType)
	}
	if err = json.Unmarshal(result, &data); err != nil {
		log.Println("Failed to unmarshal Netscaler Response!")
		return data, fmt.Errorf("Failed to unmarshal Netscaler Response:resource %s of type %s", resourceName, resourceType)
	}
	if data[resourceType] == nil {
		log.Printf("No %s %s found", resourceType, resourceName)
		return data, fmt.Errorf("No resource %s of type %s found", resourceName, resourceType)
	}
	resource := data[resourceType].([]interface{})[0] //only one resource obviously

	return resource.(map[string]interface{}), nil
}

//ResourceBindingExists returns true if the supplied binding exists
func (c *NitroClient) ResourceBindingExists(resourceType string, resourceName string, boundResourceType string, boundResourceFilterName string, boundResourceFilterValue string) bool {
	result, err := c.listBoundResources(resourceName, resourceType, boundResourceType, boundResourceFilterName, boundResourceFilterValue)
	if err != nil {
		log.Printf("No %s %s to %s %s binding found", resourceType, resourceName, boundResourceType, boundResourceFilterValue)
		return false
	}

	var data map[string]interface{}
	if err := json.Unmarshal(result, &data); err != nil {
		log.Println("Failed to unmarshal Netscaler Response!")
		return false
	}
	if data[fmt.Sprintf("%s_%s_binding", resourceType, boundResourceType)] == nil {
		return false
	}

	log.Printf("%s of type  %s is bound to %s type and name %s", resourceType, resourceName, boundResourceType, boundResourceFilterValue)
	return true
}

//Enable a list of features
func (c *NitroClient) EnableFeatures(featureNames []string) error {
	/* construct this:
	{
	        "nsfeature":
		{
		    "feature": [ "LB", ]
		}
	}
	*/
	featureStruct := make(map[string]map[string][]string)
	featureStruct["nsfeature"] = make(map[string][]string)
	featureStruct["nsfeature"]["feature"] = featureNames

	featureJSON, err := json.Marshal(featureStruct)
	if err != nil {
		log.Println("Failed to marshal features to JSON")
		return fmt.Errorf("Failed to marshal features to JSON")
	}

	_, err = c.enableFeatures(featureJSON)
	if err != nil {
		return fmt.Errorf("Failed to enable feature ", err)
	}
	return nil
}

//Determine list of features
func (c *NitroClient) ListEnabledFeatures() ([]string, error) {

	bytes, err := c.listEnabledFeatures()
	if err != nil {
		return []string{}, fmt.Errorf("Failed to enable feature ", err)
	}
	var data map[string]interface{}
	if err = json.Unmarshal(bytes, &data); err != nil {
		log.Println("Failed to unmarshal Netscaler Response!")
		return []string{}, fmt.Errorf("Failed to unmarshal Netscaler Response to list Features")
	}
	if data["nsfeature"] == nil {
		log.Printf("No features found")
		return []string{}, fmt.Errorf("No features found")
	}
	features := data["nsfeature"].(map[string]interface{})
	// since the returned JSON map mixes boolean and array values, the unmarshal fails to figure out there
	// is an array. So we have to convert it to a string and then parse it.
	// this doesn't work: return features["feature"].([]string), nil
	// convert to string: [LB CS SSL] (note: no commas)

	result := fmt.Sprintf("%v", features["feature"])
	result = strings.TrimPrefix(result, "[")
	result = strings.TrimSuffix(result, "]")
	flist := strings.Split(result, " ")
	log.Println("result: ", result, "flist: ", flist)
	return flist, nil
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "os"
    "flag"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/defaults"
    "github.com/aws/aws-sdk-go/service/ec2"
)

const TAGKEY string = "poke"

var verbose bool = false

func describe_instances(svc *ec2.EC2) *ec2.DescribeInstancesOutput {

    resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput {
        Filters: []*ec2.Filter {
            &ec2.Filter {
                Name: aws.String("tag-key"),
                Values: []*string{aws.String(TAGKEY)},
            },
        },
    })

    if err != nil {
        panic(err)
    }

    return resp
}

func status(svc *ec2.EC2) {
    resp := describe_instances(svc)

    for _, res := range resp.Reservations {
        for _, inst := range res.Instances {
            if verbose {
                fmt.Println(*inst)
            } else {
                fmt.Println("{")
                fmt.Println("      InstanceId:", *inst.InstanceId)
                if inst.PublicIpAddress != nil {
                    fmt.Println(" PublicIPAddress:", *inst.PublicIpAddress)
                }
                if inst.PrivateIpAddress != nil {
                    fmt.Println("PrivateIPAddress:", *inst.PrivateIpAddress)
                }
                fmt.Println("    InstanceType:", *inst.InstanceType)
                fmt.Println("      LaunchTime:", inst.LaunchTime)
                fmt.Println("           State:", *inst.State.Name)
                fmt.Print(  "            Tags:")
                for _, tag:= range inst.Tags {
                    fmt.Printf(" (%s, %s)", *tag.Key, *tag.Value)
                }
                fmt.Println("\n}")
            }
        }
    }
}

func boot(svc *ec2.EC2) {
    count := int64(4)
    params := &ec2.RunInstancesInput {
        ImageId: aws.String("ami-9ff7e8af"),
        InstanceType: aws.String("t2.micro"),
        MinCount: &count,
        MaxCount: &count,
    }

    res, err := svc.RunInstances(params)
    if err != nil {
        panic(err)
    }

    tag_params := ec2.CreateTagsInput {
        Resources: nil,
        Tags: []*ec2.Tag {
            &ec2.Tag {
                Key: aws.String(TAGKEY),
                Value: aws.String("production"),
            },
        },
    }

    for _, inst := range res.Instances {
        tag_params.Resources = append(tag_params.Resources, inst.InstanceId)
    }

    _, err = svc.CreateTags(&tag_params)
    if err != nil {
        panic("Failed to tag Instaces")
    }
}

func terminate(svc *ec2.EC2) {
    desc := describe_instances(svc)

    for _, res := range desc.Reservations {
        var names []*string
        for _, inst := range res.Instances {
            names = append(names, inst.InstanceId)
        }

        params := &ec2.TerminateInstancesInput {
            InstanceIds: names,
        }

        _, err := svc.TerminateInstances(params)
        if err != nil {
            panic(err)
        }
    }
}

func main() {
    flag.Usage = func() {
        fmt.Fprintf(os.Stderr, "%s: status|boot|terminate\n", os.Args[0])
        flag.PrintDefaults()
    }

    var vp = flag.Bool("v", false, "Turn on verbose log messaging.")
    flag.Parse()
    verbose = *vp

    if verbose {
        defaults.DefaultConfig.LogLevel = aws.LogLevel(aws.LogDebug)
    }
    defaults.DefaultConfig.Region = aws.String("us-west-2")

    svc := ec2.New(nil)
    for _, arg := range flag.Args() {
        switch arg {
        case "status":
            status(svc)
        case "boot":
            boot(svc)
        case "terminate":
            terminate(svc)
        default:
            fmt.Fprintf(os.Stderr, "Unknown Command: %s\n", arg)
        }
    }
}
<commit_msg>poke_ops: Make some minor style tweaks.<commit_after>package main

import (
    "flag"
    "fmt"
    "os"

    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/defaults"
    "github.com/aws/aws-sdk-go/service/ec2"
)

const TAGKEY string = "poke"

var verbose bool = false

func describe_instances(svc *ec2.EC2) *ec2.DescribeInstancesOutput {

    resp, err := svc.DescribeInstances(&ec2.DescribeInstancesInput {
        Filters: []*ec2.Filter {
            &ec2.Filter {
                Name: aws.String("tag-key"),
                Values: []*string{aws.String(TAGKEY)},
            },
        },
    })

    if err != nil {
        panic(err)
    }

    return resp
}

func status(svc *ec2.EC2) {
    resp := describe_instances(svc)

    for _, res := range resp.Reservations {
        for _, inst := range res.Instances {
            if verbose {
                fmt.Println(*inst)
            } else {
                fmt.Println("{")
                fmt.Println("      InstanceId:", *inst.InstanceId)
                if inst.PublicIpAddress != nil {
                    fmt.Println(" PublicIPAddress:", *inst.PublicIpAddress)
                }
                if inst.PrivateIpAddress != nil {
                    fmt.Println("PrivateIPAddress:", *inst.PrivateIpAddress)
                }
                fmt.Println("    InstanceType:", *inst.InstanceType)
                fmt.Println("      LaunchTime:", inst.LaunchTime)
                fmt.Println("           State:", *inst.State.Name)
                fmt.Print(  "            Tags:")
                for _, tag:= range inst.Tags {
                    fmt.Printf(" (%s, %s)", *tag.Key, *tag.Value)
                }
                fmt.Println("\n}")
            }
        }
    }
}

func boot(svc *ec2.EC2) {
    count := int64(4)
    params := &ec2.RunInstancesInput {
        ImageId: aws.String("ami-9ff7e8af"),
        InstanceType: aws.String("t2.micro"),
        MinCount: &count,
        MaxCount: &count,
    }

    res, err := svc.RunInstances(params)
    if err != nil {
        panic(err)
    }

    tag_params := ec2.CreateTagsInput {
        Resources: nil,
        Tags: []*ec2.Tag {
            &ec2.Tag {
                Key: aws.String(TAGKEY),
                Value: aws.String("production"),
            },
        },
    }

    for _, inst := range res.Instances {
        tag_params.Resources = append(tag_params.Resources, inst.InstanceId)
    }

    _, err = svc.CreateTags(&tag_params)
    if err != nil {
        panic(err)
    }
}

func terminate(svc *ec2.EC2) {
    desc := describe_instances(svc)

    for _, res := range desc.Reservations {
        var names []*string
        for _, inst := range res.Instances {
            names = append(names, inst.InstanceId)
        }

        params := &ec2.TerminateInstancesInput {
            InstanceIds: names,
        }

        _, err := svc.TerminateInstances(params)
        if err != nil {
            panic(err)
        }
    }
}

func main() {
    flag.Usage = func() {
        fmt.Fprintf(os.Stderr, "%s: status|boot|terminate\n", os.Args[0])
        flag.PrintDefaults()
    }

    var vp = flag.Bool("v", false, "Turn on verbose log messaging.")
    flag.Parse()
    verbose = *vp

    if verbose {
        defaults.DefaultConfig.LogLevel = aws.LogLevel(aws.LogDebug)
    }
    defaults.DefaultConfig.Region = aws.String("us-west-2")

    svc := ec2.New(nil)
    for _, arg := range flag.Args() {
        switch arg {
        case "status":
            status(svc)
        case "boot":
            boot(svc)
        case "terminate":
            terminate(svc)
        default:
            fmt.Fprintf(os.Stderr, "Unknown Command: %s\n", arg)
        }
    }
}
<|endoftext|>


<commit_before>package webhook

import (
	"testing"
	"log"
	"time"
	"strconv"
)

//// 测试发送文本内容结构体是否正常
//func TestTextStruct(t *testing.T) {
//
//}

// 测试封装接口
func TestSendText(t *testing.T) {
	i := time.Now().Unix()
	log.Println(i)
	SendText("单元测试 发送文本内容: 自动执行，忽略。" + strconv.FormatInt(i, 10),"11b53105e91cf109aaa16690a6a9177e5dfd148d86c6bfd8863f213952c9c985")
}<commit_msg><持续集成><测试><commit_after>package webhook

import (
	"testing"
	"log"
	"time"
	"strconv"
)

//// 测试发送文本内容结构体是否正常
//func TestTextStruct(t *testing.T) {
//
//}

// 测试封装接口
func TestSendText(t *testing.T) {
	i := time.Now().Unix()
	log.Println(i)
	SendText("单元测试 - 发送文本内容: 自动执行，忽略。" + strconv.FormatInt(i, 10),"11b53105e91cf109aaa16690a6a9177e5dfd148d86c6bfd8863f213952c9c985")
}<|endoftext|>