<commit_before>package snapshot

import (
	"context"

	"github.com/containerd/containerd"
)

// Kind identifies the kind of snapshot.
type Kind int

// definitions of snapshot kinds
const (
	KindActive Kind = iota
	KindCommitted
)

// Info provides information about a particular snapshot.
type Info struct {
	Name     string // name or key of snapshot
	Parent   string // name of parent snapshot
	Kind     Kind   // active or committed snapshot
	Readonly bool   // true if readonly, only valid for active
}

// Snapshotter defines the methods required to implement a snapshot snapshotter for
// allocating, snapshotting and mounting filesystem changesets. The model works
// by building up sets of changes with parent-child relationships.
//
// A snapshot represents a filesystem state. Every snapshot has a parent, where
// the empty parent is represented by the empty string. A diff can be taken
// between a parent and its snapshot to generate a classic layer.
//
// An active snapshot is created by calling `Prepare`. After mounting, changes
// can be made to the snapshot. The act of commiting creates a committed
// snapshot. The committed snapshot will get the parent of active snapshot. The
// committed snapshot can then be used as a parent. Active snapshots can never
// act as a parent.
//
// Snapshots are best understood by their lifecycle. Active snapshots are
// always created with Prepare or View. Committed snapshots are always created
// with Commit.  Active snapshots never become committed snapshots and vice
// versa. All snapshots may be removed.
//
// For consistency, we define the following terms to be used throughout this
// interface for snapshotter implementations:
//
// 	`key` - refers to an active snapshot
// 	`name` - refers to a committed snapshot
// 	`parent` - refers to the parent in relation
//
// Most methods take various combinations of these identifiers. Typically,
// `name` and `parent` will be used in cases where a method *only* takes
// committed snapshots. `key` will be used to refer to active snapshots in most
// cases, except where noted. All variables used to access snapshots use the
// same key space. For example, an active snapshot may not share the same key
// with a committed snapshot.
//
// We cover several examples below to demonstrate the utility of a snapshot
// snapshotter.
//
// Importing a Layer
//
// To import a layer, we simply have the Snapshotter provide a list of
// mounts to be applied such that our dst will capture a changeset. We start
// out by getting a path to the layer tar file and creating a temp location to
// unpack it to:
//
//	layerPath, tmpDir := getLayerPath(), mkTmpDir() // just a path to layer tar file.
//
// We start by using a Snapshotter to Prepare a new snapshot transaction, using a
// key and descending from the empty parent "":
//
//	mounts, err := snapshotter.Prepare(key, "")
// 	if err != nil { ... }
//
// We get back a list of mounts from Snapshotter.Prepare, with the key identifying
// the active snapshot. Mount this to the temporary location with the
// following:
//
//	if err := MountAll(mounts, tmpDir); err != nil { ... }
//
// Once the mounts are performed, our temporary location is ready to capture
// a diff. In practice, this works similar to a filesystem transaction. The
// next step is to unpack the layer. We have a special function unpackLayer
// that applies the contents of the layer to target location and calculates the
// DiffID of the unpacked layer (this is a requirement for docker
// implementation):
//
//	layer, err := os.Open(layerPath)
//	if err != nil { ... }
// 	digest, err := unpackLayer(tmpLocation, layer) // unpack into layer location
// 	if err != nil { ... }
//
// When the above completes, we should have a filesystem the represents the
// contents of the layer. Careful implementations should verify that digest
// matches the expected DiffID. When completed, we unmount the mounts:
//
//	unmount(mounts) // optional, for now
//
// Now that we've verified and unpacked our layer, we commit the active
// snapshot to a name. For this example, we are just going to use the layer
// digest, but in practice, this will probably be the ChainID:
//
//	if err := snapshotter.Commit(digest.String(), key); err != nil { ... }
//
// Now, we have a layer in the Snapshotter that can be accessed with the digest
// provided during commit. Once you have committed the snapshot, the active
// snapshot can be removed with the following:
//
// 	snapshotter.Remove(key)
//
// Importing the Next Layer
//
// Making a layer depend on the above is identical to the process described
// above except that the parent is provided as parent when calling
// Manager.Prepare, assuming a clean tmpLocation:
//
// 	mounts, err := snapshotter.Prepare(tmpLocation, parentDigest)
//
// We then mount, apply and commit, as we did above. The new snapshot will be
// based on the content of the previous one.
//
// Running a Container
//
// To run a container, we simply provide Snapshotter.Prepare the committed image
// snapshot as the parent. After mounting, the prepared path can
// be used directly as the container's filesystem:
//
// 	mounts, err := snapshotter.Prepare(containerKey, imageRootFSChainID)
//
// The returned mounts can then be passed directly to the container runtime. If
// one would like to create a new image from the filesystem, Manager.Commit is
// called:
//
// 	if err := snapshotter.Commit(newImageSnapshot, containerKey); err != nil { ... }
//
// Alternatively, for most container runs, Snapshotter.Remove will be called to
// signal the Snapshotter to abandon the changes.
type Snapshotter interface {
	// Stat returns the info for an active or committed snapshot by name or
	// key.
	//
	// Should be used for parent resolution, existence checks and to discern
	// the kind of snapshot.
	Stat(ctx context.Context, key string) (Info, error)

	// Mounts returns the mounts for the active snapshot transaction identified
	// by key. Can be called on an read-write or readonly transaction. This is
	// available only for active snapshots.
	//
	// This can be used to recover mounts after calling View or Prepare.
	Mounts(ctx context.Context, key string) ([]containerd.Mount, error)

	// Prepare creates an active snapshot identified by key descending from the
	// provided parent.  The returned mounts can be used to mount the snapshot
	// to capture changes.
	//
	// If a parent is provided, after performing the mounts, the destination
	// will start with the content of the parent. The parent must be a
	// committed snapshot. Changes to the mounted destination will be captured
	// in relation to the parent. The default parent, "", is an empty
	// directory.
	//
	// The changes may be saved to a committed snapshot by calling Commit. When
	// one is done with the transaction, Remove should be called on the key.
	//
	// Multiple calls to Prepare or View with the same key should fail.
	Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error)

	// View behaves identically to Prepare except the result may not be
	// committed back to the snapshot snapshotter. View returns a readonly view on
	// the parent, with the active snapshot being tracked by the given key.
	//
	// This method operates identically to Prepare, except that Mounts returned
	// may have the readonly flag set. Any modifications to the underlying
	// filesystem will be ignored. Implementations may perform this in a more
	// efficient manner that differs from what would be attempted with
	// `Prepare`.
	//
	// Commit may not be called on the provided key and will return an error.
	// To collect the resources associated with key, Remove must be called with
	// key as the argument.
	View(ctx context.Context, key, parent string) ([]containerd.Mount, error)

	// Commit captures the changes between key and its parent into a snapshot
	// identified by name.  The name can then be used with the snapshotter's other
	// methods to create subsequent snapshots.
	//
	// A committed snapshot will be created under name with the parent of the
	// active snapshot.
	//
	// Commit may be called multiple times on the same key. Snapshots created
	// in this manner will all reference the parent used to start the
	// transaction.
	Commit(ctx context.Context, name, key string) error

	// Remove the committed or active snapshot by the provided key.
	//
	// All resources associated with the key will be removed.
	//
	// If the snapshot is a parent of another snapshot, its children must be
	// removed before proceeding.
	Remove(ctx context.Context, key string) error

	// Walk the committed snapshots. For each snapshot in the snapshotter, the
	// function will be called.
	Walk(ctx context.Context, fn func(context.Context, Info) error) error
}
<commit_msg>snapshot: the examples need context<commit_after>package snapshot

import (
	"context"

	"github.com/containerd/containerd"
)

// Kind identifies the kind of snapshot.
type Kind int

// definitions of snapshot kinds
const (
	KindActive Kind = iota
	KindCommitted
)

// Info provides information about a particular snapshot.
type Info struct {
	Name     string // name or key of snapshot
	Parent   string // name of parent snapshot
	Kind     Kind   // active or committed snapshot
	Readonly bool   // true if readonly, only valid for active
}

// Snapshotter defines the methods required to implement a snapshot snapshotter for
// allocating, snapshotting and mounting filesystem changesets. The model works
// by building up sets of changes with parent-child relationships.
//
// A snapshot represents a filesystem state. Every snapshot has a parent, where
// the empty parent is represented by the empty string. A diff can be taken
// between a parent and its snapshot to generate a classic layer.
//
// An active snapshot is created by calling `Prepare`. After mounting, changes
// can be made to the snapshot. The act of commiting creates a committed
// snapshot. The committed snapshot will get the parent of active snapshot. The
// committed snapshot can then be used as a parent. Active snapshots can never
// act as a parent.
//
// Snapshots are best understood by their lifecycle. Active snapshots are
// always created with Prepare or View. Committed snapshots are always created
// with Commit.  Active snapshots never become committed snapshots and vice
// versa. All snapshots may be removed.
//
// For consistency, we define the following terms to be used throughout this
// interface for snapshotter implementations:
//
// 	`ctx` - refers to a context.Context
// 	`key` - refers to an active snapshot
// 	`name` - refers to a committed snapshot
// 	`parent` - refers to the parent in relation
//
// Most methods take various combinations of these identifiers. Typically,
// `name` and `parent` will be used in cases where a method *only* takes
// committed snapshots. `key` will be used to refer to active snapshots in most
// cases, except where noted. All variables used to access snapshots use the
// same key space. For example, an active snapshot may not share the same key
// with a committed snapshot.
//
// We cover several examples below to demonstrate the utility of a snapshot
// snapshotter.
//
// Importing a Layer
//
// To import a layer, we simply have the Snapshotter provide a list of
// mounts to be applied such that our dst will capture a changeset. We start
// out by getting a path to the layer tar file and creating a temp location to
// unpack it to:
//
//	layerPath, tmpDir := getLayerPath(), mkTmpDir() // just a path to layer tar file.
//
// We start by using a Snapshotter to Prepare a new snapshot transaction, using a
// key and descending from the empty parent "":
//
//	mounts, err := snapshotter.Prepare(ctx, key, "")
// 	if err != nil { ... }
//
// We get back a list of mounts from Snapshotter.Prepare, with the key identifying
// the active snapshot. Mount this to the temporary location with the
// following:
//
//	if err := containerd.MountAll(mounts, tmpDir); err != nil { ... }
//
// Once the mounts are performed, our temporary location is ready to capture
// a diff. In practice, this works similar to a filesystem transaction. The
// next step is to unpack the layer. We have a special function unpackLayer
// that applies the contents of the layer to target location and calculates the
// DiffID of the unpacked layer (this is a requirement for docker
// implementation):
//
//	layer, err := os.Open(layerPath)
//	if err != nil { ... }
// 	digest, err := unpackLayer(tmpLocation, layer) // unpack into layer location
// 	if err != nil { ... }
//
// When the above completes, we should have a filesystem the represents the
// contents of the layer. Careful implementations should verify that digest
// matches the expected DiffID. When completed, we unmount the mounts:
//
//	unmount(mounts) // optional, for now
//
// Now that we've verified and unpacked our layer, we commit the active
// snapshot to a name. For this example, we are just going to use the layer
// digest, but in practice, this will probably be the ChainID:
//
//	if err := snapshotter.Commit(ctx, digest.String(), key); err != nil { ... }
//
// Now, we have a layer in the Snapshotter that can be accessed with the digest
// provided during commit. Once you have committed the snapshot, the active
// snapshot can be removed with the following:
//
// 	snapshotter.Remove(ctx, key)
//
// Importing the Next Layer
//
// Making a layer depend on the above is identical to the process described
// above except that the parent is provided as parent when calling
// Manager.Prepare, assuming a clean, unique key identifier:
//
// 	mounts, err := snapshotter.Prepare(ctx, key, parentDigest)
//
// We then mount, apply and commit, as we did above. The new snapshot will be
// based on the content of the previous one.
//
// Running a Container
//
// To run a container, we simply provide Snapshotter.Prepare the committed image
// snapshot as the parent. After mounting, the prepared path can
// be used directly as the container's filesystem:
//
// 	mounts, err := snapshotter.Prepare(ctx, containerKey, imageRootFSChainID)
//
// The returned mounts can then be passed directly to the container runtime. If
// one would like to create a new image from the filesystem, Manager.Commit is
// called:
//
// 	if err := snapshotter.Commit(ctx, newImageSnapshot, containerKey); err != nil { ... }
//
// Alternatively, for most container runs, Snapshotter.Remove will be called to
// signal the Snapshotter to abandon the changes.
type Snapshotter interface {
	// Stat returns the info for an active or committed snapshot by name or
	// key.
	//
	// Should be used for parent resolution, existence checks and to discern
	// the kind of snapshot.
	Stat(ctx context.Context, key string) (Info, error)

	// Mounts returns the mounts for the active snapshot transaction identified
	// by key. Can be called on an read-write or readonly transaction. This is
	// available only for active snapshots.
	//
	// This can be used to recover mounts after calling View or Prepare.
	Mounts(ctx context.Context, key string) ([]containerd.Mount, error)

	// Prepare creates an active snapshot identified by key descending from the
	// provided parent.  The returned mounts can be used to mount the snapshot
	// to capture changes.
	//
	// If a parent is provided, after performing the mounts, the destination
	// will start with the content of the parent. The parent must be a
	// committed snapshot. Changes to the mounted destination will be captured
	// in relation to the parent. The default parent, "", is an empty
	// directory.
	//
	// The changes may be saved to a committed snapshot by calling Commit. When
	// one is done with the transaction, Remove should be called on the key.
	//
	// Multiple calls to Prepare or View with the same key should fail.
	Prepare(ctx context.Context, key, parent string) ([]containerd.Mount, error)

	// View behaves identically to Prepare except the result may not be
	// committed back to the snapshot snapshotter. View returns a readonly view on
	// the parent, with the active snapshot being tracked by the given key.
	//
	// This method operates identically to Prepare, except that Mounts returned
	// may have the readonly flag set. Any modifications to the underlying
	// filesystem will be ignored. Implementations may perform this in a more
	// efficient manner that differs from what would be attempted with
	// `Prepare`.
	//
	// Commit may not be called on the provided key and will return an error.
	// To collect the resources associated with key, Remove must be called with
	// key as the argument.
	View(ctx context.Context, key, parent string) ([]containerd.Mount, error)

	// Commit captures the changes between key and its parent into a snapshot
	// identified by name.  The name can then be used with the snapshotter's other
	// methods to create subsequent snapshots.
	//
	// A committed snapshot will be created under name with the parent of the
	// active snapshot.
	//
	// Commit may be called multiple times on the same key. Snapshots created
	// in this manner will all reference the parent used to start the
	// transaction.
	Commit(ctx context.Context, name, key string) error

	// Remove the committed or active snapshot by the provided key.
	//
	// All resources associated with the key will be removed.
	//
	// If the snapshot is a parent of another snapshot, its children must be
	// removed before proceeding.
	Remove(ctx context.Context, key string) error

	// Walk the committed snapshots. For each snapshot in the snapshotter, the
	// function will be called.
	Walk(ctx context.Context, fn func(context.Context, Info) error) error
}
<|endoftext|>


<commit_before>// Copyright 2015 Dmitry Vyukov. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package tar

import (
	"archive/tar"
	"bytes"
	"io"
	"io/ioutil"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
)

func Fuzz(data []byte) int {
	t := tar.NewReader(bytes.NewReader(data))
	var headers []*tar.Header
	var contents [][]byte
	for {
		hdr, err := t.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return 0
		}
		if len(hdr.Name) > 1e6 ||
			len(hdr.Linkname) > 1e6 ||
			len(hdr.Uname) > 1e6 ||
			len(hdr.Gname) > 1e6 ||
			len(hdr.Xattrs) > 1e6 {
			panic("huge header data")
		}
		if hdr.Size > 1e6 {
			panic("huge claimed file size")
		}
		fdata, err := ioutil.ReadAll(t)
		if err != nil {
			return 0
		}
		if int64(len(fdata)) > hdr.Size {
			panic("long read")
		}
		if int64(len(fdata)) < hdr.Size {
			panic("short read")
		}
		hdr1 := *hdr // make a copy to be safe
		headers = append(headers, &hdr1)
		contents = append(contents, fdata)
	}
	buf := new(bytes.Buffer)
	w := tar.NewWriter(buf)
	for i, hdr := range headers {
		err := w.WriteHeader(hdr)
		if err != nil {
			panic(err)
		}
		n, err := w.Write(contents[i])
		if err != nil {
			panic(err)
		}
		if n != len(contents[i]) {
			panic("short write")
		}
	}
	err := w.Close()
	if err != nil {
		panic(err)
	}
	t1 := tar.NewReader(buf)
	for i := 0; ; i++ {
		hdr, err := t1.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}
		fdata, err := ioutil.ReadAll(t)
		if err != nil {
			panic(err)
		}
		if !fuzz.DeepEqual(hdr, headers[i]) {
			panic("headers diffs")
		}
		if !bytes.Equal(fdata, contents[i]) {
			panic("data diffs")
		}
	}
	return 1
}
<commit_msg>fix tar test<commit_after>// Copyright 2015 Dmitry Vyukov. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package tar

import (
	"archive/tar"
	"bytes"
	"io"
	"io/ioutil"

	"github.com/dvyukov/go-fuzz/examples/fuzz"
)

func Fuzz(data []byte) int {
	t := tar.NewReader(bytes.NewReader(data))
	var headers []*tar.Header
	var contents [][]byte
	for {
		hdr, err := t.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return 0
		}
		if len(hdr.Name) > 1e6 ||
			len(hdr.Linkname) > 1e6 ||
			len(hdr.Uname) > 1e6 ||
			len(hdr.Gname) > 1e6 ||
			len(hdr.Xattrs) > 1e6 {
			panic("huge header data")
		}
		if hdr.Size > 1e6 {
			panic("huge claimed file size")
		}
		fdata, err := ioutil.ReadAll(t)
		if err != nil {
			return 0
		}
		if int64(len(fdata)) > hdr.Size {
			panic("long read")
		}
		if int64(len(fdata)) < hdr.Size {
			panic("short read")
		}
		hdr1 := *hdr // make a copy to be safe
		headers = append(headers, &hdr1)
		contents = append(contents, fdata)
	}
	buf := new(bytes.Buffer)
	w := tar.NewWriter(buf)
	for i, hdr := range headers {
		err := w.WriteHeader(hdr)
		if err != nil {
			panic(err)
		}
		n, err := w.Write(contents[i])
		if err != nil {
			panic(err)
		}
		if n != len(contents[i]) {
			panic("short write")
		}
	}
	err := w.Close()
	if err != nil {
		panic(err)
	}
	t1 := tar.NewReader(buf)
	for i := 0; ; i++ {
		hdr, err := t1.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			panic(err)
		}
		fdata, err := ioutil.ReadAll(t1)
		if err != nil {
			panic(err)
		}
		if !fuzz.DeepEqual(hdr, headers[i]) {
			panic("headers diffs")
		}
		if !bytes.Equal(fdata, contents[i]) {
			panic("data diffs")
		}
	}
	return 1
}
<|endoftext|>


<commit_before>package cache

import (
	"context"

	"github.com/docker/distribution"
	prometheus "github.com/docker/distribution/metrics"
	"github.com/opencontainers/go-digest"
)

// Metrics is used to hold metric counters
// related to the number of times a cache was
// hit or missed.
type Metrics struct {
	Requests uint64
	Hits     uint64
	Misses   uint64
}

// Logger can be provided on the MetricsTracker to log errors.
//
// Usually, this is just a proxy to dcontext.GetLogger.
type Logger interface {
	Errorf(format string, args ...interface{})
}

// MetricsTracker represents a metric tracker
// which simply counts the number of hits and misses.
type MetricsTracker interface {
	Hit()
	Miss()
	Metrics() Metrics
	Logger(context.Context) Logger
}

type cachedBlobStatter struct {
	cache   distribution.BlobDescriptorService
	backend distribution.BlobDescriptorService
	tracker MetricsTracker
}

var (
	// cacheCount is the number of total cache request received/hits/misses
	cacheCount = prometheus.StorageNamespace.NewLabeledCounter("cache", "The number of cache request received", "type")
)

// NewCachedBlobStatter creates a new statter which prefers a cache and
// falls back to a backend.
func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService) distribution.BlobDescriptorService {
	return &cachedBlobStatter{
		cache:   cache,
		backend: backend,
	}
}

// NewCachedBlobStatterWithMetrics creates a new statter which prefers a cache and
// falls back to a backend. Hits and misses will send to the tracker.
func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService, tracker MetricsTracker) distribution.BlobStatter {
	return &cachedBlobStatter{
		cache:   cache,
		backend: backend,
		tracker: tracker,
	}
}

func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
	cacheCount.WithValues("Request").Inc(1)
	desc, err := cbds.cache.Stat(ctx, dgst)
	if err != nil {
		if err != distribution.ErrBlobUnknown {
			logErrorf(ctx, cbds.tracker, "error retrieving descriptor from cache: %v", err)
		}

		goto fallback
	}
	cacheCount.WithValues("Hit").Inc(1)
	if cbds.tracker != nil {
		cbds.tracker.Hit()
	}
	return desc, nil
fallback:
	cacheCount.WithValues("Miss").Inc(1)
	if cbds.tracker != nil {
		cbds.tracker.Miss()
	}
	desc, err = cbds.backend.Stat(ctx, dgst)
	if err != nil {
		return desc, err
	}

	if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
		logErrorf(ctx, cbds.tracker, "error adding descriptor %v to cache: %v", desc.Digest, err)
	}

	return desc, err

}

func (cbds *cachedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
	err := cbds.cache.Clear(ctx, dgst)
	if err != nil {
		return err
	}

	err = cbds.backend.Clear(ctx, dgst)
	if err != nil {
		return err
	}
	return nil
}

func (cbds *cachedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
	if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
		logErrorf(ctx, cbds.tracker, "error adding descriptor %v to cache: %v", desc.Digest, err)
	}
	return nil
}

func logErrorf(ctx context.Context, tracker MetricsTracker, format string, args ...interface{}) {
	if tracker == nil {
		return
	}

	logger := tracker.Logger(ctx)
	if logger == nil {
		return
	}
	logger.Errorf(format, args...)
}
<commit_msg>fix redis caching issue<commit_after>package cache

import (
	"context"

	"github.com/docker/distribution"
	dcontext "github.com/docker/distribution/context"
	prometheus "github.com/docker/distribution/metrics"
	"github.com/opencontainers/go-digest"
)

// Metrics is used to hold metric counters
// related to the number of times a cache was
// hit or missed.
type Metrics struct {
	Requests uint64
	Hits     uint64
	Misses   uint64
}

// Logger can be provided on the MetricsTracker to log errors.
//
// Usually, this is just a proxy to dcontext.GetLogger.
type Logger interface {
	Errorf(format string, args ...interface{})
}

// MetricsTracker represents a metric tracker
// which simply counts the number of hits and misses.
type MetricsTracker interface {
	Hit()
	Miss()
	Metrics() Metrics
	Logger(context.Context) Logger
}

type cachedBlobStatter struct {
	cache   distribution.BlobDescriptorService
	backend distribution.BlobDescriptorService
	tracker MetricsTracker
}

var (
	// cacheCount is the number of total cache request received/hits/misses
	cacheCount = prometheus.StorageNamespace.NewLabeledCounter("cache", "The number of cache request received", "type")
)

// NewCachedBlobStatter creates a new statter which prefers a cache and
// falls back to a backend.
func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService) distribution.BlobDescriptorService {
	return &cachedBlobStatter{
		cache:   cache,
		backend: backend,
	}
}

// NewCachedBlobStatterWithMetrics creates a new statter which prefers a cache and
// falls back to a backend. Hits and misses will send to the tracker.
func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService, tracker MetricsTracker) distribution.BlobStatter {
	return &cachedBlobStatter{
		cache:   cache,
		backend: backend,
		tracker: tracker,
	}
}

func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
	cacheCount.WithValues("Request").Inc(1)

	// try getting from cache
	desc, cacheErr := cbds.cache.Stat(ctx, dgst)
	if cacheErr == nil {
		cacheCount.WithValues("Hit").Inc(1)
		if cbds.tracker != nil {
			cbds.tracker.Hit()
		}
		return desc, nil
	}

	// couldn't get from cache; get from backend
	desc, err := cbds.backend.Stat(ctx, dgst)
	if err != nil {
		return desc, err
	}

	if cacheErr == distribution.ErrBlobUnknown {
		// cache doesn't have info. update it with info got from backend
		cacheCount.WithValues("Miss").Inc(1)
		if cbds.tracker != nil {
			cbds.tracker.Miss()
		}
		if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
			dcontext.GetLoggerWithField(ctx, "blob", dgst).WithError(err).Error("error from cache setting desc")
		}
		// we don't need to return cache error upstream if any. continue returning value from backend
		return desc, nil
	}

	// unknown error from cache. just log and error. do not store cache as it may be trigger many set calls
	dcontext.GetLoggerWithField(ctx, "blob", dgst).WithError(cacheErr).Error("error from cache stat(ing) blob")
	cacheCount.WithValues("Error").Inc(1)

	return desc, nil
}

func (cbds *cachedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
	err := cbds.cache.Clear(ctx, dgst)
	if err != nil {
		return err
	}

	err = cbds.backend.Clear(ctx, dgst)
	if err != nil {
		return err
	}
	return nil
}

func (cbds *cachedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
	if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
		dcontext.GetLoggerWithField(ctx, "blob", dgst).WithError(err).Error("error from cache setting desc")
	}
	return nil
}

func logErrorf(ctx context.Context, tracker MetricsTracker, format string, args ...interface{}) {
	if tracker == nil {
		return
	}

	logger := tracker.Logger(ctx)
	if logger == nil {
		return
	}
	logger.Errorf(format, args...)
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package transform contains methods to transform json trees as specified by the config.
package transform

import (
	"context"
	"encoding/json"
	"fmt"
	"log"

	"github.com/golang/protobuf/proto" /* copybara-comment: proto */

	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/auth" /* copybara-comment: auth */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/builtins" /* copybara-comment: builtins */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/cloudfunction" /* copybara-comment: cloudfunction */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/errors" /* copybara-comment: errors */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/fetch" /* copybara-comment: fetch */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/harmonization/harmonizecode" /* copybara-comment: harmonizecode */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/harmonization/harmonizeunit" /* copybara-comment: harmonizeunit */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/mapping" /* copybara-comment: mapping */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/postprocess" /* copybara-comment: postprocess */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/projector" /* copybara-comment: projector */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/types" /* copybara-comment: types */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/gcsutil" /* copybara-comment: gcsutil */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/jsonutil" /* copybara-comment: jsonutil */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_language/transpiler" /* copybara-comment: transpiler */

	dhpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: data_harmonization_go_proto */
	hapb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: harmonization_go_proto */
	httppb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: http_go_proto */
	libpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: library_go_proto */
	mappb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: mapping_go_proto */

)

// Transformer contains projectors initialized for a specific config, and receiver methods to
// perform transformations.
type Transformer struct {
	Registry                *types.Registry
	dataHarmonizationConfig *dhpb.DataHarmonizationConfig
	mappingConfig           *mappb.MappingConfig
	parallel                bool
}

// TransformationConfigs contains metadata used during transformation.
type TransformationConfigs struct {
	LogTrace     bool
	SkipBundling bool
}

// Options for initializing Data Harmonization transform library
type Options struct {
	// CloudFunctions enables support for cloud functions within the transform library.
	CloudFunctions bool

	// FetchConfigs enables support for fetch configurations wtihin the transform library.
	FetchConfigs bool

	// Parallel enables support for parallelization within the transform library.
	Parallel bool

	// GCSClient is a client for downloading from GCS. If unset, the default third party GCS client will be used.
	GCSClient gcsutil.StorageClient
}

// Option is a setter function for Options.
type Option func(*Options)

// CloudFunctions initializes the CloudFunctions transform option.
func CloudFunctions(enabledCloudFunctions bool) Option {
	return func(args *Options) {
		args.CloudFunctions = enabledCloudFunctions
	}
}

// FetchConfigs initializes the FetchConfigs transform option.
func FetchConfigs(enableFetchConfig bool) Option {
	return func(args *Options) {
		args.FetchConfigs = enableFetchConfig
	}
}

// Parallel initializes the Parallel transform option.
func Parallel(enableParallel bool) Option {
	return func(args *Options) {
		args.Parallel = enableParallel
	}
}

// GCSClient sets the GCSClient in the transform option.
func GCSClient(c gcsutil.StorageClient) Option {
	return func(args *Options) {
		args.GCSClient = c
	}
}

// NewTransformer creates and initializes a transformer.
func NewTransformer(ctx context.Context, config *dhpb.DataHarmonizationConfig, setters ...Option) (*Transformer, error) {
	t := &Transformer{
		Registry:                types.NewRegistry(),
		dataHarmonizationConfig: config,
	}

	if err := builtins.RegisterAll(t.Registry); err != nil {
		return nil, err
	}

	options := &Options{}
	for _, setter := range setters {
		setter(options)
	}

	t.parallel = options.Parallel

	gcsutil.InitializeClient(options.GCSClient)

	if hc := config.GetHarmonizationConfig(); hc != nil {
		if err := harmonizecode.LoadCodeHarmonizationProjectors(t.Registry, hc); err != nil {
			// TODO: return nil, err
			log.Printf("failed to load code harmonization projectors: %v", err)
		}
	}

	if uc := config.GetUnitHarmonizationConfig(); uc != nil {
		if err := harmonizeunit.LoadUnitHarmonizationProjectors(t.Registry, uc); err != nil {
			// TODO: return nil, err
			log.Printf("failed to load unit harmonization projectors: %v", err)
		}
	}

	mpc, err := t.LoadMappingConfig(config)
	if err != nil {
		return nil, err
	}
	t.mappingConfig = mpc

	if err := t.LoadProjectors(mpc.GetProjector()); err != nil {
		return nil, err
	}

	// Load the library configurations.
	for _, lc := range config.GetLibraryConfig() {
		if err := t.LoadProjectors(lc.Projector); err != nil {
			return nil, err
		}

		if options.CloudFunctions {
			if err := cloudfunction.LoadCloudFunctionProjectors(t.Registry, lc.CloudFunction); err != nil {
				return nil, err
			}
		} else {
			if len(lc.CloudFunction) != 0 {
				return nil, &invalidCloudFunctionProjectorError{lc: lc}
			}
		}

		if err := auth.LoadServerConfigs(lc.Servers); err != nil {
			return nil, err
		}

		if options.FetchConfigs {
			if err := fetch.LoadFetchProjectors(context.Background(), t.Registry, lc.HttpQuery); err != nil {
				return nil, err
			}
		} else {
			if len(lc.HttpQuery) != 0 {
				return nil, &invalidFetchProjectorError{lc: lc}
			}
		}

		for _, lib := range lc.GetUserLibraries() {
			mpc, err := loadMappingConfig(lib.GetPath(), lib.GetType())
			if err != nil {
				return nil, err
			}
			// Custom library configs should only have projectors defined, all else is ignored.
			if err := t.LoadProjectors(mpc.GetProjector()); err != nil {
				return nil, err
			}
		}
	}

	return t, nil
}

type invalidCloudFunctionProjectorError struct {
	lc *libpb.LibraryConfig
}

type invalidFetchProjectorError struct {
	lc *libpb.LibraryConfig
}

func (e invalidCloudFunctionProjectorError) Error() string {
	var names []string
	for _, cf := range e.lc.CloudFunction {
		names = append(names, cf.Name)
	}
	return fmt.Sprintf("attempting to use disabled Cloud Function projectors feature with projectors: %v", names)
}

func (e invalidFetchProjectorError) Error() string {
	var names []string
	for _, cf := range e.lc.HttpQuery {
		names = append(names, cf.Name)
	}
	return fmt.Sprintf("attempting to use disabled Fetch projectors feature with projectors: %v", names)
}

// Project is a convenience function to call a single projector out of context.
func (t *Transformer) Project(projector string, args ...jsonutil.JSONMetaNode) (res jsonutil.JSONToken, err error) {
	pctx := types.NewContext(t.Registry)

	defer errors.RecoverWithTrace(pctx.Trace, "Project", func(e error) {
		err = e
	})

	proj, err := t.Registry.FindProjector(projector)
	if err != nil {
		return nil, err
	}

	res, err = proj(args, pctx)
	return
}

// LoadMappingConfig loads the mapping config inline or from a GCS path.
func (t *Transformer) LoadMappingConfig(config *dhpb.DataHarmonizationConfig) (*mappb.MappingConfig, error) {
	mpc := &mappb.MappingConfig{}
	if config.GetDeprecateStructureMappingConfig() != nil && config.GetStructureMappingConfig() != nil {
		return nil, fmt.Errorf("only one of DeprecateStructureMappingConfig or StructureMappingConfig can be filled")
	}
	if config.GetDeprecateStructureMappingConfig() != nil {
		return config.GetDeprecateStructureMappingConfig(), nil
	}
	if sm := config.GetStructureMappingConfig(); sm != nil {
		switch mapping := config.GetStructureMappingConfig().Mapping.(type) {
		case *hapb.StructureMappingConfig_MappingConfig:
			return mapping.MappingConfig, nil
		case *hapb.StructureMappingConfig_MappingPathConfig:
			return loadMappingConfig(mapping.MappingPathConfig.MappingConfigPath, mapping.MappingPathConfig.MappingType)
		case *hapb.StructureMappingConfig_MappingLanguageString:
			return transpiler.Transpile(mapping.MappingLanguageString)
		default:
			return nil, fmt.Errorf("unsupported structure mapping config type: %v", mapping)
		}
	}
	return mpc, nil
}

// LoadProjectors registers all given projectors.
func (t *Transformer) LoadProjectors(projectors []*mappb.ProjectorDefinition) error {
	for _, pd := range projectors {
		p := projector.FromDef(pd, t.parallel)

		if err := t.Registry.RegisterProjector(pd.Name, p); err != nil {
			return fmt.Errorf("error registering projector %s: %v", pd.Name, err)
		}
	}
	return nil
}

// Transform converts the json tree using the specified config.
func (t *Transformer) Transform(in *jsonutil.JSONContainer, tconfig TransformationConfigs) (res jsonutil.JSONToken, err error) {
	pctx := types.NewContext(t.Registry)
	defer errors.RecoverWithTrace(pctx.Trace, "Transform", func(e error) {
		err = e
	})

	pctx.Variables.Push()

	inn, err := jsonutil.TokenToNode(*in)
	if err != nil {
		return nil, fmt.Errorf("input was invalid: %v", err)
	}
	args := []jsonutil.JSONMetaNode{inn}

	if err := mapping.ProcessMappings(t.mappingConfig.RootMapping, "root", args, &pctx.Output, pctx, t.parallel); err != nil {
		return nil, pctx.Trace.AsError(err)
	}

	result, err := postprocess.Process(pctx, t.mappingConfig, tconfig.SkipBundling, t.parallel)
	if err != nil {
		return nil, pctx.Trace.AsError(err)
	}

	if tconfig.LogTrace {
		log.Printf(pctx.Trace.String())
	}

	return result, nil
}

// JSONtoJSON converts the byte array (JSON format) using the specified config.
// TODO: Refactor to use json.RawMessage instead of []byte.
func (t *Transformer) JSONtoJSON(in []byte, tconfig TransformationConfigs) ([]byte, error) {
	ji := &jsonutil.JSONContainer{}
	if err := ji.UnmarshalJSON(in); err != nil {
		return nil, err
	}

	res, err := t.Transform(ji, tconfig)
	if err != nil {
		return nil, err
	}
	return json.Marshal(res)
}

// RegisterProjector adds the given Projector to this transformer's registry.
func (t *Transformer) RegisterProjector(name string, proj types.Projector) error {
	return t.Registry.RegisterProjector(name, proj)
}

// HasPostProcessProjector returns true iff a post process projector is set.
func (t *Transformer) HasPostProcessProjector() bool {
	return t.mappingConfig.GetPostProcessProjectorDefinition() != nil || t.mappingConfig.GetPostProcessProjectorName() != ""
}

// loadMappingConfig loads a mapping config from GCS.
func loadMappingConfig(loc *httppb.Location, typ hapb.MappingType) (*mappb.MappingConfig, error) {
	mpc := &mappb.MappingConfig{}
	switch l := loc.Location.(type) {
	case *httppb.Location_GcsLocation:
		data, err := gcsutil.ReadFromGcs(context.Background(), l.GcsLocation)
		if err != nil {
			return nil, fmt.Errorf("failed to read MappingConfig from GCS, %v", err)
		}
		switch typ {
		case hapb.MappingType_RAW_PROTO:
			if err := proto.UnmarshalText(string(data), mpc); err != nil {
				return nil, err
			}
		case hapb.MappingType_MAPPING_LANGUAGE:
			lmpc, err := transpiler.Transpile(string(data))
			if err != nil {
				return nil, err
			}
			mpc = lmpc
		default:
			return nil, fmt.Errorf("invalid mapping config type %v", typ)
		}
		return mpc, nil
	case *httppb.Location_UrlPath:
		return nil, fmt.Errorf("loading from remote path %s is unsupported", l.UrlPath)
	case *httppb.Location_LocalPath:
		return nil, fmt.Errorf("loading from local path %s is unsupported", l.LocalPath)
	default:
		return nil, fmt.Errorf("location type %s is unsupported", l)
	}
}
<commit_msg>Return errors when optional configs fail to load<commit_after>// Copyright 2020 Google LLC.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package transform contains methods to transform json trees as specified by the config.
package transform

import (
	"context"
	"encoding/json"
	"fmt"
	"log"

	"github.com/golang/protobuf/proto" /* copybara-comment: proto */

	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/auth" /* copybara-comment: auth */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/builtins" /* copybara-comment: builtins */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/cloudfunction" /* copybara-comment: cloudfunction */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/errors" /* copybara-comment: errors */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/fetch" /* copybara-comment: fetch */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/harmonization/harmonizecode" /* copybara-comment: harmonizecode */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/harmonization/harmonizeunit" /* copybara-comment: harmonizeunit */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/mapping" /* copybara-comment: mapping */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/postprocess" /* copybara-comment: postprocess */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/projector" /* copybara-comment: projector */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/types" /* copybara-comment: types */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/gcsutil" /* copybara-comment: gcsutil */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/util/jsonutil" /* copybara-comment: jsonutil */
	"github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_language/transpiler" /* copybara-comment: transpiler */

	dhpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: data_harmonization_go_proto */
	hapb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: harmonization_go_proto */
	httppb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: http_go_proto */
	libpb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: library_go_proto */
	mappb "github.com/GoogleCloudPlatform/healthcare-data-harmonization/mapping_engine/proto" /* copybara-comment: mapping_go_proto */

)

// Transformer contains projectors initialized for a specific config, and receiver methods to
// perform transformations.
type Transformer struct {
	Registry                *types.Registry
	dataHarmonizationConfig *dhpb.DataHarmonizationConfig
	mappingConfig           *mappb.MappingConfig
	parallel                bool
}

// TransformationConfigs contains metadata used during transformation.
type TransformationConfigs struct {
	LogTrace     bool
	SkipBundling bool
}

// Options for initializing Data Harmonization transform library
type Options struct {
	// CloudFunctions enables support for cloud functions within the transform library.
	CloudFunctions bool

	// FetchConfigs enables support for fetch configurations wtihin the transform library.
	FetchConfigs bool

	// Parallel enables support for parallelization within the transform library.
	Parallel bool

	// GCSClient is a client for downloading from GCS. If unset, the default third party GCS client will be used.
	GCSClient gcsutil.StorageClient
}

// Option is a setter function for Options.
type Option func(*Options)

// CloudFunctions initializes the CloudFunctions transform option.
func CloudFunctions(enabledCloudFunctions bool) Option {
	return func(args *Options) {
		args.CloudFunctions = enabledCloudFunctions
	}
}

// FetchConfigs initializes the FetchConfigs transform option.
func FetchConfigs(enableFetchConfig bool) Option {
	return func(args *Options) {
		args.FetchConfigs = enableFetchConfig
	}
}

// Parallel initializes the Parallel transform option.
func Parallel(enableParallel bool) Option {
	return func(args *Options) {
		args.Parallel = enableParallel
	}
}

// GCSClient sets the GCSClient in the transform option.
func GCSClient(c gcsutil.StorageClient) Option {
	return func(args *Options) {
		args.GCSClient = c
	}
}

// NewTransformer creates and initializes a transformer.
func NewTransformer(ctx context.Context, config *dhpb.DataHarmonizationConfig, setters ...Option) (*Transformer, error) {
	t := &Transformer{
		Registry:                types.NewRegistry(),
		dataHarmonizationConfig: config,
	}

	if err := builtins.RegisterAll(t.Registry); err != nil {
		return nil, err
	}

	options := &Options{}
	for _, setter := range setters {
		setter(options)
	}

	t.parallel = options.Parallel

	gcsutil.InitializeClient(options.GCSClient)

	if hc := config.GetHarmonizationConfig(); hc != nil {
		if err := harmonizecode.LoadCodeHarmonizationProjectors(t.Registry, hc); err != nil {
			return nil, err
		}
	}

	if uc := config.GetUnitHarmonizationConfig(); uc != nil {
		if err := harmonizeunit.LoadUnitHarmonizationProjectors(t.Registry, uc); err != nil {
			return nil, err
		}
	}

	mpc, err := t.LoadMappingConfig(config)
	if err != nil {
		return nil, err
	}
	t.mappingConfig = mpc

	if err := t.LoadProjectors(mpc.GetProjector()); err != nil {
		return nil, err
	}

	// Load the library configurations.
	for _, lc := range config.GetLibraryConfig() {
		if err := t.LoadProjectors(lc.Projector); err != nil {
			return nil, err
		}

		if options.CloudFunctions {
			if err := cloudfunction.LoadCloudFunctionProjectors(t.Registry, lc.CloudFunction); err != nil {
				return nil, err
			}
		} else {
			if len(lc.CloudFunction) != 0 {
				return nil, &invalidCloudFunctionProjectorError{lc: lc}
			}
		}

		if err := auth.LoadServerConfigs(lc.Servers); err != nil {
			return nil, err
		}

		if options.FetchConfigs {
			if err := fetch.LoadFetchProjectors(context.Background(), t.Registry, lc.HttpQuery); err != nil {
				return nil, err
			}
		} else {
			if len(lc.HttpQuery) != 0 {
				return nil, &invalidFetchProjectorError{lc: lc}
			}
		}

		for _, lib := range lc.GetUserLibraries() {
			mpc, err := loadMappingConfig(lib.GetPath(), lib.GetType())
			if err != nil {
				return nil, err
			}
			// Custom library configs should only have projectors defined, all else is ignored.
			if err := t.LoadProjectors(mpc.GetProjector()); err != nil {
				return nil, err
			}
		}
	}

	return t, nil
}

type invalidCloudFunctionProjectorError struct {
	lc *libpb.LibraryConfig
}

type invalidFetchProjectorError struct {
	lc *libpb.LibraryConfig
}

func (e invalidCloudFunctionProjectorError) Error() string {
	var names []string
	for _, cf := range e.lc.CloudFunction {
		names = append(names, cf.Name)
	}
	return fmt.Sprintf("attempting to use disabled Cloud Function projectors feature with projectors: %v", names)
}

func (e invalidFetchProjectorError) Error() string {
	var names []string
	for _, cf := range e.lc.HttpQuery {
		names = append(names, cf.Name)
	}
	return fmt.Sprintf("attempting to use disabled Fetch projectors feature with projectors: %v", names)
}

// Project is a convenience function to call a single projector out of context.
func (t *Transformer) Project(projector string, args ...jsonutil.JSONMetaNode) (res jsonutil.JSONToken, err error) {
	pctx := types.NewContext(t.Registry)

	defer errors.RecoverWithTrace(pctx.Trace, "Project", func(e error) {
		err = e
	})

	proj, err := t.Registry.FindProjector(projector)
	if err != nil {
		return nil, err
	}

	res, err = proj(args, pctx)
	return
}

// LoadMappingConfig loads the mapping config inline or from a GCS path.
func (t *Transformer) LoadMappingConfig(config *dhpb.DataHarmonizationConfig) (*mappb.MappingConfig, error) {
	mpc := &mappb.MappingConfig{}
	if config.GetDeprecateStructureMappingConfig() != nil && config.GetStructureMappingConfig() != nil {
		return nil, fmt.Errorf("only one of DeprecateStructureMappingConfig or StructureMappingConfig can be filled")
	}
	if config.GetDeprecateStructureMappingConfig() != nil {
		return config.GetDeprecateStructureMappingConfig(), nil
	}
	if sm := config.GetStructureMappingConfig(); sm != nil {
		switch mapping := config.GetStructureMappingConfig().Mapping.(type) {
		case *hapb.StructureMappingConfig_MappingConfig:
			return mapping.MappingConfig, nil
		case *hapb.StructureMappingConfig_MappingPathConfig:
			return loadMappingConfig(mapping.MappingPathConfig.MappingConfigPath, mapping.MappingPathConfig.MappingType)
		case *hapb.StructureMappingConfig_MappingLanguageString:
			return transpiler.Transpile(mapping.MappingLanguageString)
		default:
			return nil, fmt.Errorf("unsupported structure mapping config type: %v", mapping)
		}
	}
	return mpc, nil
}

// LoadProjectors registers all given projectors.
func (t *Transformer) LoadProjectors(projectors []*mappb.ProjectorDefinition) error {
	for _, pd := range projectors {
		p := projector.FromDef(pd, t.parallel)

		if err := t.Registry.RegisterProjector(pd.Name, p); err != nil {
			return fmt.Errorf("error registering projector %s: %v", pd.Name, err)
		}
	}
	return nil
}

// Transform converts the json tree using the specified config.
func (t *Transformer) Transform(in *jsonutil.JSONContainer, tconfig TransformationConfigs) (res jsonutil.JSONToken, err error) {
	pctx := types.NewContext(t.Registry)
	defer errors.RecoverWithTrace(pctx.Trace, "Transform", func(e error) {
		err = e
	})

	pctx.Variables.Push()

	inn, err := jsonutil.TokenToNode(*in)
	if err != nil {
		return nil, fmt.Errorf("input was invalid: %v", err)
	}
	args := []jsonutil.JSONMetaNode{inn}

	if err := mapping.ProcessMappings(t.mappingConfig.RootMapping, "root", args, &pctx.Output, pctx, t.parallel); err != nil {
		return nil, pctx.Trace.AsError(err)
	}

	result, err := postprocess.Process(pctx, t.mappingConfig, tconfig.SkipBundling, t.parallel)
	if err != nil {
		return nil, pctx.Trace.AsError(err)
	}

	if tconfig.LogTrace {
		log.Printf(pctx.Trace.String())
	}

	return result, nil
}

// JSONtoJSON converts the byte array (JSON format) using the specified config.
// TODO: Refactor to use json.RawMessage instead of []byte.
func (t *Transformer) JSONtoJSON(in []byte, tconfig TransformationConfigs) ([]byte, error) {
	ji := &jsonutil.JSONContainer{}
	if err := ji.UnmarshalJSON(in); err != nil {
		return nil, err
	}

	res, err := t.Transform(ji, tconfig)
	if err != nil {
		return nil, err
	}
	return json.Marshal(res)
}

// RegisterProjector adds the given Projector to this transformer's registry.
func (t *Transformer) RegisterProjector(name string, proj types.Projector) error {
	return t.Registry.RegisterProjector(name, proj)
}

// HasPostProcessProjector returns true iff a post process projector is set.
func (t *Transformer) HasPostProcessProjector() bool {
	return t.mappingConfig.GetPostProcessProjectorDefinition() != nil || t.mappingConfig.GetPostProcessProjectorName() != ""
}

// loadMappingConfig loads a mapping config from GCS.
func loadMappingConfig(loc *httppb.Location, typ hapb.MappingType) (*mappb.MappingConfig, error) {
	mpc := &mappb.MappingConfig{}
	switch l := loc.Location.(type) {
	case *httppb.Location_GcsLocation:
		data, err := gcsutil.ReadFromGcs(context.Background(), l.GcsLocation)
		if err != nil {
			return nil, fmt.Errorf("failed to read MappingConfig from GCS, %v", err)
		}
		switch typ {
		case hapb.MappingType_RAW_PROTO:
			if err := proto.UnmarshalText(string(data), mpc); err != nil {
				return nil, err
			}
		case hapb.MappingType_MAPPING_LANGUAGE:
			lmpc, err := transpiler.Transpile(string(data))
			if err != nil {
				return nil, err
			}
			mpc = lmpc
		default:
			return nil, fmt.Errorf("invalid mapping config type %v", typ)
		}
		return mpc, nil
	case *httppb.Location_UrlPath:
		return nil, fmt.Errorf("loading from remote path %s is unsupported", l.UrlPath)
	case *httppb.Location_LocalPath:
		return nil, fmt.Errorf("loading from local path %s is unsupported", l.LocalPath)
	default:
		return nil, fmt.Errorf("location type %s is unsupported", l)
	}
}
<|endoftext|>


<commit_before>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debugf("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// Searches this nodes links for one to the given key,
// returns the index of said link
func FindLink(n *Node, k u.Key, found []*Node) (int, error) {
	for i, lnk := range n.Links {
		if u.Key(lnk.Hash) == k && found[i] == nil {
			return i, nil
		}
	}
	return -1, u.ErrNotFound
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			i, err := FindLink(root, blk.Key(), nodes)
			if err != nil {
				// NB: can only occur as a result of programmer error
				panic("Received block that wasnt in this nodes links!")
			}

			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				//		input data
				log.Error("Got back bad block!")
				break
			}
			nodes[i] = nd
			for { //Check for duplicate links
				ni, err := FindLink(root, blk.Key(), nodes)
				if err != nil {
					break
				}
				nodes[ni] = nd
			}

			if next == i {
				sig <- nd
				next++
				for ; next < len(nodes) && nodes[next] != nil; next++ {
					sig <- nodes[next]
				}
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<commit_msg>Update merkledag.go<commit_after>// package merkledag implements the ipfs Merkle DAG datastructures.
package merkledag

import (
	"fmt"
	"sync"
	"time"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
	blocks "github.com/jbenet/go-ipfs/blocks"
	bserv "github.com/jbenet/go-ipfs/blockservice"
	u "github.com/jbenet/go-ipfs/util"
)

var log = u.Logger("merkledag")
var ErrNotFound = fmt.Errorf("merkledag: not found")

// NodeMap maps u.Keys to Nodes.
// We cannot use []byte/Multihash for keys :(
// so have to convert Multihash bytes to string (u.Key)
type NodeMap map[u.Key]*Node

// DAGService is an IPFS Merkle DAG service.
type DAGService interface {
	Add(*Node) (u.Key, error)
	AddRecursive(*Node) error
	Get(u.Key) (*Node, error)
	Remove(*Node) error

	// GetDAG returns, in order, all the single leve child
	// nodes of the passed in node.
	GetDAG(context.Context, *Node) <-chan *Node
}

func NewDAGService(bs *bserv.BlockService) DAGService {
	return &dagService{bs}
}

// Node represents a node in the IPFS Merkle DAG.
// nodes have opaque data and a set of navigable links.
type Node struct {
	Links []*Link
	Data  []byte

	// cache encoded/marshaled value
	encoded []byte

	cached mh.Multihash
}

// Link represents an IPFS Merkle DAG Link between Nodes.
type Link struct {
	// utf string name. should be unique per object
	Name string // utf8

	// cumulative size of target object
	Size uint64

	// multihash of the target object
	Hash mh.Multihash

	// a ptr to the actual node for graph manipulation
	Node *Node
}

// MakeLink creates a link to the given node
func MakeLink(n *Node) (*Link, error) {
	s, err := n.Size()
	if err != nil {
		return nil, err
	}

	h, err := n.Multihash()
	if err != nil {
		return nil, err
	}
	return &Link{
		Size: s,
		Hash: h,
	}, nil
}

// GetNode returns the MDAG Node that this link points to
func (l *Link) GetNode(serv DAGService) (*Node, error) {
	if l.Node != nil {
		return l.Node, nil
	}

	return serv.Get(u.Key(l.Hash))
}

// AddNodeLink adds a link to another node.
func (n *Node) AddNodeLink(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name
	lnk.Node = that

	n.Links = append(n.Links, lnk)
	return nil
}

// AddNodeLink adds a link to another node. without keeping a reference to
// the child node
func (n *Node) AddNodeLinkClean(name string, that *Node) error {
	lnk, err := MakeLink(that)
	if err != nil {
		return err
	}
	lnk.Name = name

	n.Links = append(n.Links, lnk)
	return nil
}

// Remove a link on this node by the given name
func (n *Node) RemoveNodeLink(name string) error {
	for i, l := range n.Links {
		if l.Name == name {
			n.Links = append(n.Links[:i], n.Links[i+1:]...)
			return nil
		}
	}
	return ErrNotFound
}

// Copy returns a copy of the node.
// NOTE: does not make copies of Node objects in the links.
func (n *Node) Copy() *Node {
	nnode := new(Node)
	nnode.Data = make([]byte, len(n.Data))
	copy(nnode.Data, n.Data)

	nnode.Links = make([]*Link, len(n.Links))
	copy(nnode.Links, n.Links)
	return nnode
}

// Size returns the total size of the data addressed by node,
// including the total sizes of references.
func (n *Node) Size() (uint64, error) {
	b, err := n.Encoded(false)
	if err != nil {
		return 0, err
	}

	s := uint64(len(b))
	for _, l := range n.Links {
		s += l.Size
	}
	return s, nil
}

// Multihash hashes the encoded data of this node.
func (n *Node) Multihash() (mh.Multihash, error) {
	// Note: Encoded generates the hash and puts it in n.cached.
	_, err := n.Encoded(false)
	if err != nil {
		return nil, err
	}

	return n.cached, nil
}

// Key returns the Multihash as a key, for maps.
func (n *Node) Key() (u.Key, error) {
	h, err := n.Multihash()
	return u.Key(h), err
}

// dagService is an IPFS Merkle DAG service.
// - the root is virtual (like a forest)
// - stores nodes' data in a BlockService
// TODO: should cache Nodes that are in memory, and be
//       able to free some of them when vm pressure is high
type dagService struct {
	Blocks *bserv.BlockService
}

// Add adds a node to the dagService, storing the block in the BlockService
func (n *dagService) Add(nd *Node) (u.Key, error) {
	k, _ := nd.Key()
	log.Debugf("DagService Add [%s]", k)
	if n == nil {
		return "", fmt.Errorf("dagService is nil")
	}

	d, err := nd.Encoded(false)
	if err != nil {
		return "", err
	}

	b := new(blocks.Block)
	b.Data = d
	b.Multihash, err = nd.Multihash()
	if err != nil {
		return "", err
	}

	return n.Blocks.AddBlock(b)
}

// AddRecursive adds the given node and all child nodes to the BlockService
func (n *dagService) AddRecursive(nd *Node) error {
	_, err := n.Add(nd)
	if err != nil {
		log.Info("AddRecursive Error: %s\n", err)
		return err
	}

	for _, link := range nd.Links {
		if link.Node != nil {
			err := n.AddRecursive(link.Node)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

// Get retrieves a node from the dagService, fetching the block in the BlockService
func (n *dagService) Get(k u.Key) (*Node, error) {
	if n == nil {
		return nil, fmt.Errorf("dagService is nil")
	}

	ctx, _ := context.WithTimeout(context.TODO(), time.Second*5)
	b, err := n.Blocks.GetBlock(ctx, k)
	if err != nil {
		return nil, err
	}

	return Decoded(b.Data)
}

// Remove deletes the given node and all of its children from the BlockService
func (n *dagService) Remove(nd *Node) error {
	for _, l := range nd.Links {
		if l.Node != nil {
			n.Remove(l.Node)
		}
	}
	k, err := nd.Key()
	if err != nil {
		return err
	}
	return n.Blocks.DeleteBlock(k)
}

// FetchGraph asynchronously fetches all nodes that are children of the given
// node, and returns a channel that may be waited upon for the fetch to complete
func FetchGraph(ctx context.Context, root *Node, serv DAGService) chan struct{} {
	log.Warning("Untested.")
	var wg sync.WaitGroup
	done := make(chan struct{})

	for _, l := range root.Links {
		wg.Add(1)
		go func(lnk *Link) {

			// Signal child is done on way out
			defer wg.Done()
			select {
			case <-ctx.Done():
				return
			}

			nd, err := lnk.GetNode(serv)
			if err != nil {
				log.Error(err)
				return
			}

			// Wait for children to finish
			<-FetchGraph(ctx, nd, serv)
		}(l)
	}

	go func() {
		wg.Wait()
		done <- struct{}{}
	}()

	return done
}

// Searches this nodes links for one to the given key,
// returns the index of said link
func FindLink(n *Node, k u.Key, found []*Node) (int, error) {
	for i, lnk := range n.Links {
		if u.Key(lnk.Hash) == k && found[i] == nil {
			return i, nil
		}
	}
	return -1, u.ErrNotFound
}

// GetDAG will fill out all of the links of the given Node.
// It returns a channel of nodes, which the caller can receive
// all the child nodes of 'root' on, in proper order.
func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
	sig := make(chan *Node)
	go func() {
		defer close(sig)

		var keys []u.Key
		for _, lnk := range root.Links {
			keys = append(keys, u.Key(lnk.Hash))
		}
		blkchan := ds.Blocks.GetBlocks(ctx, keys)

		nodes := make([]*Node, len(root.Links))
		next := 0
		for blk := range blkchan {
			i, err := FindLink(root, blk.Key(), nodes)
			if err != nil {
				// NB: can only occur as a result of programmer error
				panic("Received block that wasnt in this nodes links!")
			}

			nd, err := Decoded(blk.Data)
			if err != nil {
				// NB: can occur in normal situations, with improperly formatted
				// input data
				log.Error("Got back bad block!")
				break
			}
			nodes[i] = nd
			for { //Check for duplicate links
				ni, err := FindLink(root, blk.Key(), nodes)
				if err != nil {
					break
				}
				nodes[ni] = nd
			}

			if next == i {
				sig <- nd
				next++
				for ; next < len(nodes) && nodes[next] != nil; next++ {
					sig <- nodes[next]
				}
			}
		}
		if next < len(nodes) {
			// TODO: bubble errors back up.
			log.Errorf("Did not receive correct number of nodes!")
		}
	}()

	return sig
}
<|endoftext|>


<commit_before>package runtime

import (
	"bytes"
	"errors"
	"fmt"
	"sync/atomic"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestByteStreamConsumer(t *testing.T) {
	cons := ByteStreamConsumer()

	expected := "the data for the stream to be sent over the wire"

	// can consume as a Writer
	var b bytes.Buffer
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &b)) {
		assert.Equal(t, expected, b.String())
	}

	// can consume as an UnmarshalBinary
	var bu binaryUnmarshalDummy
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &bu)) {
		assert.Equal(t, expected, bu.str)
	}

	// can consume as a binary slice
	var bs []byte
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &bs)) {
		assert.Equal(t, expected, string(bs))
	}
	type binarySlice []byte
	var bs2 binarySlice
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &bs2)) {
		assert.Equal(t, expected, string(bs2))
	}

	// passing in a nilslice wil result in an error
	var ns *[]byte
	assert.Error(t, cons.Consume(bytes.NewBufferString(expected), &ns))

	// passing in nil wil result in an error as well
	assert.Error(t, cons.Consume(bytes.NewBufferString(expected), nil))

	// a reader who results in an error, will make it fail
	assert.Error(t, cons.Consume(new(nopReader), &bu))
	assert.Error(t, cons.Consume(new(nopReader), &bs))

	// the readers can also not be nil
	assert.Error(t, cons.Consume(nil, &bs))
}

type binaryUnmarshalDummy struct {
	str string
}

func (b *binaryUnmarshalDummy) UnmarshalBinary(bytes []byte) error {
	if len(bytes) == 0 {
		return errors.New("no text given")
	}

	b.str = string(bytes)
	return nil
}

func TestByteStreamProducer(t *testing.T) {
	cons := ByteStreamProducer()
	expected := "the data for the stream to be sent over the wire"

	var rdr bytes.Buffer

	// can produce using a reader
	if assert.NoError(t, cons.Produce(&rdr, bytes.NewBufferString(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// can produce using a binary marshaller
	if assert.NoError(t, cons.Produce(&rdr, &binaryMarshalDummy{expected})) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// string can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, expected)) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// binary slices can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, []byte(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// errors can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, errors.New(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// structs can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, Error{Message: expected})) {
		assert.Equal(t, fmt.Sprintf(`{"message":%q}`, expected), rdr.String())
		rdr.Reset()
	}

	// struct pointers can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, &Error{Message: expected})) {
		assert.Equal(t, fmt.Sprintf(`{"message":%q}`, expected), rdr.String())
		rdr.Reset()
	}

	// slices can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, []string{expected})) {
		assert.Equal(t, fmt.Sprintf(`[%q]`, expected), rdr.String())
		rdr.Reset()
	}

	type binarySlice []byte
	if assert.NoError(t, cons.Produce(&rdr, binarySlice(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// when binaryMarshal data is used, its potential error gets propagated
	assert.Error(t, cons.Produce(&rdr, new(binaryMarshalDummy)))
	// nil data should never be accepted either
	assert.Error(t, cons.Produce(&rdr, nil))
	// nil readers should also never be acccepted
	assert.Error(t, cons.Produce(nil, bytes.NewBufferString(expected)))
}

type binaryMarshalDummy struct {
	str string
}

func (b *binaryMarshalDummy) MarshalBinary() ([]byte, error) {
	if len(b.str) == 0 {
		return nil, errors.New("no text set")
	}

	return []byte(b.str), nil
}

type closingWriter struct {
	calledClose int64
	calledWrite int64
	b           bytes.Buffer
}

func (c *closingWriter) Close() error {
	atomic.AddInt64(&c.calledClose, 1)
	return nil
}

func (c *closingWriter) Write(p []byte) (n int, err error) {
	atomic.AddInt64(&c.calledWrite, 1)
	return c.b.Write(p)
}

func (c *closingWriter) String() string {
	return c.b.String()
}

type closingReader struct {
	calledClose int64
	calledRead  int64
	b           *bytes.Buffer
}

func (c *closingReader) Close() error {
	atomic.AddInt64(&c.calledClose, 1)
	return nil
}

func (c *closingReader) Read(p []byte) (n int, err error) {
	atomic.AddInt64(&c.calledRead, 1)
	return c.b.Read(p)
}

func TestBytestreamConsumer_Close(t *testing.T) {
	cons := ByteStreamConsumer(ClosesStream)
	expected := "the data for the stream to be sent over the wire"

	// can consume as a Writer
	var b bytes.Buffer
	r := &closingReader{b: bytes.NewBufferString(expected)}
	if assert.NoError(t, cons.Consume(r, &b)) {
		assert.Equal(t, expected, b.String())
		assert.EqualValues(t, 1, r.calledClose)
	}

	// can consume as a Writer
	cons = ByteStreamConsumer()
	b.Reset()
	r = &closingReader{b: bytes.NewBufferString(expected)}
	if assert.NoError(t, cons.Consume(r, &b)) {
		assert.Equal(t, expected, b.String())
		assert.EqualValues(t, 0, r.calledClose)
	}
}

func TestBytestreamProducer_Close(t *testing.T) {
	cons := ByteStreamProducer(ClosesStream)
	expected := "the data for the stream to be sent over the wire"

	// can consume as a Writer
	r := &closingWriter{}
	// can produce using a reader
	if assert.NoError(t, cons.Produce(r, bytes.NewBufferString(expected))) {
		assert.Equal(t, expected, r.String())
		assert.EqualValues(t, 1, r.calledClose)
	}

	cons = ByteStreamProducer()
	r = &closingWriter{}
	// can produce using a reader
	if assert.NoError(t, cons.Produce(r, bytes.NewBufferString(expected))) {
		assert.Equal(t, expected, r.String())
		assert.EqualValues(t, 0, r.calledClose)
	}

	cons = ByteStreamProducer()
	r = &closingWriter{}
	data := &closingReader{b: bytes.NewBufferString(expected)}
	// can produce using a readcloser
	if assert.NoError(t, cons.Produce(r, data)) {
		assert.Equal(t, expected, r.String())
		assert.EqualValues(t, 0, r.calledClose)
		assert.EqualValues(t, 1, data.calledClose)
	}
}
<commit_msg>Add test.<commit_after>package runtime

import (
	"bytes"
	"errors"
	"fmt"
	"sync/atomic"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestByteStreamConsumer(t *testing.T) {
	cons := ByteStreamConsumer()

	expected := "the data for the stream to be sent over the wire"

	// can consume as a Writer
	var b bytes.Buffer
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &b)) {
		assert.Equal(t, expected, b.String())
	}

	//can consume as a string
	var s string
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &s)) {
		assert.Equal(t, expected, s)
	}

	// can consume as an UnmarshalBinary
	var bu binaryUnmarshalDummy
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &bu)) {
		assert.Equal(t, expected, bu.str)
	}

	// can consume as a binary slice
	var bs []byte
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &bs)) {
		assert.Equal(t, expected, string(bs))
	}
	type binarySlice []byte
	var bs2 binarySlice
	if assert.NoError(t, cons.Consume(bytes.NewBufferString(expected), &bs2)) {
		assert.Equal(t, expected, string(bs2))
	}

	// passing in a nilslice wil result in an error
	var ns *[]byte
	assert.Error(t, cons.Consume(bytes.NewBufferString(expected), &ns))

	// passing in nil wil result in an error as well
	assert.Error(t, cons.Consume(bytes.NewBufferString(expected), nil))

	// a reader who results in an error, will make it fail
	assert.Error(t, cons.Consume(new(nopReader), &bu))
	assert.Error(t, cons.Consume(new(nopReader), &bs))

	// the readers can also not be nil
	assert.Error(t, cons.Consume(nil, &bs))
}

type binaryUnmarshalDummy struct {
	str string
}

func (b *binaryUnmarshalDummy) UnmarshalBinary(bytes []byte) error {
	if len(bytes) == 0 {
		return errors.New("no text given")
	}

	b.str = string(bytes)
	return nil
}

func TestByteStreamProducer(t *testing.T) {
	cons := ByteStreamProducer()
	expected := "the data for the stream to be sent over the wire"

	var rdr bytes.Buffer

	// can produce using a reader
	if assert.NoError(t, cons.Produce(&rdr, bytes.NewBufferString(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// can produce using a binary marshaller
	if assert.NoError(t, cons.Produce(&rdr, &binaryMarshalDummy{expected})) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// string can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, expected)) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// binary slices can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, []byte(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// errors can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, errors.New(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// structs can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, Error{Message: expected})) {
		assert.Equal(t, fmt.Sprintf(`{"message":%q}`, expected), rdr.String())
		rdr.Reset()
	}

	// struct pointers can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, &Error{Message: expected})) {
		assert.Equal(t, fmt.Sprintf(`{"message":%q}`, expected), rdr.String())
		rdr.Reset()
	}

	// slices can also be used to produce
	if assert.NoError(t, cons.Produce(&rdr, []string{expected})) {
		assert.Equal(t, fmt.Sprintf(`[%q]`, expected), rdr.String())
		rdr.Reset()
	}

	type binarySlice []byte
	if assert.NoError(t, cons.Produce(&rdr, binarySlice(expected))) {
		assert.Equal(t, expected, rdr.String())
		rdr.Reset()
	}

	// when binaryMarshal data is used, its potential error gets propagated
	assert.Error(t, cons.Produce(&rdr, new(binaryMarshalDummy)))
	// nil data should never be accepted either
	assert.Error(t, cons.Produce(&rdr, nil))
	// nil readers should also never be acccepted
	assert.Error(t, cons.Produce(nil, bytes.NewBufferString(expected)))
}

type binaryMarshalDummy struct {
	str string
}

func (b *binaryMarshalDummy) MarshalBinary() ([]byte, error) {
	if len(b.str) == 0 {
		return nil, errors.New("no text set")
	}

	return []byte(b.str), nil
}

type closingWriter struct {
	calledClose int64
	calledWrite int64
	b           bytes.Buffer
}

func (c *closingWriter) Close() error {
	atomic.AddInt64(&c.calledClose, 1)
	return nil
}

func (c *closingWriter) Write(p []byte) (n int, err error) {
	atomic.AddInt64(&c.calledWrite, 1)
	return c.b.Write(p)
}

func (c *closingWriter) String() string {
	return c.b.String()
}

type closingReader struct {
	calledClose int64
	calledRead  int64
	b           *bytes.Buffer
}

func (c *closingReader) Close() error {
	atomic.AddInt64(&c.calledClose, 1)
	return nil
}

func (c *closingReader) Read(p []byte) (n int, err error) {
	atomic.AddInt64(&c.calledRead, 1)
	return c.b.Read(p)
}

func TestBytestreamConsumer_Close(t *testing.T) {
	cons := ByteStreamConsumer(ClosesStream)
	expected := "the data for the stream to be sent over the wire"

	// can consume as a Writer
	var b bytes.Buffer
	r := &closingReader{b: bytes.NewBufferString(expected)}
	if assert.NoError(t, cons.Consume(r, &b)) {
		assert.Equal(t, expected, b.String())
		assert.EqualValues(t, 1, r.calledClose)
	}

	// can consume as a Writer
	cons = ByteStreamConsumer()
	b.Reset()
	r = &closingReader{b: bytes.NewBufferString(expected)}
	if assert.NoError(t, cons.Consume(r, &b)) {
		assert.Equal(t, expected, b.String())
		assert.EqualValues(t, 0, r.calledClose)
	}

}

func TestBytestreamProducer_Close(t *testing.T) {
	cons := ByteStreamProducer(ClosesStream)
	expected := "the data for the stream to be sent over the wire"

	// can consume as a Writer
	r := &closingWriter{}
	// can produce using a reader
	if assert.NoError(t, cons.Produce(r, bytes.NewBufferString(expected))) {
		assert.Equal(t, expected, r.String())
		assert.EqualValues(t, 1, r.calledClose)
	}

	cons = ByteStreamProducer()
	r = &closingWriter{}
	// can produce using a reader
	if assert.NoError(t, cons.Produce(r, bytes.NewBufferString(expected))) {
		assert.Equal(t, expected, r.String())
		assert.EqualValues(t, 0, r.calledClose)
	}

	cons = ByteStreamProducer()
	r = &closingWriter{}
	data := &closingReader{b: bytes.NewBufferString(expected)}
	// can produce using a readcloser
	if assert.NoError(t, cons.Produce(r, data)) {
		assert.Equal(t, expected, r.String())
		assert.EqualValues(t, 0, r.calledClose)
		assert.EqualValues(t, 1, data.calledClose)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"testing"
)

var (
	testDashboard = Dashboard{}
)

func init() {
	file := "dashboard-example.json"
	fi, err := os.Open(file)
	if err != nil {
		fmt.Printf("Error opening example json file (%s)\n", file)
		os.Exit(1)
	}
	defer fi.Close()

	dec := json.NewDecoder(fi)
	dec.Decode(&testDashboard)
}

func testDashboardServer() *httptest.Server {
	f := func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if path == "/dashboard/1234" {
			switch r.Method {
			case "GET":
				ret, err := json.Marshal(testDashboard)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			case "PUT":
				defer r.Body.Close()
				b, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(b))
			case "DELETE":
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else if path == "/dashboard" {
			switch r.Method {
			case "GET":
				c := []Dashboard{testDashboard}
				ret, err := json.Marshal(c)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			case "POST":
				defer r.Body.Close()
				_, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				ret, err := json.Marshal(testDashboard)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else {
			w.WriteHeader(404)
			fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
		}
	}

	return httptest.NewServer(http.HandlerFunc(f))
}

func TestFetchDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("without CID")
	{
		cid := ""
		expectedError := errors.New("Invalid dashboard CID [none]")
		_, err := apih.FetchDashboard(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}

	t.Log("with valid CID")
	{
		cid := "/dashboard/1234"
		dashboard, err := apih.FetchDashboard(CIDType(&cid))
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboard)
		expectedType := "*api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}

		if dashboard.CID != testDashboard.CID {
			t.Fatalf("CIDs do not match: %+v != %+v\n", dashboard, testDashboard)
		}
	}

	t.Log("with invalid CID")
	{
		cid := "/invalid"
		expectedError := errors.New("Invalid dashboard CID [/invalid]")
		_, err := apih.FetchDashboard(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestFetchDashboards(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	dashboards, err := apih.FetchDashboards()
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(dashboards)
	expectedType := "*[]api.Dashboard"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}

}

func TestCreateDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	dashboard, err := apih.CreateDashboard(&testDashboard)
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(dashboard)
	expectedType := "*api.Dashboard"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}
}

func TestUpdateDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid Dashboard")
	{
		dashboard, err := apih.UpdateDashboard(&testDashboard)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboard)
		expectedType := "*api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid dashboard CID [/invalid]")
		x := &Dashboard{CID: "/invalid"}
		_, err := apih.UpdateDashboard(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestDeleteDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid Dashboard")
	{
		_, err := apih.DeleteDashboard(&testDashboard)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid dashboard CID [/invalid]")
		x := &Dashboard{CID: "/invalid"}
		_, err := apih.UpdateDashboard(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}
<commit_msg>add: search test coverage<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"reflect"
	"testing"
)

var (
	testDashboard = Dashboard{}
)

func init() {
	file := "dashboard-example.json"
	fi, err := os.Open(file)
	if err != nil {
		fmt.Printf("Error opening example json file (%s)\n", file)
		os.Exit(1)
	}
	defer fi.Close()

	dec := json.NewDecoder(fi)
	dec.Decode(&testDashboard)
}

func testDashboardServer() *httptest.Server {
	f := func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path
		if path == "/dashboard/1234" {
			switch r.Method {
			case "GET":
				ret, err := json.Marshal(testDashboard)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			case "PUT":
				defer r.Body.Close()
				b, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(b))
			case "DELETE":
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else if path == "/dashboard" {
			switch r.Method {
			case "GET":
				reqURL := r.URL.String()
				var c []Dashboard
				if reqURL == "/dashboard?search=my+dashboard" {
					c = []Dashboard{testDashboard}
				} else if reqURL == "/dashboard?f__created_gt=1483639916" {
					c = []Dashboard{testDashboard}
				} else if reqURL == "/dashboard?f__created_gt=1483639916&search=my+dashboard" {
					c = []Dashboard{testDashboard}
				} else if reqURL == "/dashboard" {
					c = []Dashboard{testDashboard}
				} else {
					c = []Dashboard{}
				}
				if len(c) > 0 {
					ret, err := json.Marshal(c)
					if err != nil {
						panic(err)
					}
					w.WriteHeader(200)
					w.Header().Set("Content-Type", "application/json")
					fmt.Fprintln(w, string(ret))
				} else {
					w.WriteHeader(404)
					fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, reqURL))
				}
			case "POST":
				defer r.Body.Close()
				_, err := ioutil.ReadAll(r.Body)
				if err != nil {
					panic(err)
				}
				ret, err := json.Marshal(testDashboard)
				if err != nil {
					panic(err)
				}
				w.WriteHeader(200)
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintln(w, string(ret))
			default:
				w.WriteHeader(404)
				fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
			}
		} else {
			w.WriteHeader(404)
			fmt.Fprintln(w, fmt.Sprintf("not found: %s %s", r.Method, path))
		}
	}

	return httptest.NewServer(http.HandlerFunc(f))
}

func TestFetchDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("without CID")
	{
		cid := ""
		expectedError := errors.New("Invalid dashboard CID [none]")
		_, err := apih.FetchDashboard(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}

	t.Log("with valid CID")
	{
		cid := "/dashboard/1234"
		dashboard, err := apih.FetchDashboard(CIDType(&cid))
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboard)
		expectedType := "*api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}

		if dashboard.CID != testDashboard.CID {
			t.Fatalf("CIDs do not match: %+v != %+v\n", dashboard, testDashboard)
		}
	}

	t.Log("with invalid CID")
	{
		cid := "/invalid"
		expectedError := errors.New("Invalid dashboard CID [/invalid]")
		_, err := apih.FetchDashboard(CIDType(&cid))
		if err == nil {
			t.Fatalf("Expected error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestFetchDashboards(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	dashboards, err := apih.FetchDashboards()
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(dashboards)
	expectedType := "*[]api.Dashboard"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}

}

func TestCreateDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	dashboard, err := apih.CreateDashboard(&testDashboard)
	if err != nil {
		t.Fatalf("Expected no error, got '%v'", err)
	}

	actualType := reflect.TypeOf(dashboard)
	expectedType := "*api.Dashboard"
	if actualType.String() != expectedType {
		t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
	}
}

func TestUpdateDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid Dashboard")
	{
		dashboard, err := apih.UpdateDashboard(&testDashboard)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboard)
		expectedType := "*api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid dashboard CID [/invalid]")
		x := &Dashboard{CID: "/invalid"}
		_, err := apih.UpdateDashboard(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestDeleteDashboard(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	t.Log("valid Dashboard")
	{
		_, err := apih.DeleteDashboard(&testDashboard)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}
	}

	t.Log("Test with invalid CID")
	{
		expectedError := errors.New("Invalid dashboard CID [/invalid]")
		x := &Dashboard{CID: "/invalid"}
		_, err := apih.UpdateDashboard(x)
		if err == nil {
			t.Fatal("Expected an error")
		}
		if err.Error() != expectedError.Error() {
			t.Fatalf("Expected %+v got '%+v'", expectedError, err)
		}
	}
}

func TestSearchDashboards(t *testing.T) {
	server := testDashboardServer()
	defer server.Close()

	var apih *API

	ac := &Config{
		TokenKey: "abc123",
		TokenApp: "test",
		URL:      server.URL,
	}
	apih, err := NewAPI(ac)
	if err != nil {
		t.Errorf("Expected no error, got '%v'", err)
	}

	search := SearchQueryType("my dashboard")
	filter := SearchFilterType(map[string][]string{"f__created_gt": []string{"1483639916"}})

	t.Log("no search, no filter")
	{
		dashboards, err := apih.SearchDashboards(nil, nil)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboards)
		expectedType := "*[]api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("search, no filter")
	{
		dashboards, err := apih.SearchDashboards(&search, nil)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboards)
		expectedType := "*[]api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("no search, filter")
	{
		dashboards, err := apih.SearchDashboards(nil, &filter)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboards)
		expectedType := "*[]api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}

	t.Log("search, filter")
	{
		dashboards, err := apih.SearchDashboards(&search, &filter)
		if err != nil {
			t.Fatalf("Expected no error, got '%v'", err)
		}

		actualType := reflect.TypeOf(dashboards)
		expectedType := "*[]api.Dashboard"
		if actualType.String() != expectedType {
			t.Fatalf("Expected %s, got %s", expectedType, actualType.String())
		}
	}
}
<|endoftext|>