<commit_before>package buildah

import (
	"bytes"
	"io"
	"time"

	"github.com/Sirupsen/logrus"
	cp "github.com/containers/image/copy"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	"github.com/containers/storage/pkg/stringid"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/projectatomic/buildah/util"
)

var (
	// gzippedEmptyLayer is a gzip-compressed version of an empty tar file (just 1024 zero bytes).  This
	// comes from github.com/docker/distribution/manifest/schema1/config_builder.go by way of
	// github.com/containers/image/image/docker_schema2.go; there is a non-zero embedded timestamp; we could
	// zero that, but that would just waste storage space in registries, so let’s use the same values.
	gzippedEmptyLayer = []byte{
		31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88,
		0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0,
	}
)

// CommitOptions can be used to alter how an image is committed.
type CommitOptions struct {
	// PreferredManifestType is the preferred type of image manifest.  The
	// image configuration format will be of a compatible type.
	PreferredManifestType string
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// AdditionalTags is a list of additional names to add to the image, if
	// the transport to which we're writing the image gives us a way to add
	// them.
	AdditionalTags []string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// HistoryTimestamp is the timestamp used when creating new items in the
	// image's history.  If unset, the current time will be used.
	HistoryTimestamp *time.Time
}

// PushOptions can be used to alter how an image is copied somewhere.
type PushOptions struct {
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// Store is the local storage store which holds the source image.
	Store storage.Store
}

// shallowCopy copies the most recent layer, the configuration, and the manifest from one image to another.
// For local storage, which doesn't care about histories and the manifest's contents, that's sufficient, but
// almost any other destination has higher expectations.
// We assume that "dest" is a reference to a local image (specifically, a containers/image/storage.storageReference),
// and will fail if it isn't.
func (b *Builder) shallowCopy(dest types.ImageReference, src types.ImageReference, systemContext *types.SystemContext) error {
	// Read the target image name.
	if dest.DockerReference() == nil {
		return errors.New("can't write to an unnamed image")
	}
	names, err := util.ExpandTags([]string{dest.DockerReference().String()})
	if err != nil {
		return err
	}
	// Make a temporary image reference.
	tmpName := stringid.GenerateRandomID() + "-tmp-" + Package + "-commit"
	tmpRef, err := is.Transport.ParseStoreReference(b.store, tmpName)
	if err != nil {
		return err
	}
	defer func() {
		if err2 := tmpRef.DeleteImage(systemContext); err2 != nil {
			logrus.Debugf("error deleting temporary image %q: %v", tmpName, err2)
		}
	}()
	// Open the source for reading and a temporary image for writing.
	srcImage, err := src.NewImage(systemContext)
	if err != nil {
		return errors.Wrapf(err, "error reading configuration to write to image %q", transports.ImageName(dest))
	}
	defer srcImage.Close()
	tmpImage, err := tmpRef.NewImageDestination(systemContext)
	if err != nil {
		return errors.Wrapf(err, "error opening temporary copy of image %q for writing", transports.ImageName(dest))
	}
	defer tmpImage.Close()
	// Write an empty filesystem layer, because the image layer requires at least one.
	_, err = tmpImage.PutBlob(bytes.NewReader(gzippedEmptyLayer), types.BlobInfo{Size: int64(len(gzippedEmptyLayer))})
	if err != nil {
		return errors.Wrapf(err, "error writing dummy layer for image %q", transports.ImageName(dest))
	}
	// Read the newly-generated configuration blob.
	config, err := srcImage.ConfigBlob()
	if err != nil {
		return errors.Wrapf(err, "error reading new configuration for image %q", transports.ImageName(dest))
	}
	if len(config) == 0 {
		return errors.Errorf("error reading new configuration for image %q: it's empty", transports.ImageName(dest))
	}
	logrus.Debugf("read configuration blob %q", string(config))
	// Write the configuration to the temporary image.
	configBlobInfo := types.BlobInfo{
		Digest: digest.Canonical.FromBytes(config),
		Size:   int64(len(config)),
	}
	_, err = tmpImage.PutBlob(bytes.NewReader(config), configBlobInfo)
	if err != nil && len(config) > 0 {
		return errors.Wrapf(err, "error writing image configuration for temporary copy of %q", transports.ImageName(dest))
	}
	// Read the newly-generated, mostly fake, manifest.
	manifest, _, err := srcImage.Manifest()
	if err != nil {
		return errors.Wrapf(err, "error reading new manifest for image %q", transports.ImageName(dest))
	}
	// Write the manifest to the temporary image.
	err = tmpImage.PutManifest(manifest)
	if err != nil {
		return errors.Wrapf(err, "error writing new manifest to temporary copy of image %q", transports.ImageName(dest))
	}
	// Save the temporary image.
	err = tmpImage.Commit()
	if err != nil {
		return errors.Wrapf(err, "error committing new image %q", transports.ImageName(dest))
	}
	// Locate the temporary image in the lower-level API.  Read its item names.
	tmpImg, err := is.Transport.GetStoreImage(b.store, tmpRef)
	if err != nil {
		return errors.Wrapf(err, "error locating temporary image %q", transports.ImageName(dest))
	}
	items, err := b.store.ListImageBigData(tmpImg.ID)
	if err != nil {
		return errors.Wrapf(err, "error reading list of named data for image %q", tmpImg.ID)
	}
	// Look up the container's read-write layer.
	container, err := b.store.Container(b.ContainerID)
	if err != nil {
		return errors.Wrapf(err, "error reading information about working container %q", b.ContainerID)
	}
	parentLayer := ""
	// Look up the container's source image's layer, if there is a source image.
	if container.ImageID != "" {
		img, err2 := b.store.Image(container.ImageID)
		if err2 != nil {
			return errors.Wrapf(err2, "error reading information about working container %q's source image", b.ContainerID)
		}
		parentLayer = img.TopLayer
	}
	// Extract the read-write layer's contents.
	layerDiff, err := b.store.Diff(parentLayer, container.LayerID)
	if err != nil {
		return errors.Wrapf(err, "error reading layer from source image %q", transports.ImageName(src))
	}
	defer layerDiff.Close()
	// Write a copy of the layer for the new image to reference.
	layer, _, err := b.store.PutLayer("", parentLayer, []string{}, "", false, layerDiff)
	if err != nil {
		return errors.Wrapf(err, "error creating new read-only layer from container %q", b.ContainerID)
	}
	// Create a low-level image record that uses the new layer.
	image, err := b.store.CreateImage("", []string{}, layer.ID, "", nil)
	if err != nil {
		err2 := b.store.DeleteLayer(layer.ID)
		if err2 != nil {
			logrus.Debugf("error removing layer %q: %v", layer, err2)
		}
		return errors.Wrapf(err, "error creating new low-level image %q", transports.ImageName(dest))
	}
	logrus.Debugf("created image ID %q", image.ID)
	defer func() {
		if err != nil {
			_, err2 := b.store.DeleteImage(image.ID, true)
			if err2 != nil {
				logrus.Debugf("error removing image %q: %v", image.ID, err2)
			}
		}
	}()
	// Copy the configuration and manifest, which are big data items, along with whatever else is there.
	for _, item := range items {
		var data []byte
		data, err = b.store.ImageBigData(tmpImg.ID, item)
		if err != nil {
			return errors.Wrapf(err, "error copying data item %q", item)
		}
		err = b.store.SetImageBigData(image.ID, item, data)
		if err != nil {
			return errors.Wrapf(err, "error copying data item %q", item)
		}
		logrus.Debugf("copied data item %q to %q", item, image.ID)
	}
	// Set low-level metadata in the new image so that the image library will accept it as a real image.
	err = b.store.SetMetadata(image.ID, "{}")
	if err != nil {
		return errors.Wrapf(err, "error assigning metadata to new image %q", transports.ImageName(dest))
	}
	// Move the target name(s) from the temporary image to the new image.
	err = util.AddImageNames(b.store, image, names)
	if err != nil {
		return errors.Wrapf(err, "error assigning names %v to new image", names)
	}
	logrus.Debugf("assigned names %v to image %q", names, image.ID)
	return nil
}

// Commit writes the contents of the container, along with its updated
// configuration, to a new image in the specified location, and if we know how,
// add any additional tags that were specified.
func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error {
	policy, err := signature.DefaultPolicy(getSystemContext(options.SignaturePolicyPath))
	if err != nil {
		return err
	}
	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return err
	}
	// Check if we're keeping everything in local storage.  If so, we can take certain shortcuts.
	_, destIsStorage := dest.Transport().(is.StoreTransport)
	exporting := !destIsStorage
	src, err := b.makeContainerImageRef(options.PreferredManifestType, exporting, options.Compression, options.HistoryTimestamp)
	if err != nil {
		return errors.Wrapf(err, "error computing layer digests and building metadata")
	}
	if exporting {
		// Copy everything.
		err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
		if err != nil {
			return errors.Wrapf(err, "error copying layers and metadata")
		}
	} else {
		// Copy only the most recent layer, the configuration, and the manifest.
		err = b.shallowCopy(dest, src, getSystemContext(options.SignaturePolicyPath))
		if err != nil {
			return errors.Wrapf(err, "error copying layer and metadata")
		}
	}
	if len(options.AdditionalTags) > 0 {
		switch dest.Transport().Name() {
		case is.Transport.Name():
			img, err := is.Transport.GetStoreImage(b.store, dest)
			if err != nil {
				return errors.Wrapf(err, "error locating just-written image %q", transports.ImageName(dest))
			}
			err = util.AddImageNames(b.store, img, options.AdditionalTags)
			if err != nil {
				return errors.Wrapf(err, "error setting image names to %v", append(img.Names, options.AdditionalTags...))
			}
			logrus.Debugf("assigned names %v to image %q", img.Names, img.ID)
		default:
			logrus.Warnf("don't know how to add tags to images stored in %q transport", dest.Transport().Name())
		}
	}
	return nil
}

// Push copies the contents of the image to a new location.
func Push(image string, dest types.ImageReference, options PushOptions) error {
	systemContext := getSystemContext(options.SignaturePolicyPath)
	policy, err := signature.DefaultPolicy(systemContext)
	if err != nil {
		return err
	}
	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return err
	}
	importOptions := ImportFromImageOptions{
		Image:               image,
		SignaturePolicyPath: options.SignaturePolicyPath,
	}
	builder, err := importBuilderFromImage(options.Store, importOptions)
	if err != nil {
		return errors.Wrap(err, "error importing builder information from image")
	}
	// Look up the image name and its layer.
	ref, err := is.Transport.ParseStoreReference(options.Store, image)
	if err != nil {
		return errors.Wrapf(err, "error parsing reference to image %q", image)
	}
	img, err := is.Transport.GetStoreImage(options.Store, ref)
	if err != nil {
		return errors.Wrapf(err, "error locating image %q", image)
	}
	// Give the image we're producing the same ancestors as its source image.
	builder.FromImage = builder.Docker.ContainerConfig.Image
	builder.FromImageID = string(builder.Docker.Parent)
	// Prep the layers and manifest for export.
	src, err := builder.makeImageImageRef(options.Compression, img.Names, img.TopLayer, nil)
	if err != nil {
		return errors.Wrapf(err, "error recomputing layer digests and building metadata")
	}
	// Copy everything.
	err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
	if err != nil {
		return errors.Wrapf(err, "error copying layers and metadata")
	}
	return nil
}
<commit_msg>Make shallowCopy() not use a temporary image<commit_after>package buildah

import (
	"bytes"
	"io"
	"time"

	"github.com/Sirupsen/logrus"
	cp "github.com/containers/image/copy"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
	"github.com/projectatomic/buildah/util"
)

var (
	// gzippedEmptyLayer is a gzip-compressed version of an empty tar file (just 1024 zero bytes).  This
	// comes from github.com/docker/distribution/manifest/schema1/config_builder.go by way of
	// github.com/containers/image/image/docker_schema2.go; there is a non-zero embedded timestamp; we could
	// zero that, but that would just waste storage space in registries, so let’s use the same values.
	gzippedEmptyLayer = []byte{
		31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88,
		0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0,
	}
)

// CommitOptions can be used to alter how an image is committed.
type CommitOptions struct {
	// PreferredManifestType is the preferred type of image manifest.  The
	// image configuration format will be of a compatible type.
	PreferredManifestType string
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// AdditionalTags is a list of additional names to add to the image, if
	// the transport to which we're writing the image gives us a way to add
	// them.
	AdditionalTags []string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// HistoryTimestamp is the timestamp used when creating new items in the
	// image's history.  If unset, the current time will be used.
	HistoryTimestamp *time.Time
}

// PushOptions can be used to alter how an image is copied somewhere.
type PushOptions struct {
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// Store is the local storage store which holds the source image.
	Store storage.Store
}

// shallowCopy copies the most recent layer, the configuration, and the manifest from one image to another.
// For local storage, which doesn't care about histories and the manifest's contents, that's sufficient, but
// almost any other destination has higher expectations.
// We assume that "dest" is a reference to a local image (specifically, a containers/image/storage.storageReference),
// and will fail if it isn't.
func (b *Builder) shallowCopy(dest types.ImageReference, src types.ImageReference, systemContext *types.SystemContext) error {
	var names []string
	// Read the target image name.
	if dest.DockerReference() != nil {
		names = []string{dest.DockerReference().String()}
	}
	// Open the source for reading and the new image for writing.
	srcImage, err := src.NewImage(systemContext)
	if err != nil {
		return errors.Wrapf(err, "error reading configuration to write to image %q", transports.ImageName(dest))
	}
	defer srcImage.Close()
	destImage, err := dest.NewImageDestination(systemContext)
	if err != nil {
		return errors.Wrapf(err, "error opening image %q for writing", transports.ImageName(dest))
	}
	// Write an empty filesystem layer, because the image layer requires at least one.
	_, err = destImage.PutBlob(bytes.NewReader(gzippedEmptyLayer), types.BlobInfo{Size: int64(len(gzippedEmptyLayer))})
	if err != nil {
		return errors.Wrapf(err, "error writing dummy layer for image %q", transports.ImageName(dest))
	}
	// Read the newly-generated configuration blob.
	config, err := srcImage.ConfigBlob()
	if err != nil {
		return errors.Wrapf(err, "error reading new configuration for image %q", transports.ImageName(dest))
	}
	if len(config) == 0 {
		return errors.Errorf("error reading new configuration for image %q: it's empty", transports.ImageName(dest))
	}
	logrus.Debugf("read configuration blob %q", string(config))
	// Write the configuration to the new image.
	configBlobInfo := types.BlobInfo{
		Digest: digest.Canonical.FromBytes(config),
		Size:   int64(len(config)),
	}
	_, err = destImage.PutBlob(bytes.NewReader(config), configBlobInfo)
	if err != nil && len(config) > 0 {
		return errors.Wrapf(err, "error writing image configuration for temporary copy of %q", transports.ImageName(dest))
	}
	// Read the newly-generated, mostly fake, manifest.
	manifest, _, err := srcImage.Manifest()
	if err != nil {
		return errors.Wrapf(err, "error reading new manifest for image %q", transports.ImageName(dest))
	}
	// Write the manifest to the new image.
	err = destImage.PutManifest(manifest)
	if err != nil {
		return errors.Wrapf(err, "error writing new manifest to image %q", transports.ImageName(dest))
	}
	// Save the new image.
	err = destImage.Commit()
	if err != nil {
		return errors.Wrapf(err, "error committing new image %q", transports.ImageName(dest))
	}
	err = destImage.Close()
	if err != nil {
		return errors.Wrapf(err, "error closing new image %q", transports.ImageName(dest))
	}
	// Locate the new image in the lower-level API.  Extract its items.
	destImg, err := is.Transport.GetStoreImage(b.store, dest)
	if err != nil {
		return errors.Wrapf(err, "error locating new image %q", transports.ImageName(dest))
	}
	items, err := b.store.ListImageBigData(destImg.ID)
	if err != nil {
		return errors.Wrapf(err, "error reading list of named data for image %q", destImg.ID)
	}
	bigdata := make(map[string][]byte)
	for _, itemName := range items {
		var data []byte
		data, err = b.store.ImageBigData(destImg.ID, itemName)
		if err != nil {
			return errors.Wrapf(err, "error reading named data %q for image %q", itemName, destImg.ID)
		}
		bigdata[itemName] = data
	}
	// Delete the image so that we can recreate it.
	_, err = b.store.DeleteImage(destImg.ID, true)
	if err != nil {
		return errors.Wrapf(err, "error deleting image %q for rewriting", destImg.ID)
	}
	// Look up the container's read-write layer.
	container, err := b.store.Container(b.ContainerID)
	if err != nil {
		return errors.Wrapf(err, "error reading information about working container %q", b.ContainerID)
	}
	parentLayer := ""
	// Look up the container's source image's layer, if there is a source image.
	if container.ImageID != "" {
		img, err2 := b.store.Image(container.ImageID)
		if err2 != nil {
			return errors.Wrapf(err2, "error reading information about working container %q's source image", b.ContainerID)
		}
		parentLayer = img.TopLayer
	}
	// Extract the read-write layer's contents.
	layerDiff, err := b.store.Diff(parentLayer, container.LayerID)
	if err != nil {
		return errors.Wrapf(err, "error reading layer %q from source image %q", container.LayerID, transports.ImageName(src))
	}
	defer layerDiff.Close()
	// Write a copy of the layer for the new image to reference.
	layer, _, err := b.store.PutLayer("", parentLayer, []string{}, "", false, layerDiff)
	if err != nil {
		return errors.Wrapf(err, "error creating new read-only layer from container %q", b.ContainerID)
	}
	// Create a low-level image record that uses the new layer, discarding the old metadata.
	image, err := b.store.CreateImage(destImg.ID, []string{}, layer.ID, "{}", nil)
	if err != nil {
		err2 := b.store.DeleteLayer(layer.ID)
		if err2 != nil {
			logrus.Debugf("error removing layer %q: %v", layer, err2)
		}
		return errors.Wrapf(err, "error creating new low-level image %q", transports.ImageName(dest))
	}
	logrus.Debugf("(re-)created image ID %q using layer %q", image.ID, layer.ID)
	defer func() {
		if err != nil {
			_, err2 := b.store.DeleteImage(image.ID, true)
			if err2 != nil {
				logrus.Debugf("error removing image %q: %v", image.ID, err2)
			}
		}
	}()
	// Store the configuration and manifest, which are big data items, along with whatever else is there.
	for itemName, data := range bigdata {
		err = b.store.SetImageBigData(image.ID, itemName, data)
		if err != nil {
			return errors.Wrapf(err, "error saving data item %q", itemName)
		}
		logrus.Debugf("saved data item %q to %q", itemName, image.ID)
	}
	// Add the target name(s) to the new image.
	if len(names) > 0 {
		err = util.AddImageNames(b.store, image, names)
		if err != nil {
			return errors.Wrapf(err, "error assigning names %v to new image", names)
		}
		logrus.Debugf("assigned names %v to image %q", names, image.ID)
	}
	return nil
}

// Commit writes the contents of the container, along with its updated
// configuration, to a new image in the specified location, and if we know how,
// add any additional tags that were specified.
func (b *Builder) Commit(dest types.ImageReference, options CommitOptions) error {
	policy, err := signature.DefaultPolicy(getSystemContext(options.SignaturePolicyPath))
	if err != nil {
		return err
	}
	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return err
	}
	// Check if we're keeping everything in local storage.  If so, we can take certain shortcuts.
	_, destIsStorage := dest.Transport().(is.StoreTransport)
	exporting := !destIsStorage
	src, err := b.makeContainerImageRef(options.PreferredManifestType, exporting, options.Compression, options.HistoryTimestamp)
	if err != nil {
		return errors.Wrapf(err, "error computing layer digests and building metadata")
	}
	if exporting {
		// Copy everything.
		err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
		if err != nil {
			return errors.Wrapf(err, "error copying layers and metadata")
		}
	} else {
		// Copy only the most recent layer, the configuration, and the manifest.
		err = b.shallowCopy(dest, src, getSystemContext(options.SignaturePolicyPath))
		if err != nil {
			return errors.Wrapf(err, "error copying layer and metadata")
		}
	}
	if len(options.AdditionalTags) > 0 {
		switch dest.Transport().Name() {
		case is.Transport.Name():
			img, err := is.Transport.GetStoreImage(b.store, dest)
			if err != nil {
				return errors.Wrapf(err, "error locating just-written image %q", transports.ImageName(dest))
			}
			err = util.AddImageNames(b.store, img, options.AdditionalTags)
			if err != nil {
				return errors.Wrapf(err, "error setting image names to %v", append(img.Names, options.AdditionalTags...))
			}
			logrus.Debugf("assigned names %v to image %q", img.Names, img.ID)
		default:
			logrus.Warnf("don't know how to add tags to images stored in %q transport", dest.Transport().Name())
		}
	}
	return nil
}

// Push copies the contents of the image to a new location.
func Push(image string, dest types.ImageReference, options PushOptions) error {
	systemContext := getSystemContext(options.SignaturePolicyPath)
	policy, err := signature.DefaultPolicy(systemContext)
	if err != nil {
		return err
	}
	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return err
	}
	importOptions := ImportFromImageOptions{
		Image:               image,
		SignaturePolicyPath: options.SignaturePolicyPath,
	}
	builder, err := importBuilderFromImage(options.Store, importOptions)
	if err != nil {
		return errors.Wrap(err, "error importing builder information from image")
	}
	// Look up the image name and its layer.
	ref, err := is.Transport.ParseStoreReference(options.Store, image)
	if err != nil {
		return errors.Wrapf(err, "error parsing reference to image %q", image)
	}
	img, err := is.Transport.GetStoreImage(options.Store, ref)
	if err != nil {
		return errors.Wrapf(err, "error locating image %q", image)
	}
	// Give the image we're producing the same ancestors as its source image.
	builder.FromImage = builder.Docker.ContainerConfig.Image
	builder.FromImageID = string(builder.Docker.Parent)
	// Prep the layers and manifest for export.
	src, err := builder.makeImageImageRef(options.Compression, img.Names, img.TopLayer, nil)
	if err != nil {
		return errors.Wrapf(err, "error recomputing layer digests and building metadata")
	}
	// Copy everything.
	err = cp.Image(policyContext, dest, src, getCopyOptions(options.ReportWriter))
	if err != nil {
		return errors.Wrapf(err, "error copying layers and metadata")
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
)

var (
	ErrECIDUndefined   = errors.New("ECID type undefined")
	ErrUnknownECBEntry = errors.New("Unknown Entry Credit Block Entry type")
)

// ECID defines the type of an Entry Credit Block Entry
type ECID byte

// Available ECID types
const (
	ECIDServerIndexNumber ECID = iota // 0
	ECIDMinuteNumber                  // 1
	ECIDChainCommit                   // 2
	ECIDEntryCommit                   // 3
	ECIDBalanceIncrease               // 4
)

func (id ECID) String() string {
	switch id {
	case ECIDServerIndexNumber:
		return "ServerIndexNumber"
	case ECIDMinuteNumber:
		return "MinuteNumber"
	case ECIDChainCommit:
		return "ChainCommit"
	case ECIDEntryCommit:
		return "EntryCommit"
	case ECIDBalanceIncrease:
		return "BalanceIncrease"
	default:
		return "ECIDUndefined"
	}
}

// ECBlock (Entry Credit Block) holds transactions that create Chains and
// Entries, and fund Entry Credit Addresses.
type ECBlock struct {
	Header struct {
		BodyHash            string `json:"bodyhash"`
		PrevHeaderHash      string `json:"prevheaderhash"`
		PrevFullHash        string `json:"prevfullhash"`
		DBHeight            int64  `json:"dbheight"`
		HeaderExpansionArea []byte `json:"headerexpansionarea,omitempty"`
	} `json:"header"`
	HeaderHash string     `json:"headerhash"`
	FullHash   string     `json:"fullhash"`
	Entries    []ECBEntry `json:"body"`
}

func (e *ECBlock) String() string {
	var s string

	s += fmt.Sprintln("HeaderHash:", e.HeaderHash)
	s += fmt.Sprintln("PrevHeaderHash:", e.Header.PrevHeaderHash)
	s += fmt.Sprintln("FullHash:", e.FullHash)
	s += fmt.Sprintln("PrevFullHash:", e.Header.PrevFullHash)
	s += fmt.Sprintln("BodyHash:", e.Header.BodyHash)
	s += fmt.Sprintln("DBHeight:", e.Header.DBHeight)
	if e.Header.HeaderExpansionArea != nil {
		s += fmt.Sprintf("HeaderExpansionArea: %x\n", e.Header.HeaderExpansionArea)
	}

	s += fmt.Sprintln("Entries:")
	for _, v := range e.Entries {
		s += fmt.Sprintln(v)
	}

	return s
}

func (e *ECBlock) UnmarshalJSON(js []byte) error {
	tmp := new(struct {
		Header struct {
			BodyHash       string `json:"bodyhash"`
			PrevHeaderHash string `json:"prevheaderhash"`
			PrevFullHash   string `json:"prevfullhash"`
			DBHeight       int64  `json:"dbheight"`
		} `json:"header"`
		HeaderHash string `json:"headerhash"`
		FullHash   string `json:"fullhash"`
		Body       struct {
			Entries []json.RawMessage `json:"entries"`
		} `json:"body"`
	})

	err := json.Unmarshal(js, tmp)
	if err != nil {
		return err
	}

	e.Header.BodyHash = tmp.Header.BodyHash
	e.Header.PrevHeaderHash = tmp.Header.PrevHeaderHash
	e.Header.PrevFullHash = tmp.Header.PrevFullHash
	e.Header.DBHeight = tmp.Header.DBHeight
	e.HeaderHash = tmp.HeaderHash
	e.FullHash = tmp.FullHash

	// the entry block entry type is not specified in the json data, so detect
	// the entry type by regex and umarshal into the correct type.
	for _, v := range tmp.Body.Entries {
		switch {
		case regexp.MustCompile(`"serverindexnumber":`).MatchString(string(v)):
			a := new(ECServerIndexNumber)
			err := json.Unmarshal(v, a)
			if err != nil {
				return err
			}
			e.Entries = append(e.Entries, a)
		case regexp.MustCompile(`"number":`).MatchString(string(v)):
			a := new(ECMinuteNumber)
			err := json.Unmarshal(v, a)
			if err != nil {
				return err
			}
			e.Entries = append(e.Entries, a)
		case regexp.MustCompile(`"entryhash":`).MatchString(string(v)):
			if regexp.MustCompile(`"chainidhash":`).MatchString(string(v)) {
				a := new(ECChainCommit)
				err := json.Unmarshal(v, a)
				if err != nil {
					return err
				}
				e.Entries = append(e.Entries, a)

			} else {
				a := new(ECEntryCommit)
				err := json.Unmarshal(v, a)
				if err != nil {
					return err
				}
				e.Entries = append(e.Entries, a)
			}
		default:
			return ErrUnknownECBEntry
		}
	}

	return nil
}

// Entry Credit Block Entries are individual members of the Entry Credit Block.
type ECBEntry interface {
	Type() ECID
	String() string
}

// ECServerIndexNumber shows the index of the server that acknowledged the
// following ECBEntries.
type ECServerIndexNumber struct {
	ServerIndexNumber int `json:"serverindexnumber"`
}

func (i *ECServerIndexNumber) Type() ECID {
	return ECIDServerIndexNumber
}

func (i *ECServerIndexNumber) String() string {
	return fmt.Sprintln("ServerIndexNumber:", i.ServerIndexNumber)
}

// ECMinuteNumber represents the end of a minute minute [1-10] in the order of
// the ECBEntries.
type ECMinuteNumber struct {
	Number int `json:"number"`
}

func (m *ECMinuteNumber) Type() ECID {
	return ECIDMinuteNumber
}

func (m *ECMinuteNumber) String() string {
	return fmt.Sprintln("MinuteNumber:", m.Number)
}

// ECChainCommit pays for and reserves a new chain in Factom.
type ECChainCommit struct {
	Version     int    `json:"version"`
	MilliTime   int64  `json:"millitime"`
	ChainIDHash string `json:"chainidhash"`
	Weld        string `json:"weld"`
	EntryHash   string `json:"entryhash"`
	Credits     int    `json:"credits"`
	ECPubKey    string `json:"ecpubkey"`
	Sig         string `json:"sig"`
}

func (c *ECChainCommit) UnmarshalJSON(js []byte) error {
	tmp := new(struct {
		Version     int    `json:"version"`
		MilliTime   string `json:"millitime"`
		ChainIDHash string `json:"chainidhash"`
		Weld        string `json:"weld"`
		EntryHash   string `json:"entryhash"`
		Credits     int    `json:"credits"`
		ECPubKey    string `json:"ecpubkey"`
		Sig         string `json:"sig"`
	})

	err := json.Unmarshal(js, tmp)
	if err != nil {
		return err
	}

	// convert 6 byte MilliTime into int64
	m := make([]byte, 8)
	if p, err := hex.DecodeString(tmp.MilliTime); err != nil {
		return err
	} else {
		// copy p into the last 6 bytes
		copy(m[2:], p)
	}
	c.MilliTime = int64(binary.BigEndian.Uint64(m))

	c.Version = tmp.Version
	c.ChainIDHash = tmp.ChainIDHash
	c.Weld = tmp.Weld
	c.EntryHash = tmp.EntryHash
	c.Credits = tmp.Credits
	c.ECPubKey = tmp.ECPubKey
	c.Sig = tmp.Sig

	return nil
}

func (c *ECChainCommit) Type() ECID {
	return ECIDChainCommit
}

func (c *ECChainCommit) String() string {
	var s string

	s += fmt.Sprintln("ChainCommit {")
	s += fmt.Sprintln("	Version:", c.Version)
	s += fmt.Sprintln("	Millitime:", c.MilliTime)
	s += fmt.Sprintln("	ChainIDHash:", c.ChainIDHash)
	s += fmt.Sprintln("	Weld:", c.Weld)
	s += fmt.Sprintln("	EntryHash:", c.EntryHash)
	s += fmt.Sprintln("	Credits:", c.Credits)
	s += fmt.Sprintln("	ECPubKey:", c.ECPubKey)
	s += fmt.Sprintln("	Signature:", c.Sig)
	s += fmt.Sprintln("}")

	return s
}

// ECEntryCommit pays for and reserves a new entry in Factom.
type ECEntryCommit struct {
	Version   int    `json:"version"`
	MilliTime int64  `json:"millitime"`
	EntryHash string `json:"entryhash"`
	Credits   int    `json:"credits"`
	ECPubKey  string `json:"ecpubkey"`
	Sig       string `json:"sig"`
}

func (e *ECEntryCommit) UnmarshalJSON(js []byte) error {
	tmp := new(struct {
		Version   int    `json:"version"`
		MilliTime string `json:"millitime"`
		EntryHash string `json:"entryhash"`
		Credits   int    `json:"credits"`
		ECPubKey  string `json:"ecpubkey"`
		Sig       string `json:"sig"`
	})

	err := json.Unmarshal(js, tmp)
	if err != nil {
		return err
	}

	// convert 6 byte MilliTime into int64
	m := make([]byte, 8)
	if p, err := hex.DecodeString(tmp.MilliTime); err != nil {
		return err
	} else {
		// copy p into the last 6 bytes
		copy(m[2:], p)
	}
	e.MilliTime = int64(binary.BigEndian.Uint64(m))

	e.Version = tmp.Version
	e.EntryHash = tmp.EntryHash
	e.Credits = tmp.Credits
	e.ECPubKey = tmp.ECPubKey
	e.Sig = tmp.Sig

	return nil
}

func (e *ECEntryCommit) Type() ECID {
	return ECIDEntryCommit
}

func (e *ECEntryCommit) String() string {
	var s string

	s += fmt.Sprintln("EntryCommit {")
	s += fmt.Sprintln("	Version:", e.Version)
	s += fmt.Sprintln("	Millitime:", e.MilliTime)
	s += fmt.Sprintln("	EntryHash:", e.EntryHash)
	s += fmt.Sprintln("	Credits:", e.Credits)
	s += fmt.Sprintln("	ECPubKey:", e.ECPubKey)
	s += fmt.Sprintln("	Signature:", e.Sig)
	s += fmt.Sprintln("}")

	return s
}

// GetECBlock requests a specified Entry Credit Block from the factomd API.
func GetECBlock(keymr string) (ecblock *ECBlock, raw []byte, err error) {
	params := keyMRRequest{KeyMR: keymr}
	req := NewJSON2Request("entrycredit-block", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	// create a wraper construct for the ECBlock API return
	wrap := new(struct {
		ECBlock *ECBlock `json:"ecblock"`
		RawData string   `json:"rawdata"`
	})
	err = json.Unmarshal(resp.JSONResult(), wrap)
	if err != nil {
		return
	}

	raw, err = hex.DecodeString(wrap.RawData)
	if err != nil {
		return
	}

	return wrap.ECBlock, raw, nil
}
<commit_msg>minor formatting<commit_after>// Copyright 2016 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package factom

import (
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
)

var (
	ErrECIDUndefined   = errors.New("ECID type undefined")
	ErrUnknownECBEntry = errors.New("Unknown Entry Credit Block Entry type")
)

// ECID defines the type of an Entry Credit Block Entry
type ECID byte

// Available ECID types
const (
	ECIDServerIndexNumber ECID = iota // 0
	ECIDMinuteNumber                  // 1
	ECIDChainCommit                   // 2
	ECIDEntryCommit                   // 3
	ECIDBalanceIncrease               // 4
)

func (id ECID) String() string {
	switch id {
	case ECIDServerIndexNumber:
		return "ServerIndexNumber"
	case ECIDMinuteNumber:
		return "MinuteNumber"
	case ECIDChainCommit:
		return "ChainCommit"
	case ECIDEntryCommit:
		return "EntryCommit"
	case ECIDBalanceIncrease:
		return "BalanceIncrease"
	default:
		return "ECIDUndefined"
	}
}

// ECBlock (Entry Credit Block) holds transactions that create Chains and
// Entries, and fund Entry Credit Addresses.
type ECBlock struct {
	Header struct {
		BodyHash            string `json:"bodyhash"`
		PrevHeaderHash      string `json:"prevheaderhash"`
		PrevFullHash        string `json:"prevfullhash"`
		DBHeight            int64  `json:"dbheight"`
		HeaderExpansionArea []byte `json:"headerexpansionarea,omitempty"`
	} `json:"header"`
	HeaderHash string     `json:"headerhash"`
	FullHash   string     `json:"fullhash"`
	Entries    []ECBEntry `json:"body"`
}

func (e *ECBlock) String() string {
	var s string

	s += fmt.Sprintln("HeaderHash:", e.HeaderHash)
	s += fmt.Sprintln("PrevHeaderHash:", e.Header.PrevHeaderHash)
	s += fmt.Sprintln("FullHash:", e.FullHash)
	s += fmt.Sprintln("PrevFullHash:", e.Header.PrevFullHash)
	s += fmt.Sprintln("BodyHash:", e.Header.BodyHash)
	s += fmt.Sprintln("DBHeight:", e.Header.DBHeight)
	if e.Header.HeaderExpansionArea != nil {
		s += fmt.Sprintf("HeaderExpansionArea: %x\n", e.Header.HeaderExpansionArea)
	}

	s += fmt.Sprintln("Entries:")
	for _, v := range e.Entries {
		s += fmt.Sprintln(v)
	}

	return s
}

func (e *ECBlock) UnmarshalJSON(js []byte) error {
	tmp := new(struct {
		Header struct {
			BodyHash       string `json:"bodyhash"`
			PrevHeaderHash string `json:"prevheaderhash"`
			PrevFullHash   string `json:"prevfullhash"`
			DBHeight       int64  `json:"dbheight"`
		} `json:"header"`
		HeaderHash string `json:"headerhash"`
		FullHash   string `json:"fullhash"`
		Body       struct {
			Entries []json.RawMessage `json:"entries"`
		} `json:"body"`
	})

	err := json.Unmarshal(js, tmp)
	if err != nil {
		return err
	}

	e.Header.BodyHash = tmp.Header.BodyHash
	e.Header.PrevHeaderHash = tmp.Header.PrevHeaderHash
	e.Header.PrevFullHash = tmp.Header.PrevFullHash
	e.Header.DBHeight = tmp.Header.DBHeight
	e.HeaderHash = tmp.HeaderHash
	e.FullHash = tmp.FullHash

	// the entry block entry type is not specified in the json data, so detect
	// the entry type by regex and umarshal into the correct type.
	for _, v := range tmp.Body.Entries {
		switch {
		case regexp.MustCompile(`"serverindexnumber":`).MatchString(string(v)):
			a := new(ECServerIndexNumber)
			err := json.Unmarshal(v, a)
			if err != nil {
				return err
			}
			e.Entries = append(e.Entries, a)
		case regexp.MustCompile(`"number":`).MatchString(string(v)):
			a := new(ECMinuteNumber)
			err := json.Unmarshal(v, a)
			if err != nil {
				return err
			}
			e.Entries = append(e.Entries, a)
		case regexp.MustCompile(`"entryhash":`).MatchString(string(v)):
			if regexp.MustCompile(`"chainidhash":`).MatchString(string(v)) {
				a := new(ECChainCommit)
				err := json.Unmarshal(v, a)
				if err != nil {
					return err
				}
				e.Entries = append(e.Entries, a)
			} else {
				a := new(ECEntryCommit)
				err := json.Unmarshal(v, a)
				if err != nil {
					return err
				}
				e.Entries = append(e.Entries, a)
			}
		default:
			return ErrUnknownECBEntry
		}
	}

	return nil
}

// Entry Credit Block Entries are individual members of the Entry Credit Block.
type ECBEntry interface {
	Type() ECID
	String() string
}

// ECServerIndexNumber shows the index of the server that acknowledged the
// following ECBEntries.
type ECServerIndexNumber struct {
	ServerIndexNumber int `json:"serverindexnumber"`
}

func (i *ECServerIndexNumber) Type() ECID {
	return ECIDServerIndexNumber
}

func (i *ECServerIndexNumber) String() string {
	return fmt.Sprintln("ServerIndexNumber:", i.ServerIndexNumber)
}

// ECMinuteNumber represents the end of a minute minute [1-10] in the order of
// the ECBEntries.
type ECMinuteNumber struct {
	Number int `json:"number"`
}

func (m *ECMinuteNumber) Type() ECID {
	return ECIDMinuteNumber
}

func (m *ECMinuteNumber) String() string {
	return fmt.Sprintln("MinuteNumber:", m.Number)
}

// ECChainCommit pays for and reserves a new chain in Factom.
type ECChainCommit struct {
	Version     int    `json:"version"`
	MilliTime   int64  `json:"millitime"`
	ChainIDHash string `json:"chainidhash"`
	Weld        string `json:"weld"`
	EntryHash   string `json:"entryhash"`
	Credits     int    `json:"credits"`
	ECPubKey    string `json:"ecpubkey"`
	Sig         string `json:"sig"`
}

func (c *ECChainCommit) UnmarshalJSON(js []byte) error {
	tmp := new(struct {
		Version     int    `json:"version"`
		MilliTime   string `json:"millitime"`
		ChainIDHash string `json:"chainidhash"`
		Weld        string `json:"weld"`
		EntryHash   string `json:"entryhash"`
		Credits     int    `json:"credits"`
		ECPubKey    string `json:"ecpubkey"`
		Sig         string `json:"sig"`
	})

	err := json.Unmarshal(js, tmp)
	if err != nil {
		return err
	}

	// convert 6 byte MilliTime into int64
	m := make([]byte, 8)
	if p, err := hex.DecodeString(tmp.MilliTime); err != nil {
		return err
	} else {
		// copy p into the last 6 bytes
		copy(m[2:], p)
	}
	c.MilliTime = int64(binary.BigEndian.Uint64(m))

	c.Version = tmp.Version
	c.ChainIDHash = tmp.ChainIDHash
	c.Weld = tmp.Weld
	c.EntryHash = tmp.EntryHash
	c.Credits = tmp.Credits
	c.ECPubKey = tmp.ECPubKey
	c.Sig = tmp.Sig

	return nil
}

func (c *ECChainCommit) Type() ECID {
	return ECIDChainCommit
}

func (c *ECChainCommit) String() string {
	var s string

	s += fmt.Sprintln("ChainCommit {")
	s += fmt.Sprintln("	Version:", c.Version)
	s += fmt.Sprintln("	Millitime:", c.MilliTime)
	s += fmt.Sprintln("	ChainIDHash:", c.ChainIDHash)
	s += fmt.Sprintln("	Weld:", c.Weld)
	s += fmt.Sprintln("	EntryHash:", c.EntryHash)
	s += fmt.Sprintln("	Credits:", c.Credits)
	s += fmt.Sprintln("	ECPubKey:", c.ECPubKey)
	s += fmt.Sprintln("	Signature:", c.Sig)
	s += fmt.Sprintln("}")

	return s
}

// ECEntryCommit pays for and reserves a new entry in Factom.
type ECEntryCommit struct {
	Version   int    `json:"version"`
	MilliTime int64  `json:"millitime"`
	EntryHash string `json:"entryhash"`
	Credits   int    `json:"credits"`
	ECPubKey  string `json:"ecpubkey"`
	Sig       string `json:"sig"`
}

func (e *ECEntryCommit) UnmarshalJSON(js []byte) error {
	tmp := new(struct {
		Version   int    `json:"version"`
		MilliTime string `json:"millitime"`
		EntryHash string `json:"entryhash"`
		Credits   int    `json:"credits"`
		ECPubKey  string `json:"ecpubkey"`
		Sig       string `json:"sig"`
	})

	err := json.Unmarshal(js, tmp)
	if err != nil {
		return err
	}

	// convert 6 byte MilliTime into int64
	m := make([]byte, 8)
	if p, err := hex.DecodeString(tmp.MilliTime); err != nil {
		return err
	} else {
		// copy p into the last 6 bytes
		copy(m[2:], p)
	}
	e.MilliTime = int64(binary.BigEndian.Uint64(m))

	e.Version = tmp.Version
	e.EntryHash = tmp.EntryHash
	e.Credits = tmp.Credits
	e.ECPubKey = tmp.ECPubKey
	e.Sig = tmp.Sig

	return nil
}

func (e *ECEntryCommit) Type() ECID {
	return ECIDEntryCommit
}

func (e *ECEntryCommit) String() string {
	var s string

	s += fmt.Sprintln("EntryCommit {")
	s += fmt.Sprintln("	Version:", e.Version)
	s += fmt.Sprintln("	Millitime:", e.MilliTime)
	s += fmt.Sprintln("	EntryHash:", e.EntryHash)
	s += fmt.Sprintln("	Credits:", e.Credits)
	s += fmt.Sprintln("	ECPubKey:", e.ECPubKey)
	s += fmt.Sprintln("	Signature:", e.Sig)
	s += fmt.Sprintln("}")

	return s
}

// GetECBlock requests a specified Entry Credit Block from the factomd API.
func GetECBlock(keymr string) (ecblock *ECBlock, raw []byte, err error) {
	params := keyMRRequest{KeyMR: keymr}
	req := NewJSON2Request("entrycredit-block", APICounter(), params)
	resp, err := factomdRequest(req)
	if err != nil {
		return
	}
	if resp.Error != nil {
		return nil, nil, resp.Error
	}

	// create a wraper construct for the ECBlock API return
	wrap := new(struct {
		ECBlock *ECBlock `json:"ecblock"`
		RawData string   `json:"rawdata"`
	})
	err = json.Unmarshal(resp.JSONResult(), wrap)
	if err != nil {
		return
	}

	raw, err = hex.DecodeString(wrap.RawData)
	if err != nil {
		return
	}

	return wrap.ECBlock, raw, nil
}
<|endoftext|>


<commit_before>package minhash

import (
	"encoding/binary"
	"math"
	"strconv"
	"sync"
)

// MinWise is a data structure for generating a parametric family of
// hash functions of the form h1 + i*h2 for i=1, ..., k to compute
// a MinHash signature.  Each instance is tied to a single
// streamed set and hence single signature.  As it ingests
// elements it will update the current signature.

// The ith element of the signature is the current minimal value
// of the ith hash function.
type MinWise struct {
	minimums Signature
	h1       HashFunc
	h2       HashFunc
}

// NewMinWise constructs a new MinWise instance that will
// compute a signature of the specified size.
func NewMinWise(h1, h2 HashFunc, size int) *MinWise {
	mw := MinWise{
		minimums: defaultSignature(size), // running set of min values
		h1:       h1,
		h2:       h2,
	}
	return &mw
}

// Push updates the set's signature.  It hashes the input
// with each function in the family and compares these values
// with the current set of minimums, replacing them as necessary.
func (m *MinWise) Push(b []byte) {

	v1 := m.h1(b)
	v2 := m.h2(b)

	// Compare minimal values
	for i, min := range m.minimums {
		// Compute hi(b) for ith hash function hi
		hb := v1 + uint64(i)*v2
		if hb < min {
			m.minimums[i] = hb
		}
	}
}

// PushStringInt converts a string representation of an integer.
func (m *MinWise) PushStringInt(s string) {
	m.Push(stringIntToByte(s))
}

// PushGeneric deals with generic data by handling byte conversion.
func (m *MinWise) PushGeneric(x interface{}) {
	m.Push(toBytes(x))
}

func (m *MinWise) Signature() Signature {
	return m.minimums
}

func (m *MinWise) Similarity(m2 *MinWise) float64 {
	return MinWiseSimilarity(m.Signature(), m2.Signature())
}

// MinWiseSimilarity computes an estimate for the
// Jaccard similarity of two sets given their MinWise signatures.
func MinWiseSimilarity(sig1, sig2 Signature) float64 {
	if len(sig1) != len(sig2) {
		panic("signature size mismatch")
	}

	intersect := 0 // counter for number of elements in intersection

	for i := range sig1 {
		if sig1[i] == sig2[i] {
			intersect++
		}
	}

	return float64(intersect) / float64(len(sig1))
}

// Merge combines the signatures of the second set,
// creating the signature of their union.
func (m *MinWise) Merge(m2 *MinWise) {

	for i, v := range m2.minimums {

		if v < m.minimums[i] {
			m.minimums[i] = v
		}
	}
}

// Cardinality estimates the cardinality of the set.
func (m *MinWise) Cardinality() int {

	// http://www.cohenwang.com/edith/Papers/tcest.pdf

	sum := 0.0

	for _, v := range m.minimums {
		sum += -math.Log(float64(infinity-v) / float64(infinity))
	}

	return int(float64(len(m.minimums)-1) / sum)
}

// Similarity computes an estimate for the similarity between the two sets.
func (m *MinWise) Similarity(m2 *MinWise) float64 {

	if len(m.minimums) != len(m2.minimums) {
		panic("minhash minimums size mismatch")
	}

	intersect := 0

	for i := range m.minimums {
		if m.minimums[i] == m2.minimums[i] {
			intersect++
		}
	}

	return float64(intersect) / float64(len(m.minimums))
}

// SignatureBbit returns a b-bit reduction of the signature.  This will result in unused bits at the high-end of the words if b does not divide 64 evenly.
func (m *MinWise) SignatureBbit(b uint) []uint64 {

	var sig []uint64 // full signature
	var w uint64     // current word
	bits := uint(64) // bits free in current word

	mask := uint64(1<<b) - 1

	for _, v := range m.minimums {
		if bits >= b {
			w <<= b
			w |= v & mask
			bits -= b
		} else {
			sig = append(sig, w)
			w = 0
			bits = 64
		}
	}

	if bits != 64 {
		sig = append(sig, w)
	}

	return sig
}

// SimilarityBbit computes an estimate for the similarity between two b-bit signatures
func SimilarityBbit(sig1, sig2 []uint64, b uint) float64 {

	if len(sig1) != len(sig2) {
		panic("signature size mismatch")
	}

	intersect := 0
	count := 0

	mask := uint64(1<<b) - 1

	for i := range sig1 {
		w1 := sig1[i]
		w2 := sig2[i]

		bits := uint(64)

		for bits >= b {
			v1 := (w1 & mask)
			v2 := (w2 & mask)

			count++
			if v1 == v2 {
				intersect++
			}

			bits -= b
			w1 >>= b
			w2 >>= b
		}
	}

	return float64(intersect) / float64(count)
}
<commit_msg>Add MinWise string ints constructor.<commit_after>package minhash

import (
	"encoding/binary"
	"math"
	"strconv"
	"sync"
)

// MinWise is a data structure for generating a parametric family of
// hash functions of the form h1 + i*h2 for i=1, ..., k to compute
// a MinHash signature.  Each instance is tied to a single
// streamed set and hence single signature.  As it ingests
// elements it will update the current signature.

// The ith element of the signature is the current minimal value
// of the ith hash function.
type MinWise struct {
	minimums Signature
	h1       HashFunc
	h2       HashFunc
}

// NewMinWise constructs a new MinWise instance initialized with
// the empty set.
func NewMinWise(h1, h2 HashFunc, size int) *MinWise {
	mw := MinWise{
		minimums: defaultSignature(size), // running set of min values
		h1:       h1,
		h2:       h2,
	}
	return &mw
}

func InitMinWise(h1, h2 HashFunc, size int) *MinWise {

}

// Push updates the set's signature.  It hashes the input
// with each function in the family and compares these values
// with the current set of minimums, replacing them as necessary.
func (m *MinWise) Push(b []byte) {

	v1 := m.h1(b)
	v2 := m.h2(b)

	// Compare minimal values
	for i, min := range m.minimums {
		// Compute hi(b) for ith hash function hi
		hb := v1 + uint64(i)*v2
		if hb < min {
			m.minimums[i] = hb
		}
	}
}

// PushStringInt first converts a string into a uint64 before pushing.
func (m *MinWise) PushStringInt(s string) {
	m.Push(stringIntToByte(s))
}

// PushGeneric deals with generic data by handling byte conversion.
func (m *MinWise) PushGeneric(x interface{}) {
	m.Push(toBytes(x))
}

// Signature returns the current signature.
func (m *MinWise) Signature() []uint64 {
	return m.minimums
}

// Similarity computes the similarity of two signatures represented
// as MinWise instances.  This estimates the Jaccard index of the
// two underlying sets.
func (m *MinWise) Similarity(m2 *MinWise) float64 {
	return MinWiseSimilarity(m.Signature(), m2.Signature())
}

// MinWiseSimilarity computes an estimate for the
// Jaccard similarity of two sets given their MinWise signatures.
func MinWiseSimilarity(sig1, sig2 []uint64) float64 {
	if len(sig1) != len(sig2) {
		panic("Signature size mismatch.")
	}

	intersect := 0 // counter for number of elements in intersection

	for i := range sig1 {
		if sig1[i] == sig2[i] {
			intersect++
		}
	}

	return float64(intersect) / float64(len(sig1))
}

// Merge combines the signatures of the second set,
// creating the signature of their union.
func (m *MinWise) Merge(m2 *MinWise) {

	for i, v := range m2.minimums {

		if v < m.minimums[i] {
			m.minimums[i] = v
		}
	}
}

// Cardinality estimates the cardinality of the set.
func (m *MinWise) Cardinality() int {

	// http://www.cohenwang.com/edith/Papers/tcest.pdf

	sum := 0.0

	for _, v := range m.minimums {
		sum += -math.Log(float64(infinity-v) / float64(infinity))
	}

	return int(float64(len(m.minimums)-1) / sum)
}

// Similarity computes an estimate for the similarity between the two sets.
func (m *MinWise) Similarity(m2 *MinWise) float64 {

	if len(m.minimums) != len(m2.minimums) {
		panic("minhash minimums size mismatch")
	}

	intersect := 0

	for i := range m.minimums {
		if m.minimums[i] == m2.minimums[i] {
			intersect++
		}
	}

	return float64(intersect) / float64(len(m.minimums))
}

// SignatureBbit returns a b-bit reduction of the signature.  This will result in unused bits at the high-end of the words if b does not divide 64 evenly.
func (m *MinWise) SignatureBbit(b uint) []uint64 {

	var sig []uint64 // full signature
	var w uint64     // current word
	bits := uint(64) // bits free in current word

	mask := uint64(1<<b) - 1

	for _, v := range m.minimums {
		if bits >= b {
			w <<= b
			w |= v & mask
			bits -= b
		} else {
			sig = append(sig, w)
			w = 0
			bits = 64
		}
	}

	if bits != 64 {
		sig = append(sig, w)
	}

	return sig
}

// SimilarityBbit computes an estimate for the similarity between two b-bit signatures
func SimilarityBbit(sig1, sig2 []uint64, b uint) float64 {

	if len(sig1) != len(sig2) {
		panic("signature size mismatch")
	}

	intersect := 0
	count := 0

	mask := uint64(1<<b) - 1

	for i := range sig1 {
		w1 := sig1[i]
		w2 := sig2[i]

		bits := uint(64)

		for bits >= b {
			v1 := (w1 & mask)
			v2 := (w2 & mask)

			count++
			if v1 == v2 {
				intersect++
			}

			bits -= b
			w1 >>= b
			w2 >>= b
		}
	}

	return float64(intersect) / float64(count)
}
<|endoftext|>


<commit_before>// Copyright 2011 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twittergo

import (
	"compress/gzip"
	"fmt"
	"github.com/kurrik/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	H_LIMIT              = "X-Rate-Limit-Limit"
	H_LIMIT_REMAIN       = "X-Rate-Limit-Remaining"
	H_LIMIT_RESET        = "X-Rate-Limit-Reset"
	H_MEDIA_LIMIT        = "X-MediaRateLimit-Limit"
	H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining"
	H_MEDIA_LIMIT_RESET  = "X-MediaRateLimit-Reset"
)

const (
	STATUS_INVALID      = 400
	STATUS_UNAUTHORIZED = 401
	STATUS_FORBIDDEN    = 403
	STATUS_NOTFOUND     = 404
	STATUS_LIMIT        = 429
	STATUS_GATEWAY      = 502
)

type Error map[string]interface{}

func (e Error) Code() int64 {
	return e["code"].(int64)
}

func (e Error) Message() string {
	return e["message"].(string)
}

func (e Error) Error() string {
	msg := "Error %v: %v"
	return fmt.Sprintf(msg, e.Code(), e.Message())
}

type Errors map[string]interface{}

func (e Errors) Error() string {
	var (
		msg string = ""
		err Error
		ok  bool
	)
	for _, val := range e["errors"].([]interface{}) {
		if err, ok = val.(map[string]interface{}); ok {
			msg += err.Error() + ". "
		}
	}
	return msg
}

func (e Errors) String() string {
	return e.Error()
}

type RateLimitError struct {
	Limit     uint32
	Remaining uint32
	Reset     time.Time
}

func (e RateLimitError) Error() string {
	msg := "Rate limit: %v, Remaining: %v, Reset: %v"
	return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset)
}

type APIResponse http.Response

func (r APIResponse) HasRateLimit() bool {
	return r.Header.Get(H_LIMIT) != ""
}

func (r APIResponse) RateLimit() uint32 {
	h := r.Header.Get(H_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitRemaining() uint32 {
	h := r.Header.Get(H_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitReset() time.Time {
	h := r.Header.Get(H_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) HasMediaRateLimit() bool {
	return r.Header.Get(H_MEDIA_LIMIT) != ""
}

func (r APIResponse) MediaRateLimit() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitRemaining() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitReset() time.Time {
	h := r.Header.Get(H_MEDIA_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) readBody() (b []byte, err error) {
	var (
		header string
		reader io.Reader
	)
	header = strings.ToLower(r.Header.Get("Content-Encoding"))
	if header == "" || strings.Index(header, "gzip") == -1 {
		reader = r.Body
		defer r.Body.Close()
	} else {
		if reader, err = gzip.NewReader(r.Body); err != nil {
			return
		}
	}
	b, err = ioutil.ReadAll(reader)
	return
}

func (r APIResponse) ReadBody() string {
	var (
		b   []byte
		err error
	)
	if b, err = r.readBody(); err != nil {
		return ""
	}
	return string(b)
}

// Parses a JSON encoded HTTP response into the supplied interface.
func (r APIResponse) Parse(out interface{}) (err error) {
	var b []byte
	switch r.StatusCode {
	case STATUS_UNAUTHORIZED:
		fallthrough
	case STATUS_NOTFOUND:
		fallthrough
	case STATUS_GATEWAY:
		fallthrough
	case STATUS_FORBIDDEN:
		fallthrough
	case STATUS_INVALID:
		e := &Errors{}
		if b, err = r.readBody(); err != nil {
			return
		}
		err = json.Unmarshal(b, e)
		err = *e
		return
	case STATUS_LIMIT:
		err = RateLimitError{
			Limit:     r.RateLimit(),
			Remaining: r.RateLimitRemaining(),
			Reset:     r.RateLimitReset(),
		}
		return
	}
	if b, err = r.readBody(); err != nil {
		return
	}
	err = json.Unmarshal(b, out)
	if err == io.EOF {
		err = nil
	}
	return
}

// It's a user!
type User map[string]interface{}

func (u User) Id() uint64 {
	id, _ := strconv.ParseUint(u["id_str"].(string), 10, 64)
	return id
}

func (u User) IdStr() string {
	return u["id_str"].(string)
}

func (u User) Name() string {
	return u["name"].(string)
}

func (u User) ScreenName() string {
	return u["screen_name"].(string)
}

// It's a Tweet! (Adorably referred to by the API as a "status").
type Tweet map[string]interface{}

func (t Tweet) Id() (id uint64) {
	var (
		err error
		src = t["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (t Tweet) IdStr() string {
	return t["id_str"].(string)
}

func (t Tweet) Text() string {
	return t["text"].(string)
}

func (t Tweet) User() User {
	return User(t["user"].(map[string]interface{}))
}

func (t Tweet) CreatedAt() (out time.Time) {
	var (
		err error
		src = t["created_at"].(string)
	)
	if out, err = time.Parse(time.RubyDate, src); err != nil {
		panic(fmt.Sprintf("Could not parse time: %v", err))
	}
	return
}

// It's a less structured list of Tweets!
type Timeline []Tweet

// It's a structured list of Tweets!
type SearchResults map[string]interface{}

func (sr SearchResults) Statuses() []Tweet {
	var a []interface{} = sr["statuses"].([]interface{})
	b := make([]Tweet, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

func (sr SearchResults) SearchMetadata() map[string]interface{} {
	a := sr["search_metadata"].(map[string]interface{})
	return a
}

func (sr SearchResults) NextQuery() (val url.Values, err error) {
	var (
		sm   map[string]interface{}
		n    interface{}
		next string
		ok   bool
	)
	sm = sr.SearchMetadata()
	if n, ok = sm["next_results"]; !ok {
		err = fmt.Errorf("Could not get next_results from search")
		return
	}
	if next, ok = n.(string); !ok {
		err = fmt.Errorf("Could not convert next_results to str: %v", n)
		return
	}
	if next[0] == '?' {
		next = next[1:]
	}
	val, err = url.ParseQuery(next)
	return
}

// A List!
type List map[string]interface{}

func (l List) User() User {
	return User(l["user"].(map[string]interface{}))
}

func (l List) Id() (id uint64) {
	var (
		err error
		src = l["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (l List) IdStr() string {
	return l["id_str"].(string)
}

func (l List) Mode() string {
	return l["mode"].(string)
}

func (l List) Name() string {
	return l["name"].(string)
}

func (l List) Slug() string {
	return l["slug"].(string)
}

func (l List) SubscriberCount() int64 {
	return l["subscriber_count"].(int64)
}

func (l List) MemberCount() int64 {
	return l["member_count"].(int64)
}

// It's a less structured list of Lists!
type Lists []List

// It's a cursored list of Lists!

type CursoredLists map[string]interface{}

func (cl CursoredLists) NextCursorStr() string {
	return cl["next_cursor_str"].(string)
}

func (cl CursoredLists) PreviousCursorStr() string {
	return cl["previous_cursor_str"].(string)
}

func (cl CursoredLists) Lists() Lists {
	var a []interface{} = cl["lists"].([]interface{})
	b := make([]List, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}
<commit_msg>Get access to a slice of underlying errors for server error responses.<commit_after>// Copyright 2011 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package twittergo

import (
	"compress/gzip"
	"fmt"
	"github.com/kurrik/json"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

const (
	H_LIMIT              = "X-Rate-Limit-Limit"
	H_LIMIT_REMAIN       = "X-Rate-Limit-Remaining"
	H_LIMIT_RESET        = "X-Rate-Limit-Reset"
	H_MEDIA_LIMIT        = "X-MediaRateLimit-Limit"
	H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining"
	H_MEDIA_LIMIT_RESET  = "X-MediaRateLimit-Reset"
)

const (
	STATUS_INVALID      = 400
	STATUS_UNAUTHORIZED = 401
	STATUS_FORBIDDEN    = 403
	STATUS_NOTFOUND     = 404
	STATUS_LIMIT        = 429
	STATUS_GATEWAY      = 502
)

type Error map[string]interface{}

func (e Error) Code() int64 {
	return e["code"].(int64)
}

func (e Error) Message() string {
	return e["message"].(string)
}

func (e Error) Error() string {
	msg := "Error %v: %v"
	return fmt.Sprintf(msg, e.Code(), e.Message())
}

type Errors map[string]interface{}

func (e Errors) Error() string {
	var (
		msg string = ""
		err Error
		ok  bool
	)
	for _, val := range e["errors"].([]interface{}) {
		if err, ok = val.(map[string]interface{}); ok {
			msg += err.Error() + ". "
		}
	}
	return msg
}

func (e Errors) String() string {
	return e.Error()
}

func (e Errors) Errors() []Error {
	var errs = e["errors"].([]interface{})
	var out = make([]Error, len(errs))
	for i, val := range errs {
		out[i] = Error(val.(map[string]interface{}))
	}
	return out
}

type RateLimitError struct {
	Limit     uint32
	Remaining uint32
	Reset     time.Time
}

func (e RateLimitError) Error() string {
	msg := "Rate limit: %v, Remaining: %v, Reset: %v"
	return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset)
}

type APIResponse http.Response

func (r APIResponse) HasRateLimit() bool {
	return r.Header.Get(H_LIMIT) != ""
}

func (r APIResponse) RateLimit() uint32 {
	h := r.Header.Get(H_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitRemaining() uint32 {
	h := r.Header.Get(H_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) RateLimitReset() time.Time {
	h := r.Header.Get(H_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) HasMediaRateLimit() bool {
	return r.Header.Get(H_MEDIA_LIMIT) != ""
}

func (r APIResponse) MediaRateLimit() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitRemaining() uint32 {
	h := r.Header.Get(H_MEDIA_LIMIT_REMAIN)
	i, _ := strconv.ParseUint(h, 10, 32)
	return uint32(i)
}

func (r APIResponse) MediaRateLimitReset() time.Time {
	h := r.Header.Get(H_MEDIA_LIMIT_RESET)
	i, _ := strconv.ParseUint(h, 10, 32)
	t := time.Unix(int64(i), 0)
	return t
}

func (r APIResponse) readBody() (b []byte, err error) {
	var (
		header string
		reader io.Reader
	)
	header = strings.ToLower(r.Header.Get("Content-Encoding"))
	if header == "" || strings.Index(header, "gzip") == -1 {
		reader = r.Body
		defer r.Body.Close()
	} else {
		if reader, err = gzip.NewReader(r.Body); err != nil {
			return
		}
	}
	b, err = ioutil.ReadAll(reader)
	return
}

func (r APIResponse) ReadBody() string {
	var (
		b   []byte
		err error
	)
	if b, err = r.readBody(); err != nil {
		return ""
	}
	return string(b)
}

// Parses a JSON encoded HTTP response into the supplied interface.
func (r APIResponse) Parse(out interface{}) (err error) {
	var b []byte
	switch r.StatusCode {
	case STATUS_UNAUTHORIZED:
		fallthrough
	case STATUS_NOTFOUND:
		fallthrough
	case STATUS_GATEWAY:
		fallthrough
	case STATUS_FORBIDDEN:
		fallthrough
	case STATUS_INVALID:
		e := &Errors{}
		if b, err = r.readBody(); err != nil {
			return
		}
		err = json.Unmarshal(b, e)
		err = *e
		return
	case STATUS_LIMIT:
		err = RateLimitError{
			Limit:     r.RateLimit(),
			Remaining: r.RateLimitRemaining(),
			Reset:     r.RateLimitReset(),
		}
		return
	}
	if b, err = r.readBody(); err != nil {
		return
	}
	err = json.Unmarshal(b, out)
	if err == io.EOF {
		err = nil
	}
	return
}

// It's a user!
type User map[string]interface{}

func (u User) Id() uint64 {
	id, _ := strconv.ParseUint(u["id_str"].(string), 10, 64)
	return id
}

func (u User) IdStr() string {
	return u["id_str"].(string)
}

func (u User) Name() string {
	return u["name"].(string)
}

func (u User) ScreenName() string {
	return u["screen_name"].(string)
}

// It's a Tweet! (Adorably referred to by the API as a "status").
type Tweet map[string]interface{}

func (t Tweet) Id() (id uint64) {
	var (
		err error
		src = t["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (t Tweet) IdStr() string {
	return t["id_str"].(string)
}

func (t Tweet) Text() string {
	return t["text"].(string)
}

func (t Tweet) User() User {
	return User(t["user"].(map[string]interface{}))
}

func (t Tweet) CreatedAt() (out time.Time) {
	var (
		err error
		src = t["created_at"].(string)
	)
	if out, err = time.Parse(time.RubyDate, src); err != nil {
		panic(fmt.Sprintf("Could not parse time: %v", err))
	}
	return
}

// It's a less structured list of Tweets!
type Timeline []Tweet

// It's a structured list of Tweets!
type SearchResults map[string]interface{}

func (sr SearchResults) Statuses() []Tweet {
	var a []interface{} = sr["statuses"].([]interface{})
	b := make([]Tweet, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}

func (sr SearchResults) SearchMetadata() map[string]interface{} {
	a := sr["search_metadata"].(map[string]interface{})
	return a
}

func (sr SearchResults) NextQuery() (val url.Values, err error) {
	var (
		sm   map[string]interface{}
		n    interface{}
		next string
		ok   bool
	)
	sm = sr.SearchMetadata()
	if n, ok = sm["next_results"]; !ok {
		err = fmt.Errorf("Could not get next_results from search")
		return
	}
	if next, ok = n.(string); !ok {
		err = fmt.Errorf("Could not convert next_results to str: %v", n)
		return
	}
	if next[0] == '?' {
		next = next[1:]
	}
	val, err = url.ParseQuery(next)
	return
}

// A List!
type List map[string]interface{}

func (l List) User() User {
	return User(l["user"].(map[string]interface{}))
}

func (l List) Id() (id uint64) {
	var (
		err error
		src = l["id_str"].(string)
	)
	if id, err = strconv.ParseUint(src, 10, 64); err != nil {
		panic(fmt.Sprintf("Could not parse ID: %v", err))
	}
	return
}

func (l List) IdStr() string {
	return l["id_str"].(string)
}

func (l List) Mode() string {
	return l["mode"].(string)
}

func (l List) Name() string {
	return l["name"].(string)
}

func (l List) Slug() string {
	return l["slug"].(string)
}

func (l List) SubscriberCount() int64 {
	return l["subscriber_count"].(int64)
}

func (l List) MemberCount() int64 {
	return l["member_count"].(int64)
}

// It's a less structured list of Lists!
type Lists []List

// It's a cursored list of Lists!

type CursoredLists map[string]interface{}

func (cl CursoredLists) NextCursorStr() string {
	return cl["next_cursor_str"].(string)
}

func (cl CursoredLists) PreviousCursorStr() string {
	return cl["previous_cursor_str"].(string)
}

func (cl CursoredLists) Lists() Lists {
	var a []interface{} = cl["lists"].([]interface{})
	b := make([]List, len(a))
	for i, v := range a {
		b[i] = v.(map[string]interface{})
	}
	return b
}
<|endoftext|>


<commit_before>package controllers

import (
	"net/http"
	"strings"

	"github.com/zenazn/goji/web"
)

func Slack(c web.C, w http.ResponseWriter, r *http.Request) {
	params := strings.Fields(r.PostFormValue("text"))

	if params[0] == "check" {
		Check(c, w, r)
	} else if params[0] == "portfolio" {
		Portfolio(c, w, r)
	} else if params[0] == "buy" {
		Buy(c, w, r)
	} else if params[0] == "sell" {
		Sell(c, w, r)
	}
}
<commit_msg>Adding command shortcuts because I'm lazy when testing in Slack<commit_after>package controllers

import (
	"net/http"
	"strings"

	"github.com/zenazn/goji/web"
)

func Slack(c web.C, w http.ResponseWriter, r *http.Request) {
	params := strings.Fields(r.PostFormValue("text"))

	if params[0] == "check" || params[0] == "c" {
		Check(c, w, r)
	} else if params[0] == "portfolio" || params[0] == "p" {
		Portfolio(c, w, r)
	} else if params[0] == "buy" || params[0] == "b" {
		Buy(c, w, r)
	} else if params[0] == "sell" || params[0] == "s" {
		Sell(c, w, r)
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"bytes"
	"github.com/aphistic/gomol"
	"github.com/quakkels/goshipit/images"
	"github.com/quakkels/goshipit/slack"
	"net/http"
	"strconv"
)

func slash(w http.ResponseWriter, req *http.Request) {
	if req.Method == "POST" {
		slashCommand := slack.NewSlashCommandModel(req)
		gomol.Infof("slash command received: %f", slashCommand)

		w.Header().Add("Content Type", "text/plain")

		if slashCommand.Text == "categories" {
			cats := images.Images.GetCategories()
			var catBuffer bytes.Buffer
			for key, count := range cats {
				catBuffer.WriteString(key +
					" has " +
					strconv.Itoa(count) +
					" shipit squirrels.\n")
			}

			catBuffer.WriteTo(w)
		} else if slashCommand.Text == "" {
			image, err := images.Images.Take()
			if err != nil {
				gomol.Err("Failed to .Take() image. " + err.Error())
			}

			incomingWebhook := slack.IncomingWebhook{}
			incomingWebhook.Username = slashCommand.UserName
			incomingWebhook.Channel = slashCommand.ChannelName
			incomingWebhook.Text = slack.GetImageMarkup(image)

		} else {
			b := bytes.NewBufferString("Command not recognized.")
			b.WriteTo(w)
		}
	}
}
<commit_msg>calling send command from slack controller<commit_after>package controllers

import (
	"bytes"
	"github.com/aphistic/gomol"
	"github.com/quakkels/goshipit/images"
	"github.com/quakkels/goshipit/slack"
	"net/http"
	"strconv"
)

func slash(w http.ResponseWriter, req *http.Request) {
	if req.Method == "POST" {
		slashCommand := slack.NewSlashCommandModel(req)
		gomol.Infof("slash command received: %f", slashCommand)

		w.Header().Add("Content Type", "text/plain")

		if slashCommand.Text == "categories" {
			cats := images.Images.GetCategories()
			var catBuffer bytes.Buffer
			for key, count := range cats {
				catBuffer.WriteString(key +
					" has " +
					strconv.Itoa(count) +
					" shipit squirrels.\n")
			}

			catBuffer.WriteTo(w)
		} else if slashCommand.Text == "" {
			image, err := images.Images.Take()
			if err != nil {
				gomol.Err("Failed to .Take() image. " + err.Error())
			}

			incomingWebhook := slack.IncomingWebhook{}
			incomingWebhook.Username = slashCommand.UserName
			incomingWebhook.Channel = slashCommand.ChannelName
			incomingWebhook.Text = slack.GetImageMarkup(image)

			slack.SendIncomingWebhook(incomingWebhook)
		} else {
			b := bytes.NewBufferString("Command not recognized.")
			b.WriteTo(w)
		}
	}
}
<|endoftext|>


<commit_before>package mithril

import (
	"fmt"
	"regexp"
	"strings"
)

const (
	// Parser is the tags regex parser
	Parser = `(?:(^|#|\.)([^#\.\[\]]+))|(\[.+?\])`
	// AttrParser is the attribute regex parser
	AttrParser = `\[(.+?)(?:=("|'|)(.*?)("|'|))?\]`
)

// M returns VirtualElement
func M(selector string, opts ...interface{}) *VirtualElement {
	// new VirtualElement channel
	c := make(chan *VirtualElement)
	// create element in go routine
	go func() {
		// declare query string
		query := "div"
		// replace query as div if selector does not existed
		if selector != "" {
			query = selector
		}
		// create virtual element
		element := &VirtualElement{"div", []Attribute{}, nil}
		// map options
		for i, opt := range opts {
			switch obj := opt.(type) {
			case []Attribute:
				if i == 0 {
					element.Attrs = obj
				}
			case string, bool, int, *VirtualElement, []interface{}:
				element.Children = obj
			}
		}
		// match all results
		matches := regexp.MustCompile(Parser)
		for _, res := range matches.FindAllStringSubmatch(query, -1) {
			if res[1] == "" && len(res[2]) > 0 {
				element.Tag = strings.ToLower(res[2])
			} else if res[1] == "#" {
				element.Attr("id", res[2])
			} else if res[1] == "." {
				if !element.HasAttr("class") {
					element.Attrs = append(element.Attrs, Class(res[2]))
				} else {
					element.Attr("class", res[2])
				}
			} else if string(res[3][0]) == "[" {
				matches := regexp.MustCompile(AttrParser)
				for _, pair := range matches.FindAllStringSubmatch(res[3], -1) {
					if len(pair[1]) > 0 {
						element.Attrs = append(element.Attrs, NewStringAttr(pair[1], pair[3]))
					}
				}
			}
		}
		c <- element
	}()
	// return virtual element
	return <-c
}

// Render returns HTML string
func Render(elements ...interface{}) string {
	html := ""
	for _, element := range elements {
		switch obj := element.(type) {
		case Component:
			obj.Controller()
			html += fmt.Sprintf("%s", obj.View())
		case []interface{}:
			for _, item := range obj {
				html += Render(item)
			}
		case *VirtualElement:
			html += obj.String()
		case string:
			html += obj
		case int, int8, int32, int64, uint:
			html += fmt.Sprintf("%d", obj)
		case float32, float64:
			html += fmt.Sprintf("%f", obj)
		}
	}
	return html
}
<commit_msg>update pass-in data type list<commit_after>package mithril

import (
	"fmt"
	"regexp"
	"strings"
)

const (
	// Parser is the tags regex parser
	Parser = `(?:(^|#|\.)([^#\.\[\]]+))|(\[.+?\])`
	// AttrParser is the attribute regex parser
	AttrParser = `\[(.+?)(?:=("|'|)(.*?)("|'|))?\]`
)

// M returns VirtualElement
func M(selector string, opts ...interface{}) *VirtualElement {
	// new VirtualElement channel
	c := make(chan *VirtualElement)
	// create element in go routine
	go func() {
		// declare query string
		query := "div"
		// replace query as div if selector does not existed
		if selector != "" {
			query = selector
		}
		// create virtual element
		element := &VirtualElement{"div", []Attribute{}, nil}
		// map options
		for i, opt := range opts {
			switch obj := opt.(type) {
			case []Attribute:
				if i == 0 {
					element.Attrs = obj
				}
			case string, bool, int, int8, int32, int64, uint, uint8, uint32, uint64, float32, float64, complex64, *VirtualElement, *Component, []interface{}:
				element.Children = obj
			}
		}
		// match all results
		matches := regexp.MustCompile(Parser)
		for _, res := range matches.FindAllStringSubmatch(query, -1) {
			if res[1] == "" && len(res[2]) > 0 {
				element.Tag = strings.ToLower(res[2])
			} else if res[1] == "#" {
				element.Attr("id", res[2])
			} else if res[1] == "." {
				if !element.HasAttr("class") {
					element.Attrs = append(element.Attrs, Class(res[2]))
				} else {
					element.Attr("class", res[2])
				}
			} else if string(res[3][0]) == "[" {
				matches := regexp.MustCompile(AttrParser)
				for _, pair := range matches.FindAllStringSubmatch(res[3], -1) {
					if len(pair[1]) > 0 {
						element.Attrs = append(element.Attrs, NewStringAttr(pair[1], pair[3]))
					}
				}
			}
		}
		c <- element
	}()
	// return virtual element
	return <-c
}

// Render returns HTML string
func Render(elements ...interface{}) string {
	html := ""
	for _, element := range elements {
		switch obj := element.(type) {
		case Component:
			obj.Controller()
			html += fmt.Sprintf("%s", obj.View())
		case []interface{}:
			for _, item := range obj {
				html += Render(item)
			}
		case *VirtualElement:
			html += obj.String()
		case string:
			html += obj
		case bool:
			html += fmt.Sprintf("%t", obj)
		case int, int8, int32, int64, uint, uint8, uint32, uint64:
			html += fmt.Sprintf("%d", obj)
		case float32, float64, complex64:
			html += fmt.Sprintf("%f", obj)
		}
	}
	return html
}
<|endoftext|>


<commit_before>//
// Copyright (C) 2013 The Docker Cloud authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockercloud

import (
	"code.google.com/p/goauth2/oauth"
	compute "code.google.com/p/google-api-go-client/compute/v1beta16"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

var (
	instanceType = flag.String("instance-type",
		"/zones/us-central1-a/machineTypes/n1-standard-1",
		"The reference to the instance type to create.")
	image = flag.String("image",
		"https://www.googleapis.com/compute/v1beta16/projects/debian-cloud/global/images/debian-7-wheezy-v20131120",
		"The GCE image to boot from.")
)

const startup = `#!/bin/bash
sysctl -w net.ipv4.ip_forward=1
wget -qO- https://get.docker.io/ | sh
/etc/init.d/docker stop
docker -d -H=tcp://127.0.0.1:8000 &
`

// A Google Compute Engine implementation of the Cloud interface
type GCECloud struct {
	service   *compute.Service
	projectId string
}

// Create a GCE Cloud instance.  'clientId', 'clientSecret' and 'scope' are used to ask for a client
// credential.  'code' is optional and is only used if a cached credential can not be found.
// 'projectId' is the Google Cloud project name.
func NewCloudGce(clientId string, clientSecret string, scope string, code string, projectId string) *GCECloud {
	// Set up a configuration.
	config := &oauth.Config{
		ClientId:     clientId,
		ClientSecret: clientSecret,
		RedirectURL:  "oob",
		Scope:        scope,
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
		// TODO(bburns) : This prob. won't work on Windows
		TokenCache: oauth.CacheFile(os.Getenv("HOME") + "/cache.json"),
	}

	// Set up a Transport using the config.
	// transport := &oauth.Transport{Config: config,
	//         Transport: &LogTransport{http.DefaultTransport},}
	transport := &oauth.Transport{Config: config, Transport: http.DefaultTransport}

	// Try to pull the token from the cache; if this fails, we need to get one.
	token, err := config.TokenCache.Token()
	if err != nil {
		if clientId == "" || clientSecret == "" {
			flag.Usage()
			fmt.Fprint(os.Stderr, "Client id and secret are required.")
			os.Exit(2)
		}
		if code == "" {
			// Get an authorization code from the data provider.
			// ("Please ask the user if I can access this resource.")
			url := config.AuthCodeURL("")
			fmt.Println("Visit this URL to get a code, then run again with -code=YOUR_CODE\n")
			fmt.Println(url)
			// The below doesn't work for some reason.  Not sure why.  I get 404's
			// fmt.Print("Enter code: ")
			// bio := bufio.NewReader(os.Stdin)
			// code, err = bio.ReadString('\n')
			// if err != nil {
			//        log.Fatal("input: ", err)
			// }
			return nil
		}
		// Exchange the authorization code for an access token.
		// ("Here's the code you gave the user, now give me a token!")
		// TODO(bburns) : Put up a separate web end point to do the oauth dance, so a user can just go to a web page.
		token, err = transport.Exchange(code)
		if err != nil {
			log.Fatal("Exchange:", err)
		}
		// (The Exchange method will automatically cache the token.)
		fmt.Printf("Token is cached in %v\n", config.TokenCache)
	}

	// Make the actual request using the cached token to authenticate.
	// ("Here's the token, let me in!")
	transport.Token = token

	svc, err := compute.New(transport.Client())
	if err != nil {
		log.Printf("Error creating service: %v", err)
	}
	return &GCECloud{
		service:   svc,
		projectId: projectId,
	}
}

// Implementation of the Cloud interface
func (cloud GCECloud) GetPublicIPAddress(name string, zone string) (string, error) {
	instance, err := cloud.service.Instances.Get(cloud.projectId, zone, name).Do()
	if err != nil {
		return "", err
	}
	// Found the instance, we're good.
	return instance.NetworkInterfaces[0].AccessConfigs[0].NatIP, nil
}

// Implementation of the Cloud interface
func (cloud GCECloud) CreateInstance(name string, zone string) (string, error) {
	prefix := "https://www.googleapis.com/compute/v1beta16/projects/" + cloud.projectId
	instance := &compute.Instance{
		Name:        name,
		Description: "Docker on GCE",
		MachineType: prefix + *instanceType,
		Image:       *image,
		NetworkInterfaces: []*compute.NetworkInterface{
			&compute.NetworkInterface{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{Type: "ONE_TO_ONE_NAT"},
				},
				Network: prefix + "/global/networks/default",
			},
		},
		Metadata: &compute.Metadata{
			Items: []*compute.MetadataItems{
				&compute.MetadataItems{
					Key:   "startup-script",
					Value: startup,
				},
			},
		},
	}
	op, err := cloud.service.Instances.Insert(cloud.projectId, zone, instance).Do()
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		return instance.NetworkInterfaces[0].AccessConfigs[0].NatIP, err
	}
	err = cloud.waitForOp(op, zone)
	// Wait for docker to come up
	// TODO(bburns) : Use metadata instead to signal that docker is up and read.
	time.Sleep(60 * time.Second)

	return instance.NetworkInterfaces[0].AccessConfigs[0].NatIP, err
}

// Implementation of the Cloud interface
func (cloud GCECloud) DeleteInstance(name string, zone string) error {
	op, err := cloud.service.Instances.Delete(cloud.projectId, zone, name).Do()
	if err != nil {
		log.Printf("Got compute.Operation, err: %#v, %v", op, err)
		return err
	}
	return cloud.waitForOp(op, zone)
}

func (cloud GCECloud) OpenSecureTunnel(name string, zone string, localPort int, remotePort int) (*os.Process, error) {
	ip, err := cloud.GetPublicIPAddress(name, zone)
	if err != nil {
		return nil, err
	}
	username := os.Getenv("USER")
	homedir := os.Getenv("HOME")

	sshCommand := fmt.Sprintf("-o LogLevel=quiet -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i %s/.ssh/google_compute_engine -A -p 22 %s@%s -f -N -L %d:localhost:%d", homedir, username, ip, localPort, remotePort)
	fmt.Printf("Running %s\n", sshCommand)
	cmd := exec.Command("ssh", strings.Split(sshCommand, " ")...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd.Run()
	return cmd.Process, nil
}

// Implementation of the Cloud interface
func (cloud GCECloud) RunRemoteCommand(name string, zone string, outstream io.Writer, errstream io.Writer, instream io.Reader, args ...string) error {
	ip, err := cloud.GetPublicIPAddress(name, zone)
	if err != nil {
		return err
	}
	username := os.Getenv("USER")
	homedir := os.Getenv("HOME")

	sshCommand := fmt.Sprintf("-o LogLevel=quiet -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i %s/.ssh/google_compute_engine -A -p 22 %s@%s %s", homedir, username, ip, strings.Join(args, " "))
	fmt.Printf("Running %s\n", sshCommand)
	cmd := exec.Command("ssh", strings.Split(sshCommand, " ")...)
	cmd.Stdout = outstream
	cmd.Stderr = errstream
	if instream != nil {
		cmd.Stdin = os.Stdin
	}
	cmd.Run()
	fmt.Printf("Run...")
	return nil
}

// Wait for a compute operation to finish.
//   op The operation
//   zone The zone for the operation
// Returns an error if one occurs, or nil
func (cloud GCECloud) waitForOp(op *compute.Operation, zone string) error {
	op, err := cloud.service.ZoneOperations.Get(cloud.projectId, zone, op.Name).Do()
	for op.Status != "DONE" {
		time.Sleep(5 * time.Second)
		op, err = cloud.service.ZoneOperations.Get(cloud.projectId, zone, op.Name).Do()
		if err != nil {
			log.Printf("Got compute.Operation, err: %#v, %v", op, err)
		}
		if op.Status != "PENDING" && op.Status != "RUNNING" && op.Status != "DONE" {
			log.Printf("Error waiting for operation: %s\n", op)
			return errors.New(fmt.Sprintf("Bad operation: %s", op))
		}
	}
	return err
}
<commit_msg>dockercloud/gce: merge<commit_after>//
// Copyright (C) 2013 The Docker Cloud authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dockercloud

import (
	"code.google.com/p/goauth2/oauth"
	compute "code.google.com/p/google-api-go-client/compute/v1beta16"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"
)

var (
	instanceType = flag.String("instance-type",
		"/zones/us-central1-a/machineTypes/n1-standard-1",
		"The reference to the instance type to create.")
	image = flag.String("image",
		"https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-7-wheezy-v20131120",
		"The GCE image to boot from.")
)

const startup = `#!/bin/bash
sysctl -w net.ipv4.ip_forward=1
wget -qO- https://get.docker.io/ | sh
/etc/init.d/docker stop
docker -d -H=tcp://127.0.0.1:8000 &
`

// A Google Compute Engine implementation of the Cloud interface
type GCECloud struct {
	service   *compute.Service
	projectId string
}

// Create a GCE Cloud instance.  'clientId', 'clientSecret' and 'scope' are used to ask for a client
// credential.  'code' is optional and is only used if a cached credential can not be found.
// 'projectId' is the Google Cloud project name.
func NewCloudGce(clientId string, clientSecret string, scope string, code string, projectId string) *GCECloud {
	// Set up a configuration.
	config := &oauth.Config{
		ClientId:     clientId,
		ClientSecret: clientSecret,
		RedirectURL:  "oob",
		Scope:        scope,
		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
		TokenURL:     "https://accounts.google.com/o/oauth2/token",
		// TODO(bburns) : This prob. won't work on Windows
		TokenCache: oauth.CacheFile(os.Getenv("HOME") + "/cache.json"),
	}

	// Set up a Transport using the config.
	// transport := &oauth.Transport{Config: config,
	//         Transport: &LogTransport{http.DefaultTransport},}
	transport := &oauth.Transport{Config: config, Transport: http.DefaultTransport}

	// Try to pull the token from the cache; if this fails, we need to get one.
	token, err := config.TokenCache.Token()
	if err != nil {
		if clientId == "" || clientSecret == "" {
			flag.Usage()
			fmt.Fprint(os.Stderr, "Client id and secret are required.")
			os.Exit(2)
		}
		if code == "" {
			// Get an authorization code from the data provider.
			// ("Please ask the user if I can access this resource.")
			url := config.AuthCodeURL("")
			fmt.Println("Visit this URL to get a code, then run again with -code=YOUR_CODE\n")
			fmt.Println(url)
			// The below doesn't work for some reason.  Not sure why.  I get 404's
			// fmt.Print("Enter code: ")
			// bio := bufio.NewReader(os.Stdin)
			// code, err = bio.ReadString('\n')
			// if err != nil {
			//        log.Fatal("input: ", err)
			// }
			return nil
		}
		// Exchange the authorization code for an access token.
		// ("Here's the code you gave the user, now give me a token!")
		// TODO(bburns) : Put up a separate web end point to do the oauth dance, so a user can just go to a web page.
		token, err = transport.Exchange(code)
		if err != nil {
			log.Fatal("Exchange:", err)
		}
		// (The Exchange method will automatically cache the token.)
		fmt.Printf("Token is cached in %v\n", config.TokenCache)
	}

	// Make the actual request using the cached token to authenticate.
	// ("Here's the token, let me in!")
	transport.Token = token

	svc, err := compute.New(transport.Client())
	if err != nil {
		log.Printf("Error creating service: %v", err)
	}
	return &GCECloud{
		service:   svc,
		projectId: projectId,
	}
}

// Implementation of the Cloud interface
func (cloud GCECloud) GetPublicIPAddress(name string, zone string) (string, error) {
	instance, err := cloud.service.Instances.Get(cloud.projectId, zone, name).Do()
	if err != nil {
		return "", err
	}
	// Found the instance, we're good.
	return instance.NetworkInterfaces[0].AccessConfigs[0].NatIP, nil
}

// Implementation of the Cloud interface
func (cloud GCECloud) CreateInstance(name string, zone string) (string, error) {
	prefix := "https://www.googleapis.com/compute/v1beta16/projects/" + cloud.projectId
	instance := &compute.Instance{
		Name:        name,
		Description: "Docker on GCE",
		MachineType: prefix + *instanceType,
		Image:       *image,
		NetworkInterfaces: []*compute.NetworkInterface{
			&compute.NetworkInterface{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{Type: "ONE_TO_ONE_NAT"},
				},
				Network: prefix + "/global/networks/default",
			},
		},
		Metadata: &compute.Metadata{
			Items: []*compute.MetadataItems{
				&compute.MetadataItems{
					Key:   "startup-script",
					Value: startup,
				},
			},
		},
	}
	op, err := cloud.service.Instances.Insert(cloud.projectId, zone, instance).Do()
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		return instance.NetworkInterfaces[0].AccessConfigs[0].NatIP, err
	}
	err = cloud.waitForOp(op, zone)
	// Wait for docker to come up
	// TODO(bburns) : Use metadata instead to signal that docker is up and read.
	time.Sleep(60 * time.Second)

	return instance.NetworkInterfaces[0].AccessConfigs[0].NatIP, err
}

// Implementation of the Cloud interface
func (cloud GCECloud) DeleteInstance(name string, zone string) error {
	op, err := cloud.service.Instances.Delete(cloud.projectId, zone, name).Do()
	if err != nil {
		log.Printf("Got compute.Operation, err: %#v, %v", op, err)
		return err
	}
	return cloud.waitForOp(op, zone)
}

func (cloud GCECloud) OpenSecureTunnel(name string, zone string, localPort int, remotePort int) (*os.Process, error) {
	ip, err := cloud.GetPublicIPAddress(name, zone)
	if err != nil {
		return nil, err
	}
	username := os.Getenv("USER")
	homedir := os.Getenv("HOME")

	sshCommand := fmt.Sprintf("-o LogLevel=quiet -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i %s/.ssh/google_compute_engine -A -p 22 %s@%s -f -N -L %d:localhost:%d", homedir, username, ip, localPort, remotePort)
	fmt.Printf("Running %s\n", sshCommand)
	cmd := exec.Command("ssh", strings.Split(sshCommand, " ")...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd.Run()
	return cmd.Process, nil
}

// Implementation of the Cloud interface
func (cloud GCECloud) RunRemoteCommand(name string, zone string, outstream io.Writer, errstream io.Writer, instream io.Reader, args ...string) error {
	ip, err := cloud.GetPublicIPAddress(name, zone)
	if err != nil {
		return err
	}
	username := os.Getenv("USER")
	homedir := os.Getenv("HOME")

	sshCommand := fmt.Sprintf("-o LogLevel=quiet -o UserKnownHostsFile=/dev/null -o CheckHostIP=no -o StrictHostKeyChecking=no -i %s/.ssh/google_compute_engine -A -p 22 %s@%s %s", homedir, username, ip, strings.Join(args, " "))
	fmt.Printf("Running %s\n", sshCommand)
	cmd := exec.Command("ssh", strings.Split(sshCommand, " ")...)
	cmd.Stdout = outstream
	cmd.Stderr = errstream
	if instream != nil {
		cmd.Stdin = os.Stdin
	}
	cmd.Run()
	fmt.Printf("Run...")
	return nil
}

// Wait for a compute operation to finish.
//   op The operation
//   zone The zone for the operation
// Returns an error if one occurs, or nil
func (cloud GCECloud) waitForOp(op *compute.Operation, zone string) error {
	op, err := cloud.service.ZoneOperations.Get(cloud.projectId, zone, op.Name).Do()
	for op.Status != "DONE" {
		time.Sleep(5 * time.Second)
		op, err = cloud.service.ZoneOperations.Get(cloud.projectId, zone, op.Name).Do()
		if err != nil {
			log.Printf("Got compute.Operation, err: %#v, %v", op, err)
		}
		if op.Status != "PENDING" && op.Status != "RUNNING" && op.Status != "DONE" {
			log.Printf("Error waiting for operation: %s\n", op)
			return errors.New(fmt.Sprintf("Bad operation: %s", op))
		}
	}
	return err
}
<|endoftext|>


<commit_before>package eval

import (
	"bytecode"
	"bytecode/ir"
)

type evaluator struct {
	trace evaluatorTracer
	*bytecode.Object
	stack stack
	argc  int
}

func (ev *evaluator) eval() {
	ev.trace.PreEval(ev)
	for index, bb := range ev.Blocks {
		ev.trace.PreBlock(index, bb.Name)
		ev.evalBlock(bb)
	}
	ev.trace.PostEval(ev)

	ev.StackUsage = ev.stack.maxLen
}

func (ev *evaluator) evalBlock(bb *bytecode.BasicBlock) {
	for i, instr := range bb.Instrs {
		switch instr.Op {
		case ir.OpPanic:
			ev.stack.Drop(2)
			bb.Instrs[i] = ir.Call(1)

		case ir.OpCall:
			ev.stack.Drop(int(instr.Data) + 1)
			ev.stack.PushUnknown()

		case ir.OpConstRef:
			ev.stack.Push(ev.ConstPool.Get(instr.Data))

		case ir.OpLocalRef:
			index := ev.stack.FindLocal(instr.Data)
			bb.Instrs[i] = ir.StackRef(ev.stack.Len() - index - 1)
			ev.stack.Copy(index)

		case ir.OpLocalBind:
			ev.stack.BindLocal(instr.Data)
			bb.Instrs[i] = ir.Empty

		case ir.OpScopeExit:
			ev.stack.Drop(int(instr.Data))
			bb.Instrs[i] = ir.Empty

		case ir.OpReturn, ir.OpJmpNil:
			ev.stack.Drop(1)

		case ir.OpNumAdd, ir.OpNumSub, ir.OpNumMul, ir.OpNumQuo:
			fallthrough
		case ir.OpNumGt, ir.OpNumLt, ir.OpNumEq:
			ev.stack.Drop(2)
			ev.stack.PushUnknown()
		}

		ev.trace.PostInstr(ev, instr)
	}
}
<commit_msg>adding missing cases for evaluator<commit_after>package eval

import (
	"bytecode"
	"bytecode/ir"
)

type evaluator struct {
	trace evaluatorTracer
	*bytecode.Object
	stack stack
	argc  int
}

func (ev *evaluator) eval() {
	ev.trace.PreEval(ev)
	for index, bb := range ev.Blocks {
		ev.trace.PreBlock(index, bb.Name)
		ev.evalBlock(bb)
	}
	ev.trace.PostEval(ev)

	ev.StackUsage = ev.stack.maxLen
}

func (ev *evaluator) evalBlock(bb *bytecode.BasicBlock) {
	for i, instr := range bb.Instrs {
		switch instr.Op {
		case ir.OpPanic:
			ev.stack.Drop(2)
			bb.Instrs[i] = ir.Call(1)

		case ir.OpIsCons, ir.OpIsInt, ir.OpIsNum, ir.OpIsString:
			ev.stack.Drop(1)
			ev.stack.PushUnknown()

		case ir.OpCall:
			ev.stack.Drop(int(instr.Data) + 1)
			ev.stack.PushUnknown()

		case ir.OpConstRef:
			ev.stack.Push(ev.ConstPool.Get(instr.Data))

		case ir.OpStackRef:
			ev.stack.Copy(ev.stack.Len() - int(instr.Data) - 1)

		case ir.OpLocalRef:
			index := ev.stack.FindLocal(instr.Data)
			bb.Instrs[i] = ir.StackRef(ev.stack.Len() - index - 1)
			ev.stack.Copy(index)

		case ir.OpLocalBind:
			ev.stack.BindLocal(instr.Data)
			bb.Instrs[i] = ir.Empty

		case ir.OpScopeExit:
			ev.stack.Drop(int(instr.Data))
			bb.Instrs[i] = ir.Empty

		case ir.OpReturn, ir.OpJmpNil, ir.OpJmpNotNil:
			ev.stack.Drop(1)

		case ir.OpNumAdd, ir.OpNumSub, ir.OpNumMul, ir.OpNumQuo:
			fallthrough
		case ir.OpNumGt, ir.OpNumLt, ir.OpNumEq:
			ev.stack.Drop(2)
			ev.stack.PushUnknown()
		}

		ev.trace.PostInstr(ev, instr)
	}
}
<|endoftext|>


<commit_before><commit_msg>adding pt<commit_after><|endoftext|>