<commit_before>package image

import (
	"io"
	"os"
	"strings"
	"syscall"

	cp "github.com/containers/image/copy"
	"github.com/containers/image/docker/tarfile"
	"github.com/containers/image/manifest"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports/alltransports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	"github.com/kubernetes-incubator/cri-o/libkpod/common"
	"github.com/pkg/errors"
)

const (
	// DefaultRegistry is a prefix that we apply to an image name
	// to check docker hub first for the image
	DefaultRegistry = "docker://"
)

// CopyOptions contains the options given when pushing or pulling images
type CopyOptions struct {
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// Store is the local storage store which holds the source image.
	Store storage.Store
	// DockerRegistryOptions encapsulates settings that affect how we
	// connect or authenticate to a remote registry to which we want to
	// push the image.
	common.DockerRegistryOptions
	// SigningOptions encapsulates settings that control whether or not we
	// strip or add signatures to the image when pushing (uploading) the
	// image to a registry.
	common.SigningOptions
}

// PushImage pushes the src image to the destination
func PushImage(srcName, destName string, options CopyOptions) error {
	if srcName == "" || destName == "" {
		return errors.Wrapf(syscall.EINVAL, "source and destination image names must be specified")
	}

	// Get the destination Image Reference
	dest, err := alltransports.ParseImageName(destName)
	if err != nil {
		return errors.Wrapf(err, "error getting destination imageReference for %q", destName)
	}

	policyContext, err := common.GetPolicyContext(options.SignaturePolicyPath)
	if err != nil {
		return errors.Wrapf(err, "Could not get default policy context for signature policy path %q", options.SignaturePolicyPath)
	}
	defer policyContext.Destroy()
	// Look up the image name and its layer, then build the imagePushData from
	// the image
	img, err := FindImage(options.Store, srcName)
	if err != nil {
		return errors.Wrapf(err, "error locating image %q for importing settings", srcName)
	}
	systemContext := common.GetSystemContext(options.SignaturePolicyPath)
	cd, err := ImportCopyDataFromImage(options.Store, systemContext, img.ID, "", "")
	if err != nil {
		return err
	}
	// Give the image we're producing the same ancestors as its source image
	cd.FromImage = cd.Docker.ContainerConfig.Image
	cd.FromImageID = string(cd.Docker.Parent)

	// Prep the layers and manifest for export
	src, err := cd.MakeImageRef(manifest.GuessMIMEType(cd.Manifest), options.Compression, img.Names, img.TopLayer, nil)
	if err != nil {
		return errors.Wrapf(err, "error copying layers and metadata")
	}

	copyOptions := common.GetCopyOptions(options.ReportWriter, options.SignaturePolicyPath, nil, &options.DockerRegistryOptions, options.SigningOptions)

	// Copy the image to the remote destination
	err = cp.Image(policyContext, dest, src, copyOptions)
	if err != nil {
		return errors.Wrapf(err, "Error copying image to the remote destination")
	}
	return nil
}

// PullImage copies the image from the source to the destination
func PullImage(store storage.Store, imgName string, allTags, quiet bool, sc *types.SystemContext) error {
	var (
		images []string
		output io.Writer
	)

	if quiet {
		output = nil
	} else {
		output = os.Stdout
	}

	srcRef, err := alltransports.ParseImageName(imgName)
	if err != nil {
		defaultName := DefaultRegistry + imgName
		srcRef2, err2 := alltransports.ParseImageName(defaultName)
		if err2 != nil {
			return errors.Errorf("error parsing image name %q: %v", defaultName, err2)
		}
		srcRef = srcRef2
	}

	splitArr := strings.Split(imgName, ":")

	// supports pulling from docker-archive, oci, and registries
	if splitArr[0] == "docker-archive" {
		tarSource := tarfile.NewSource(splitArr[len(splitArr)-1])
		manifest, err := tarSource.LoadTarManifest()
		if err != nil {
			return errors.Errorf("error retrieving manifest.json: %v", err)
		}
		// to pull all the images stored in one tar file
		for i := range manifest {
			images = append(images, manifest[i].RepoTags[0])
		}
	} else if splitArr[0] == "oci" {
		// needs to be implemented in future
		return errors.Errorf("oci not supported")
	} else {
		images = append(images, imgName)
	}

	policy, err := signature.DefaultPolicy(sc)
	if err != nil {
		return err
	}

	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return err
	}
	defer policyContext.Destroy()

	copyOptions := common.GetCopyOptions(output, "", nil, nil, common.SigningOptions{})

	for _, image := range images {
		destRef, err := is.Transport.ParseStoreReference(store, image)
		if err != nil {
			return errors.Errorf("error parsing dest reference name: %v", err)
		}
		if err = cp.Image(policyContext, destRef, srcRef, copyOptions); err != nil {
			return errors.Errorf("error loading image %q: %v", image, err)
		}
	}
	return nil
}
<commit_msg>Fix 'kpod load' nil issue raised by QE<commit_after>package image

import (
	"io"
	"os"
	"strings"
	"syscall"

	cp "github.com/containers/image/copy"
	"github.com/containers/image/docker/tarfile"
	"github.com/containers/image/manifest"
	"github.com/containers/image/signature"
	is "github.com/containers/image/storage"
	"github.com/containers/image/transports/alltransports"
	"github.com/containers/image/types"
	"github.com/containers/storage"
	"github.com/containers/storage/pkg/archive"
	"github.com/kubernetes-incubator/cri-o/libkpod/common"
	"github.com/pkg/errors"
)

const (
	// DefaultRegistry is a prefix that we apply to an image name
	// to check docker hub first for the image
	DefaultRegistry = "docker://"
)

// CopyOptions contains the options given when pushing or pulling images
type CopyOptions struct {
	// Compression specifies the type of compression which is applied to
	// layer blobs.  The default is to not use compression, but
	// archive.Gzip is recommended.
	Compression archive.Compression
	// SignaturePolicyPath specifies an override location for the signature
	// policy which should be used for verifying the new image as it is
	// being written.  Except in specific circumstances, no value should be
	// specified, indicating that the shared, system-wide default policy
	// should be used.
	SignaturePolicyPath string
	// ReportWriter is an io.Writer which will be used to log the writing
	// of the new image.
	ReportWriter io.Writer
	// Store is the local storage store which holds the source image.
	Store storage.Store
	// DockerRegistryOptions encapsulates settings that affect how we
	// connect or authenticate to a remote registry to which we want to
	// push the image.
	common.DockerRegistryOptions
	// SigningOptions encapsulates settings that control whether or not we
	// strip or add signatures to the image when pushing (uploading) the
	// image to a registry.
	common.SigningOptions
}

// PushImage pushes the src image to the destination
func PushImage(srcName, destName string, options CopyOptions) error {
	if srcName == "" || destName == "" {
		return errors.Wrapf(syscall.EINVAL, "source and destination image names must be specified")
	}

	// Get the destination Image Reference
	dest, err := alltransports.ParseImageName(destName)
	if err != nil {
		return errors.Wrapf(err, "error getting destination imageReference for %q", destName)
	}

	policyContext, err := common.GetPolicyContext(options.SignaturePolicyPath)
	if err != nil {
		return errors.Wrapf(err, "Could not get default policy context for signature policy path %q", options.SignaturePolicyPath)
	}
	defer policyContext.Destroy()
	// Look up the image name and its layer, then build the imagePushData from
	// the image
	img, err := FindImage(options.Store, srcName)
	if err != nil {
		return errors.Wrapf(err, "error locating image %q for importing settings", srcName)
	}
	systemContext := common.GetSystemContext(options.SignaturePolicyPath)
	cd, err := ImportCopyDataFromImage(options.Store, systemContext, img.ID, "", "")
	if err != nil {
		return err
	}
	// Give the image we're producing the same ancestors as its source image
	cd.FromImage = cd.Docker.ContainerConfig.Image
	cd.FromImageID = string(cd.Docker.Parent)

	// Prep the layers and manifest for export
	src, err := cd.MakeImageRef(manifest.GuessMIMEType(cd.Manifest), options.Compression, img.Names, img.TopLayer, nil)
	if err != nil {
		return errors.Wrapf(err, "error copying layers and metadata")
	}

	copyOptions := common.GetCopyOptions(options.ReportWriter, options.SignaturePolicyPath, nil, &options.DockerRegistryOptions, options.SigningOptions)

	// Copy the image to the remote destination
	err = cp.Image(policyContext, dest, src, copyOptions)
	if err != nil {
		return errors.Wrapf(err, "Error copying image to the remote destination")
	}
	return nil
}

// PullImage copies the image from the source to the destination
func PullImage(store storage.Store, imgName string, allTags, quiet bool, sc *types.SystemContext) error {
	var (
		images []string
		output io.Writer
	)

	if quiet {
		output = nil
	} else {
		output = os.Stdout
	}

	srcRef, err := alltransports.ParseImageName(imgName)
	if err != nil {
		defaultName := DefaultRegistry + imgName
		srcRef2, err2 := alltransports.ParseImageName(defaultName)
		if err2 != nil {
			return errors.Errorf("error parsing image name %q: %v", defaultName, err2)
		}
		srcRef = srcRef2
	}

	splitArr := strings.Split(imgName, ":")

	// supports pulling from docker-archive, oci, and registries
	if splitArr[0] == "docker-archive" {
		tarSource := tarfile.NewSource(splitArr[len(splitArr)-1])
		manifest, err := tarSource.LoadTarManifest()
		if err != nil {
			return errors.Errorf("error retrieving manifest.json: %v", err)
		}
		// to pull all the images stored in one tar file
		for i := range manifest {
			if manifest[i].RepoTags != nil {
				images = append(images, manifest[i].RepoTags[0])
			} else {
				// create an image object and use the hex value of the digest as the image ID
				// for parsing the store reference
				newImg, err := srcRef.NewImage(sc)
				if err != nil {
					return err
				}
				defer newImg.Close()
				digest := newImg.ConfigInfo().Digest
				if err := digest.Validate(); err == nil {
					images = append(images, "@"+digest.Hex())
				} else {
					return errors.Wrapf(err, "error getting config info")
				}
			}
		}
	} else if splitArr[0] == "oci" {
		// needs to be implemented in future
		return errors.Errorf("oci not supported")
	} else {
		images = append(images, imgName)
	}

	policy, err := signature.DefaultPolicy(sc)
	if err != nil {
		return err
	}

	policyContext, err := signature.NewPolicyContext(policy)
	if err != nil {
		return err
	}
	defer policyContext.Destroy()

	copyOptions := common.GetCopyOptions(output, "", nil, nil, common.SigningOptions{})

	for _, image := range images {
		destRef, err := is.Transport.ParseStoreReference(store, image)
		if err != nil {
			return errors.Errorf("error parsing dest reference name: %v", err)
		}
		if err = cp.Image(policyContext, destRef, srcRef, copyOptions); err != nil {
			return errors.Errorf("error loading image %q: %v", image, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package herd

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/hash"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"path"
	"syscall"
	"time"
)

type state struct {
	sub                     *Sub
	subFS                   *filesystem.FileSystem
	requiredFS              *filesystem.FileSystem
	requiredInodeToSubInode map[uint64]uint64
	inodesChanged           map[uint64]bool   // Required inode number.
	inodesCreated           map[uint64]string // Required inode number.
	subObjectCacheUsage     map[hash.Hash]uint64
}

// Returns true if no update needs to be performed.
func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) (
	bool, bool) {
	sub.herd.computeSemaphore <- struct{}{}
	defer func() { <-sub.herd.computeSemaphore }()
	var state state
	state.sub = sub
	state.subFS = sub.fileSystem
	requiredImage := sub.herd.getImageNoError(sub.requiredImage)
	state.requiredFS = requiredImage.FileSystem
	filter := requiredImage.Filter
	request.Triggers = requiredImage.Triggers
	state.requiredInodeToSubInode = make(map[uint64]uint64)
	state.inodesChanged = make(map[uint64]bool)
	state.inodesCreated = make(map[uint64]string)
	state.subObjectCacheUsage = make(map[hash.Hash]uint64, len(sub.objectCache))
	var rusageStart, rusageStop syscall.Rusage
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusageStart)
	// Populate subObjectCacheUsage.
	for _, hash := range sub.objectCache {
		state.subObjectCacheUsage[hash] = 0
	}
	if !filesystem.CompareDirectoriesMetadata(&state.subFS.DirectoryInode,
		&state.requiredFS.DirectoryInode, nil) {
		makeDirectory(request, &state.requiredFS.DirectoryInode, "/", false)
	}
	if !compareDirectories(request, &state,
		&state.subFS.DirectoryInode, &state.requiredFS.DirectoryInode,
		"/", filter) {
		return false, true
	}
	// Look for multiply used objects and tell the sub.
	for obj, useCount := range state.subObjectCacheUsage {
		if useCount > 1 {
			if request.MultiplyUsedObjects == nil {
				request.MultiplyUsedObjects = make(map[hash.Hash]uint64)
			}
			request.MultiplyUsedObjects[obj] = useCount
		}
	}
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusageStop)
	sub.lastComputeUpdateCpuDuration = time.Duration(
		rusageStop.Utime.Sec)*time.Second +
		time.Duration(rusageStop.Utime.Usec)*time.Microsecond -
		time.Duration(rusageStart.Utime.Sec)*time.Second -
		time.Duration(rusageStart.Utime.Usec)*time.Microsecond
	computeCpuTimeDistribution.Add(sub.lastComputeUpdateCpuDuration)
	if len(request.FilesToCopyToCache) > 0 ||
		len(request.InodesToMake) > 0 ||
		len(request.HardlinksToMake) > 0 ||
		len(request.PathsToDelete) > 0 ||
		len(request.DirectoriesToMake) > 0 ||
		len(request.InodesToChange) > 0 {
		sub.herd.logger.Printf(
			"buildUpdateRequest(%s) took: %s user CPU time\n",
			sub.hostname, sub.lastComputeUpdateCpuDuration)
		return false, false
	}
	return true, false
}

func compareDirectories(request *subproto.UpdateRequest, state *state,
	subDirectory, requiredDirectory *filesystem.DirectoryInode,
	myPathName string, filter *filter.Filter) bool {
	// First look for entries that should be deleted.
	if filter != nil && subDirectory != nil {
		for name := range subDirectory.EntriesByName {
			pathname := path.Join(myPathName, name)
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
			}
		}
	}
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(myPathName, name)
		if filter != nil && filter.Match(pathname) {
			continue
		}
		var subEntry *filesystem.DirectoryEntry
		if subDirectory != nil {
			if se, ok := subDirectory.EntriesByName[name]; ok {
				subEntry = se
			}
		}
		requiredInode := requiredEntry.Inode()
		if _, ok := requiredInode.(*filesystem.ComputedRegularInode); ok {
			// Replace with computed file.
			inode, ok := state.sub.computedInodes[pathname]
			if !ok {
				state.sub.herd.logger.Printf(
					"compareDirectories(%s): missing computed file: %s\n",
					state.sub.hostname, pathname)
				return true
			}
			newEntry := new(filesystem.DirectoryEntry)
			newEntry.Name = name
			newEntry.InodeNumber = requiredEntry.InodeNumber
			newEntry.SetInode(inode)
			requiredEntry = newEntry
		}
		if subEntry == nil {
			addEntry(request, state, requiredEntry, pathname)
		} else {
			compareEntries(request, state, subEntry, requiredEntry, pathname)
		}
		// If a directory: descend (possibly with the directory for the sub).
		if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
			var subInode *filesystem.DirectoryInode
			if subEntry != nil {
				if si, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
					subInode = si
				}
			}
			compareDirectories(request, state, subInode, requiredInode,
				pathname, filter)
		}
	}
	return false
}

func addEntry(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	requiredInode := requiredEntry.Inode()
	if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
		makeDirectory(request, requiredInode, myPathName, true)
	} else {
		addInode(request, state, requiredEntry, myPathName)
	}
}

func compareEntries(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	subInode := subEntry.Inode()
	requiredInode := requiredEntry.Inode()
	sameType, sameMetadata, sameData := filesystem.CompareInodes(
		subInode, requiredInode, nil)
	if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
		if sameMetadata {
			return
		}
		if sameType {
			makeDirectory(request, requiredInode, myPathName, false)
		} else {
			makeDirectory(request, requiredInode, myPathName, true)
		}
		return
	}
	if sameType && sameData && sameMetadata {
		relink(request, state, subEntry, requiredEntry, myPathName)
		return
	}
	if sameType && sameData {
		updateMetadata(request, state, requiredEntry, myPathName)
		relink(request, state, subEntry, requiredEntry, myPathName)
		return
	}
	addInode(request, state, requiredEntry, myPathName)
}

func relink(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	subInum, ok := state.requiredInodeToSubInode[requiredEntry.InodeNumber]
	if !ok {
		state.requiredInodeToSubInode[requiredEntry.InodeNumber] =
			subEntry.InodeNumber
		return
	}
	if subInum == subEntry.InodeNumber {
		return
	}
	makeHardlink(request,
		myPathName, state.subFS.InodeToFilenamesTable()[subInum][0])
}

func makeHardlink(request *subproto.UpdateRequest, newLink, target string) {
	var hardlink subproto.Hardlink
	hardlink.NewLink = newLink
	hardlink.Target = target
	request.HardlinksToMake = append(request.HardlinksToMake, hardlink)
}

func updateMetadata(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	if state.inodesChanged[requiredEntry.InodeNumber] {
		return
	}
	var inode subproto.Inode
	inode.Name = myPathName
	inode.GenericInode = requiredEntry.Inode()
	request.InodesToChange = append(request.InodesToChange, inode)
	state.inodesChanged[requiredEntry.InodeNumber] = true
}

func makeDirectory(request *subproto.UpdateRequest,
	requiredInode *filesystem.DirectoryInode, pathName string, create bool) {
	var newInode subproto.Inode
	newInode.Name = pathName
	var newDirectoryInode filesystem.DirectoryInode
	newDirectoryInode.Mode = requiredInode.Mode
	newDirectoryInode.Uid = requiredInode.Uid
	newDirectoryInode.Gid = requiredInode.Gid
	newInode.GenericInode = &newDirectoryInode
	if create {
		request.DirectoriesToMake = append(request.DirectoriesToMake, newInode)
	} else {
		request.InodesToChange = append(request.InodesToChange, newInode)
	}
}

func addInode(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	requiredInode := requiredEntry.Inode()
	if name, ok := state.inodesCreated[requiredEntry.InodeNumber]; ok {
		makeHardlink(request, myPathName, name)
		return
	}
	// Try to find a sibling inode.
	names := state.requiredFS.InodeToFilenamesTable()[requiredEntry.InodeNumber]
	if len(names) > 1 {
		var sameDataInode filesystem.GenericInode
		var sameDataName string
		for _, name := range names {
			if inum, found := state.subFS.FilenameToInodeTable()[name]; found {
				subInode := state.subFS.InodeTable[inum]
				_, sameMetadata, sameData := filesystem.CompareInodes(
					subInode, requiredInode, nil)
				if sameMetadata && sameData {
					makeHardlink(request, myPathName, name)
					return
				}
				if sameData {
					sameDataInode = subInode
					sameDataName = name
				}
			}
		}
		if sameDataInode != nil {
			updateMetadata(request, state, requiredEntry, sameDataName)
			makeHardlink(request, myPathName, sameDataName)
			return
		}
	}
	if inode, ok := requiredEntry.Inode().(*filesystem.RegularInode); ok {
		if inode.Size > 0 {
			if _, ok := state.subObjectCacheUsage[inode.Hash]; ok {
				state.subObjectCacheUsage[inode.Hash]++
			} else {
				// Not in object cache: grab it from file-system.
				if inos, ok := state.subFS.HashToInodesTable()[inode.Hash]; ok {
					var fileToCopy subproto.FileToCopyToCache
					fileToCopy.Name =
						state.subFS.InodeToFilenamesTable()[inos[0]][0]
					fileToCopy.Hash = inode.Hash
					request.FilesToCopyToCache = append(
						request.FilesToCopyToCache, fileToCopy)
					state.subObjectCacheUsage[inode.Hash] = 1
				} else {
					panic("No object in cache for: " + myPathName)
				}
			}
		}
	}
	var inode subproto.Inode
	inode.Name = myPathName
	inode.GenericInode = requiredEntry.Inode()
	request.InodesToMake = append(request.InodesToMake, inode)
	state.inodesCreated[requiredEntry.InodeNumber] = myPathName
}
<commit_msg>Fix bug in dominator: falsely reported missing computed file.<commit_after>package herd

import (
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	"github.com/Symantec/Dominator/lib/hash"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"path"
	"syscall"
	"time"
)

type state struct {
	sub                     *Sub
	subFS                   *filesystem.FileSystem
	requiredFS              *filesystem.FileSystem
	requiredInodeToSubInode map[uint64]uint64
	inodesChanged           map[uint64]bool   // Required inode number.
	inodesCreated           map[uint64]string // Required inode number.
	subObjectCacheUsage     map[hash.Hash]uint64
}

// Returns true if no update needs to be performed.
func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) (
	bool, bool) {
	sub.herd.computeSemaphore <- struct{}{}
	defer func() { <-sub.herd.computeSemaphore }()
	var state state
	state.sub = sub
	state.subFS = sub.fileSystem
	requiredImage := sub.herd.getImageNoError(sub.requiredImage)
	state.requiredFS = requiredImage.FileSystem
	filter := requiredImage.Filter
	request.Triggers = requiredImage.Triggers
	state.requiredInodeToSubInode = make(map[uint64]uint64)
	state.inodesChanged = make(map[uint64]bool)
	state.inodesCreated = make(map[uint64]string)
	state.subObjectCacheUsage = make(map[hash.Hash]uint64, len(sub.objectCache))
	var rusageStart, rusageStop syscall.Rusage
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusageStart)
	// Populate subObjectCacheUsage.
	for _, hash := range sub.objectCache {
		state.subObjectCacheUsage[hash] = 0
	}
	if !filesystem.CompareDirectoriesMetadata(&state.subFS.DirectoryInode,
		&state.requiredFS.DirectoryInode, nil) {
		makeDirectory(request, &state.requiredFS.DirectoryInode, "/", false)
	}
	if compareDirectories(request, &state,
		&state.subFS.DirectoryInode, &state.requiredFS.DirectoryInode,
		"/", filter) {
		return false, true
	}
	// Look for multiply used objects and tell the sub.
	for obj, useCount := range state.subObjectCacheUsage {
		if useCount > 1 {
			if request.MultiplyUsedObjects == nil {
				request.MultiplyUsedObjects = make(map[hash.Hash]uint64)
			}
			request.MultiplyUsedObjects[obj] = useCount
		}
	}
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusageStop)
	sub.lastComputeUpdateCpuDuration = time.Duration(
		rusageStop.Utime.Sec)*time.Second +
		time.Duration(rusageStop.Utime.Usec)*time.Microsecond -
		time.Duration(rusageStart.Utime.Sec)*time.Second -
		time.Duration(rusageStart.Utime.Usec)*time.Microsecond
	computeCpuTimeDistribution.Add(sub.lastComputeUpdateCpuDuration)
	if len(request.FilesToCopyToCache) > 0 ||
		len(request.InodesToMake) > 0 ||
		len(request.HardlinksToMake) > 0 ||
		len(request.PathsToDelete) > 0 ||
		len(request.DirectoriesToMake) > 0 ||
		len(request.InodesToChange) > 0 {
		sub.herd.logger.Printf(
			"buildUpdateRequest(%s) took: %s user CPU time\n",
			sub.hostname, sub.lastComputeUpdateCpuDuration)
		return false, false
	}
	return true, false
}

func compareDirectories(request *subproto.UpdateRequest, state *state,
	subDirectory, requiredDirectory *filesystem.DirectoryInode,
	myPathName string, filter *filter.Filter) bool {
	// First look for entries that should be deleted.
	if filter != nil && subDirectory != nil {
		for name := range subDirectory.EntriesByName {
			pathname := path.Join(myPathName, name)
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
			}
		}
	}
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(myPathName, name)
		if filter != nil && filter.Match(pathname) {
			continue
		}
		var subEntry *filesystem.DirectoryEntry
		if subDirectory != nil {
			if se, ok := subDirectory.EntriesByName[name]; ok {
				subEntry = se
			}
		}
		requiredInode := requiredEntry.Inode()
		if _, ok := requiredInode.(*filesystem.ComputedRegularInode); ok {
			// Replace with computed file.
			inode, ok := state.sub.computedInodes[pathname]
			if !ok {
				state.sub.herd.logger.Printf(
					"compareDirectories(%s): missing computed file: %s\n",
					state.sub.hostname, pathname)
				return true
			}
			newEntry := new(filesystem.DirectoryEntry)
			newEntry.Name = name
			newEntry.InodeNumber = requiredEntry.InodeNumber
			newEntry.SetInode(inode)
			requiredEntry = newEntry
		}
		if subEntry == nil {
			addEntry(request, state, requiredEntry, pathname)
		} else {
			compareEntries(request, state, subEntry, requiredEntry, pathname)
		}
		// If a directory: descend (possibly with the directory for the sub).
		if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
			var subInode *filesystem.DirectoryInode
			if subEntry != nil {
				if si, ok := subEntry.Inode().(*filesystem.DirectoryInode); ok {
					subInode = si
				}
			}
			compareDirectories(request, state, subInode, requiredInode,
				pathname, filter)
		}
	}
	return false
}

func addEntry(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	requiredInode := requiredEntry.Inode()
	if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
		makeDirectory(request, requiredInode, myPathName, true)
	} else {
		addInode(request, state, requiredEntry, myPathName)
	}
}

func compareEntries(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	subInode := subEntry.Inode()
	requiredInode := requiredEntry.Inode()
	sameType, sameMetadata, sameData := filesystem.CompareInodes(
		subInode, requiredInode, nil)
	if requiredInode, ok := requiredInode.(*filesystem.DirectoryInode); ok {
		if sameMetadata {
			return
		}
		if sameType {
			makeDirectory(request, requiredInode, myPathName, false)
		} else {
			makeDirectory(request, requiredInode, myPathName, true)
		}
		return
	}
	if sameType && sameData && sameMetadata {
		relink(request, state, subEntry, requiredEntry, myPathName)
		return
	}
	if sameType && sameData {
		updateMetadata(request, state, requiredEntry, myPathName)
		relink(request, state, subEntry, requiredEntry, myPathName)
		return
	}
	addInode(request, state, requiredEntry, myPathName)
}

func relink(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	subInum, ok := state.requiredInodeToSubInode[requiredEntry.InodeNumber]
	if !ok {
		state.requiredInodeToSubInode[requiredEntry.InodeNumber] =
			subEntry.InodeNumber
		return
	}
	if subInum == subEntry.InodeNumber {
		return
	}
	makeHardlink(request,
		myPathName, state.subFS.InodeToFilenamesTable()[subInum][0])
}

func makeHardlink(request *subproto.UpdateRequest, newLink, target string) {
	var hardlink subproto.Hardlink
	hardlink.NewLink = newLink
	hardlink.Target = target
	request.HardlinksToMake = append(request.HardlinksToMake, hardlink)
}

func updateMetadata(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	if state.inodesChanged[requiredEntry.InodeNumber] {
		return
	}
	var inode subproto.Inode
	inode.Name = myPathName
	inode.GenericInode = requiredEntry.Inode()
	request.InodesToChange = append(request.InodesToChange, inode)
	state.inodesChanged[requiredEntry.InodeNumber] = true
}

func makeDirectory(request *subproto.UpdateRequest,
	requiredInode *filesystem.DirectoryInode, pathName string, create bool) {
	var newInode subproto.Inode
	newInode.Name = pathName
	var newDirectoryInode filesystem.DirectoryInode
	newDirectoryInode.Mode = requiredInode.Mode
	newDirectoryInode.Uid = requiredInode.Uid
	newDirectoryInode.Gid = requiredInode.Gid
	newInode.GenericInode = &newDirectoryInode
	if create {
		request.DirectoriesToMake = append(request.DirectoriesToMake, newInode)
	} else {
		request.InodesToChange = append(request.InodesToChange, newInode)
	}
}

func addInode(request *subproto.UpdateRequest, state *state,
	requiredEntry *filesystem.DirectoryEntry, myPathName string) {
	requiredInode := requiredEntry.Inode()
	if name, ok := state.inodesCreated[requiredEntry.InodeNumber]; ok {
		makeHardlink(request, myPathName, name)
		return
	}
	// Try to find a sibling inode.
	names := state.requiredFS.InodeToFilenamesTable()[requiredEntry.InodeNumber]
	if len(names) > 1 {
		var sameDataInode filesystem.GenericInode
		var sameDataName string
		for _, name := range names {
			if inum, found := state.subFS.FilenameToInodeTable()[name]; found {
				subInode := state.subFS.InodeTable[inum]
				_, sameMetadata, sameData := filesystem.CompareInodes(
					subInode, requiredInode, nil)
				if sameMetadata && sameData {
					makeHardlink(request, myPathName, name)
					return
				}
				if sameData {
					sameDataInode = subInode
					sameDataName = name
				}
			}
		}
		if sameDataInode != nil {
			updateMetadata(request, state, requiredEntry, sameDataName)
			makeHardlink(request, myPathName, sameDataName)
			return
		}
	}
	if inode, ok := requiredEntry.Inode().(*filesystem.RegularInode); ok {
		if inode.Size > 0 {
			if _, ok := state.subObjectCacheUsage[inode.Hash]; ok {
				state.subObjectCacheUsage[inode.Hash]++
			} else {
				// Not in object cache: grab it from file-system.
				if inos, ok := state.subFS.HashToInodesTable()[inode.Hash]; ok {
					var fileToCopy subproto.FileToCopyToCache
					fileToCopy.Name =
						state.subFS.InodeToFilenamesTable()[inos[0]][0]
					fileToCopy.Hash = inode.Hash
					request.FilesToCopyToCache = append(
						request.FilesToCopyToCache, fileToCopy)
					state.subObjectCacheUsage[inode.Hash] = 1
				} else {
					panic("No object in cache for: " + myPathName)
				}
			}
		}
	}
	var inode subproto.Inode
	inode.Name = myPathName
	inode.GenericInode = requiredEntry.Inode()
	request.InodesToMake = append(request.InodesToMake, inode)
	state.inodesCreated[requiredEntry.InodeNumber] = myPathName
}
<|endoftext|>


<commit_before>package linreg

import "testing"

func TestNewLinearRegression(t *testing.T) {
	if lr := NewLinearRegression(); lr == nil {
		t.Errorf("got nil linear regression")
	}
}

func TestInitialize(t *testing.T) {
	lr := NewLinearRegression()

	lr.Initialize()

	if len(lr.Xn) != len(lr.Yn) {
		t.Errorf("got different size of vectors Xn Yn, wants same size")
	}

	if len(lr.Xn[0]) != len(lr.Wn) {
		t.Errorf("got different size of vectors Xn Wn, wants same size")
	}

	if len(lr.Xn) != lr.TrainingPoints {
		t.Errorf("got different size of vectors Xn and training points, wants same number")
	}

	for i := 0; i < len(lr.Xn); i++ {
		for j := 0; j < len(lr.Xn[0]); j++ {
			if lr.Xn[i][j] < lr.Interval.Min ||
				lr.Xn[i][j] > lr.Interval.Max {
				t.Errorf("got value of Xn[%d][%d] = %v, want it between %v and %v", i, j, lr.Xn[i][j], lr.Interval.Min, lr.Interval.Max)
			}
		}
	}

	for i := 0; i < len(lr.Yn); i++ {
		if lr.Yn[i] != float64(-1) && lr.Yn[i] != float64(1) {
			t.Errorf("got value of Yn[%v] = %v, want it equal to -1 or 1", i, lr.Yn[i])
		}
	}

}

func TestFlip(t *testing.T) {
	lr := NewLinearRegression()
	lr.Noise = 0
	for i := 0; i < 100; i++ {
		if v := lr.flip(); v != float64(1) {
			t.Errorf("got flip value = -1 wants 1")
		}
	}
	lr.Noise = 1
	for i := 0; i < 100; i++ {
		if v := lr.flip(); v != float64(-1) {
			t.Errorf("got flip value = 1 wants -1")
		}
	}

	lr.Noise = 0.5
	for i := 0; i < 100; i++ {
		if v := lr.flip(); v != float64(-1) && v != float64(1) {
			t.Errorf("got flip value = %v wants value equal to 1 or -1", v)
		}
	}
}

func TestInitializeFromFile(t *testing.T) {
	// todo(santiaago): make this test.
}

func TestInitializeFromData(t *testing.T) {
	data := [][]float64{
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
	}
	lr := NewLinearRegression()
	if err := lr.InitializeFromData(data); err != nil {
		t.Errorf("%v", err)
	}
	if len(lr.Xn) != len(data) || len(lr.Yn) != len(data) {
		t.Errorf("got difference in size of Xn or Yn and data")
	}

	if len(lr.Xn) != lr.TrainingPoints {
		t.Errorf("got difference in size of Xn or TrainingPoints and data")
	}

	if len(lr.Xn[0]) != len(lr.Wn) {
		t.Errorf("got different size of vectors Xn Wn, wants same size")
	}

	if len(lr.Xn[0]) != lr.VectorSize || len(data[0]) != lr.VectorSize {
		t.Errorf("got difference in size of Xn[0] or data[0] with VectorSize")
	}
}

func TestInitializeValidationFromData(t *testing.T) {
	//todo(santiaago): test this
}
<commit_msg>add test TestApplyTransformation<commit_after>package linreg

import "testing"

func TestNewLinearRegression(t *testing.T) {
	if lr := NewLinearRegression(); lr == nil {
		t.Errorf("got nil linear regression")
	}
}

func TestInitialize(t *testing.T) {
	lr := NewLinearRegression()

	lr.Initialize()

	if len(lr.Xn) != len(lr.Yn) {
		t.Errorf("got different size of vectors Xn Yn, wants same size")
	}

	if len(lr.Xn[0]) != len(lr.Wn) {
		t.Errorf("got different size of vectors Xn Wn, wants same size")
	}

	if len(lr.Xn) != lr.TrainingPoints {
		t.Errorf("got different size of vectors Xn and training points, wants same number")
	}

	for i := 0; i < len(lr.Xn); i++ {
		for j := 0; j < len(lr.Xn[0]); j++ {
			if lr.Xn[i][j] < lr.Interval.Min ||
				lr.Xn[i][j] > lr.Interval.Max {
				t.Errorf("got value of Xn[%d][%d] = %v, want it between %v and %v", i, j, lr.Xn[i][j], lr.Interval.Min, lr.Interval.Max)
			}
		}
	}

	for i := 0; i < len(lr.Yn); i++ {
		if lr.Yn[i] != float64(-1) && lr.Yn[i] != float64(1) {
			t.Errorf("got value of Yn[%v] = %v, want it equal to -1 or 1", i, lr.Yn[i])
		}
	}

}

func TestFlip(t *testing.T) {
	lr := NewLinearRegression()
	lr.Noise = 0
	for i := 0; i < 100; i++ {
		if v := lr.flip(); v != float64(1) {
			t.Errorf("got flip value = -1 wants 1")
		}
	}
	lr.Noise = 1
	for i := 0; i < 100; i++ {
		if v := lr.flip(); v != float64(-1) {
			t.Errorf("got flip value = 1 wants -1")
		}
	}

	lr.Noise = 0.5
	for i := 0; i < 100; i++ {
		if v := lr.flip(); v != float64(-1) && v != float64(1) {
			t.Errorf("got flip value = %v wants value equal to 1 or -1", v)
		}
	}
}

func TestInitializeFromFile(t *testing.T) {
	// todo(santiaago): make this test.
}

func TestInitializeFromData(t *testing.T) {
	data := [][]float64{
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
	}
	lr := NewLinearRegression()
	if err := lr.InitializeFromData(data); err != nil {
		t.Errorf("%v", err)
	}
	if len(lr.Xn) != len(data) || len(lr.Yn) != len(data) {
		t.Errorf("got difference in size of Xn or Yn and data")
	}

	if len(lr.Xn) != lr.TrainingPoints {
		t.Errorf("got difference in size of Xn or TrainingPoints and data")
	}

	if len(lr.Xn[0]) != len(lr.Wn) {
		t.Errorf("got different size of vectors Xn Wn, wants same size")
	}

	if len(lr.Xn[0]) != lr.VectorSize || len(data[0]) != lr.VectorSize {
		t.Errorf("got difference in size of Xn[0] or data[0] with VectorSize")
	}
}

func TestInitializeValidationFromData(t *testing.T) {
	//todo(santiaago): test this
}

func TestApplyTransformation(t *testing.T) {

	tf := func(a []float64) []float64 {
		for i := 0; i < len(a); i++ {
			a[i] = -a[i]
		}
		return a
	}

	data := [][]float64{
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
		{1, 1, 1},
	}

	lr := NewLinearRegression()
	lr.InitializeFromData(data)
	lr.TransformFunction = tf
	lr.ApplyTransformation()

	for i := 0; i < lr.TrainingPoints; i++ {
		for j := 0; j < len(lr.Xn[i]); j++ {
			if lr.Xn[i][j] != -1 {
				t.Errorf("got %v wants -1", lr.Xn[i][j])
			}
		}
	}

}
<|endoftext|>


<commit_before>package lnutil

import (
	"bytes"
	"crypto/sha256"
	"testing"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
)

// OutPointsEqual
// test 2 patterns x 2 patterns, a.Hash compared with b.Hash and a.Index compared with b.Index
func TestOutPointsEqual(t *testing.T) {
	var hash1 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}
	var hash2 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x02}

	var u1 uint32 = 1
	var u2 uint32 = 2

	tests := []struct {
		in_a wire.OutPoint
		in_b wire.OutPoint
		want bool
	}{
		//
		// normal situation
		//
		// hash of in_a is not the same as hash of in_b
		// index of in_a is not the same as index of in_b
		{
			wire.OutPoint{hash1, u2}, // in_a
			wire.OutPoint{hash2, u1}, // in_b
			false,
		},

		// hash of in_a is not the same as hash of in_b
		// index of in_a is the same as index of in_b
		{
			wire.OutPoint{hash1, u1}, // in_a
			wire.OutPoint{hash2, u1}, // in_b
			false,
		},

		// hash of in_a is the same as hash of in_b
		// index of in_a is not the same as index of in_b
		{
			wire.OutPoint{hash1, u1}, // in_a
			wire.OutPoint{hash1, u2}, // in_b
			false,
		},

		// hash of in_a is the same as hash of in_b
		// index of in_a is the same as index of in_b
		{
			wire.OutPoint{hash1, u1}, // in_a
			wire.OutPoint{hash1, u1}, // in_b
			true,
		},

		//
		// anomaly situation
		//
		// index of OutPoint is uint32 and the index can not be nil
		// so no test for nil index

		// hash of in_a is just initialized
		{
			wire.OutPoint{[32]byte{}, u1}, // in_a
			wire.OutPoint{hash1, u1},      // in_b
			false,
		},

		// hash of in_b is just initialized
		{
			wire.OutPoint{hash1, u1},      // in_a
			wire.OutPoint{[32]byte{}, u1}, // in_b
			false,
		},

		// hash of in_a is just initialized
		// hash of in_b is just initialized
		{
			wire.OutPoint{[32]byte{}, u1}, // in_a
			wire.OutPoint{[32]byte{}, u1}, // in_b
			true,
		},
	}

	// compare with want
	for i, test := range tests {
		result := OutPointsEqual(test.in_a, test.in_b)
		if test.want != result {
			t.Fatalf("test failed at %d th test", i+1)
			continue
		}
	}
}

// OutPointToBytes
// test some byte arrays
func TestOutPointToBytes(t *testing.T) {
	// test for a normal situation
	// input: [32]bytes array, uint32
	// output: [36]bytes array
	//
	// no problem if input equals to output

	// OutPoint to be an input for OutPointToBytes
	var hash32 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}
	var u uint32 = 1
	op := wire.OutPoint{hash32, u}

	// a bytes array to be compared
	b36 := [36]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x01}

	if OutPointToBytes(op) != b36 {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// OutPointFromBytes
// test some bytes arrays of 36 byte long
// this depends on a test of NewOutPoint() in btcsuite/btcd/wire/msgtx.go
// and OutPointsEqual() in this file
func TestOutPointFromBytes(t *testing.T) {
	// test for a normal situation
	// input: [36]bytes array
	// output: Outpoint created by wire.NewOutPoint()
	//
	// no problem if input equals to output

	// a bytes array to be an input for OutPointFromBytes
	b36 := [36]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x01}

	// OutPoint to be compared
	var hash32 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}
	var u uint32 = 1
	op := wire.NewOutPoint(&hash32, u)

	// compare an output of OutPointFromBytes() and op
	if !OutPointsEqual(*OutPointFromBytes(b36), *op) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// P2WSHify
// test some simple script bytes
func TestP2WSHify(t *testing.T) {
	// test for a normal situation
	// input: []bytes slice, []byte{0x00, 0x01}
	// output: []bytes slice, script bytes
	//
	// no problem if input equals to output

	// a bytes slice to be an input for P2WSHify()
	var inB2 = []byte{0x00, 0x01}

	// script to be compared
	// sha256.Sum256() needs a bytes slice
	sha256InB2 := sha256.Sum256(inB2) // use a different built-in function from fastsha26.Sum256()

	bScript := []byte{txscript.OP_0}
	bScript = append(bScript, txscript.OP_DATA_32) // data for sha256
	for _, b := range sha256InB2 {
		bScript = append(bScript, byte(b))
	}

	// compare an output of P2WSHify() and bScript
	if !bytes.Equal(P2WSHify(inB2), bScript) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// DirectWPKHScript
// test some public keys
// this depends on a test of Hash160() in btcsuite/btcutil
func TestDirectWPKHScript(t *testing.T) {
	// test for a normal situation
	// input: [33]bytes array, public key
	// output: []bytes slice, script bytes
	//
	// no problem if input equals to output

	// a bytes array to be an input for P2WSHify()
	var inB33 = [33]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x01}

	// script to be compared
	bScript := []byte{txscript.OP_0}
	bScript = append(bScript, txscript.OP_DATA_20) // data for hash160

	hash160InB33 := btcutil.Hash160(inB33[:])
	for _, b := range hash160InB33 {
		bScript = append(bScript, byte(b))
	}

	// compare an output of DirectWPKHScript() and bScript
	if !bytes.Equal(DirectWPKHScript(inB33), bScript) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// DirectWPKHScriptFromPKH
// test some public key hashes
func TestDirectWPKHScriptFromPKH(t *testing.T) {
	// test for a normal situation
	// input: [20]bytes array, public key hash
	// output: []bytes slice, script bytes
	//
	// no problem if input equals to output

	// a bytes array to be an input for DirectWPKHScriptFromPKH()
	var inB20 = [20]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}

	// Script to be compared
	bScript := []byte{txscript.OP_0}
	bScript = append(bScript, txscript.OP_DATA_20) // public key hash
	for _, b := range inB20 {
		bScript = append(bScript, byte(b))
	}

	// compare an output of DirectWPKHScriptFromPKH() and bScript
	if !bytes.Equal(DirectWPKHScriptFromPKH(inB20), bScript) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}
<commit_msg>add comments for nil and modify variable names in btclib_test.go<commit_after>package lnutil

import (
	"bytes"
	"crypto/sha256"
	"testing"

	"github.com/btcsuite/btcd/chaincfg/chainhash"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
)

// OutPointsEqual
// test 2 patterns x 2 patterns, a.Hash compared with b.Hash and a.Index compared with b.Index
func TestOutPointsEqual(t *testing.T) {
	var hash1 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}
	var hash2 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x02}

	var u1 uint32 = 1
	var u2 uint32 = 2

	tests := []struct {
		inA  wire.OutPoint
		inB  wire.OutPoint
		want bool
	}{
		//
		// normal situation
		//
		// hash of inA is not the same as hash of inB
		// index of inA is not the same as index of inB
		{
			wire.OutPoint{hash1, u2}, // inA
			wire.OutPoint{hash2, u1}, // inB
			false,
		},

		// hash of inA is not the same as hash of inB
		// index of inA is the same as index of inB
		{
			wire.OutPoint{hash1, u1}, // inA
			wire.OutPoint{hash2, u1}, // inB
			false,
		},

		// hash of inA is the same as hash of inB
		// index of inA is not the same as index of inB
		{
			wire.OutPoint{hash1, u1}, // inA
			wire.OutPoint{hash1, u2}, // inB
			false,
		},

		// hash of inA is the same as hash of inB
		// index of inA is the same as index of inB
		{
			wire.OutPoint{hash1, u1}, // inA
			wire.OutPoint{hash1, u1}, // inB
			true,
		},

		//
		// anomaly situation
		//
		// index of OutPoint is uint32 and the index can not be nil
		// so no test for nil index
		// inA and inB can not be nil so test just initialized one

		// hash of inA is just initialized
		{
			wire.OutPoint{[32]byte{}, u1}, // inA
			wire.OutPoint{hash1, u1},      // inB
			false,
		},

		// hash of inB is just initialized
		{
			wire.OutPoint{hash1, u1},      // inA
			wire.OutPoint{[32]byte{}, u1}, // inB
			false,
		},

		// hash of inA is just initialized
		// hash of inB is just initialized
		{
			wire.OutPoint{[32]byte{}, u1}, // inA
			wire.OutPoint{[32]byte{}, u1}, // inB
			true,
		},
	}

	// compare with want
	for i, test := range tests {
		result := OutPointsEqual(test.inA, test.inB)
		if test.want != result {
			t.Fatalf("test failed at %d th test", i+1)
			continue
		}
	}
}

// OutPointToBytes
// test some byte arrays
func TestOutPointToBytes(t *testing.T) {
	// test for a normal situation
	// input: [32]bytes array, uint32
	// output: [36]bytes array
	//
	// no problem if input equals to output

	// OutPoint to be an input for OutPointToBytes
	var hash32 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}
	var u uint32 = 1
	op := wire.OutPoint{hash32, u}

	// a bytes array to be compared
	b36 := [36]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x01}

	if OutPointToBytes(op) != b36 {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// OutPointFromBytes
// test some bytes arrays of 36 byte long
// this depends on a test of NewOutPoint() in btcsuite/btcd/wire/msgtx.go
// and OutPointsEqual() in this file
func TestOutPointFromBytes(t *testing.T) {
	// test for a normal situation
	// input: [36]bytes array
	// output: Outpoint created by wire.NewOutPoint()
	//
	// no problem if input equals to output

	// a bytes array to be an input for OutPointFromBytes
	b36 := [36]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x01}

	// OutPoint to be compared
	var hash32 chainhash.Hash = [32]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}
	var u uint32 = 1
	op := wire.NewOutPoint(&hash32, u)

	// compare an output of OutPointFromBytes() and op
	if !OutPointsEqual(*OutPointFromBytes(b36), *op) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// P2WSHify
// test some simple script bytes
func TestP2WSHify(t *testing.T) {
	// test for a normal situation
	// input: []bytes slice, []byte{0x00, 0x01}
	// output: []bytes slice, script bytes
	//
	// no problem if input equals to output

	// a bytes slice to be an input for P2WSHify()
	var inB2 = []byte{0x00, 0x01}

	// script to be compared
	// sha256.Sum256() needs a bytes slice
	sha256InB2 := sha256.Sum256(inB2) // use a different built-in function from fastsha26.Sum256()

	bScript := []byte{txscript.OP_0}
	bScript = append(bScript, txscript.OP_DATA_32) // data for sha256
	for _, b := range sha256InB2 {
		bScript = append(bScript, byte(b))
	}

	// compare an output of P2WSHify() and bScript
	if !bytes.Equal(P2WSHify(inB2), bScript) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// DirectWPKHScript
// test some public keys
// this depends on a test of Hash160() in btcsuite/btcutil
func TestDirectWPKHScript(t *testing.T) {
	// test for a normal situation
	// input: [33]bytes array, public key
	// output: []bytes slice, script bytes
	//
	// no problem if input equals to output

	// a bytes array to be an input for P2WSHify()
	var inB33 = [33]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x01}

	// script to be compared
	bScript := []byte{txscript.OP_0}
	bScript = append(bScript, txscript.OP_DATA_20) // data for hash160

	hash160InB33 := btcutil.Hash160(inB33[:])
	for _, b := range hash160InB33 {
		bScript = append(bScript, byte(b))
	}

	// compare an output of DirectWPKHScript() and bScript
	if !bytes.Equal(DirectWPKHScript(inB33), bScript) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}

// DirectWPKHScriptFromPKH
// test some public key hashes
func TestDirectWPKHScriptFromPKH(t *testing.T) {
	// test for a normal situation
	// input: [20]bytes array, public key hash
	// output: []bytes slice, script bytes
	//
	// no problem if input equals to output

	// a bytes array to be an input for DirectWPKHScriptFromPKH()
	var inB20 = [20]byte{0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x01}

	// Script to be compared
	bScript := []byte{txscript.OP_0}
	bScript = append(bScript, txscript.OP_DATA_20) // public key hash
	for _, b := range inB20 {
		bScript = append(bScript, byte(b))
	}

	// compare an output of DirectWPKHScriptFromPKH() and bScript
	if !bytes.Equal(DirectWPKHScriptFromPKH(inB20), bScript) {
		t.Fatalf("it needs to be equal")
	}

	// TODO: one more test case
}
<|endoftext|>


<commit_before>package api

// StorageVolumesPost represents the fields of a new LXD storage pool volume
//
// swagger:model
//
// API extension: storage.
type StorageVolumesPost struct {
	StorageVolumePut `yaml:",inline"`

	// Volume name
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// Volume type (container, custom, image or virtual-machine)
	// Example: custom
	Type string `json:"type" yaml:"type"`

	// Migration source
	//
	// API extension: storage_api_local_volume_handling
	Source StorageVolumeSource `json:"source" yaml:"source"`

	// Volume content type (filesystem or block)
	// Example: filesystem
	//
	// API extension: custom_block_volumes
	ContentType string `json:"content_type" yaml:"content_type"`
}

// StorageVolumePost represents the fields required to rename a LXD storage pool volume
//
// swagger:model
//
// API extension: storage_api_volume_rename.
type StorageVolumePost struct {
	// New volume name
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// New storage pool
	// Example: remote
	//
	// API extension: storage_api_local_volume_handling
	Pool string `json:"pool,omitempty" yaml:"pool,omitempty"`

	// Initiate volume migration
	// Example: false
	//
	// API extension: storage_api_remote_volume_handling
	Migration bool `json:"migration" yaml:"migration"`

	// Migration target (for push mode)
	//
	// API extension: storage_api_remote_volume_handling
	Target *StorageVolumePostTarget `json:"target" yaml:"target"`

	// Whether snapshots should be discarded (migration only)
	// Example: false
	//
	// API extension: storage_api_remote_volume_snapshots
	VolumeOnly bool `json:"volume_only" yaml:"volume_only"`

	// New project name
	// Example: foo
	//
	// API extension: storage_volume_project_move
	Project string `json:"project,omitempty" yaml:"project,omitempty"`
}

// StorageVolumePostTarget represents the migration target host and operation
//
// swagger:model
//
// API extension: storage_api_remote_volume_handling.
type StorageVolumePostTarget struct {
	// The certificate of the migration target
	// Example: X509 PEM certificate
	Certificate string `json:"certificate" yaml:"certificate"`

	// Remote operation URL (for migration)
	// Example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1
	Operation string `json:"operation,omitempty" yaml:"operation,omitempty"`

	// Migration websockets credentials
	// Example: {"migration": "random-string"}
	Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"`
}

// StorageVolume represents the fields of a LXD storage volume.
//
// swagger:model
//
// API extension: storage.
type StorageVolume struct {
	StorageVolumePut `yaml:",inline"`

	// Volume name
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// Volume type
	// Example: custom
	Type string `json:"type" yaml:"type"`

	// List of URLs of objects using this storage volume
	// Example: ["/1.0/instances/blah"]
	UsedBy []string `json:"used_by" yaml:"used_by"`

	// What cluster member this record was found on
	// Example: lxd01
	//
	// API extension: clustering
	Location string `json:"location" yaml:"location"`

	// Volume content type (filesystem or block)
	// Example: filesystem
	//
	// API extension: custom_block_volumes
	ContentType string `json:"content_type" yaml:"content_type"`
}

// URL returns the URL for the volume.
func (v *StorageVolume) URL(apiVersion string, poolName string, projectName string) *URL {
	u := NewURL()

	volName, snapName, isSnap := GetParentAndSnapshotName(v.Name)
	if isSnap {
		u = u.Path(apiVersion, "storage-pools", poolName, "volumes", v.Type, volName, "snapshots", snapName)
	} else {
		u = u.Path(apiVersion, "storage-pools", poolName, "volumes", v.Type, volName)
	}

	return u.Project(projectName).Target(v.Location)
}

// StorageVolumePut represents the modifiable fields of a LXD storage volume
//
// swagger:model
//
// API extension: storage.
type StorageVolumePut struct {
	// Storage volume configuration map (refer to doc/storage.md)
	// Example: {"zfs.remove_snapshots": "true", "size": "50GiB"}
	Config map[string]string `json:"config" yaml:"config"`

	// Description of the storage volume
	// Example: My custom volume
	//
	// API extension: entity_description
	Description string `json:"description" yaml:"description"`

	// Name of a snapshot to restore
	// Example: snap0
	//
	// API extension: storage_api_volume_snapshots
	Restore string `json:"restore,omitempty" yaml:"restore,omitempty"`
}

// StorageVolumeSource represents the creation source for a new storage volume
//
// swagger:model
//
// API extension: storage_api_local_volume_handling.
type StorageVolumeSource struct {
	// Source volume name (for copy)
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// Source type (copy or migration)
	// Example: copy
	Type string `json:"type" yaml:"type"`

	// Source storage pool (for copy)
	// Example: local
	Pool string `json:"pool" yaml:"pool"`

	// Certificate (for migration)
	// Example: X509 PEM certificate
	//
	// API extension: storage_api_remote_volume_handling
	Certificate string `json:"certificate" yaml:"certificate"`

	// Whether to use pull or push mode (for migration)
	// Example: pull
	//
	// API extension: storage_api_remote_volume_handling
	Mode string `json:"mode,omitempty" yaml:"mode,omitempty"`

	// Remote operation URL (for migration)
	// Example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1
	//
	// API extension: storage_api_remote_volume_handling
	Operation string `json:"operation,omitempty" yaml:"operation,omitempty"`

	// Map of migration websockets (for migration)
	// Example: {"rsync": "RANDOM-STRING"}
	//
	// API extension: storage_api_remote_volume_handling
	Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"`

	// Whether snapshots should be discarded (for migration)
	// Example: false
	//
	// API extension: storage_api_volume_snapshots
	VolumeOnly bool `json:"volume_only" yaml:"volume_only"`

	// Whether existing destination volume should be refreshed
	// Example: false
	//
	// API extension: custom_volume_refresh
	Refresh bool `json:"refresh" yaml:"refresh"`

	// Source project name
	// Example: foo
	//
	// API extension: storage_api_project
	Project string `json:"project,omitempty" yaml:"project,omitempty"`
}

// Writable converts a full StorageVolume struct into a StorageVolumePut struct (filters read-only fields).
func (storageVolume *StorageVolume) Writable() StorageVolumePut {
	return storageVolume.StorageVolumePut
}
<commit_msg>shared/api: Add Project field to StorageVolume<commit_after>package api

// StorageVolumesPost represents the fields of a new LXD storage pool volume
//
// swagger:model
//
// API extension: storage.
type StorageVolumesPost struct {
	StorageVolumePut `yaml:",inline"`

	// Volume name
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// Volume type (container, custom, image or virtual-machine)
	// Example: custom
	Type string `json:"type" yaml:"type"`

	// Migration source
	//
	// API extension: storage_api_local_volume_handling
	Source StorageVolumeSource `json:"source" yaml:"source"`

	// Volume content type (filesystem or block)
	// Example: filesystem
	//
	// API extension: custom_block_volumes
	ContentType string `json:"content_type" yaml:"content_type"`
}

// StorageVolumePost represents the fields required to rename a LXD storage pool volume
//
// swagger:model
//
// API extension: storage_api_volume_rename.
type StorageVolumePost struct {
	// New volume name
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// New storage pool
	// Example: remote
	//
	// API extension: storage_api_local_volume_handling
	Pool string `json:"pool,omitempty" yaml:"pool,omitempty"`

	// Initiate volume migration
	// Example: false
	//
	// API extension: storage_api_remote_volume_handling
	Migration bool `json:"migration" yaml:"migration"`

	// Migration target (for push mode)
	//
	// API extension: storage_api_remote_volume_handling
	Target *StorageVolumePostTarget `json:"target" yaml:"target"`

	// Whether snapshots should be discarded (migration only)
	// Example: false
	//
	// API extension: storage_api_remote_volume_snapshots
	VolumeOnly bool `json:"volume_only" yaml:"volume_only"`

	// New project name
	// Example: foo
	//
	// API extension: storage_volume_project_move
	Project string `json:"project,omitempty" yaml:"project,omitempty"`
}

// StorageVolumePostTarget represents the migration target host and operation
//
// swagger:model
//
// API extension: storage_api_remote_volume_handling.
type StorageVolumePostTarget struct {
	// The certificate of the migration target
	// Example: X509 PEM certificate
	Certificate string `json:"certificate" yaml:"certificate"`

	// Remote operation URL (for migration)
	// Example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1
	Operation string `json:"operation,omitempty" yaml:"operation,omitempty"`

	// Migration websockets credentials
	// Example: {"migration": "random-string"}
	Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"`
}

// StorageVolume represents the fields of a LXD storage volume.
//
// swagger:model
//
// API extension: storage.
type StorageVolume struct {
	StorageVolumePut `yaml:",inline"`

	// Volume name
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// Volume type
	// Example: custom
	Type string `json:"type" yaml:"type"`

	// List of URLs of objects using this storage volume
	// Example: ["/1.0/instances/blah"]
	UsedBy []string `json:"used_by" yaml:"used_by"`

	// What cluster member this record was found on
	// Example: lxd01
	//
	// API extension: clustering
	Location string `json:"location" yaml:"location"`

	// Volume content type (filesystem or block)
	// Example: filesystem
	//
	// API extension: custom_block_volumes
	ContentType string `json:"content_type" yaml:"content_type"`

	// Project containing the volume.
	// Example: default
	//
	// API extension: storage_volumes_all_projects
	Project string `json:"project" yaml:"project"`
}

// URL returns the URL for the volume.
func (v *StorageVolume) URL(apiVersion string, poolName string, projectName string) *URL {
	u := NewURL()

	volName, snapName, isSnap := GetParentAndSnapshotName(v.Name)
	if isSnap {
		u = u.Path(apiVersion, "storage-pools", poolName, "volumes", v.Type, volName, "snapshots", snapName)
	} else {
		u = u.Path(apiVersion, "storage-pools", poolName, "volumes", v.Type, volName)
	}

	return u.Project(projectName).Target(v.Location)
}

// StorageVolumePut represents the modifiable fields of a LXD storage volume
//
// swagger:model
//
// API extension: storage.
type StorageVolumePut struct {
	// Storage volume configuration map (refer to doc/storage.md)
	// Example: {"zfs.remove_snapshots": "true", "size": "50GiB"}
	Config map[string]string `json:"config" yaml:"config"`

	// Description of the storage volume
	// Example: My custom volume
	//
	// API extension: entity_description
	Description string `json:"description" yaml:"description"`

	// Name of a snapshot to restore
	// Example: snap0
	//
	// API extension: storage_api_volume_snapshots
	Restore string `json:"restore,omitempty" yaml:"restore,omitempty"`
}

// StorageVolumeSource represents the creation source for a new storage volume
//
// swagger:model
//
// API extension: storage_api_local_volume_handling.
type StorageVolumeSource struct {
	// Source volume name (for copy)
	// Example: foo
	Name string `json:"name" yaml:"name"`

	// Source type (copy or migration)
	// Example: copy
	Type string `json:"type" yaml:"type"`

	// Source storage pool (for copy)
	// Example: local
	Pool string `json:"pool" yaml:"pool"`

	// Certificate (for migration)
	// Example: X509 PEM certificate
	//
	// API extension: storage_api_remote_volume_handling
	Certificate string `json:"certificate" yaml:"certificate"`

	// Whether to use pull or push mode (for migration)
	// Example: pull
	//
	// API extension: storage_api_remote_volume_handling
	Mode string `json:"mode,omitempty" yaml:"mode,omitempty"`

	// Remote operation URL (for migration)
	// Example: https://1.2.3.4:8443/1.0/operations/1721ae08-b6a8-416a-9614-3f89302466e1
	//
	// API extension: storage_api_remote_volume_handling
	Operation string `json:"operation,omitempty" yaml:"operation,omitempty"`

	// Map of migration websockets (for migration)
	// Example: {"rsync": "RANDOM-STRING"}
	//
	// API extension: storage_api_remote_volume_handling
	Websockets map[string]string `json:"secrets,omitempty" yaml:"secrets,omitempty"`

	// Whether snapshots should be discarded (for migration)
	// Example: false
	//
	// API extension: storage_api_volume_snapshots
	VolumeOnly bool `json:"volume_only" yaml:"volume_only"`

	// Whether existing destination volume should be refreshed
	// Example: false
	//
	// API extension: custom_volume_refresh
	Refresh bool `json:"refresh" yaml:"refresh"`

	// Source project name
	// Example: foo
	//
	// API extension: storage_api_project
	Project string `json:"project,omitempty" yaml:"project,omitempty"`
}

// Writable converts a full StorageVolume struct into a StorageVolumePut struct (filters read-only fields).
func (storageVolume *StorageVolume) Writable() StorageVolumePut {
	return storageVolume.StorageVolumePut
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2017 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package starttls

import (
	"crypto/tls"
	"fmt"
	"net/smtp"

	"github.com/square/certigo/starttls/mysql"
	"github.com/square/certigo/starttls/psql"
)

func tlsConfigForConnect(connectName, connectCert, connectKey string) (*tls.Config, error) {
	conf := &tls.Config{
		// We verify later manually so we can print results
		InsecureSkipVerify: true,
		ServerName:         connectName,
	}

	if connectCert != "" {
		keyFile := connectCert
		if connectKey != "" {
			keyFile = connectKey
		}

		cert, err := tls.LoadX509KeyPair(connectCert, keyFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read client certificate/key: %s\n", err)
		}

		conf.Certificates = []tls.Certificate{cert}
	}

	return conf, nil
}

// GetConnectionState connects to a TLS server, returning the connection state.
// Currently, connectStartTLS can be one of "mysql", "postgres" or "psql", or the empty string, which does a normal TLS
// connection.  connectTo specifies the address to connect to. connectName sets SNI.  connectCert and connectKey are
// client certs
func GetConnectionState(connectStartTLS, connectName, connectTo, connectCert, connectKey string) (*tls.ConnectionState, error) {
	var state *tls.ConnectionState
	var err error
	var tlsConfig *tls.Config

	switch connectStartTLS {
	case "postgres", "psql":
		// No tlsConfig needed for postgres, but all others do.
	default:
		tlsConfig, err = tlsConfigForConnect(connectName, connectCert, connectKey)
		if err != nil {
			return nil, err
		}
	}

	switch connectStartTLS {
	case "":
		conn, err := tls.Dial("tcp", connectTo, tlsConfig)
		if err != nil {
			return nil, fmt.Errorf("error connecting: %v\n", err)
		}
		defer conn.Close()
		s := conn.ConnectionState()
		state = &s
	case "mysql":
		mysql.RegisterTLSConfig("certigo", tlsConfig)
		state, err = mysql.DumpTLS(fmt.Sprintf("certigo@tcp(%s)/?tls=certigo", connectTo))
	case "postgres", "psql":
		// Setting sslmode to "require" skips verification.
		url := fmt.Sprintf("postgres://certigo@%s/?sslmode=require", connectTo)
		if connectCert != "" {
			url += fmt.Sprintf("&sslcert=%s", connectCert)
		}
		if connectKey != "" {
			url += fmt.Sprintf("&sslkey=%s", connectCert)
		}
		state, err = pq.DumpTLS(url)
	case "smtp":
		client, err := smtp.Dial(connectTo)
		if err != nil {
			return nil, err
		}
		err = client.StartTLS(tlsConfig)
		if err != nil {
			return nil, err
		}
		smtpState, ok := client.TLSConnectionState()
		if !ok {
			panic("SMTP Connection isn't TLS after we successfully called StartTLS")
		}
		state = &smtpState
	default:
		return nil, fmt.Errorf("error connecting: unknown StartTLS protocol '%s'\n", connectStartTLS)
	}

	if err != nil {
		return nil, fmt.Errorf("error connecting: %v\n", err)
	}

	return state, nil
}
<commit_msg>Change some variable names in the starttls package<commit_after>/*-
 * Copyright 2017 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package starttls

import (
	"crypto/tls"
	"fmt"
	"net/smtp"

	"github.com/square/certigo/starttls/mysql"
	"github.com/square/certigo/starttls/psql"
)

func tlsConfigForConnect(connectName, clientCert, clientKey string) (*tls.Config, error) {
	conf := &tls.Config{
		// We verify later manually so we can print results
		InsecureSkipVerify: true,
		ServerName:         connectName,
	}

	if clientCert != "" {
		keyFile := clientCert
		if clientKey != "" {
			keyFile = clientKey
		}

		cert, err := tls.LoadX509KeyPair(clientCert, keyFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read client certificate/key: %s\n", err)
		}

		conf.Certificates = []tls.Certificate{cert}
	}

	return conf, nil
}

// GetConnectionState connects to a TLS server, returning the connection state.
// Currently, startTLSType can be one of "mysql", "postgres" or "psql", or the empty string, which does a normal TLS
// connection.  connectTo specifies the address to connect to. connectName sets SNI.  connectCert and connectKey are
// client certs
func GetConnectionState(startTLSType, connectName, connectTo, clientCert, clientKey string) (*tls.ConnectionState, error) {
	var state *tls.ConnectionState
	var err error
	var tlsConfig *tls.Config

	switch startTLSType {
	case "postgres", "psql":
		// No tlsConfig needed for postgres, but all others do.
	default:
		tlsConfig, err = tlsConfigForConnect(connectName, clientCert, clientKey)
		if err != nil {
			return nil, err
		}
	}

	switch startTLSType {
	case "":
		conn, err := tls.Dial("tcp", connectTo, tlsConfig)
		if err != nil {
			return nil, fmt.Errorf("error connecting: %v\n", err)
		}
		defer conn.Close()
		s := conn.ConnectionState()
		state = &s
	case "mysql":
		mysql.RegisterTLSConfig("certigo", tlsConfig)
		state, err = mysql.DumpTLS(fmt.Sprintf("certigo@tcp(%s)/?tls=certigo", connectTo))
	case "postgres", "psql":
		// Setting sslmode to "require" skips verification.
		url := fmt.Sprintf("postgres://certigo@%s/?sslmode=require", connectTo)
		if clientCert != "" {
			url += fmt.Sprintf("&sslcert=%s", clientCert)
		}
		if clientKey != "" {
			url += fmt.Sprintf("&sslkey=%s", clientCert)
		}
		state, err = pq.DumpTLS(url)
	case "smtp":
		client, err := smtp.Dial(connectTo)
		if err != nil {
			return nil, err
		}
		err = client.StartTLS(tlsConfig)
		if err != nil {
			return nil, err
		}
		smtpState, ok := client.TLSConnectionState()
		if !ok {
			panic("SMTP Connection isn't TLS after we successfully called StartTLS")
		}
		state = &smtpState
	default:
		return nil, fmt.Errorf("error connecting: unknown StartTLS protocol '%s'\n", startTLSType)
	}

	if err != nil {
		return nil, fmt.Errorf("error connecting: %v\n", err)
	}

	return state, nil
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/zhirsch/destinykioskstatus/db"
	"github.com/zhirsch/oauth2"
	"golang.org/x/net/context"

	"github.com/cenkalti/backoff"
)

type Client struct {
	AuthConfig *oauth2.Config
}

func (c *Client) GetCurrentBungieAccount(tok *oauth2.Token) *GetCurrentBungieAccountResponse {
	req := &GetCurrentBungieAccountRequest{}
	resp := new(GetCurrentBungieAccountResponse)
	c.get(tok, req, resp)
	return resp
}

func (c *Client) MyCharacterVendorData(tok *oauth2.Token, membershipType db.DestinyMembershipType, characterID db.DestinyCharacterID, vendorHash uint32) *MyCharacterVendorDataResponse {
	req := &MyCharacterVendorDataRequest{int64(membershipType), string(characterID), vendorHash}
	resp := new(MyCharacterVendorDataResponse)
	c.get(tok, req, resp)
	return resp
}

func (c *Client) GetAllVendorsForCurrentCharacter(tok *oauth2.Token, membershipType db.DestinyMembershipType, characterID db.DestinyCharacterID) *GetAllVendorsForCurrentCharacterResponse {
	req := &GetAllVendorsForCurrentCharacterRequest{int64(membershipType), string(characterID)}
	resp := new(GetAllVendorsForCurrentCharacterResponse)
	c.get(tok, req, resp)
	return resp
}

func (c *Client) get(tok *oauth2.Token, req Request, resp Response) {
	httpReq, err := http.NewRequest("GET", req.URL(), nil)
	if err != nil {
		panic(err)
	}
	httpReq.Header.Add("X-API-Key", c.AuthConfig.ClientID)

	client := c.AuthConfig.Client(context.TODO(), tok)
	err = backoff.RetryNotify(
		func() error {
			httpResp, err := client.Do(httpReq)
			if err != nil {
				return err
			}
			if httpResp.StatusCode != http.StatusOK {
				return fmt.Errorf("bad response: %v", httpResp.StatusCode)
			}
			if err := json.NewDecoder(httpResp.Body).Decode(resp); err != nil {
				return err
			}
			if resp.GetHeader().ErrorCode != 1 {
				return fmt.Errorf("bad message for %v: %+v", req.URL(), resp)
			}
			return nil
		},
		backoff.NewExponentialBackOff(),
		func(err error, dur time.Duration) {
			log.Printf("retrying: %v", err)
		},
	)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Fix file descriptor leak.<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/zhirsch/destinykioskstatus/db"
	"github.com/zhirsch/oauth2"
	"golang.org/x/net/context"

	"github.com/cenkalti/backoff"
)

type Client struct {
	AuthConfig *oauth2.Config
}

func (c *Client) GetCurrentBungieAccount(tok *oauth2.Token) *GetCurrentBungieAccountResponse {
	req := &GetCurrentBungieAccountRequest{}
	resp := new(GetCurrentBungieAccountResponse)
	c.get(tok, req, resp)
	return resp
}

func (c *Client) MyCharacterVendorData(tok *oauth2.Token, membershipType db.DestinyMembershipType, characterID db.DestinyCharacterID, vendorHash uint32) *MyCharacterVendorDataResponse {
	req := &MyCharacterVendorDataRequest{int64(membershipType), string(characterID), vendorHash}
	resp := new(MyCharacterVendorDataResponse)
	c.get(tok, req, resp)
	return resp
}

func (c *Client) GetAllVendorsForCurrentCharacter(tok *oauth2.Token, membershipType db.DestinyMembershipType, characterID db.DestinyCharacterID) *GetAllVendorsForCurrentCharacterResponse {
	req := &GetAllVendorsForCurrentCharacterRequest{int64(membershipType), string(characterID)}
	resp := new(GetAllVendorsForCurrentCharacterResponse)
	c.get(tok, req, resp)
	return resp
}

func (c *Client) get(tok *oauth2.Token, req Request, resp Response) {
	httpReq, err := http.NewRequest("GET", req.URL(), nil)
	if err != nil {
		panic(err)
	}
	httpReq.Header.Add("X-API-Key", c.AuthConfig.ClientID)

	client := c.AuthConfig.Client(context.TODO(), tok)
	err = backoff.RetryNotify(
		func() error {
			httpResp, err := client.Do(httpReq)
			if err != nil {
				return err
			}
			defer httpResp.Body.Close();
			if httpResp.StatusCode != http.StatusOK {
				return fmt.Errorf("bad response: %v", httpResp.StatusCode)
			}
			if err := json.NewDecoder(httpResp.Body).Decode(resp); err != nil {
				return err
			}
			if resp.GetHeader().ErrorCode != 1 {
				return fmt.Errorf("bad message for %v: %+v", req.URL(), resp)
			}
			return nil
		},
		backoff.NewExponentialBackOff(),
		func(err error, dur time.Duration) {
			log.Printf("retrying: %v", err)
		},
	)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package api

type AuthScope int
type AuthScopes map[AuthScope]string

type ServiceMethod int
type EventKey string

type ObjectFactory func() interface{}

type HttpMethod string
type QueryDefault interface{}
type UrlQueries map[string]QueryDefault
type HttpHeaders map[string]string

var (
	GET       HttpMethod = HttpMethod("GET")
	POST      HttpMethod = HttpMethod("POST")
	PUT       HttpMethod = HttpMethod("PUT")
	DELETE    HttpMethod = HttpMethod("DELETE")
	MULTIPART HttpMethod = HttpMethod("POST")
)

type MethodSpec struct {
	Doc                  string
	UrlRoute             string
	HttpHeaders          HttpHeaders
	HttpMethod           HttpMethod
	UrlQueries           UrlQueries
	ContentTypes         []string
	RequestBody          ObjectFactory
	ResponseBody         ObjectFactory
	CallbackEvent        EventKey
	CallbackBodyTemplate string
	AuthScope            string
}

type ServiceMethods map[ServiceMethod]MethodSpec
<commit_msg>Change in API RequestBody/ ObjectFactory signature to allow different prototype objects based on http request (content-type)<commit_after>package api

import (
	"net/http"
)

type AuthScope int
type AuthScopes map[AuthScope]string

type ServiceMethod int
type EventKey string

type ObjectFactory func(*http.Request) interface{}

type HttpMethod string
type QueryDefault interface{}
type UrlQueries map[string]QueryDefault
type HttpHeaders map[string]string

var (
	GET       HttpMethod = HttpMethod("GET")
	POST      HttpMethod = HttpMethod("POST")
	PUT       HttpMethod = HttpMethod("PUT")
	DELETE    HttpMethod = HttpMethod("DELETE")
	MULTIPART HttpMethod = HttpMethod("POST")
)

type MethodSpec struct {
	Doc                  string
	UrlRoute             string
	HttpHeaders          HttpHeaders
	HttpMethod           HttpMethod
	UrlQueries           UrlQueries
	ContentTypes         []string
	RequestBody          ObjectFactory
	ResponseBody         ObjectFactory
	CallbackEvent        EventKey
	CallbackBodyTemplate string
	AuthScope            string
}

type ServiceMethods map[ServiceMethod]MethodSpec
<|endoftext|>