<commit_before>package image // import "github.com/docker/docker/image"

import (
	"encoding/json"
	"errors"
	"io"
	"runtime"
	"strings"
	"time"

	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/layer"
	digest "github.com/opencontainers/go-digest"
)

// ID is the content-addressable ID of an image.
type ID digest.Digest

func (id ID) String() string {
	return id.Digest().String()
}

// Digest converts ID into a digest
func (id ID) Digest() digest.Digest {
	return digest.Digest(id)
}

// IDFromDigest creates an ID from a digest
func IDFromDigest(digest digest.Digest) ID {
	return ID(digest)
}

// V1Image stores the V1 image configuration.
type V1Image struct {
	// ID is a unique 64 character identifier of the image
	ID string `json:"id,omitempty"`
	// Parent is the ID of the parent image
	Parent string `json:"parent,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Container is the id of the container used to commit
	Container string `json:"container,omitempty"`
	// ContainerConfig is the configuration of the container that is committed into the image
	ContainerConfig container.Config `json:"container_config,omitempty"`
	// DockerVersion specifies the version of Docker that was used to build the image
	DockerVersion string `json:"docker_version,omitempty"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// Config is the configuration of the container received from the client
	Config *container.Config `json:"config,omitempty"`
	// Architecture is the hardware that the image is built and runs on
	Architecture string `json:"architecture,omitempty"`
	// OS is the operating system used to build and run the image
	OS string `json:"os,omitempty"`
	// Size is the total size of the image including all layers it is composed of
	Size int64 `json:",omitempty"`
}

// Image stores the image configuration
type Image struct {
	V1Image
	Parent     ID        `json:"parent,omitempty"`
	RootFS     *RootFS   `json:"rootfs,omitempty"`
	History    []History `json:"history,omitempty"`
	OSVersion  string    `json:"os.version,omitempty"`
	OSFeatures []string  `json:"os.features,omitempty"`

	// rawJSON caches the immutable JSON associated with this image.
	rawJSON []byte

	// computedID is the ID computed from the hash of the image config.
	// Not to be confused with the legacy V1 ID in V1Image.
	computedID ID
}

// RawJSON returns the immutable JSON associated with the image.
func (img *Image) RawJSON() []byte {
	return img.rawJSON
}

// ID returns the image's content-addressable ID.
func (img *Image) ID() ID {
	return img.computedID
}

// ImageID stringifies ID.
func (img *Image) ImageID() string {
	return img.ID().String()
}

// RunConfig returns the image's container config.
func (img *Image) RunConfig() *container.Config {
	return img.Config
}

// BaseImgArch returns the image's architecture. If not populated, defaults to the host runtime arch.
func (img *Image) BaseImgArch() string {
	arch := img.Architecture
	if arch == "" {
		arch = runtime.GOARCH
	}
	return arch
}

// OperatingSystem returns the image's operating system. If not populated, defaults to the host runtime OS.
func (img *Image) OperatingSystem() string {
	os := img.OS
	if os == "" {
		os = runtime.GOOS
	}
	return os
}

// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
// that JSON that's been manipulated by a push/pull cycle with a legacy
// registry won't end up with a different key order.
func (img *Image) MarshalJSON() ([]byte, error) {
	type MarshalImage Image

	pass1, err := json.Marshal(MarshalImage(*img))
	if err != nil {
		return nil, err
	}

	var c map[string]*json.RawMessage
	if err := json.Unmarshal(pass1, &c); err != nil {
		return nil, err
	}
	return json.Marshal(c)
}

// ChildConfig is the configuration to apply to an Image to create a new
// Child image. Other properties of the image are copied from the parent.
type ChildConfig struct {
	ContainerID     string
	Author          string
	Comment         string
	DiffID          layer.DiffID
	ContainerConfig *container.Config
	Config          *container.Config
}

// NewChildImage creates a new Image as a child of this image.
func NewChildImage(img *Image, child ChildConfig, os string) *Image {
	isEmptyLayer := layer.IsEmpty(child.DiffID)
	var rootFS *RootFS
	if img.RootFS != nil {
		rootFS = img.RootFS.Clone()
	} else {
		rootFS = NewRootFS()
	}

	if !isEmptyLayer {
		rootFS.Append(child.DiffID)
	}
	imgHistory := NewHistory(
		child.Author,
		child.Comment,
		strings.Join(child.ContainerConfig.Cmd, " "),
		isEmptyLayer)

	return &Image{
		V1Image: V1Image{
			DockerVersion:   dockerversion.Version,
			Config:          child.Config,
			Architecture:    img.BaseImgArch(),
			OS:              os,
			Container:       child.ContainerID,
			ContainerConfig: *child.ContainerConfig,
			Author:          child.Author,
			Created:         imgHistory.Created,
		},
		RootFS:     rootFS,
		History:    append(img.History, imgHistory),
		OSFeatures: img.OSFeatures,
		OSVersion:  img.OSVersion,
	}
}

// History stores build commands that were used to create an image
type History struct {
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// CreatedBy keeps the Dockerfile command used while building the image
	CreatedBy string `json:"created_by,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// EmptyLayer is set to true if this history item did not generate a
	// layer. Otherwise, the history item is associated with the next
	// layer in the RootFS section.
	EmptyLayer bool `json:"empty_layer,omitempty"`
}

// NewHistory creates a new history struct from arguments, and sets the created
// time to the current time in UTC
func NewHistory(author, comment, createdBy string, isEmptyLayer bool) History {
	return History{
		Author:     author,
		Created:    time.Now().UTC(),
		CreatedBy:  createdBy,
		Comment:    comment,
		EmptyLayer: isEmptyLayer,
	}
}

// Exporter provides interface for loading and saving images
type Exporter interface {
	Load(io.ReadCloser, io.Writer, bool) error
	// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
	Save([]string, io.Writer) error
}

// NewFromJSON creates an Image configuration from json.
func NewFromJSON(src []byte) (*Image, error) {
	img := &Image{}

	if err := json.Unmarshal(src, img); err != nil {
		return nil, err
	}
	if img.RootFS == nil {
		return nil, errors.New("invalid image JSON, no RootFS key")
	}

	img.rawJSON = src

	return img, nil
}
<commit_msg>govet: suppress warning about Parent field that repeated json tag "parent"<commit_after>package image // import "github.com/docker/docker/image"

import (
	"encoding/json"
	"errors"
	"io"
	"runtime"
	"strings"
	"time"

	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/layer"
	digest "github.com/opencontainers/go-digest"
)

// ID is the content-addressable ID of an image.
type ID digest.Digest

func (id ID) String() string {
	return id.Digest().String()
}

// Digest converts ID into a digest
func (id ID) Digest() digest.Digest {
	return digest.Digest(id)
}

// IDFromDigest creates an ID from a digest
func IDFromDigest(digest digest.Digest) ID {
	return ID(digest)
}

// V1Image stores the V1 image configuration.
type V1Image struct {
	// ID is a unique 64 character identifier of the image
	ID string `json:"id,omitempty"`
	// Parent is the ID of the parent image
	Parent string `json:"parent,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Container is the id of the container used to commit
	Container string `json:"container,omitempty"`
	// ContainerConfig is the configuration of the container that is committed into the image
	ContainerConfig container.Config `json:"container_config,omitempty"`
	// DockerVersion specifies the version of Docker that was used to build the image
	DockerVersion string `json:"docker_version,omitempty"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// Config is the configuration of the container received from the client
	Config *container.Config `json:"config,omitempty"`
	// Architecture is the hardware that the image is built and runs on
	Architecture string `json:"architecture,omitempty"`
	// OS is the operating system used to build and run the image
	OS string `json:"os,omitempty"`
	// Size is the total size of the image including all layers it is composed of
	Size int64 `json:",omitempty"`
}

// Image stores the image configuration
type Image struct {
	V1Image
	Parent     ID        `json:"parent,omitempty"` //nolint:govet
	RootFS     *RootFS   `json:"rootfs,omitempty"`
	History    []History `json:"history,omitempty"`
	OSVersion  string    `json:"os.version,omitempty"`
	OSFeatures []string  `json:"os.features,omitempty"`

	// rawJSON caches the immutable JSON associated with this image.
	rawJSON []byte

	// computedID is the ID computed from the hash of the image config.
	// Not to be confused with the legacy V1 ID in V1Image.
	computedID ID
}

// RawJSON returns the immutable JSON associated with the image.
func (img *Image) RawJSON() []byte {
	return img.rawJSON
}

// ID returns the image's content-addressable ID.
func (img *Image) ID() ID {
	return img.computedID
}

// ImageID stringifies ID.
func (img *Image) ImageID() string {
	return img.ID().String()
}

// RunConfig returns the image's container config.
func (img *Image) RunConfig() *container.Config {
	return img.Config
}

// BaseImgArch returns the image's architecture. If not populated, defaults to the host runtime arch.
func (img *Image) BaseImgArch() string {
	arch := img.Architecture
	if arch == "" {
		arch = runtime.GOARCH
	}
	return arch
}

// OperatingSystem returns the image's operating system. If not populated, defaults to the host runtime OS.
func (img *Image) OperatingSystem() string {
	os := img.OS
	if os == "" {
		os = runtime.GOOS
	}
	return os
}

// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
// that JSON that's been manipulated by a push/pull cycle with a legacy
// registry won't end up with a different key order.
func (img *Image) MarshalJSON() ([]byte, error) {
	type MarshalImage Image

	pass1, err := json.Marshal(MarshalImage(*img))
	if err != nil {
		return nil, err
	}

	var c map[string]*json.RawMessage
	if err := json.Unmarshal(pass1, &c); err != nil {
		return nil, err
	}
	return json.Marshal(c)
}

// ChildConfig is the configuration to apply to an Image to create a new
// Child image. Other properties of the image are copied from the parent.
type ChildConfig struct {
	ContainerID     string
	Author          string
	Comment         string
	DiffID          layer.DiffID
	ContainerConfig *container.Config
	Config          *container.Config
}

// NewChildImage creates a new Image as a child of this image.
func NewChildImage(img *Image, child ChildConfig, os string) *Image {
	isEmptyLayer := layer.IsEmpty(child.DiffID)
	var rootFS *RootFS
	if img.RootFS != nil {
		rootFS = img.RootFS.Clone()
	} else {
		rootFS = NewRootFS()
	}

	if !isEmptyLayer {
		rootFS.Append(child.DiffID)
	}
	imgHistory := NewHistory(
		child.Author,
		child.Comment,
		strings.Join(child.ContainerConfig.Cmd, " "),
		isEmptyLayer)

	return &Image{
		V1Image: V1Image{
			DockerVersion:   dockerversion.Version,
			Config:          child.Config,
			Architecture:    img.BaseImgArch(),
			OS:              os,
			Container:       child.ContainerID,
			ContainerConfig: *child.ContainerConfig,
			Author:          child.Author,
			Created:         imgHistory.Created,
		},
		RootFS:     rootFS,
		History:    append(img.History, imgHistory),
		OSFeatures: img.OSFeatures,
		OSVersion:  img.OSVersion,
	}
}

// History stores build commands that were used to create an image
type History struct {
	// Created is the timestamp at which the image was created
	Created time.Time `json:"created"`
	// Author is the name of the author that was specified when committing the image
	Author string `json:"author,omitempty"`
	// CreatedBy keeps the Dockerfile command used while building the image
	CreatedBy string `json:"created_by,omitempty"`
	// Comment is the commit message that was set when committing the image
	Comment string `json:"comment,omitempty"`
	// EmptyLayer is set to true if this history item did not generate a
	// layer. Otherwise, the history item is associated with the next
	// layer in the RootFS section.
	EmptyLayer bool `json:"empty_layer,omitempty"`
}

// NewHistory creates a new history struct from arguments, and sets the created
// time to the current time in UTC
func NewHistory(author, comment, createdBy string, isEmptyLayer bool) History {
	return History{
		Author:     author,
		Created:    time.Now().UTC(),
		CreatedBy:  createdBy,
		Comment:    comment,
		EmptyLayer: isEmptyLayer,
	}
}

// Exporter provides interface for loading and saving images
type Exporter interface {
	Load(io.ReadCloser, io.Writer, bool) error
	// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
	Save([]string, io.Writer) error
}

// NewFromJSON creates an Image configuration from json.
func NewFromJSON(src []byte) (*Image, error) {
	img := &Image{}

	if err := json.Unmarshal(src, img); err != nil {
		return nil, err
	}
	if img.RootFS == nil {
		return nil, errors.New("invalid image JSON, no RootFS key")
	}

	img.rawJSON = src

	return img, nil
}
<|endoftext|>


<commit_before>package gocsi

import (
	"sync"

	"github.com/codedellemc/gocsi/csi"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"

	"golang.org/x/net/context"
)

// IdempotencyProvider is the interface that works with a server-side,
// gRPC interceptor to provide serial access and idempotency for CSI's
// volume resources.
type IdempotencyProvider interface {
	// GetVolumeName should return the name of the volume specified
	// by the provided volume ID. If the volume does not exist then
	// an empty string should be returned.
	GetVolumeName(id *csi.VolumeID) (string, error)

	// GetVolumeInfo should return information about the volume
	// specified by the provided volume name. If the volume does not
	// exist then a nil value should be returned.
	GetVolumeInfo(name string) (*csi.VolumeInfo, error)

	// IsControllerPublished should return publication info about
	// the volume specified by the provided volume name or ID. If the
	// volume does not exist then a nil value should be returned.
	IsControllerPublished(id *csi.VolumeID) (*csi.PublishVolumeInfo, error)

	// IsNodePublished should return a flag indicating whether or
	// not the volume exists and is published on the current host.
	IsNodePublished(id *csi.VolumeID, targetPath string) (bool, error)
}

// NewIdempotentInterceptor returns a new server-side, gRPC interceptor
// that can be used in conjunction with an IdempotencyProvider to
// provide serialized, idempotent access to the following CSI RPCs:
//
//  * CreateVolume
//  * DeleteVolume
//  * ControllerPublishVolume
//  * ControllerUnpublishVolume
//  * NodePublishVolume
//  * NodeUnpublishVolume
func NewIdempotentInterceptor(
	p IdempotencyProvider) grpc.UnaryServerInterceptor {

	i := &idempotencyInterceptor{
		p:        p,
		volLocks: map[string]MutexWithTryLock{},
	}

	return i.handle
}

type idempotencyInterceptor struct {
	sync.Mutex
	p        IdempotencyProvider
	volLocks map[string]MutexWithTryLock
}

func (i *idempotencyInterceptor) lockWithName(name string) MutexWithTryLock {
	i.Lock()
	defer i.Unlock()
	lock := i.volLocks[name]
	if lock == nil {
		lock = NewMutexWithTryLock()
		i.volLocks[name] = lock
	}
	return lock
}

func (i *idempotencyInterceptor) handle(
	ctx context.Context,
	req interface{},
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	switch treq := req.(type) {
	case *csi.ControllerPublishVolumeRequest:
		return i.controllerPublishVolume(ctx, treq, info, handler)
	case *csi.ControllerUnpublishVolumeRequest:
		return i.controllerUnpublishVolume(ctx, treq, info, handler)
	case *csi.CreateVolumeRequest:
		return i.createVolume(ctx, treq, info, handler)
	case *csi.DeleteVolumeRequest:
		return i.deleteVolume(ctx, treq, info, handler)
	case *csi.NodePublishVolumeRequest:
		return i.nodePublishVolume(ctx, treq, info, handler)
	case *csi.NodeUnpublishVolumeRequest:
		return i.nodeUnpublishVolume(ctx, treq, info, handler)
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) controllerPublishVolume(
	ctx context.Context,
	req *csi.ControllerPublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrControllerPublishVolume(
			csi.Error_ControllerPublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrControllerPublishVolume(
			csi.Error_ControllerPublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	pubInfo, err := i.p.IsControllerPublished(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if pubInfo != nil {
		log.WithField("name", name).Info("idempotent controller publish")
		return &csi.ControllerPublishVolumeResponse{
			Reply: &csi.ControllerPublishVolumeResponse_Result_{
				Result: &csi.ControllerPublishVolumeResponse_Result{
					PublishVolumeInfo: pubInfo,
				},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) controllerUnpublishVolume(
	ctx context.Context,
	req *csi.ControllerUnpublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrControllerUnpublishVolume(
			csi.Error_ControllerUnpublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrControllerUnpublishVolume(
			csi.Error_ControllerUnpublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	pubInfo, err := i.p.IsControllerPublished(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if pubInfo == nil {
		log.WithField("name", name).Info("idempotent controller publish")
		return &csi.ControllerUnpublishVolumeResponse{
			Reply: &csi.ControllerUnpublishVolumeResponse_Result_{
				Result: &csi.ControllerUnpublishVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) createVolume(
	ctx context.Context,
	req *csi.CreateVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	lock := i.lockWithName(req.Name)
	if !lock.TryLock(tryLockNow) {
		return ErrCreateVolume(
			csi.Error_CreateVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	volInfo, err := i.p.GetVolumeInfo(req.Name)
	if err != nil {
		return nil, err
	}

	if volInfo != nil {
		log.WithField("name", req.Name).Info("idempotent create")
		return &csi.CreateVolumeResponse{
			Reply: &csi.CreateVolumeResponse_Result_{
				Result: &csi.CreateVolumeResponse_Result{
					VolumeInfo: volInfo,
				},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) deleteVolume(
	ctx context.Context,
	req *csi.DeleteVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrDeleteVolume(
			csi.Error_DeleteVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrDeleteVolume(
			csi.Error_DeleteVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	volInfo, err := i.p.GetVolumeInfo(name)
	if err != nil {
		return nil, err
	}

	if volInfo == nil {
		log.WithField("name", name).Info("idempotent delete")
		return &csi.DeleteVolumeResponse{
			Reply: &csi.DeleteVolumeResponse_Result_{
				Result: &csi.DeleteVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) nodePublishVolume(
	ctx context.Context,
	req *csi.NodePublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrNodePublishVolume(
			csi.Error_NodePublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrNodePublishVolume(
			csi.Error_NodePublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	ok, err := i.p.IsNodePublished(req.VolumeId, req.TargetPath)
	if err != nil {
		return nil, err
	}
	if ok {
		log.WithField("name", name).Info("idempotent node publish")
		return &csi.NodePublishVolumeResponse{
			Reply: &csi.NodePublishVolumeResponse_Result_{
				Result: &csi.NodePublishVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) nodeUnpublishVolume(
	ctx context.Context,
	req *csi.NodeUnpublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrNodeUnpublishVolume(
			csi.Error_NodeUnpublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrNodeUnpublishVolume(
			csi.Error_NodeUnpublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	ok, err := i.p.IsNodePublished(req.VolumeId, req.TargetPath)
	if err != nil {
		return nil, err
	}
	if !ok {
		log.WithField("name", name).Info("idempotent node unpublish")
		return &csi.NodeUnpublishVolumeResponse{
			Reply: &csi.NodeUnpublishVolumeResponse_Result_{
				Result: &csi.NodeUnpublishVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}
<commit_msg>Update Idempotency Provider Godoc<commit_after>package gocsi

import (
	"sync"

	"github.com/codedellemc/gocsi/csi"
	log "github.com/sirupsen/logrus"
	"google.golang.org/grpc"

	"golang.org/x/net/context"
)

// IdempotencyProvider is the interface that works with a server-side,
// gRPC interceptor to provide serial access and idempotency for CSI's
// volume resources.
type IdempotencyProvider interface {
	// GetVolumeName should return the name of the volume specified
	// by the provided volume ID. If the volume does not exist then
	// an empty string should be returned.
	GetVolumeName(id *csi.VolumeID) (string, error)

	// GetVolumeInfo should return information about the volume
	// specified by the provided volume name. If the volume does not
	// exist then a nil value should be returned.
	GetVolumeInfo(name string) (*csi.VolumeInfo, error)

	// IsControllerPublished should return publication info about
	// the volume specified by the provided volume name or ID.
	IsControllerPublished(id *csi.VolumeID) (*csi.PublishVolumeInfo, error)

	// IsNodePublished should return a flag indicating whether or
	// not the volume exists and is published on the current host.
	IsNodePublished(id *csi.VolumeID, targetPath string) (bool, error)
}

// NewIdempotentInterceptor returns a new server-side, gRPC interceptor
// that can be used in conjunction with an IdempotencyProvider to
// provide serialized, idempotent access to the following CSI RPCs:
//
//  * CreateVolume
//  * DeleteVolume
//  * ControllerPublishVolume
//  * ControllerUnpublishVolume
//  * NodePublishVolume
//  * NodeUnpublishVolume
func NewIdempotentInterceptor(
	p IdempotencyProvider) grpc.UnaryServerInterceptor {

	i := &idempotencyInterceptor{
		p:        p,
		volLocks: map[string]MutexWithTryLock{},
	}

	return i.handle
}

type idempotencyInterceptor struct {
	sync.Mutex
	p        IdempotencyProvider
	volLocks map[string]MutexWithTryLock
}

func (i *idempotencyInterceptor) lockWithName(name string) MutexWithTryLock {
	i.Lock()
	defer i.Unlock()
	lock := i.volLocks[name]
	if lock == nil {
		lock = NewMutexWithTryLock()
		i.volLocks[name] = lock
	}
	return lock
}

func (i *idempotencyInterceptor) handle(
	ctx context.Context,
	req interface{},
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	switch treq := req.(type) {
	case *csi.ControllerPublishVolumeRequest:
		return i.controllerPublishVolume(ctx, treq, info, handler)
	case *csi.ControllerUnpublishVolumeRequest:
		return i.controllerUnpublishVolume(ctx, treq, info, handler)
	case *csi.CreateVolumeRequest:
		return i.createVolume(ctx, treq, info, handler)
	case *csi.DeleteVolumeRequest:
		return i.deleteVolume(ctx, treq, info, handler)
	case *csi.NodePublishVolumeRequest:
		return i.nodePublishVolume(ctx, treq, info, handler)
	case *csi.NodeUnpublishVolumeRequest:
		return i.nodeUnpublishVolume(ctx, treq, info, handler)
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) controllerPublishVolume(
	ctx context.Context,
	req *csi.ControllerPublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrControllerPublishVolume(
			csi.Error_ControllerPublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrControllerPublishVolume(
			csi.Error_ControllerPublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	pubInfo, err := i.p.IsControllerPublished(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if pubInfo != nil {
		log.WithField("name", name).Info("idempotent controller publish")
		return &csi.ControllerPublishVolumeResponse{
			Reply: &csi.ControllerPublishVolumeResponse_Result_{
				Result: &csi.ControllerPublishVolumeResponse_Result{
					PublishVolumeInfo: pubInfo,
				},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) controllerUnpublishVolume(
	ctx context.Context,
	req *csi.ControllerUnpublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrControllerUnpublishVolume(
			csi.Error_ControllerUnpublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrControllerUnpublishVolume(
			csi.Error_ControllerUnpublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	pubInfo, err := i.p.IsControllerPublished(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if pubInfo == nil {
		log.WithField("name", name).Info("idempotent controller publish")
		return &csi.ControllerUnpublishVolumeResponse{
			Reply: &csi.ControllerUnpublishVolumeResponse_Result_{
				Result: &csi.ControllerUnpublishVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) createVolume(
	ctx context.Context,
	req *csi.CreateVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	lock := i.lockWithName(req.Name)
	if !lock.TryLock(tryLockNow) {
		return ErrCreateVolume(
			csi.Error_CreateVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	volInfo, err := i.p.GetVolumeInfo(req.Name)
	if err != nil {
		return nil, err
	}

	if volInfo != nil {
		log.WithField("name", req.Name).Info("idempotent create")
		return &csi.CreateVolumeResponse{
			Reply: &csi.CreateVolumeResponse_Result_{
				Result: &csi.CreateVolumeResponse_Result{
					VolumeInfo: volInfo,
				},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) deleteVolume(
	ctx context.Context,
	req *csi.DeleteVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrDeleteVolume(
			csi.Error_DeleteVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrDeleteVolume(
			csi.Error_DeleteVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	volInfo, err := i.p.GetVolumeInfo(name)
	if err != nil {
		return nil, err
	}

	if volInfo == nil {
		log.WithField("name", name).Info("idempotent delete")
		return &csi.DeleteVolumeResponse{
			Reply: &csi.DeleteVolumeResponse_Result_{
				Result: &csi.DeleteVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) nodePublishVolume(
	ctx context.Context,
	req *csi.NodePublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrNodePublishVolume(
			csi.Error_NodePublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrNodePublishVolume(
			csi.Error_NodePublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	ok, err := i.p.IsNodePublished(req.VolumeId, req.TargetPath)
	if err != nil {
		return nil, err
	}
	if ok {
		log.WithField("name", name).Info("idempotent node publish")
		return &csi.NodePublishVolumeResponse{
			Reply: &csi.NodePublishVolumeResponse_Result_{
				Result: &csi.NodePublishVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}

func (i *idempotencyInterceptor) nodeUnpublishVolume(
	ctx context.Context,
	req *csi.NodeUnpublishVolumeRequest,
	info *grpc.UnaryServerInfo,
	handler grpc.UnaryHandler) (interface{}, error) {

	name, err := i.p.GetVolumeName(req.VolumeId)
	if err != nil {
		return nil, err
	}
	if name == "" {
		return ErrNodeUnpublishVolume(
			csi.Error_NodeUnpublishVolumeError_VOLUME_DOES_NOT_EXIST,
			""), nil
	}

	lock := i.lockWithName(name)
	if !lock.TryLock(tryLockNow) {
		return ErrNodeUnpublishVolume(
			csi.Error_NodeUnpublishVolumeError_OPERATION_PENDING_FOR_VOLUME,
			""), nil
	}
	defer lock.Unlock()

	ok, err := i.p.IsNodePublished(req.VolumeId, req.TargetPath)
	if err != nil {
		return nil, err
	}
	if !ok {
		log.WithField("name", name).Info("idempotent node unpublish")
		return &csi.NodeUnpublishVolumeResponse{
			Reply: &csi.NodeUnpublishVolumeResponse_Result_{
				Result: &csi.NodeUnpublishVolumeResponse_Result{},
			},
		}, nil
	}

	return handler(ctx, req)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Source: https://github.com/golang/crypto/blob/master/ssh/terminal/util_windows.go
// Revision: 30ad74476e5c37a080ab5999ce9ab4465a2ecfac
// License: BSD

// +build windows

package imported

import (
	"syscall"
	"unsafe"
)

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
	var st uint32
	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	return r != 0 && e == 0
}
<commit_msg>Fix windows compilation problem.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Source: https://github.com/golang/crypto/blob/master/ssh/terminal/util_windows.go
// Revision: 30ad74476e5c37a080ab5999ce9ab4465a2ecfac
// License: BSD

// +build windows

package imported

import (
	"syscall"
	"unsafe"
)

var kernel32 = syscall.NewLazyDLL("kernel32.dll")
var procGetConsoleMode = kernel32.NewProc("GetConsoleMode")

// IsTerminal returns true if the given file descriptor is a terminal.
func IsTerminal(fd int) bool {
	var st uint32
	r, _, e := syscall.Syscall(procGetConsoleMode.Addr(), 2, uintptr(fd), uintptr(unsafe.Pointer(&st)), 0)
	return r != 0 && e == 0
}
<|endoftext|>


<commit_before>package gapi

import (
	"testing"

	"github.com/gobs/pretty"
)

const (
	getLibraryPanelNameResponse = `{
		"result": [
	    {
				"id": 25,
				"orgId": 1,
				"folderId": 0,
				"uid": "V--OrYHnz",
				"name": "API docs Example",
				"kind": 1,
				"model": {
					"description": "",
					"type": ""
				},
				"version": 1,
				"meta": {
					"folderName": "General",
					"folderUid": "",
					"connectedDashboards": 1,
					"created": "2021-09-27T09:56:17+02:00",
					"updated": "2021-09-27T09:56:17+02:00",
					"createdBy": {
						"id": 1,
						"name": "admin",
						"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
					},
					"updatedBy": {
						"id": 1,
						"name": "admin",
						"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
					}
				}
	 		}
		]
	}`

	getLibraryPanelUIDResponse = `{
		"result": {
			"id": 25,
			"orgId": 1,
			"folderId": 0,
			"uid": "V--OrYHnz",
			"name": "API docs Example",
			"kind": 1,
			"model": {
				"description": "",
				"type": ""
			},
			"version": 1,
			"meta": {
				"folderName": "General",
				"folderUid": "",
				"connectedDashboards": 1,
				"created": "2021-09-27T09:56:17+02:00",
				"updated": "2021-09-27T09:56:17+02:00",
				"createdBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				},
				"updatedBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				}
			}
	 	}
	}`

	patchLibraryPanelResponse = `{
		"result": {
			"id": 25,
			"orgId": 1,
			"folderId": 0,
			"uid": "V--OrYHnz",
			"name": "Updated library panel name",
			"kind": 1,
			"model": {
				"description": "new description",
				"type": ""
			},
			"version": 1,
			"meta": {
				"folderName": "General",
				"folderUid": "",
				"connectedDashboards": 1,
				"created": "2021-09-27T09:56:17+02:00",
				"updated": "2021-09-27T09:56:17+02:00",
				"createdBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				},
				"updatedBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				}
			}
	 	}
	}`

	deleteLibraryPanelResponse = `{
		"message": "Library element deleted",
		"id": 28
	}`

	getLibraryPanelConnectionsResponse = `{
    "result": [
      {
        "id": 148,
        "kind": 1,
        "elementId": 25,
        "connectionId": 527,
        "created": "2021-09-27T10:00:07+02:00",
        "createdBy": {
            "id": 1,
            "name": "admin",
            "avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
        }
      }
    ]
	}`

	getLibraryPanelConnectedDashboardsResponse = `[
		{
			"id":1,
			"uid": "cIBgcSjkk",
			"title":"Production Overview",
			"url": "/d/cIBgcSjkk/production-overview",
			"type":"dash-db",
			"tags":["prod"],
			"isStarred":true,
			"uri":"db/production-overview"
		},
		{
			"id":2,
			"uid": "SjkkcIBgc",
			"title":"Production Overview 2",
			"url": "/d/SjkkcIBgc/production-overview-2",
			"type":"dash-db",
			"tags":["prod"],
			"isStarred":true,
			"uri":"db/production-overview"
		}
	]`
)

func TestLibraryPanelCreate(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelUIDResponse)
	defer server.Close()

	panel := LibraryPanel{
		Folder: 0,
		Name:   "API docs Example",
		Model:  map[string]interface{}{"description": "", "type": ""},
	}

	resp, err := client.NewLibraryPanel(panel)
	if err != nil {
		t.Fatal(err)
	}

	t.Log(pretty.PrettyFormat(resp))

	if resp.UID != "V--OrYHnz" {
		t.Errorf("Invalid uid - %s, Expected %s", resp.UID, "V--OrYHnz")
	}

	for _, code := range []int{400, 401, 403} {
		server.code = code
		_, err = client.NewLibraryPanel(panel)
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestLibraryPanelGetByName(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelNameResponse)
	defer server.Close()

	resp, err := client.LibraryPanelByName("API docs Example")
	if err != nil {
		t.Error(err)
	}
	if resp.UID != "V--OrYHnz" {
		t.Errorf("Invalid uid - %s, Expected %s", resp.UID, "V--OrYHnz")
	}

	for _, code := range []int{401, 403, 404} {
		server.code = code
		_, err = client.LibraryPanelByName("test")
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestLibraryPanelGetByUID(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelUIDResponse)
	defer server.Close()

	resp, err := client.LibraryPanelByUID("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}
	if resp.Name != "API docs Example" {
		t.Fatalf("Invalid Name - %s, Expected %s", resp.Name, "API docs Example")
	}

	for _, code := range []int{401, 403, 404} {
		server.code = code
		_, err = client.LibraryPanelByUID("V--OrYHnz")
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestPatchLibraryPanel(t *testing.T) {
	server, client := gapiTestTools(t, 200, patchLibraryPanelResponse)
	defer server.Close()

	panel := LibraryPanel{
		Folder: 1,
		Name:   "Updated library panel name",
		Model:  map[string]interface{}{"description": "new description", "type": ""},
	}
	resp, err := client.PatchLibraryPanel("V--OrYHnz", panel)
	if err != nil {
		t.Fatal(err)
	}

	if resp.Name != "Updated library panel name" {
		t.Fatalf("Invalid Name - %s, Expected %s", resp.Name, "Updated library panel name")
	}
	if resp.Model["description"] != "new description" {
		t.Fatalf("Invalid panel JSON description - %s, Expected %s", resp.Name, "Updated library panel name")
	}

	for _, code := range []int{401, 403, 404} {
		server.code = code

		_, err := client.LibraryPanelByUID("V--OrYHnz")
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestLibraryPanelGetConnections(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelConnectionsResponse)
	defer server.Close()

	resp, err := client.LibraryPanelConnections("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}

	if (*resp)[0].ID != int64(148) {
		t.Fatalf("Invalid connection id - %d, Expected %d", (*resp)[0].ID, 148)
	}
}

func TestLibraryPanelConnectedDashboards(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelConnectionsResponse)
	defer server.Close()

	connections, err := client.LibraryPanelConnections("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}

	var dashboardIds []int64
	for _, connection := range *connections {
		dashboardIds = append(dashboardIds, connection.DashboardID)
	}

	_, client = gapiTestTools(t, 200, getLibraryPanelConnectedDashboardsResponse)
	dashboards, err := client.DashboardsByIDs(dashboardIds)
	if err != nil {
		t.Fatal(err)
	}

	if dashboards[0].Title != "Production Overview" {
		t.Fatalf("Invalid title from connected dashboard 0 - %s, Expected %s", dashboards[0].Title, "Production Overview")
	}
}

func TestLibraryPanelDelete(t *testing.T) {
	server, client := gapiTestTools(t, 200, deleteLibraryPanelResponse)
	defer server.Close()

	resp, err := client.DeleteLibraryPanel("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}

	if resp.Message != "Library element deleted" {
		t.Error("Failed to delete. Response should contain the correct response message")
	}
}
<commit_msg>remove server.code = code<commit_after>package gapi

import (
	"testing"

	"github.com/gobs/pretty"
)

const (
	getLibraryPanelNameResponse = `{
		"result": [
	    {
				"id": 25,
				"orgId": 1,
				"folderId": 0,
				"uid": "V--OrYHnz",
				"name": "API docs Example",
				"kind": 1,
				"model": {
					"description": "",
					"type": ""
				},
				"version": 1,
				"meta": {
					"folderName": "General",
					"folderUid": "",
					"connectedDashboards": 1,
					"created": "2021-09-27T09:56:17+02:00",
					"updated": "2021-09-27T09:56:17+02:00",
					"createdBy": {
						"id": 1,
						"name": "admin",
						"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
					},
					"updatedBy": {
						"id": 1,
						"name": "admin",
						"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
					}
				}
	 		}
		]
	}`

	getLibraryPanelUIDResponse = `{
		"result": {
			"id": 25,
			"orgId": 1,
			"folderId": 0,
			"uid": "V--OrYHnz",
			"name": "API docs Example",
			"kind": 1,
			"model": {
				"description": "",
				"type": ""
			},
			"version": 1,
			"meta": {
				"folderName": "General",
				"folderUid": "",
				"connectedDashboards": 1,
				"created": "2021-09-27T09:56:17+02:00",
				"updated": "2021-09-27T09:56:17+02:00",
				"createdBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				},
				"updatedBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				}
			}
	 	}
	}`

	patchLibraryPanelResponse = `{
		"result": {
			"id": 25,
			"orgId": 1,
			"folderId": 0,
			"uid": "V--OrYHnz",
			"name": "Updated library panel name",
			"kind": 1,
			"model": {
				"description": "new description",
				"type": ""
			},
			"version": 1,
			"meta": {
				"folderName": "General",
				"folderUid": "",
				"connectedDashboards": 1,
				"created": "2021-09-27T09:56:17+02:00",
				"updated": "2021-09-27T09:56:17+02:00",
				"createdBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				},
				"updatedBy": {
					"id": 1,
					"name": "admin",
					"avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
				}
			}
	 	}
	}`

	deleteLibraryPanelResponse = `{
		"message": "Library element deleted",
		"id": 28
	}`

	getLibraryPanelConnectionsResponse = `{
    "result": [
      {
        "id": 148,
        "kind": 1,
        "elementId": 25,
        "connectionId": 527,
        "created": "2021-09-27T10:00:07+02:00",
        "createdBy": {
            "id": 1,
            "name": "admin",
            "avatarUrl": "/avatar/46d229b033af06a191ff2267bca9ae56"
        }
      }
    ]
	}`

	getLibraryPanelConnectedDashboardsResponse = `[
		{
			"id":1,
			"uid": "cIBgcSjkk",
			"title":"Production Overview",
			"url": "/d/cIBgcSjkk/production-overview",
			"type":"dash-db",
			"tags":["prod"],
			"isStarred":true,
			"uri":"db/production-overview"
		},
		{
			"id":2,
			"uid": "SjkkcIBgc",
			"title":"Production Overview 2",
			"url": "/d/SjkkcIBgc/production-overview-2",
			"type":"dash-db",
			"tags":["prod"],
			"isStarred":true,
			"uri":"db/production-overview"
		}
	]`
)

func TestLibraryPanelCreate(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelUIDResponse)
	defer server.Close()

	panel := LibraryPanel{
		Folder: 0,
		Name:   "API docs Example",
		Model:  map[string]interface{}{"description": "", "type": ""},
	}

	resp, err := client.NewLibraryPanel(panel)
	if err != nil {
		t.Fatal(err)
	}

	t.Log(pretty.PrettyFormat(resp))

	if resp.UID != "V--OrYHnz" {
		t.Errorf("Invalid uid - %s, Expected %s", resp.UID, "V--OrYHnz")
	}

	for _, code := range []int{400, 401, 403} {
		_, err = client.NewLibraryPanel(panel)
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestLibraryPanelGetByName(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelNameResponse)
	defer server.Close()

	resp, err := client.LibraryPanelByName("API docs Example")
	if err != nil {
		t.Error(err)
	}
	if resp.UID != "V--OrYHnz" {
		t.Errorf("Invalid uid - %s, Expected %s", resp.UID, "V--OrYHnz")
	}

	for _, code := range []int{401, 403, 404} {
		_, err = client.LibraryPanelByName("test")
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestLibraryPanelGetByUID(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelUIDResponse)
	defer server.Close()

	resp, err := client.LibraryPanelByUID("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}
	if resp.Name != "API docs Example" {
		t.Fatalf("Invalid Name - %s, Expected %s", resp.Name, "API docs Example")
	}

	for _, code := range []int{401, 403, 404} {
		_, err = client.LibraryPanelByUID("V--OrYHnz")
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestPatchLibraryPanel(t *testing.T) {
	server, client := gapiTestTools(t, 200, patchLibraryPanelResponse)
	defer server.Close()

	panel := LibraryPanel{
		Folder: 1,
		Name:   "Updated library panel name",
		Model:  map[string]interface{}{"description": "new description", "type": ""},
	}
	resp, err := client.PatchLibraryPanel("V--OrYHnz", panel)
	if err != nil {
		t.Fatal(err)
	}

	if resp.Name != "Updated library panel name" {
		t.Fatalf("Invalid Name - %s, Expected %s", resp.Name, "Updated library panel name")
	}
	if resp.Model["description"] != "new description" {
		t.Fatalf("Invalid panel JSON description - %s, Expected %s", resp.Name, "Updated library panel name")
	}

	for _, code := range []int{401, 403, 404} {
		_, err := client.LibraryPanelByUID("V--OrYHnz")
		if err == nil {
			t.Errorf("%d not detected", code)
		}
	}
}

func TestLibraryPanelGetConnections(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelConnectionsResponse)
	defer server.Close()

	resp, err := client.LibraryPanelConnections("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}

	if (*resp)[0].ID != int64(148) {
		t.Fatalf("Invalid connection id - %d, Expected %d", (*resp)[0].ID, 148)
	}
}

func TestLibraryPanelConnectedDashboards(t *testing.T) {
	server, client := gapiTestTools(t, 200, getLibraryPanelConnectionsResponse)
	defer server.Close()

	connections, err := client.LibraryPanelConnections("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}

	var dashboardIds []int64
	for _, connection := range *connections {
		dashboardIds = append(dashboardIds, connection.DashboardID)
	}

	_, client = gapiTestTools(t, 200, getLibraryPanelConnectedDashboardsResponse)
	dashboards, err := client.DashboardsByIDs(dashboardIds)
	if err != nil {
		t.Fatal(err)
	}

	if dashboards[0].Title != "Production Overview" {
		t.Fatalf("Invalid title from connected dashboard 0 - %s, Expected %s", dashboards[0].Title, "Production Overview")
	}
}

func TestLibraryPanelDelete(t *testing.T) {
	server, client := gapiTestTools(t, 200, deleteLibraryPanelResponse)
	defer server.Close()

	resp, err := client.DeleteLibraryPanel("V--OrYHnz")
	if err != nil {
		t.Fatal(err)
	}

	if resp.Message != "Library element deleted" {
		t.Error("Failed to delete. Response should contain the correct response message")
	}
}
<|endoftext|>


<commit_before>package index

import (
	"database/sql"
	"encoding/json"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/pkg/errors"
)

type Index struct {
	db *sql.DB

	insertFetchQ, insertDepQ *sql.Stmt
	selectQ, latestQ         *sql.Stmt

	insertBlacklistQ, selectBlacklistQ *sql.Stmt
}

func Open(dataSourceName string) (*Index, error) {
	db, err := sql.Open("mysql", dataSourceName+"?parseTime=true")
	if err != nil {
		return nil, err
	}

	i := &Index{db: db}

	query := `CREATE TABLE IF NOT EXISTS Fetches (
		Name VARCHAR(255) NOT NULL, INDEX (Name), Parent VARCHAR(255),
		Timestamp DATETIME, Refs JSON,
		PackID BIGINT UNIQUE KEY AUTO_INCREMENT, PackRef VARCHAR(255))`
	if _, err = db.Exec(query); err != nil {
		return nil, errors.Wrap(err, "failed to create Fetches")
	}

	query = `CREATE TABLE IF NOT EXISTS PackDeps (ID BIGINT, INDEX (ID), Dep BIGINT)`
	if _, err = db.Exec(query); err != nil {
		return nil, errors.Wrap(err, "failed to create PackDeps")
	}

	query = `CREATE TABLE IF NOT EXISTS Blacklist (
		Name VARCHAR(255) NOT NULL, INDEX (Name), Whitelisted BOOLEAN NOT NULL DEFAULT 0)`
	if _, err = db.Exec(query); err != nil {
		return nil, errors.Wrap(err, "failed to create Blacklist")
	}

	prepStmts := []struct {
		name **sql.Stmt
		sql  string
	}{
		{
			&i.insertFetchQ,
			`INSERT INTO Fetches (Name, Parent, Timestamp, Refs, PackRef) VALUES (?, ?, ?, ?, ?)`,
		},
		{
			&i.insertDepQ,
			`INSERT INTO PackDeps (ID, Dep) VALUES (?, ?)`,
		},
		{
			&i.latestQ,
			`SELECT Timestamp FROM Fetches WHERE Name = ? ORDER BY Timestamp DESC LIMIT 1`,
		},
		{
			&i.selectQ,
			`SELECT Parent, Refs, PackID FROM Fetches WHERE Name = ? ORDER BY Timestamp DESC LIMIT 1`,
		},
		{
			&i.insertBlacklistQ,
			`INSERT INTO Blacklist (Name) VALUES (?)`,
		},
		{
			&i.selectBlacklistQ,
			`SELECT Whitelisted FROM Blacklist WHERE Name = ?`,
		},
	}

	for _, x := range prepStmts {
		stmt, err := db.Prepare(x.sql)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to prepare '%s'", x.sql)
		}
		*x.name = stmt
	}

	return i, nil
}

func (i *Index) AddFetch(name, parent string, timestamp time.Time,
	refs map[string]string, packRef string, packDeps []string) error {
	r, err := json.Marshal(refs)
	if err != nil {
		return err
	}
	res, err := i.insertFetchQ.Exec(name, parent, timestamp, r, packRef)
	if err != nil {
		return err
	}
	packID, err := res.LastInsertId()
	if err != nil {
		return err
	}
	for _, dep := range packDeps {
		_, err := i.insertDepQ.Exec(packID, dep)
		if err != nil {
			return err
		}
	}
	return nil
}

func (i *Index) GetLatest(name string) (timestamp time.Time, err error) {
	err = i.latestQ.QueryRow(name).Scan(&timestamp)
	if err == sql.ErrNoRows {
		timestamp = time.Time{}
		err = nil
	}
	return
}

func (i *Index) GetHaves(name string) (haves map[string]struct{}, deps []string, err error) {
	var parent, packID string
	var refs []byte
	err = i.selectQ.QueryRow(name).Scan(&parent, &refs, &packID)
	if err == sql.ErrNoRows {
		err = nil
		return
	}
	if err != nil {
		return
	}
	var r map[string]string
	err = json.Unmarshal(refs, &r)
	if err != nil {
		return
	}

	haves = make(map[string]struct{})
	for _, ref := range r {
		haves[ref] = struct{}{}
	}
	deps = append(deps, packID)

	if parent != "" {
		err = i.selectQ.QueryRow(parent).Scan(&parent, &refs, &packID)
		if err == sql.ErrNoRows {
			err = nil
			return
		}
		if err != nil {
			return
		}
		err = json.Unmarshal(refs, &r)
		if err != nil {
			return
		}

		for ref := range r {
			haves[ref] = struct{}{}
		}
		deps = append(deps, packID)
	}

	return
}

func (i *Index) AddBlacklist(name string) error {
	_, err := i.insertBlacklistQ.Exec(name)
	return err
}

type BlacklistState int

const (
	Blacklisted BlacklistState = iota
	Whitelisted
	Neutral
)

func (i *Index) BlacklistState(name string) (BlacklistState, error) {
	var whitelisted bool
	err := i.selectBlacklistQ.QueryRow(name).Scan(&whitelisted)
	if err == sql.ErrNoRows {
		return Neutral, nil
	}
	if err != nil {
		return 0, err
	}
	if whitelisted {
		return Whitelisted, nil
	}
	return Blacklisted, nil
}

func (i *Index) Close() error {
	return i.db.Close()
}
<commit_msg>index: fix the ref name vs. ref bug again (now in the parent)<commit_after>package index

import (
	"database/sql"
	"encoding/json"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/pkg/errors"
)

type Index struct {
	db *sql.DB

	insertFetchQ, insertDepQ *sql.Stmt
	selectQ, latestQ         *sql.Stmt

	insertBlacklistQ, selectBlacklistQ *sql.Stmt
}

func Open(dataSourceName string) (*Index, error) {
	db, err := sql.Open("mysql", dataSourceName+"?parseTime=true")
	if err != nil {
		return nil, err
	}

	i := &Index{db: db}

	query := `CREATE TABLE IF NOT EXISTS Fetches (
		Name VARCHAR(255) NOT NULL, INDEX (Name), Parent VARCHAR(255),
		Timestamp DATETIME, Refs JSON,
		PackID BIGINT UNIQUE KEY AUTO_INCREMENT, PackRef VARCHAR(255))`
	if _, err = db.Exec(query); err != nil {
		return nil, errors.Wrap(err, "failed to create Fetches")
	}

	query = `CREATE TABLE IF NOT EXISTS PackDeps (ID BIGINT, INDEX (ID), Dep BIGINT)`
	if _, err = db.Exec(query); err != nil {
		return nil, errors.Wrap(err, "failed to create PackDeps")
	}

	query = `CREATE TABLE IF NOT EXISTS Blacklist (
		Name VARCHAR(255) NOT NULL, INDEX (Name), Whitelisted BOOLEAN NOT NULL DEFAULT 0)`
	if _, err = db.Exec(query); err != nil {
		return nil, errors.Wrap(err, "failed to create Blacklist")
	}

	prepStmts := []struct {
		name **sql.Stmt
		sql  string
	}{
		{
			&i.insertFetchQ,
			`INSERT INTO Fetches (Name, Parent, Timestamp, Refs, PackRef) VALUES (?, ?, ?, ?, ?)`,
		},
		{
			&i.insertDepQ,
			`INSERT INTO PackDeps (ID, Dep) VALUES (?, ?)`,
		},
		{
			&i.latestQ,
			`SELECT Timestamp FROM Fetches WHERE Name = ? ORDER BY Timestamp DESC LIMIT 1`,
		},
		{
			&i.selectQ,
			`SELECT Parent, Refs, PackID FROM Fetches WHERE Name = ? ORDER BY Timestamp DESC LIMIT 1`,
		},
		{
			&i.insertBlacklistQ,
			`INSERT INTO Blacklist (Name) VALUES (?)`,
		},
		{
			&i.selectBlacklistQ,
			`SELECT Whitelisted FROM Blacklist WHERE Name = ?`,
		},
	}

	for _, x := range prepStmts {
		stmt, err := db.Prepare(x.sql)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to prepare '%s'", x.sql)
		}
		*x.name = stmt
	}

	return i, nil
}

func (i *Index) AddFetch(name, parent string, timestamp time.Time,
	refs map[string]string, packRef string, packDeps []string) error {
	r, err := json.Marshal(refs)
	if err != nil {
		return err
	}
	res, err := i.insertFetchQ.Exec(name, parent, timestamp, r, packRef)
	if err != nil {
		return err
	}
	packID, err := res.LastInsertId()
	if err != nil {
		return err
	}
	for _, dep := range packDeps {
		_, err := i.insertDepQ.Exec(packID, dep)
		if err != nil {
			return err
		}
	}
	return nil
}

func (i *Index) GetLatest(name string) (timestamp time.Time, err error) {
	err = i.latestQ.QueryRow(name).Scan(&timestamp)
	if err == sql.ErrNoRows {
		timestamp = time.Time{}
		err = nil
	}
	return
}

func (i *Index) GetHaves(name string) (haves map[string]struct{}, deps []string, err error) {
	var parent, packID string
	var refs []byte
	err = i.selectQ.QueryRow(name).Scan(&parent, &refs, &packID)
	if err == sql.ErrNoRows {
		err = nil
		return
	}
	if err != nil {
		return
	}
	var r map[string]string
	err = json.Unmarshal(refs, &r)
	if err != nil {
		return
	}

	haves = make(map[string]struct{})
	for _, ref := range r {
		haves[ref] = struct{}{}
	}
	deps = append(deps, packID)

	if parent != "" {
		err = i.selectQ.QueryRow(parent).Scan(&parent, &refs, &packID)
		if err == sql.ErrNoRows {
			err = nil
			return
		}
		if err != nil {
			return
		}
		err = json.Unmarshal(refs, &r)
		if err != nil {
			return
		}

		for _, ref := range r {
			haves[ref] = struct{}{}
		}
		deps = append(deps, packID)
	}

	return
}

func (i *Index) AddBlacklist(name string) error {
	_, err := i.insertBlacklistQ.Exec(name)
	return err
}

type BlacklistState int

const (
	Blacklisted BlacklistState = iota
	Whitelisted
	Neutral
)

func (i *Index) BlacklistState(name string) (BlacklistState, error) {
	var whitelisted bool
	err := i.selectBlacklistQ.QueryRow(name).Scan(&whitelisted)
	if err == sql.ErrNoRows {
		return Neutral, nil
	}
	if err != nil {
		return 0, err
	}
	if whitelisted {
		return Whitelisted, nil
	}
	return Blacklisted, nil
}

func (i *Index) Close() error {
	return i.db.Close()
}
<|endoftext|>


<commit_before>package indicators

import (
	"errors"
	"github.com/thetruetrade/gotrade"
)

// An Average Directional Index (ADX), no storage
type ADXWithoutStorage struct {
	*baseIndicator
	*baseFloatBounds

	// private variables
	valueAvailableAction ValueAvailableActionFloat
	periodCounter        int
	dx                   *DXWithoutStorage
	currentDX            float64
	sumDX                float64
	previousADX          float64
	timePeriod           int
}

// NewADXWithoutStorage creates an Average Directional Index (ADX) without storage
func NewADXWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *ADXWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// // the minimum timeperiod for an ADX indicator is 2
	// if timePeriod < 2 {
	// 	return nil, errors.New("timePeriod is less than the minimum (2)")
	// }

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := (2 * timePeriod) - 1
	ind := ADXWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseFloatBounds:      newBaseFloatBounds(),
		timePeriod:           timePeriod,
		periodCounter:        timePeriod * -1,
		currentDX:            0.0,
		sumDX:                0.0,
		previousADX:          0.0,
		valueAvailableAction: valueAvailableAction,
	}

	ind.dx, _ = NewDXWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {

		ind.currentDX = dataItem

		ind.periodCounter += 1
		if ind.periodCounter < 0 {
			ind.sumDX += ind.currentDX
		} else if ind.periodCounter == 0 {
			ind.dataLength += 1

			if ind.validFromBar == -1 {
				ind.validFromBar = streamBarIndex
			}

			ind.sumDX += ind.currentDX
			result := ind.sumDX / float64(ind.GetTimePeriod())
			if result > ind.maxValue {
				ind.maxValue = result
			}

			if result < ind.minValue {
				ind.minValue = result
			}
			ind.valueAvailableAction(result, streamBarIndex)
			ind.previousADX = result

		} else {

			ind.dataLength += 1

			result := (ind.previousADX*float64(ind.GetTimePeriod()-1) + ind.currentDX) / float64(ind.GetTimePeriod())
			if result > ind.maxValue {
				ind.maxValue = result
			}

			if result < ind.minValue {
				ind.minValue = result
			}
			ind.valueAvailableAction(result, streamBarIndex)
			ind.previousADX = result
		}

	})

	return &ind, nil
}

// A Directional Movement Indicator (ADX)
type ADX struct {
	*ADXWithoutStorage

	// public variables
	Data []float64
}

// NewADX creates an Average Directional Index (ADX) for online usage
func NewADX(timePeriod int) (indicator *ADX, err error) {

	ind := ADX{}
	ind.ADXWithoutStorage, err = NewADXWithoutStorage(timePeriod,
		func(dataItem float64, streamBarIndex int) {
			ind.Data = append(ind.Data, dataItem)
		})

	return &ind, err
}

// NewDefaultADX creates an Average Directional Index (ADX) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultADX() (indicator *ADX, err error) {
	timePeriod := 14
	return NewADX(timePeriod)
}

// NewADXWithKnownSourceLength creates an Average Directional Index (ADX) for offline usage
func NewADXWithKnownSourceLength(sourceLength int, timePeriod int) (indicator *ADX, err error) {
	ind, err := NewADX(timePeriod)
	ind.Data = make([]float64, 0, sourceLength)

	return ind, err
}

// NewDefaultADXWithKnownSourceLength creates an Average Directional Index (ADX) for offline usage with default parameters
func NewDefaultADXWithKnownSourceLength(sourceLength int) (indicator *ADX, err error) {

	ind, err := NewDefaultADX()
	ind.Data = make([]float64, 0, sourceLength)
	return ind, err
}

// NewADXForStream creates an Average Directional Index (ADX) for online usage with a source data stream
func NewADXForStream(priceStream *gotrade.DOHLCVStream, timePeriod int) (indicator *ADX, err error) {
	ind, err := NewADX(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewADefaultDXForStream creates an Average Directional Index (ADX) for online usage with a source data stream
func NewDefaultADXForStream(priceStream *gotrade.DOHLCVStream) (indicator *ADX, err error) {
	ind, err := NewDefaultADX()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewADXForStreamWithKnownSourceLength creates an Average Directional Index (ADX) for offline usage with a source data stream
func NewADXForStreamWithKnownSourceLength(sourceLength int, priceStream *gotrade.DOHLCVStream, timePeriod int) (indicator *ADX, err error) {
	ind, err := NewADXWithKnownSourceLength(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultADXForStreamWithKnownSourceLength creates an Average Directional Index (ADX) for offline usage with a source data stream
func NewDefaultADXForStreamWithKnownSourceLength(sourceLength int, priceStream *gotrade.DOHLCVStream) (indicator *ADX, err error) {
	ind, err := NewDefaultADXWithKnownSourceLength(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// GetTimePeriod returns the configured ADX timePeriod
func (ind *ADXWithoutStorage) GetTimePeriod() int {
	return ind.timePeriod
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *ADXWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	ind.dx.ReceiveDOHLCVTick(tickData, streamBarIndex)
}
<commit_msg>updated comments, formatting and standardised capitalisation<commit_after>package indicators

import (
	"errors"
	"github.com/thetruetrade/gotrade"
)

// An Average Directional Index (Adx), no storage, for use in other indicators
type AdxWithoutStorage struct {
	*baseIndicator
	*baseFloatBounds

	// private variables
	valueAvailableAction ValueAvailableActionFloat
	periodCounter        int
	dx                   *DXWithoutStorage
	currentDX            float64
	sumDX                float64
	previousAdx          float64
	timePeriod           int
}

// NewAdxWithoutStorage creates an Average Directional Index (Adx) without storage
func NewAdxWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *AdxWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for an Adx indicator is 2
	if timePeriod < 2 {
		return nil, errors.New("timePeriod is less than the minimum (2)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := (2 * timePeriod) - 1
	ind := AdxWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseFloatBounds:      newBaseFloatBounds(),
		timePeriod:           timePeriod,
		periodCounter:        timePeriod * -1,
		currentDX:            0.0,
		sumDX:                0.0,
		previousAdx:          0.0,
		valueAvailableAction: valueAvailableAction,
	}

	ind.dx, err = NewDXWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {

		ind.currentDX = dataItem

		ind.periodCounter += 1
		if ind.periodCounter < 0 {
			ind.sumDX += ind.currentDX
		} else if ind.periodCounter == 0 {
			ind.dataLength += 1

			if ind.validFromBar == -1 {
				// set the streamBarIndex from which this indicator returns valid results
				ind.validFromBar = streamBarIndex
			}

			ind.sumDX += ind.currentDX
			result := ind.sumDX / float64(ind.GetTimePeriod())

			// update the maximum result value
			if result > ind.maxValue {
				ind.maxValue = result
			}

			// update the minimum result value
			if result < ind.minValue {
				ind.minValue = result
			}
			ind.valueAvailableAction(result, streamBarIndex)
			ind.previousAdx = result

		} else {

			ind.dataLength += 1

			result := (ind.previousAdx*float64(ind.GetTimePeriod()-1) + ind.currentDX) / float64(ind.GetTimePeriod())

			// update the maximum result value
			if result > ind.maxValue {
				ind.maxValue = result
			}

			// update the minimum result value
			if result < ind.minValue {
				ind.minValue = result
			}
			// notify of a new result value though the value available action
			ind.valueAvailableAction(result, streamBarIndex)

			ind.previousAdx = result
		}

	})

	return &ind, err
}

// A Directional Movement Indicator (Adx)
type Adx struct {
	*AdxWithoutStorage

	// public variables
	Data []float64
}

// NewAdx creates an Average Directional Index (Adx) for online usage
func NewAdx(timePeriod int) (indicator *Adx, err error) {

	ind := Adx{}
	ind.AdxWithoutStorage, err = NewAdxWithoutStorage(timePeriod,
		func(dataItem float64, streamBarIndex int) {
			ind.Data = append(ind.Data, dataItem)
		})

	return &ind, err
}

// NewDefaultAdx creates an Average Directional Index (Adx) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultAdx() (indicator *Adx, err error) {
	timePeriod := 14
	return NewAdx(timePeriod)
}

// NewAdxWithKnownSourceLength creates an Average Directional Index (Adx) for offline usage
func NewAdxWithKnownSourceLength(sourceLength int, timePeriod int) (indicator *Adx, err error) {
	ind, err := NewAdx(timePeriod)
	ind.Data = make([]float64, 0, sourceLength)

	return ind, err
}

// NewDefaultAdxWithKnownSourceLength creates an Average Directional Index (Adx) for offline usage with default parameters
func NewDefaultAdxWithKnownSourceLength(sourceLength int) (indicator *Adx, err error) {

	ind, err := NewDefaultAdx()
	ind.Data = make([]float64, 0, sourceLength)
	return ind, err
}

// NewAdxForStream creates an Average Directional Index (Adx) for online usage with a source data stream
func NewAdxForStream(priceStream *gotrade.DOHLCVStream, timePeriod int) (indicator *Adx, err error) {
	ind, err := NewAdx(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewADefaultDXForStream creates an Average Directional Index (Adx) for online usage with a source data stream
func NewDefaultAdxForStream(priceStream *gotrade.DOHLCVStream) (indicator *Adx, err error) {
	ind, err := NewDefaultAdx()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewAdxForStreamWithKnownSourceLength creates an Average Directional Index (Adx) for offline usage with a source data stream
func NewAdxForStreamWithKnownSourceLength(sourceLength int, priceStream *gotrade.DOHLCVStream, timePeriod int) (indicator *Adx, err error) {
	ind, err := NewAdxWithKnownSourceLength(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultAdxForStreamWithKnownSourceLength creates an Average Directional Index (Adx) for offline usage with a source data stream
func NewDefaultAdxForStreamWithKnownSourceLength(sourceLength int, priceStream *gotrade.DOHLCVStream) (indicator *Adx, err error) {
	ind, err := NewDefaultAdxWithKnownSourceLength(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// GetTimePeriod returns the configured Adx timePeriod
func (ind *AdxWithoutStorage) GetTimePeriod() int {
	return ind.timePeriod
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *AdxWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {
	ind.dx.ReceiveDOHLCVTick(tickData, streamBarIndex)
}
<|endoftext|>


<commit_before>// Copyright (C) 2019-2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"go.uber.org/zap"

	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/versiondb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/utils/timer"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

const (
	// StatusUpdateFrequency is how many containers should be processed between
	// logs
	StatusUpdateFrequency = 2500
)

// Jobs tracks a series of jobs that form a DAG of dependencies.
type Jobs struct {
	// db ensures that database updates are atomically updated.
	db *versiondb.Database
	// state writes the job queue to [db].
	state *state
	// Measures the ETA until bootstrapping finishes in nanoseconds.
	etaMetric prometheus.Gauge
}

// New attempts to create a new job queue from the provided database.
func New(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*Jobs, error) {
	vdb := versiondb.New(db)
	state, err := newState(vdb, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, fmt.Errorf("couldn't create new jobs state: %w", err)
	}

	etaMetric := prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: metricsNamespace,
		Name:      "eta_execution_complete",
		Help:      "ETA in nanoseconds until execution phase of bootstrapping finishes",
	})

	return &Jobs{
		db:        vdb,
		state:     state,
		etaMetric: etaMetric,
	}, metricsRegisterer.Register(etaMetric)
}

// SetParser tells this job queue how to parse jobs from the database.
func (j *Jobs) SetParser(parser Parser) error { j.state.parser = parser; return nil }

func (j *Jobs) Has(jobID ids.ID) (bool, error) { return j.state.HasJob(jobID) }

// Returns how many pending jobs are waiting in the queue.
func (j *Jobs) PendingJobs() uint64 { return j.state.numJobs }

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (j *Jobs) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := j.state.HasJob(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := j.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := j.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := j.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

func (j *Jobs) ExecuteAll(ctx *snow.ConsensusContext, halter common.Haltable, restarted bool, acceptors ...snow.Acceptor) (int, error) {
	ctx.Executing(true)
	defer ctx.Executing(false)

	numExecuted := 0
	numToExecute := j.state.numJobs
	startTime := time.Now()

	// Disable and clear state caches to prevent us from attempting to execute
	// a vertex that was previously parsed, but not saved to the VM. Some VMs
	// may only persist containers when they are accepted. This is a stop-gap
	// measure to ensure the job will be re-parsed before executing until the VM
	// provides a more explicit interface for freeing parsed blocks.
	// TODO remove DisableCaching when VM provides better interface for freeing
	// blocks.
	j.state.DisableCaching()
	for {
		if halter.Halted() {
			ctx.Log.Info("interrupted execution",
				zap.Int("numExecuted", numExecuted),
			)
			return numExecuted, nil
		}

		job, err := j.state.RemoveRunnableJob()
		if err == database.ErrNotFound {
			break
		}
		if err != nil {
			return 0, fmt.Errorf("failed to removing runnable job with %w", err)
		}

		jobID := job.ID()
		ctx.Log.Debug("executing",
			zap.Stringer("jobID", jobID),
		)
		jobBytes := job.Bytes()
		// Note that acceptor.Accept must be called before executing [job] to
		// honor Acceptor.Accept's invariant.
		for _, acceptor := range acceptors {
			if err := acceptor.Accept(ctx, jobID, jobBytes); err != nil {
				return numExecuted, err
			}
		}
		if err := job.Execute(); err != nil {
			return 0, fmt.Errorf("failed to execute job %s due to %w", jobID, err)
		}

		dependentIDs, err := j.state.RemoveDependencies(jobID)
		if err != nil {
			return 0, fmt.Errorf("failed to remove blocking jobs for %s due to %w", jobID, err)
		}

		for _, dependentID := range dependentIDs {
			job, err := j.state.GetJob(dependentID)
			if err != nil {
				return 0, fmt.Errorf("failed to get job %s from blocking jobs due to %w", dependentID, err)
			}
			hasMissingDeps, err := job.HasMissingDependencies()
			if err != nil {
				return 0, fmt.Errorf("failed to get missing dependencies for %s due to %w", dependentID, err)
			}
			if hasMissingDeps {
				continue
			}
			if err := j.state.AddRunnableJob(dependentID); err != nil {
				return 0, fmt.Errorf("failed to add %s as a runnable job due to %w", dependentID, err)
			}
		}
		if err := j.Commit(); err != nil {
			return 0, err
		}

		numExecuted++
		if numExecuted%StatusUpdateFrequency == 0 { // Periodically print progress
			eta := timer.EstimateETA(
				startTime,
				uint64(numExecuted),
				numToExecute,
			)
			j.etaMetric.Set(float64(eta))

			if !restarted {
				ctx.Log.Info("executing operations",
					zap.Int("numExecuted", numExecuted),
					zap.Uint64("numToExecute", numToExecute),
					zap.Duration("eta", eta),
				)
			} else {
				ctx.Log.Debug("executing operations",
					zap.Int("numExecuted", numExecuted),
					zap.Uint64("numToExecute", numToExecute),
					zap.Duration("eta", eta),
				)
			}
		}
	}

	// Now that executing has finished, zero out the ETA.
	j.etaMetric.Set(0)

	if !restarted {
		ctx.Log.Info("executed operations",
			zap.Int("numExecuted", numExecuted),
		)
	} else {
		ctx.Log.Debug("executed operations",
			zap.Int("numExecuted", numExecuted),
		)
	}
	return numExecuted, nil
}

func (j *Jobs) Clear() error {
	return j.state.Clear()
}

// Commit the versionDB to the underlying database.
func (j *Jobs) Commit() error {
	return j.db.Commit()
}

type JobsWithMissing struct {
	*Jobs

	// keep the missing ID set in memory to avoid unnecessary database reads and
	// writes.
	missingIDs                            ids.Set
	removeFromMissingIDs, addToMissingIDs ids.Set
}

func NewWithMissing(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*JobsWithMissing, error) {
	innerJobs, err := New(db, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, err
	}

	jobs := &JobsWithMissing{
		Jobs: innerJobs,
	}

	missingIDs, err := jobs.state.MissingJobIDs()
	jobs.missingIDs.Add(missingIDs...)
	return jobs, err
}

// SetParser tells this job queue how to parse jobs from the database.
func (jm *JobsWithMissing) SetParser(parser Parser) error {
	jm.state.parser = parser
	return jm.cleanRunnableStack()
}

func (jm *JobsWithMissing) Clear() error {
	if err := jm.state.RemoveMissingJobIDs(jm.missingIDs); err != nil {
		return err
	}

	jm.missingIDs.Clear()
	jm.addToMissingIDs.Clear()
	jm.removeFromMissingIDs.Clear()

	return jm.Jobs.Clear()
}

func (jm *JobsWithMissing) Has(jobID ids.ID) (bool, error) {
	if jm.missingIDs.Contains(jobID) {
		return false, nil
	}

	return jm.Jobs.Has(jobID)
}

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (jm *JobsWithMissing) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := jm.Has(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := jm.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := jm.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

// AddMissingID adds [jobID] to missingIDs
func (jm *JobsWithMissing) AddMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if !jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Add(jobID)
			jm.addToMissingIDs.Add(jobID)
			jm.removeFromMissingIDs.Remove(jobID)
		}
	}
}

// RemoveMissingID removes [jobID] from missingIDs
func (jm *JobsWithMissing) RemoveMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Remove(jobID)
			jm.addToMissingIDs.Remove(jobID)
			jm.removeFromMissingIDs.Add(jobID)
		}
	}
}

func (jm *JobsWithMissing) MissingIDs() []ids.ID { return jm.missingIDs.List() }

func (jm *JobsWithMissing) NumMissingIDs() int { return jm.missingIDs.Len() }

// Commit the versionDB to the underlying database.
func (jm *JobsWithMissing) Commit() error {
	if jm.addToMissingIDs.Len() != 0 {
		if err := jm.state.AddMissingJobIDs(jm.addToMissingIDs); err != nil {
			return err
		}
		jm.addToMissingIDs.Clear()
	}
	if jm.removeFromMissingIDs.Len() != 0 {
		if err := jm.state.RemoveMissingJobIDs(jm.removeFromMissingIDs); err != nil {
			return err
		}
		jm.removeFromMissingIDs.Clear()
	}
	return jm.Jobs.Commit()
}

// cleanRunnableStack iterates over the jobs on the runnable stack and resets any job
// that has missing dependencies to block on those dependencies.
// Note: the jobs queue ensures that no job with missing dependencies will be placed
// on the runnable stack in the first place.
// However, for specific VM implementations blocks may be committed via a two stage commit
// (ex. platformvm Proposal and Commit/Abort blocks). This can cause an issue where if the first stage
// is executed immediately before the node dies, it will be removed from the runnable stack
// without writing the state transition to the VM's database. When the node restarts, the
// VM will not have marked the first block (the proposal block as accepted), but it could
// have already been removed from the jobs queue. cleanRunnableStack handles this case.
func (jm *JobsWithMissing) cleanRunnableStack() error {
	runnableJobsIter := jm.state.runnableJobIDs.NewIterator()
	defer runnableJobsIter.Release()

	for runnableJobsIter.Next() {
		jobIDBytes := runnableJobsIter.Key()
		jobID, err := ids.ToID(jobIDBytes)
		if err != nil {
			return fmt.Errorf("failed to convert jobID bytes into ID due to: %w", err)
		}

		job, err := jm.state.GetJob(jobID)
		if err != nil {
			return fmt.Errorf("failed to retrieve job on runnnable stack due to: %w", err)
		}
		deps, err := job.MissingDependencies()
		if err != nil {
			return fmt.Errorf("failed to retrieve missing dependencies of job on runnable stack due to: %w", err)
		}
		if deps.Len() == 0 {
			continue
		}

		// If the job has missing dependencies, remove it from the runnable stack
		if err := jm.state.runnableJobIDs.Delete(jobIDBytes); err != nil {
			return fmt.Errorf("failed to delete jobID from runnable stack due to: %w", err)
		}

		// Add the missing dependencies to the set that needs to be fetched.
		jm.AddMissingID(deps.List()...)
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return fmt.Errorf("failed to add blocking for depID %s, jobID %s while cleaning the runnable stack", depID, jobID)
			}
		}
	}

	errs := wrappers.Errs{}
	errs.Add(
		runnableJobsIter.Error(),
		jm.Commit(),
	)
	return errs.Err
}
<commit_msg>Display bootstrapping progress logs every 30s (#2131)<commit_after>// Copyright (C) 2019-2022, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"go.uber.org/zap"

	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/versiondb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/utils/timer"
	"github.com/ava-labs/avalanchego/utils/wrappers"
)

const progressUpdateFrequency = 30 * time.Second

// Jobs tracks a series of jobs that form a DAG of dependencies.
type Jobs struct {
	// db ensures that database updates are atomically updated.
	db *versiondb.Database
	// state writes the job queue to [db].
	state *state
	// Measures the ETA until bootstrapping finishes in nanoseconds.
	etaMetric prometheus.Gauge
}

// New attempts to create a new job queue from the provided database.
func New(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*Jobs, error) {
	vdb := versiondb.New(db)
	state, err := newState(vdb, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, fmt.Errorf("couldn't create new jobs state: %w", err)
	}

	etaMetric := prometheus.NewGauge(prometheus.GaugeOpts{
		Namespace: metricsNamespace,
		Name:      "eta_execution_complete",
		Help:      "ETA in nanoseconds until execution phase of bootstrapping finishes",
	})

	return &Jobs{
		db:        vdb,
		state:     state,
		etaMetric: etaMetric,
	}, metricsRegisterer.Register(etaMetric)
}

// SetParser tells this job queue how to parse jobs from the database.
func (j *Jobs) SetParser(parser Parser) error { j.state.parser = parser; return nil }

func (j *Jobs) Has(jobID ids.ID) (bool, error) { return j.state.HasJob(jobID) }

// Returns how many pending jobs are waiting in the queue.
func (j *Jobs) PendingJobs() uint64 { return j.state.numJobs }

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (j *Jobs) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := j.state.HasJob(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := j.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := j.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := j.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

func (j *Jobs) ExecuteAll(ctx *snow.ConsensusContext, halter common.Haltable, restarted bool, acceptors ...snow.Acceptor) (int, error) {
	ctx.Executing(true)
	defer ctx.Executing(false)

	numExecuted := 0
	numToExecute := j.state.numJobs
	startTime := time.Now()
	lastProgressUpdate := startTime

	// Disable and clear state caches to prevent us from attempting to execute
	// a vertex that was previously parsed, but not saved to the VM. Some VMs
	// may only persist containers when they are accepted. This is a stop-gap
	// measure to ensure the job will be re-parsed before executing until the VM
	// provides a more explicit interface for freeing parsed blocks.
	// TODO remove DisableCaching when VM provides better interface for freeing
	// blocks.
	j.state.DisableCaching()
	for {
		if halter.Halted() {
			ctx.Log.Info("interrupted execution",
				zap.Int("numExecuted", numExecuted),
			)
			return numExecuted, nil
		}

		job, err := j.state.RemoveRunnableJob()
		if err == database.ErrNotFound {
			break
		}
		if err != nil {
			return 0, fmt.Errorf("failed to removing runnable job with %w", err)
		}

		jobID := job.ID()
		ctx.Log.Debug("executing",
			zap.Stringer("jobID", jobID),
		)
		jobBytes := job.Bytes()
		// Note that acceptor.Accept must be called before executing [job] to
		// honor Acceptor.Accept's invariant.
		for _, acceptor := range acceptors {
			if err := acceptor.Accept(ctx, jobID, jobBytes); err != nil {
				return numExecuted, err
			}
		}
		if err := job.Execute(); err != nil {
			return 0, fmt.Errorf("failed to execute job %s due to %w", jobID, err)
		}

		dependentIDs, err := j.state.RemoveDependencies(jobID)
		if err != nil {
			return 0, fmt.Errorf("failed to remove blocking jobs for %s due to %w", jobID, err)
		}

		for _, dependentID := range dependentIDs {
			job, err := j.state.GetJob(dependentID)
			if err != nil {
				return 0, fmt.Errorf("failed to get job %s from blocking jobs due to %w", dependentID, err)
			}
			hasMissingDeps, err := job.HasMissingDependencies()
			if err != nil {
				return 0, fmt.Errorf("failed to get missing dependencies for %s due to %w", dependentID, err)
			}
			if hasMissingDeps {
				continue
			}
			if err := j.state.AddRunnableJob(dependentID); err != nil {
				return 0, fmt.Errorf("failed to add %s as a runnable job due to %w", dependentID, err)
			}
		}
		if err := j.Commit(); err != nil {
			return 0, err
		}

		numExecuted++
		if time.Since(lastProgressUpdate) > progressUpdateFrequency { // Periodically print progress
			eta := timer.EstimateETA(
				startTime,
				uint64(numExecuted),
				numToExecute,
			)
			j.etaMetric.Set(float64(eta))

			if !restarted {
				ctx.Log.Info("executing operations",
					zap.Int("numExecuted", numExecuted),
					zap.Uint64("numToExecute", numToExecute),
					zap.Duration("eta", eta),
				)
			} else {
				ctx.Log.Debug("executing operations",
					zap.Int("numExecuted", numExecuted),
					zap.Uint64("numToExecute", numToExecute),
					zap.Duration("eta", eta),
				)
			}

			lastProgressUpdate = time.Now()
		}
	}

	// Now that executing has finished, zero out the ETA.
	j.etaMetric.Set(0)

	if !restarted {
		ctx.Log.Info("executed operations",
			zap.Int("numExecuted", numExecuted),
		)
	} else {
		ctx.Log.Debug("executed operations",
			zap.Int("numExecuted", numExecuted),
		)
	}
	return numExecuted, nil
}

func (j *Jobs) Clear() error {
	return j.state.Clear()
}

// Commit the versionDB to the underlying database.
func (j *Jobs) Commit() error {
	return j.db.Commit()
}

type JobsWithMissing struct {
	*Jobs

	// keep the missing ID set in memory to avoid unnecessary database reads and
	// writes.
	missingIDs                            ids.Set
	removeFromMissingIDs, addToMissingIDs ids.Set
}

func NewWithMissing(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*JobsWithMissing, error) {
	innerJobs, err := New(db, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, err
	}

	jobs := &JobsWithMissing{
		Jobs: innerJobs,
	}

	missingIDs, err := jobs.state.MissingJobIDs()
	jobs.missingIDs.Add(missingIDs...)
	return jobs, err
}

// SetParser tells this job queue how to parse jobs from the database.
func (jm *JobsWithMissing) SetParser(parser Parser) error {
	jm.state.parser = parser
	return jm.cleanRunnableStack()
}

func (jm *JobsWithMissing) Clear() error {
	if err := jm.state.RemoveMissingJobIDs(jm.missingIDs); err != nil {
		return err
	}

	jm.missingIDs.Clear()
	jm.addToMissingIDs.Clear()
	jm.removeFromMissingIDs.Clear()

	return jm.Jobs.Clear()
}

func (jm *JobsWithMissing) Has(jobID ids.ID) (bool, error) {
	if jm.missingIDs.Contains(jobID) {
		return false, nil
	}

	return jm.Jobs.Has(jobID)
}

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (jm *JobsWithMissing) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := jm.Has(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := jm.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := jm.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

// AddMissingID adds [jobID] to missingIDs
func (jm *JobsWithMissing) AddMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if !jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Add(jobID)
			jm.addToMissingIDs.Add(jobID)
			jm.removeFromMissingIDs.Remove(jobID)
		}
	}
}

// RemoveMissingID removes [jobID] from missingIDs
func (jm *JobsWithMissing) RemoveMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Remove(jobID)
			jm.addToMissingIDs.Remove(jobID)
			jm.removeFromMissingIDs.Add(jobID)
		}
	}
}

func (jm *JobsWithMissing) MissingIDs() []ids.ID { return jm.missingIDs.List() }

func (jm *JobsWithMissing) NumMissingIDs() int { return jm.missingIDs.Len() }

// Commit the versionDB to the underlying database.
func (jm *JobsWithMissing) Commit() error {
	if jm.addToMissingIDs.Len() != 0 {
		if err := jm.state.AddMissingJobIDs(jm.addToMissingIDs); err != nil {
			return err
		}
		jm.addToMissingIDs.Clear()
	}
	if jm.removeFromMissingIDs.Len() != 0 {
		if err := jm.state.RemoveMissingJobIDs(jm.removeFromMissingIDs); err != nil {
			return err
		}
		jm.removeFromMissingIDs.Clear()
	}
	return jm.Jobs.Commit()
}

// cleanRunnableStack iterates over the jobs on the runnable stack and resets any job
// that has missing dependencies to block on those dependencies.
// Note: the jobs queue ensures that no job with missing dependencies will be placed
// on the runnable stack in the first place.
// However, for specific VM implementations blocks may be committed via a two stage commit
// (ex. platformvm Proposal and Commit/Abort blocks). This can cause an issue where if the first stage
// is executed immediately before the node dies, it will be removed from the runnable stack
// without writing the state transition to the VM's database. When the node restarts, the
// VM will not have marked the first block (the proposal block as accepted), but it could
// have already been removed from the jobs queue. cleanRunnableStack handles this case.
func (jm *JobsWithMissing) cleanRunnableStack() error {
	runnableJobsIter := jm.state.runnableJobIDs.NewIterator()
	defer runnableJobsIter.Release()

	for runnableJobsIter.Next() {
		jobIDBytes := runnableJobsIter.Key()
		jobID, err := ids.ToID(jobIDBytes)
		if err != nil {
			return fmt.Errorf("failed to convert jobID bytes into ID due to: %w", err)
		}

		job, err := jm.state.GetJob(jobID)
		if err != nil {
			return fmt.Errorf("failed to retrieve job on runnnable stack due to: %w", err)
		}
		deps, err := job.MissingDependencies()
		if err != nil {
			return fmt.Errorf("failed to retrieve missing dependencies of job on runnable stack due to: %w", err)
		}
		if deps.Len() == 0 {
			continue
		}

		// If the job has missing dependencies, remove it from the runnable stack
		if err := jm.state.runnableJobIDs.Delete(jobIDBytes); err != nil {
			return fmt.Errorf("failed to delete jobID from runnable stack due to: %w", err)
		}

		// Add the missing dependencies to the set that needs to be fetched.
		jm.AddMissingID(deps.List()...)
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return fmt.Errorf("failed to add blocking for depID %s, jobID %s while cleaning the runnable stack", depID, jobID)
			}
		}
	}

	errs := wrappers.Errs{}
	errs.Add(
		runnableJobsIter.Error(),
		jm.Commit(),
	)
	return errs.Err
}
<|endoftext|>