<commit_before>/*
Copyright 2018 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"context"
	"fmt"
	"os"

	csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
	"github.com/ceph/ceph-csi/pkg/util"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"k8s.io/klog"
)

// NodeServer struct of ceph CSI driver with supported methods of CSI
// node server spec.
type NodeServer struct {
	*csicommon.DefaultNodeServer
}

var (
	nodeVolumeIDLocker = util.NewIDLocker()
)

func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*util.Credentials, error) {
	var (
		cr      *util.Credentials
		secrets = req.GetSecrets()
	)

	if volOptions.ProvisionVolume {
		// The volume is provisioned dynamically, get the credentials directly from Ceph

		// First, get admin credentials - those are needed for retrieving the user credentials

		adminCr, err := util.GetAdminCredentials(secrets)
		if err != nil {
			return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
		}

		// Then get the ceph user

		entity, err := getCephUser(volOptions, adminCr, volID)
		if err != nil {
			return nil, fmt.Errorf("failed to get ceph user: %v", err)
		}

		cr = entity.toCredentials()
	} else {
		// The volume is pre-made, credentials are in node stage secrets

		userCr, err := util.GetUserCredentials(req.GetSecrets())
		if err != nil {
			return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
		}

		cr = userCr
	}

	return cr, nil
}

// NodeStageVolume mounts the volume to a staging path on the node.
func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
	var (
		volOptions *volumeOptions
		vid        *volumeIdentifier
	)
	if err := util.ValidateNodeStageVolumeRequest(req); err != nil {
		return nil, err
	}

	// Configuration

	stagingTargetPath := req.GetStagingTargetPath()
	volID := volumeID(req.GetVolumeId())

	volOptions, vid, err := newVolumeOptionsFromVolID(string(volID), req.GetVolumeContext(), req.GetSecrets())
	if err != nil {
		if _, ok := err.(ErrInvalidVolID); !ok {
			return nil, status.Error(codes.Internal, err.Error())
		}

		// check for pre-provisioned volumes (plugin versions > 1.0.0)
		volOptions, vid, err = newVolumeOptionsFromStaticVolume(string(volID), req.GetVolumeContext())
		if err != nil {
			if _, ok := err.(ErrNonStaticVolume); !ok {
				return nil, status.Error(codes.Internal, err.Error())
			}

			// check for volumes from plugin versions <= 1.0.0
			volOptions, vid, err = newVolumeOptionsFromVersion1Context(string(volID), req.GetVolumeContext(),
				req.GetSecrets())
			if err != nil {
				return nil, status.Error(codes.Internal, err.Error())
			}
		}
	}

	if err = util.CreateMountPoint(stagingTargetPath); err != nil {
		klog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volID, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	idLk := nodeVolumeIDLocker.Lock(string(volID))
	defer nodeVolumeIDLocker.Unlock(idLk, string(volID))

	// Check if the volume is already mounted

	isMnt, err := util.IsMountPoint(stagingTargetPath)

	if err != nil {
		klog.Errorf("stat failed: %v", err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	if isMnt {
		klog.Infof("cephfs: volume %s is already mounted to %s, skipping", volID, stagingTargetPath)
		return &csi.NodeStageVolumeResponse{}, nil
	}

	// It's not, mount now
	if err = ns.mount(volOptions, req, vid); err != nil {
		return nil, err
	}

	klog.Infof("cephfs: successfully mounted volume %s to %s", volID, stagingTargetPath)

	return &csi.NodeStageVolumeResponse{}, nil
}

func (*NodeServer) mount(volOptions *volumeOptions, req *csi.NodeStageVolumeRequest, vid *volumeIdentifier) error {
	stagingTargetPath := req.GetStagingTargetPath()
	volID := volumeID(req.GetVolumeId())

	cr, err := getCredentialsForVolume(volOptions, volumeID(vid.FsSubvolName), req)
	if err != nil {
		klog.Errorf("failed to get ceph credentials for volume %s: %v", volID, err)
		return status.Error(codes.Internal, err.Error())
	}

	m, err := newMounter(volOptions)
	if err != nil {
		klog.Errorf("failed to create mounter for volume %s: %v", volID, err)
		return status.Error(codes.Internal, err.Error())
	}

	klog.V(4).Infof("cephfs: mounting volume %s with %s", volID, m.name())

	if err = m.mount(stagingTargetPath, cr, volOptions); err != nil {
		klog.Errorf("failed to mount volume %s: %v", volID, err)
		return status.Error(codes.Internal, err.Error())
	}
	if err := volumeMountCache.nodeStageVolume(req.GetVolumeId(), stagingTargetPath, volOptions.Mounter, req.GetSecrets()); err != nil {
		klog.Warningf("mount-cache: failed to stage volume %s %s: %v", volID, stagingTargetPath, err)
	}
	return nil
}

// NodePublishVolume mounts the volume mounted to the staging path to the target
// path
func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {

	mountOptions := []string{"bind"}
	if err := util.ValidateNodePublishVolumeRequest(req); err != nil {
		return nil, err
	}

	// Configuration

	targetPath := req.GetTargetPath()
	volID := req.GetVolumeId()

	if err := util.CreateMountPoint(targetPath); err != nil {
		klog.Errorf("failed to create mount point at %s: %v", targetPath, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	volCap := req.GetVolumeCapability()

	if req.GetReadonly() {
		mountOptions = append(mountOptions, "ro")
	}

	if m := volCap.GetMount(); m != nil {
		hasOption := func(options []string, opt string) bool {
			for _, o := range options {
				if o == opt {
					return true
				}
			}
			return false
		}
		for _, f := range m.MountFlags {
			if !hasOption(mountOptions, f) {
				mountOptions = append(mountOptions, f)
			}
		}
	}

	// Check if the volume is already mounted

	isMnt, err := util.IsMountPoint(targetPath)

	if err != nil {
		klog.Errorf("stat failed: %v", err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	if isMnt {
		klog.Infof("cephfs: volume %s is already bind-mounted to %s", volID, targetPath)
		return &csi.NodePublishVolumeResponse{}, nil
	}

	// It's not, mount now

	if err = bindMount(req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly(), mountOptions); err != nil {
		klog.Errorf("failed to bind-mount volume %s: %v", volID, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	if err = volumeMountCache.nodePublishVolume(volID, targetPath, req.GetReadonly()); err != nil {
		klog.Warningf("mount-cache: failed to publish volume %s %s: %v", volID, targetPath, err)
	}

	klog.Infof("cephfs: successfully bind-mounted volume %s to %s", volID, targetPath)

	err = os.Chmod(targetPath, 0777)
	if err != nil {
		klog.Errorf("failed to change targetpath permission for volume %s: %v", volID, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	return &csi.NodePublishVolumeResponse{}, nil
}

// NodeUnpublishVolume unmounts the volume from the target path
func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
	var err error
	if err = util.ValidateNodeUnpublishVolumeRequest(req); err != nil {
		return nil, err
	}

	targetPath := req.GetTargetPath()

	volID := req.GetVolumeId()
	if err = volumeMountCache.nodeUnPublishVolume(volID, targetPath); err != nil {
		klog.Warningf("mount-cache: failed to unpublish volume %s %s: %v", volID, targetPath, err)
	}

	// Unmount the bind-mount
	if err = unmountVolume(targetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	if err = os.Remove(targetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	klog.Infof("cephfs: successfully unbinded volume %s from %s", req.GetVolumeId(), targetPath)

	return &csi.NodeUnpublishVolumeResponse{}, nil
}

// NodeUnstageVolume unstages the volume from the staging path
func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
	var err error
	if err = util.ValidateNodeUnstageVolumeRequest(req); err != nil {
		return nil, err
	}

	stagingTargetPath := req.GetStagingTargetPath()

	volID := req.GetVolumeId()
	if err = volumeMountCache.nodeUnStageVolume(volID); err != nil {
		klog.Warningf("mount-cache: failed to unstage volume %s %s: %v", volID, stagingTargetPath, err)
	}

	// Unmount the volume
	if err = unmountVolume(stagingTargetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	if err = os.Remove(stagingTargetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	klog.Infof("cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath)

	return &csi.NodeUnstageVolumeResponse{}, nil
}

// NodeGetCapabilities returns the supported capabilities of the node server
func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
	return &csi.NodeGetCapabilitiesResponse{
		Capabilities: []*csi.NodeServiceCapability{
			{
				Type: &csi.NodeServiceCapability_Rpc{
					Rpc: &csi.NodeServiceCapability_RPC{
						Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
					},
				},
			},
		},
	}, nil
}
<commit_msg>Implement metrics for CephFS CSI driver<commit_after>/*
Copyright 2018 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"context"
	"fmt"
	"os"

	csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
	"github.com/ceph/ceph-csi/pkg/util"
	csipbv1 "github.com/container-storage-interface/spec/lib/go/csi"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"k8s.io/klog"
	"k8s.io/kubernetes/pkg/volume"
)

// NodeServer struct of ceph CSI driver with supported methods of CSI
// node server spec.
type NodeServer struct {
	*csicommon.DefaultNodeServer
}

var (
	nodeVolumeIDLocker = util.NewIDLocker()
)

func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*util.Credentials, error) {
	var (
		cr      *util.Credentials
		secrets = req.GetSecrets()
	)

	if volOptions.ProvisionVolume {
		// The volume is provisioned dynamically, get the credentials directly from Ceph

		// First, get admin credentials - those are needed for retrieving the user credentials

		adminCr, err := util.GetAdminCredentials(secrets)
		if err != nil {
			return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
		}

		// Then get the ceph user

		entity, err := getCephUser(volOptions, adminCr, volID)
		if err != nil {
			return nil, fmt.Errorf("failed to get ceph user: %v", err)
		}

		cr = entity.toCredentials()
	} else {
		// The volume is pre-made, credentials are in node stage secrets

		userCr, err := util.GetUserCredentials(req.GetSecrets())
		if err != nil {
			return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
		}

		cr = userCr
	}

	return cr, nil
}

// NodeStageVolume mounts the volume to a staging path on the node.
func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
	var (
		volOptions *volumeOptions
		vid        *volumeIdentifier
	)
	if err := util.ValidateNodeStageVolumeRequest(req); err != nil {
		return nil, err
	}

	// Configuration

	stagingTargetPath := req.GetStagingTargetPath()
	volID := volumeID(req.GetVolumeId())

	volOptions, vid, err := newVolumeOptionsFromVolID(string(volID), req.GetVolumeContext(), req.GetSecrets())
	if err != nil {
		if _, ok := err.(ErrInvalidVolID); !ok {
			return nil, status.Error(codes.Internal, err.Error())
		}

		// check for pre-provisioned volumes (plugin versions > 1.0.0)
		volOptions, vid, err = newVolumeOptionsFromStaticVolume(string(volID), req.GetVolumeContext())
		if err != nil {
			if _, ok := err.(ErrNonStaticVolume); !ok {
				return nil, status.Error(codes.Internal, err.Error())
			}

			// check for volumes from plugin versions <= 1.0.0
			volOptions, vid, err = newVolumeOptionsFromVersion1Context(string(volID), req.GetVolumeContext(),
				req.GetSecrets())
			if err != nil {
				return nil, status.Error(codes.Internal, err.Error())
			}
		}
	}

	if err = util.CreateMountPoint(stagingTargetPath); err != nil {
		klog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volID, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	idLk := nodeVolumeIDLocker.Lock(string(volID))
	defer nodeVolumeIDLocker.Unlock(idLk, string(volID))

	// Check if the volume is already mounted

	isMnt, err := util.IsMountPoint(stagingTargetPath)

	if err != nil {
		klog.Errorf("stat failed: %v", err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	if isMnt {
		klog.Infof("cephfs: volume %s is already mounted to %s, skipping", volID, stagingTargetPath)
		return &csi.NodeStageVolumeResponse{}, nil
	}

	// It's not, mount now
	if err = ns.mount(volOptions, req, vid); err != nil {
		return nil, err
	}

	klog.Infof("cephfs: successfully mounted volume %s to %s", volID, stagingTargetPath)

	return &csi.NodeStageVolumeResponse{}, nil
}

func (*NodeServer) mount(volOptions *volumeOptions, req *csi.NodeStageVolumeRequest, vid *volumeIdentifier) error {
	stagingTargetPath := req.GetStagingTargetPath()
	volID := volumeID(req.GetVolumeId())

	cr, err := getCredentialsForVolume(volOptions, volumeID(vid.FsSubvolName), req)
	if err != nil {
		klog.Errorf("failed to get ceph credentials for volume %s: %v", volID, err)
		return status.Error(codes.Internal, err.Error())
	}

	m, err := newMounter(volOptions)
	if err != nil {
		klog.Errorf("failed to create mounter for volume %s: %v", volID, err)
		return status.Error(codes.Internal, err.Error())
	}

	klog.V(4).Infof("cephfs: mounting volume %s with %s", volID, m.name())

	if err = m.mount(stagingTargetPath, cr, volOptions); err != nil {
		klog.Errorf("failed to mount volume %s: %v", volID, err)
		return status.Error(codes.Internal, err.Error())
	}
	if err := volumeMountCache.nodeStageVolume(req.GetVolumeId(), stagingTargetPath, volOptions.Mounter, req.GetSecrets()); err != nil {
		klog.Warningf("mount-cache: failed to stage volume %s %s: %v", volID, stagingTargetPath, err)
	}
	return nil
}

// NodePublishVolume mounts the volume mounted to the staging path to the target
// path
func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {

	mountOptions := []string{"bind"}
	if err := util.ValidateNodePublishVolumeRequest(req); err != nil {
		return nil, err
	}

	// Configuration

	targetPath := req.GetTargetPath()
	volID := req.GetVolumeId()

	if err := util.CreateMountPoint(targetPath); err != nil {
		klog.Errorf("failed to create mount point at %s: %v", targetPath, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	volCap := req.GetVolumeCapability()

	if req.GetReadonly() {
		mountOptions = append(mountOptions, "ro")
	}

	if m := volCap.GetMount(); m != nil {
		hasOption := func(options []string, opt string) bool {
			for _, o := range options {
				if o == opt {
					return true
				}
			}
			return false
		}
		for _, f := range m.MountFlags {
			if !hasOption(mountOptions, f) {
				mountOptions = append(mountOptions, f)
			}
		}
	}

	// Check if the volume is already mounted

	isMnt, err := util.IsMountPoint(targetPath)

	if err != nil {
		klog.Errorf("stat failed: %v", err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	if isMnt {
		klog.Infof("cephfs: volume %s is already bind-mounted to %s", volID, targetPath)
		return &csi.NodePublishVolumeResponse{}, nil
	}

	// It's not, mount now

	if err = bindMount(req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly(), mountOptions); err != nil {
		klog.Errorf("failed to bind-mount volume %s: %v", volID, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	if err = volumeMountCache.nodePublishVolume(volID, targetPath, req.GetReadonly()); err != nil {
		klog.Warningf("mount-cache: failed to publish volume %s %s: %v", volID, targetPath, err)
	}

	klog.Infof("cephfs: successfully bind-mounted volume %s to %s", volID, targetPath)

	err = os.Chmod(targetPath, 0777)
	if err != nil {
		klog.Errorf("failed to change targetpath permission for volume %s: %v", volID, err)
		return nil, status.Error(codes.Internal, err.Error())
	}

	return &csi.NodePublishVolumeResponse{}, nil
}

// NodeUnpublishVolume unmounts the volume from the target path
func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
	var err error
	if err = util.ValidateNodeUnpublishVolumeRequest(req); err != nil {
		return nil, err
	}

	targetPath := req.GetTargetPath()

	volID := req.GetVolumeId()
	if err = volumeMountCache.nodeUnPublishVolume(volID, targetPath); err != nil {
		klog.Warningf("mount-cache: failed to unpublish volume %s %s: %v", volID, targetPath, err)
	}

	// Unmount the bind-mount
	if err = unmountVolume(targetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	if err = os.Remove(targetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	klog.Infof("cephfs: successfully unbinded volume %s from %s", req.GetVolumeId(), targetPath)

	return &csi.NodeUnpublishVolumeResponse{}, nil
}

// NodeUnstageVolume unstages the volume from the staging path
func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
	var err error
	if err = util.ValidateNodeUnstageVolumeRequest(req); err != nil {
		return nil, err
	}

	stagingTargetPath := req.GetStagingTargetPath()

	volID := req.GetVolumeId()
	if err = volumeMountCache.nodeUnStageVolume(volID); err != nil {
		klog.Warningf("mount-cache: failed to unstage volume %s %s: %v", volID, stagingTargetPath, err)
	}

	// Unmount the volume
	if err = unmountVolume(stagingTargetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	if err = os.Remove(stagingTargetPath); err != nil {
		return nil, status.Error(codes.Internal, err.Error())
	}

	klog.Infof("cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath)

	return &csi.NodeUnstageVolumeResponse{}, nil
}

func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) {

	var err error
	targetPath := req.GetVolumePath()
	if targetPath == "" {
		err = fmt.Errorf("targetpath %v is empty", targetPath)
		return nil, status.Error(codes.InvalidArgument, err.Error())
	}
	/*
		 volID := req.GetVolumeId()

		 TODO: Map the volumeID to the targetpath.

		we need secret to connect to the ceph cluster to get the volumeID from volume
		Name, however `secret` field/option is not available  in NodeGetVolumeStats spec,
		Below issue covers this request and once its available, we can do the validation
		as per the spec.
		https://github.com/container-storage-interface/spec/issues/371

	*/

	isMnt, err := util.IsMountPoint(targetPath)

	if err != nil {
		if os.IsNotExist(err) {
			return nil, status.Errorf(codes.InvalidArgument, "targetpath %s doesnot exist", targetPath)
		}
		return nil, err
	}
	if !isMnt {
		return nil, status.Errorf(codes.InvalidArgument, "targetpath %s is not mounted", targetPath)
	}

	cephfsProvider := volume.NewMetricsStatFS(targetPath)
	volMetrics, volMetErr := cephfsProvider.GetMetrics()
	if volMetErr != nil {
		return nil, status.Error(codes.Internal, volMetErr.Error())
	}

	available, ok := (*(volMetrics.Available)).AsInt64()
	if !ok {
		klog.Errorf("failed to fetch available bytes")
	}
	capacity, ok := (*(volMetrics.Capacity)).AsInt64()
	if !ok {
		klog.Errorf("failed to fetch capacity bytes")
		return nil, status.Error(codes.Unknown, "failed to fetch capacity bytes")
	}
	used, ok := (*(volMetrics.Used)).AsInt64()
	if !ok {
		klog.Errorf("failed to fetch used bytes")
	}
	inodes, ok := (*(volMetrics.Inodes)).AsInt64()
	if !ok {
		klog.Errorf("failed to fetch available inodes")
		return nil, status.Error(codes.Unknown, "failed to fetch available inodes")

	}
	inodesFree, ok := (*(volMetrics.InodesFree)).AsInt64()
	if !ok {
		klog.Errorf("failed to fetch free inodes")
	}

	inodesUsed, ok := (*(volMetrics.InodesUsed)).AsInt64()
	if !ok {
		klog.Errorf("failed to fetch used inodes")
	}
	return &csi.NodeGetVolumeStatsResponse{
		Usage: []*csi.VolumeUsage{
			{
				Available: available,
				Total:     capacity,
				Used:      used,
				Unit:      csipbv1.VolumeUsage_BYTES,
			},
			{
				Available: inodesFree,
				Total:     inodes,
				Used:      inodesUsed,
				Unit:      csipbv1.VolumeUsage_INODES,
			},
		},
	}, nil

}

// NodeGetCapabilities returns the supported capabilities of the node server
func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
	return &csi.NodeGetCapabilitiesResponse{
		Capabilities: []*csi.NodeServiceCapability{
			{
				Type: &csi.NodeServiceCapability_Rpc{
					Rpc: &csi.NodeServiceCapability_RPC{
						Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME,
					},
				},
			},
			{
				Type: &csi.NodeServiceCapability_Rpc{
					Rpc: &csi.NodeServiceCapability_RPC{
						Type: csi.NodeServiceCapability_RPC_GET_VOLUME_STATS,
					},
				},
			},
		},
	}, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 gRPC authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package defaults

const (
	// LoadTestLabel is a label which contains the test's unique name.
	LoadTestLabel = "loadtest"

	// RoleLabel is a label with the role  of a test component. For
	// example, "loadtest-role=server" indicates a server component.
	RoleLabel = "loadtest-role"

	// ComponentNameLabel is a label used to distinguish between test
	// components with the same role.
	ComponentNameLabel = "loadtest-component"

	// ServerRole is the value the controller expects for the RoleLabel
	// on a server component.
	ServerRole = "server"

	// ClientRole is the value the controller expects for the RoleLabel
	// on a client component.
	ClientRole = "client"

	// DriverRole is the value the controller expects for the RoleLabel
	// on a driver component.
	DriverRole = "driver"
)

// LanguageDefault defines a programming language, as well as its
// default container images.
type LanguageDefault struct {
	// Language uniquely identifies a programming language. When the
	// system encounters this name, it will select the build image and
	// run image as the defaults.
	Language string `json:"name"`

	// BuildImage specifies the default container image for building or
	// assembling an executable or bundle for a language. This image
	// likely contains a compiler and any required libraries for
	// compilation.
	BuildImage string `json:"buildImage"`

	// RunImage specifies the default container image for the
	// environment for the runtime of the test. It should provide any
	// necessary interpreters or dependencies to run or use the output
	// of the build image.
	RunImage string `json:"runImage"`
}

// Defaults defines the default settings for the system.
type Defaults struct {
	// DriverPool is the name of a pool where driver components should
	// be scheduled by default.
	DriverPool string `json:"driverPool"`

	// WorkerPool is the name of a pool where server and client
	// components should be scheduled by default.
	WorkerPool string `json:"workerPool"`

	// DriverPort is the port through which the driver and workers
	// communicate.
	DriverPort int32 `json:"driverPort"`

	// ServerPort is the port through which a server component accepts
	// traffic from a client component.
	ServerPort int32 `json:"serverPort"`

	// CloneImage specifies the default container image to use for
	// cloning Git repositories at a specific snapshot.
	CloneImage string `json:"cloneImage"`

	// DriverImage specifies a default driver image. This image will
	// be used to orchestrate a test.
	DriverImage string `json:"driverImage"`

	// Languages specifies the default build and run container images
	// for each known language.
	Languages []LanguageDefault `json:"languages,omitempty"`
}
<commit_msg>Fix wrong tag name in defaults<commit_after>/*
Copyright 2020 gRPC authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package defaults

const (
	// LoadTestLabel is a label which contains the test's unique name.
	LoadTestLabel = "loadtest"

	// RoleLabel is a label with the role  of a test component. For
	// example, "loadtest-role=server" indicates a server component.
	RoleLabel = "loadtest-role"

	// ComponentNameLabel is a label used to distinguish between test
	// components with the same role.
	ComponentNameLabel = "loadtest-component"

	// ServerRole is the value the controller expects for the RoleLabel
	// on a server component.
	ServerRole = "server"

	// ClientRole is the value the controller expects for the RoleLabel
	// on a client component.
	ClientRole = "client"

	// DriverRole is the value the controller expects for the RoleLabel
	// on a driver component.
	DriverRole = "driver"
)

// LanguageDefault defines a programming language, as well as its
// default container images.
type LanguageDefault struct {
	// Language uniquely identifies a programming language. When the
	// system encounters this name, it will select the build image and
	// run image as the defaults.
	Language string `json:"language"`

	// BuildImage specifies the default container image for building or
	// assembling an executable or bundle for a language. This image
	// likely contains a compiler and any required libraries for
	// compilation.
	BuildImage string `json:"buildImage"`

	// RunImage specifies the default container image for the
	// environment for the runtime of the test. It should provide any
	// necessary interpreters or dependencies to run or use the output
	// of the build image.
	RunImage string `json:"runImage"`
}

// Defaults defines the default settings for the system.
type Defaults struct {
	// DriverPool is the name of a pool where driver components should
	// be scheduled by default.
	DriverPool string `json:"driverPool"`

	// WorkerPool is the name of a pool where server and client
	// components should be scheduled by default.
	WorkerPool string `json:"workerPool"`

	// DriverPort is the port through which the driver and workers
	// communicate.
	DriverPort int32 `json:"driverPort"`

	// ServerPort is the port through which a server component accepts
	// traffic from a client component.
	ServerPort int32 `json:"serverPort"`

	// CloneImage specifies the default container image to use for
	// cloning Git repositories at a specific snapshot.
	CloneImage string `json:"cloneImage"`

	// DriverImage specifies a default driver image. This image will
	// be used to orchestrate a test.
	DriverImage string `json:"driverImage"`

	// Languages specifies the default build and run container images
	// for each known language.
	Languages []LanguageDefault `json:"languages,omitempty"`
}
<|endoftext|>


<commit_before>package diagrams

import (
	"bytes"
	"compress/zlib"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/pkg/errors"
	"github.com/spf13/afero"
)

func OutputPlantuml(output, plantuml, umlInput string, fs afero.Fs) error {
	mode := path.Ext(output)
	mode = strings.Replace(mode, ".", "", 1)
	var encoded string
	var out []byte
	var err error
	encoded, err = DeflateAndEncode([]byte(umlInput))
	if err != nil {
		return err
	}

	switch mode {
	case "png", "svg":
		plantuml = fmt.Sprintf("%s/%s/%s", plantuml, mode, encoded)
		out, err = sendHTTPRequest(plantuml)
		if err != nil {
			return err
		}
	case "html":
		plantuml = fmt.Sprintf("%s/%s/%s", plantuml, "svg", encoded)
		out = []byte(fmt.Sprintf(`<img src="%s" alt="plantuml">`, plantuml))
	case "link":
		plantuml = fmt.Sprintf("%s/%s/%s", plantuml, "svg", encoded)
		out = []byte(plantuml)
	case "puml", "uml", "plantuml":
		out = []byte(umlInput)
	default:
		return fmt.Errorf("extension must be .svg, .png, .uml, .puml, .plantuml, .html or .link, not %#v", mode)
	}
	return errors.Wrapf(afero.WriteFile(fs, output, append(out, byte('\n')), os.ModePerm), "writing %q", output)
}

func sendHTTPRequest(url string) ([]byte, error) {
	resp, err := http.Get(url) //nolint:gosec
	if err != nil {
		return nil, errors.Errorf("Unable to create http request to %s, Error:%s", url, err.Error())
	}
	defer resp.Body.Close()

	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, errors.Errorf("Unable to read body.")
	}
	return out, nil
}

// The functions below ported from https://github.com/dougn/python-plantuml/blob/master/plantuml.py
func DeflateAndEncode(text []byte) (string, error) {
	var buf bytes.Buffer
	zw, err := zlib.NewWriterLevel(&buf, zlib.BestCompression)
	if err != nil {
		return "", err
	}
	if _, err := zw.Write(text); err != nil {
		return "", err
	}
	if err := zw.Close(); err != nil {
		return "", err
	}
	return encode(buf.Bytes()), nil
}

func encode(data []byte) string {
	var buf bytes.Buffer
	i := 0
	for wholeTripleBytes := len(data) / 3 * 3; i < wholeTripleBytes; i += 3 {
		encode3bytes(&buf, data[i], data[i+1], data[i+2])
	}
	switch len(data) - i {
	case 1:
		encode3bytes(&buf, data[i], 0, 0)
	case 2:
		encode3bytes(&buf, data[i], data[i+1], 0)
	}
	return buf.String()
}

// 3 bytes takes 24 bits. This splits 24 bits into 4 bytes of which lower 6-bit takes account.
func encode3bytes(w io.ByteWriter, b1, b2, b3 byte) {
	if err := w.WriteByte(encode6bit(0x3F & (b1 >> 2))); err != nil {
		panic(err)
	}
	if err := w.WriteByte(encode6bit(0x3F & (((b1 & 0x3) << 4) | (b2 >> 4)))); err != nil {
		panic(err)
	}
	if err := w.WriteByte(encode6bit(0x3F & (((b2 & 0xF) << 2) | (b3 >> 6)))); err != nil {
		panic(err)
	}
	if err := w.WriteByte(encode6bit(0x3F & b3)); err != nil {
		panic(err)
	}
}

func encode6bit(b byte) byte {
	// 6 bit makes value 0 to 63. The func maps 0-63 to characters
	// '0'-'9', 'A'-'Z', 'a'-'z', '-', '_'. '?' should never be reached.
	if b > 63 {
		panic("unexpected character!")
	}
	return "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"[b]
}
<commit_msg>Fix PlantUML diagram content header (#906)<commit_after>package diagrams

import (
	"bytes"
	"compress/zlib"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/pkg/errors"
	"github.com/spf13/afero"
)

func OutputPlantuml(output, plantuml, umlInput string, fs afero.Fs) error {
	mode := path.Ext(output)
	mode = strings.Replace(mode, ".", "", 1)
	var encoded string
	var out []byte
	var err error
	encoded, err = DeflateAndEncode([]byte(umlInput))
	if err != nil {
		return err
	}

	switch mode {
	case "png", "svg":
		plantuml = fmt.Sprintf("%s/%s/~1%s", plantuml, mode, encoded)
		out, err = sendHTTPRequest(plantuml)
		if err != nil {
			return err
		}
	case "html":
		plantuml = fmt.Sprintf("%s/%s/%s", plantuml, "svg", encoded)
		out = []byte(fmt.Sprintf(`<img src="%s" alt="plantuml">`, plantuml))
	case "link":
		plantuml = fmt.Sprintf("%s/%s/%s", plantuml, "svg", encoded)
		out = []byte(plantuml)
	case "puml", "uml", "plantuml":
		out = []byte(umlInput)
	default:
		return fmt.Errorf("extension must be .svg, .png, .uml, .puml, .plantuml, .html or .link, not %#v", mode)
	}
	return errors.Wrapf(afero.WriteFile(fs, output, append(out, byte('\n')), os.ModePerm), "writing %q", output)
}

func sendHTTPRequest(url string) ([]byte, error) {
	resp, err := http.Get(url) //nolint:gosec
	if err != nil {
		return nil, errors.Errorf("Unable to create http request to %s, Error:%s", url, err.Error())
	}
	defer resp.Body.Close()

	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, errors.Errorf("Unable to read body.")
	}
	return out, nil
}

// The functions below ported from https://github.com/dougn/python-plantuml/blob/master/plantuml.py
func DeflateAndEncode(text []byte) (string, error) {
	var buf bytes.Buffer
	zw, err := zlib.NewWriterLevel(&buf, zlib.BestCompression)
	if err != nil {
		return "", err
	}
	if _, err := zw.Write(text); err != nil {
		return "", err
	}
	if err := zw.Close(); err != nil {
		return "", err
	}
	return encode(buf.Bytes()), nil
}

func encode(data []byte) string {
	var buf bytes.Buffer
	i := 0
	for wholeTripleBytes := len(data) / 3 * 3; i < wholeTripleBytes; i += 3 {
		encode3bytes(&buf, data[i], data[i+1], data[i+2])
	}
	switch len(data) - i {
	case 1:
		encode3bytes(&buf, data[i], 0, 0)
	case 2:
		encode3bytes(&buf, data[i], data[i+1], 0)
	}
	return buf.String()
}

// 3 bytes takes 24 bits. This splits 24 bits into 4 bytes of which lower 6-bit takes account.
func encode3bytes(w io.ByteWriter, b1, b2, b3 byte) {
	if err := w.WriteByte(encode6bit(0x3F & (b1 >> 2))); err != nil {
		panic(err)
	}
	if err := w.WriteByte(encode6bit(0x3F & (((b1 & 0x3) << 4) | (b2 >> 4)))); err != nil {
		panic(err)
	}
	if err := w.WriteByte(encode6bit(0x3F & (((b2 & 0xF) << 2) | (b3 >> 6)))); err != nil {
		panic(err)
	}
	if err := w.WriteByte(encode6bit(0x3F & b3)); err != nil {
		panic(err)
	}
}

func encode6bit(b byte) byte {
	// 6 bit makes value 0 to 63. The func maps 0-63 to characters
	// '0'-'9', 'A'-'Z', 'a'-'z', '-', '_'. '?' should never be reached.
	if b > 63 {
		panic("unexpected character!")
	}
	return "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-_"[b]
}
<|endoftext|>


<commit_before>// Copyright 2019 The GCR Cleaner Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcrcleaner

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"
)

const (
	contentTypeHeader = "Content-Type"
	contentTypeJSON   = "application/json"
)

// Server is a cleaning server.
type Server struct {
	cleaner *Cleaner
	logger  *Logger
}

// NewServer creates a new server for handler functions.
func NewServer(cleaner *Cleaner) (*Server, error) {
	if cleaner == nil {
		return nil, fmt.Errorf("missing cleaner")
	}

	return &Server{
		cleaner: cleaner,
		logger:  NewLogger(os.Stdout, os.Stderr),
	}, nil
}

// PubSubHandler is an http handler that invokes the cleaner from a pubsub
// request. Unlike an HTTP request, the pubsub endpoint always returns a success
// unless the pubsub message is malformed.
func (s *Server) PubSubHandler(cache Cache) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		var m pubsubMessage
		if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
			err = fmt.Errorf("failed to decode pubsub message: %w", err)
			s.handleError(w, err, 400)
			return
		}

		// PubSub is "at least once" delivery. The cleaner is idempotent, but
		// let's try to prevent unnecessary work by not processing messages we've
		// already received.
		msgID := m.Subscription + "/" + m.Message.ID
		if exists := cache.Insert(msgID); exists {
			s.logger.Info("already processed message", "id", msgID)
			w.WriteHeader(204)
			return
		}

		if len(m.Message.Data) == 0 {
			err := fmt.Errorf("missing data in pubsub payload")
			s.handleError(w, err, 400)
			return
		}

		// Start a goroutine to delete the images
		body := ioutil.NopCloser(bytes.NewReader(m.Message.Data))
		go func() {
			if _, _, err := s.clean(ctx, body); err != nil {
				s.logger.Error("failed to clean", "error", err)
			}
		}()

		w.WriteHeader(204)
	}
}

// HTTPHandler is an http handler that invokes the cleaner with the given
// parameters.
func (s *Server) HTTPHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		deleted, status, err := s.clean(ctx, r.Body)
		if err != nil {
			s.handleError(w, err, status)
			return
		}

		refs := make([]string, 0, 16)
		for _, v := range deleted {
			refs = append(refs, v...)
		}
		sort.Strings(refs)

		b, err := json.Marshal(&cleanResp{
			Count:      len(deleted),
			Refs:       refs,
			RefsByRepo: deleted,
		})
		if err != nil {
			err = fmt.Errorf("failed to marshal JSON errors: %w", err)
			s.handleError(w, err, 500)
			return
		}

		w.WriteHeader(200)
		w.Header().Set(contentTypeHeader, contentTypeJSON)
		fmt.Fprint(w, string(b))
	}
}

// clean reads the given body as JSON and starts a cleaner instance.
func (s *Server) clean(ctx context.Context, r io.ReadCloser) (map[string][]string, int, error) {
	var p Payload
	if err := json.NewDecoder(r).Decode(&p); err != nil {
		return nil, 500, fmt.Errorf("failed to decode payload as JSON: %w", err)
	}

	// Convert duration to a negative value, since we're about to "add" it to the
	// since time.
	sub := time.Duration(p.Grace)
	if p.Grace > 0 {
		sub = sub * -1
	}

	since := time.Now().UTC().Add(sub)
	tagFilter, err := BuildTagFilter(p.TagFilterFirst, p.TagFilterAny, p.TagFilterAll)
	if err != nil {
		return nil, http.StatusBadRequest, fmt.Errorf("failed to build tag filter: %w", err)
	}

	// Gather all the repositories.
	repos := make([]string, 0, len(p.Repos))
	for _, v := range p.Repos {
		if t := strings.TrimSpace(v); t != "" {
			repos = append(repos, t)
		}
	}
	if t := strings.TrimSpace(p.Repo); t != "" {
		s.logger.Warn("specifying \"repo\" is deprecated - switch to \"repos\"")
		repos = append(repos, t)
	}
	if p.Recursive {
		for _, repo := range repos {
			childRepos, err := s.cleaner.ListChildRepositories(ctx, repo)
			if err != nil {
				return nil, http.StatusBadRequest, fmt.Errorf("failed to list child repositories for %q: %w", p.Repo, err)
			}
			repos = append(repos, childRepos...)
		}
	}

	s.logger.Info("deleting refs",
		"since", since,
		"repos", repos)

	// Do the deletion.
	deleted := make(map[string][]string, len(repos))
	for _, repo := range repos {
		s.logger.Debug("deleting refs for repo", "repo", repo)

		childrenDeleted, err := s.cleaner.Clean(repo, since, p.Keep, tagFilter, p.DryRun)
		if err != nil {
			return nil, http.StatusBadRequest, fmt.Errorf("failed to clean repo %q: %w", repo, err)
		}

		if len(childrenDeleted) > 0 {
			s.logger.Info("deleted refs", "repo", repo, "refs", childrenDeleted)
			deleted[repo] = append(deleted[repo], childrenDeleted...)
		}
	}

	s.logger.Info("deleted refs", "refs", deleted)

	return deleted, http.StatusOK, nil
}

// handleError returns a JSON-formatted error message
func (s *Server) handleError(w http.ResponseWriter, err error, status int) {
	s.logger.Error(err.Error(), "error", err)

	b, err := json.Marshal(&errorResp{Error: err.Error()})
	if err != nil {
		err = fmt.Errorf("failed to marshal JSON errors: %w", err)
		http.Error(w, err.Error(), 500)
		return
	}

	w.WriteHeader(status)
	w.Header().Set(contentTypeHeader, contentTypeJSON)
	fmt.Fprint(w, string(b))
}

// Payload is the expected incoming payload format.
type Payload struct {
	// Repo is the name of the repo to clean.
	//
	// Deprecated: Use Repos instead.
	Repo string `json:"repo"`

	// Repos is the list of repositories to clean.
	Repos sortedStringSlice `json:"repos"`

	// Grace is a time.Duration value indicating how much grade period should be
	// given to new, untagged layers. The default is no grace.
	Grace duration `json:"grace"`

	// Keep is the minimum number of images to keep.
	Keep int `json:"keep"`

	// TagFilterAny is the tags pattern to be allowed removing. If given, any
	// image with at least one tag that matches this given regular expression will
	// be deleted. The image will be deleted even if it has other tags that do not
	// match the given regular expression.
	TagFilterAny string `json:"tag_filter_any"`

	// TagFilterAll is the tags pattern to be allowed removing. If given, any
	// image where all tags match this given regular expression will be deleted.
	// The image will not be delete if it has other tags that do not match the
	// given regular expression.
	TagFilterAll string `json:"tag_filter_all"`

	// DryRun instructs the server to not perform actual cleaning. The response
	// will include repositories that would have been deleted.
	DryRun bool `json:"dry_run"`

	// Recursive enables cleaning all child repositories.
	Recursive bool `json:"recursive"`

	// TagFilterFirst is the tags pattern to be allowed removing. If specified, any
	// images where the first tag matches the given regular expression will be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead.
	TagFilterFirst string `json:"tag_filter"`

	// AllowTagged is a Boolean value determine if tagged images are allowed to be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead. Setting either of
	// these values enables deleting tagged images.
	AllowTagged bool `json:"allow_tagged"`
}

type pubsubMessage struct {
	Message struct {
		Data []byte `json:"data"`
		ID   string `json:"message_id"`
	} `json:"message"`
	Subscription string `json:"subscription"`
}

type cleanResp struct {
	Count      int                 `json:"count"`
	Refs       []string            `json:"refs"`
	RefsByRepo map[string][]string `json:"refs_by_repo"`
}

type errorResp struct {
	Error string `json:"error"`
}

type sortedStringSlice []string

func (s sortedStringSlice) MarshalJSON() ([]byte, error) {
	return json.Marshal([]string(s))
}

func (s *sortedStringSlice) UnmarshalJSON(b []byte) error {
	var v interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	m := make(map[string]struct{}, 4)

	switch val := v.(type) {
	case string:
		if t := strings.TrimSpace(val); t != "" {
			m[t] = struct{}{}
		}
	case []interface{}:
		for i, v := range val {
			s, ok := v.(string)
			if !ok {
				return fmt.Errorf("list must contain only strings (got %T at index %d)", v, i)
			}
			if t := strings.TrimSpace(s); t != "" {
				m[t] = struct{}{}
			}
		}
	case []string:
		for _, v := range val {
			if t := strings.TrimSpace(v); t != "" {
				m[t] = struct{}{}
			}
		}
	default:
		return fmt.Errorf("invalid list type %T", val)
	}

	list := make([]string, 0, len(m))
	for v := range m {
		list = append(list, v)
	}
	sort.Strings(list)
	*s = list

	return nil
}

type duration time.Duration

func (d duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(time.Duration(d).String())
}

func (d *duration) UnmarshalJSON(b []byte) error {
	var v interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	switch val := v.(type) {
	case float64:
		*d = duration(time.Duration(val))
		return nil
	case string:
		s, err := time.ParseDuration(val)
		if err != nil {
			return err
		}
		*d = duration(s)
		return nil
	default:
		return fmt.Errorf("invalid duration type %T", val)
	}
}
<commit_msg>Use context.Background() on PubSub endpoint (#71)<commit_after>// Copyright 2019 The GCR Cleaner Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcrcleaner

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"
)

const (
	contentTypeHeader = "Content-Type"
	contentTypeJSON   = "application/json"
)

// Server is a cleaning server.
type Server struct {
	cleaner *Cleaner
	logger  *Logger
}

// NewServer creates a new server for handler functions.
func NewServer(cleaner *Cleaner) (*Server, error) {
	if cleaner == nil {
		return nil, fmt.Errorf("missing cleaner")
	}

	return &Server{
		cleaner: cleaner,
		logger:  NewLogger(os.Stdout, os.Stderr),
	}, nil
}

// PubSubHandler is an http handler that invokes the cleaner from a pubsub
// request. Unlike an HTTP request, the pubsub endpoint always returns a success
// unless the pubsub message is malformed.
func (s *Server) PubSubHandler(cache Cache) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		var m pubsubMessage
		if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
			err = fmt.Errorf("failed to decode pubsub message: %w", err)
			s.handleError(w, err, 400)
			return
		}

		// PubSub is "at least once" delivery. The cleaner is idempotent, but
		// let's try to prevent unnecessary work by not processing messages we've
		// already received.
		msgID := m.Subscription + "/" + m.Message.ID
		if exists := cache.Insert(msgID); exists {
			s.logger.Info("already processed message", "id", msgID)
			w.WriteHeader(204)
			return
		}

		if len(m.Message.Data) == 0 {
			err := fmt.Errorf("missing data in pubsub payload")
			s.handleError(w, err, 400)
			return
		}

		// Start a goroutine to delete the images
		body := ioutil.NopCloser(bytes.NewReader(m.Message.Data))
		go func() {
			// Intentionally don't use the request context, since it terminates but
			// the background job should still be processing.
			ctx := context.Background()
			if _, _, err := s.clean(ctx, body); err != nil {
				s.logger.Error("failed to clean", "error", err)
			}
		}()

		w.WriteHeader(204)
	}
}

// HTTPHandler is an http handler that invokes the cleaner with the given
// parameters.
func (s *Server) HTTPHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()

		deleted, status, err := s.clean(ctx, r.Body)
		if err != nil {
			s.handleError(w, err, status)
			return
		}

		refs := make([]string, 0, 16)
		for _, v := range deleted {
			refs = append(refs, v...)
		}
		sort.Strings(refs)

		b, err := json.Marshal(&cleanResp{
			Count:      len(deleted),
			Refs:       refs,
			RefsByRepo: deleted,
		})
		if err != nil {
			err = fmt.Errorf("failed to marshal JSON errors: %w", err)
			s.handleError(w, err, 500)
			return
		}

		w.WriteHeader(200)
		w.Header().Set(contentTypeHeader, contentTypeJSON)
		fmt.Fprint(w, string(b))
	}
}

// clean reads the given body as JSON and starts a cleaner instance.
func (s *Server) clean(ctx context.Context, r io.ReadCloser) (map[string][]string, int, error) {
	var p Payload
	if err := json.NewDecoder(r).Decode(&p); err != nil {
		return nil, 500, fmt.Errorf("failed to decode payload as JSON: %w", err)
	}

	// Convert duration to a negative value, since we're about to "add" it to the
	// since time.
	sub := time.Duration(p.Grace)
	if p.Grace > 0 {
		sub = sub * -1
	}

	since := time.Now().UTC().Add(sub)
	tagFilter, err := BuildTagFilter(p.TagFilterFirst, p.TagFilterAny, p.TagFilterAll)
	if err != nil {
		return nil, http.StatusBadRequest, fmt.Errorf("failed to build tag filter: %w", err)
	}

	// Gather all the repositories.
	repos := make([]string, 0, len(p.Repos))
	for _, v := range p.Repos {
		if t := strings.TrimSpace(v); t != "" {
			repos = append(repos, t)
		}
	}
	if t := strings.TrimSpace(p.Repo); t != "" {
		s.logger.Warn("specifying \"repo\" is deprecated - switch to \"repos\"")
		repos = append(repos, t)
	}
	if p.Recursive {
		for _, repo := range repos {
			childRepos, err := s.cleaner.ListChildRepositories(ctx, repo)
			if err != nil {
				return nil, http.StatusBadRequest, fmt.Errorf("failed to list child repositories for %q: %w", p.Repo, err)
			}
			repos = append(repos, childRepos...)
		}
	}

	s.logger.Info("deleting refs",
		"since", since,
		"repos", repos)

	// Do the deletion.
	deleted := make(map[string][]string, len(repos))
	for _, repo := range repos {
		s.logger.Debug("deleting refs for repo", "repo", repo)

		childrenDeleted, err := s.cleaner.Clean(repo, since, p.Keep, tagFilter, p.DryRun)
		if err != nil {
			return nil, http.StatusBadRequest, fmt.Errorf("failed to clean repo %q: %w", repo, err)
		}

		if len(childrenDeleted) > 0 {
			s.logger.Info("deleted refs", "repo", repo, "refs", childrenDeleted)
			deleted[repo] = append(deleted[repo], childrenDeleted...)
		}
	}

	s.logger.Info("deleted refs", "refs", deleted)

	return deleted, http.StatusOK, nil
}

// handleError returns a JSON-formatted error message
func (s *Server) handleError(w http.ResponseWriter, err error, status int) {
	s.logger.Error(err.Error(), "error", err)

	b, err := json.Marshal(&errorResp{Error: err.Error()})
	if err != nil {
		err = fmt.Errorf("failed to marshal JSON errors: %w", err)
		http.Error(w, err.Error(), 500)
		return
	}

	w.WriteHeader(status)
	w.Header().Set(contentTypeHeader, contentTypeJSON)
	fmt.Fprint(w, string(b))
}

// Payload is the expected incoming payload format.
type Payload struct {
	// Repo is the name of the repo to clean.
	//
	// Deprecated: Use Repos instead.
	Repo string `json:"repo"`

	// Repos is the list of repositories to clean.
	Repos sortedStringSlice `json:"repos"`

	// Grace is a time.Duration value indicating how much grade period should be
	// given to new, untagged layers. The default is no grace.
	Grace duration `json:"grace"`

	// Keep is the minimum number of images to keep.
	Keep int `json:"keep"`

	// TagFilterAny is the tags pattern to be allowed removing. If given, any
	// image with at least one tag that matches this given regular expression will
	// be deleted. The image will be deleted even if it has other tags that do not
	// match the given regular expression.
	TagFilterAny string `json:"tag_filter_any"`

	// TagFilterAll is the tags pattern to be allowed removing. If given, any
	// image where all tags match this given regular expression will be deleted.
	// The image will not be delete if it has other tags that do not match the
	// given regular expression.
	TagFilterAll string `json:"tag_filter_all"`

	// DryRun instructs the server to not perform actual cleaning. The response
	// will include repositories that would have been deleted.
	DryRun bool `json:"dry_run"`

	// Recursive enables cleaning all child repositories.
	Recursive bool `json:"recursive"`

	// TagFilterFirst is the tags pattern to be allowed removing. If specified, any
	// images where the first tag matches the given regular expression will be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead.
	TagFilterFirst string `json:"tag_filter"`

	// AllowTagged is a Boolean value determine if tagged images are allowed to be
	// deleted.
	//
	// Deprecated: Use tag_filter_all or tag_filter_any instead. Setting either of
	// these values enables deleting tagged images.
	AllowTagged bool `json:"allow_tagged"`
}

type pubsubMessage struct {
	Message struct {
		Data []byte `json:"data"`
		ID   string `json:"message_id"`
	} `json:"message"`
	Subscription string `json:"subscription"`
}

type cleanResp struct {
	Count      int                 `json:"count"`
	Refs       []string            `json:"refs"`
	RefsByRepo map[string][]string `json:"refs_by_repo"`
}

type errorResp struct {
	Error string `json:"error"`
}

type sortedStringSlice []string

func (s sortedStringSlice) MarshalJSON() ([]byte, error) {
	return json.Marshal([]string(s))
}

func (s *sortedStringSlice) UnmarshalJSON(b []byte) error {
	var v interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	m := make(map[string]struct{}, 4)

	switch val := v.(type) {
	case string:
		if t := strings.TrimSpace(val); t != "" {
			m[t] = struct{}{}
		}
	case []interface{}:
		for i, v := range val {
			s, ok := v.(string)
			if !ok {
				return fmt.Errorf("list must contain only strings (got %T at index %d)", v, i)
			}
			if t := strings.TrimSpace(s); t != "" {
				m[t] = struct{}{}
			}
		}
	case []string:
		for _, v := range val {
			if t := strings.TrimSpace(v); t != "" {
				m[t] = struct{}{}
			}
		}
	default:
		return fmt.Errorf("invalid list type %T", val)
	}

	list := make([]string, 0, len(m))
	for v := range m {
		list = append(list, v)
	}
	sort.Strings(list)
	*s = list

	return nil
}

type duration time.Duration

func (d duration) MarshalJSON() ([]byte, error) {
	return json.Marshal(time.Duration(d).String())
}

func (d *duration) UnmarshalJSON(b []byte) error {
	var v interface{}
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	switch val := v.(type) {
	case float64:
		*d = duration(time.Duration(val))
		return nil
	case string:
		s, err := time.ParseDuration(val)
		if err != nil {
			return err
		}
		*d = duration(s)
		return nil
	default:
		return fmt.Errorf("invalid duration type %T", val)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"time"

	"github.com/jonboulle/clockwork"
	"github.com/renstrom/dedent"
	"github.com/spf13/cobra"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/kubectl"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/resource"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

// ApplyOptions stores cmd.Flag values for apply.  As new fields are added,
// add them here instead of referencing the cmd.Flags()
type ApplyOptions struct {
	Filenames []string
	Recursive bool
}

const (
	// maxPatchRetry is the maximum number of conflicts retry for during a patch operation before returning failure
	maxPatchRetry = 5
	// backOffPeriod is the period to back off when apply patch resutls in error.
	backOffPeriod = 1 * time.Second
	// how many times we can retry before back off
	triesBeforeBackOff = 1
)

var (
	apply_long = dedent.Dedent(`
		Apply a configuration to a resource by filename or stdin.
		This resource will be created if it doesn't exist yet.
		To use 'apply', always create the resource initially with either 'apply' or 'create --save-config'.

		JSON and YAML formats are accepted.`)

	apply_example = dedent.Dedent(`
		# Apply the configuration in pod.json to a pod.
		kubectl apply -f ./pod.json

		# Apply the JSON passed into stdin to a pod.
		cat pod.json | kubectl apply -f -`)
)

func NewCmdApply(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	options := &ApplyOptions{}

	cmd := &cobra.Command{
		Use:     "apply -f FILENAME",
		Short:   "Apply a configuration to a resource by filename or stdin",
		Long:    apply_long,
		Example: apply_example,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(validateArgs(cmd, args))
			cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd))
			cmdutil.CheckErr(RunApply(f, cmd, out, options))
		},
	}

	usage := "Filename, directory, or URL to file that contains the configuration to apply"
	kubectl.AddJsonFilenameFlag(cmd, &options.Filenames, usage)
	cmd.MarkFlagRequired("filename")
	cmdutil.AddValidateFlags(cmd)
	cmdutil.AddRecursiveFlag(cmd, &options.Recursive)
	cmdutil.AddOutputFlagsForMutation(cmd)
	cmdutil.AddRecordFlag(cmd)
	cmdutil.AddInclude3rdPartyFlags(cmd)
	return cmd
}

func validateArgs(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return cmdutil.UsageError(cmd, "Unexpected args: %v", args)
	}

	return nil
}

func RunApply(f *cmdutil.Factory, cmd *cobra.Command, out io.Writer, options *ApplyOptions) error {
	shortOutput := cmdutil.GetFlagString(cmd, "output") == "name"
	schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir"))
	if err != nil {
		return err
	}

	cmdNamespace, enforceNamespace, err := f.DefaultNamespace()
	if err != nil {
		return err
	}

	mapper, typer := f.Object(cmdutil.GetIncludeThirdPartyAPIs(cmd))
	r := resource.NewBuilder(mapper, typer, resource.ClientMapperFunc(f.ClientForMapping), f.Decoder(true)).
		Schema(schema).
		ContinueOnError().
		NamespaceParam(cmdNamespace).DefaultNamespace().
		FilenameParam(enforceNamespace, options.Recursive, options.Filenames...).
		Flatten().
		Do()
	err = r.Err()
	if err != nil {
		return err
	}

	encoder := f.JSONEncoder()
	decoder := f.Decoder(false)

	count := 0
	err = r.Visit(func(info *resource.Info, err error) error {
		// In this method, info.Object contains the object retrieved from the server
		// and info.VersionedObject contains the object decoded from the input source.
		if err != nil {
			return err
		}

		// Get the modified configuration of the object. Embed the result
		// as an annotation in the modified configuration, so that it will appear
		// in the patch sent to the server.
		modified, err := kubectl.GetModifiedConfiguration(info, true, encoder)
		if err != nil {
			return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving modified configuration from:\n%v\nfor:", info), info.Source, err)
		}

		if err := info.Get(); err != nil {
			if !errors.IsNotFound(err) {
				return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving current configuration of:\n%v\nfrom server for:", info), info.Source, err)
			}
			// Create the resource if it doesn't exist
			// First, update the annotation used by kubectl apply
			if err := kubectl.CreateApplyAnnotation(info, encoder); err != nil {
				return cmdutil.AddSourceToErr("creating", info.Source, err)
			}

			if cmdutil.ShouldRecord(cmd, info) {
				if err := cmdutil.RecordChangeCause(info.Object, f.Command()); err != nil {
					return cmdutil.AddSourceToErr("creating", info.Source, err)
				}
			}

			// Then create the resource and skip the three-way merge
			if err := createAndRefresh(info); err != nil {
				return cmdutil.AddSourceToErr("creating", info.Source, err)
			}
			count++
			cmdutil.PrintSuccess(mapper, shortOutput, out, info.Mapping.Resource, info.Name, "created")
			return nil
		}

		helper := resource.NewHelper(info.Client, info.Mapping)
		patcher := NewPatcher(encoder, decoder, info.Mapping, helper)

		patchBytes, err := patcher.patch(info.Object, modified, info.Source, info.Namespace, info.Name)
		if err != nil {
			return cmdutil.AddSourceToErr(fmt.Sprintf("applying patch:\n%s\nto:\n%v\nfor:", patchBytes, info), info.Source, err)
		}

		if cmdutil.ShouldRecord(cmd, info) {
			patch, err := cmdutil.ChangeResourcePatch(info, f.Command())
			if err != nil {
				return err
			}
			_, err = helper.Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch)
			if err != nil {
				return cmdutil.AddSourceToErr(fmt.Sprintf("applying patch:\n%s\nto:\n%v\nfor:", patch, info), info.Source, err)
			}
		}

		count++
		cmdutil.PrintSuccess(mapper, shortOutput, out, info.Mapping.Resource, info.Name, "configured")
		return nil
	})

	if err != nil {
		return err
	}

	if count == 0 {
		return fmt.Errorf("no objects passed to apply")
	}

	return nil
}

type patcher struct {
	encoder runtime.Encoder
	decoder runtime.Decoder

	mapping *meta.RESTMapping
	helper  *resource.Helper

	backOff clockwork.Clock
}

func NewPatcher(encoder runtime.Encoder, decoder runtime.Decoder, mapping *meta.RESTMapping, helper *resource.Helper) *patcher {
	return &patcher{
		encoder: encoder,
		decoder: decoder,
		mapping: mapping,
		helper:  helper,
		backOff: clockwork.NewRealClock(),
	}
}

func (p *patcher) patchSimple(obj runtime.Object, modified []byte, source, namespace, name string) ([]byte, error) {
	// Serialize the current configuration of the object from the server.
	current, err := runtime.Encode(p.encoder, obj)
	if err != nil {
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf("serializing current configuration from:\n%v\nfor:", obj), source, err)
	}

	// Retrieve the original configuration of the object from the annotation.
	original, err := kubectl.GetOriginalConfiguration(p.mapping, obj)
	if err != nil {
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf("retrieving original configuration from:\n%v\nfor:", obj), source, err)
	}

	// Create the versioned struct from the original from the server for
	// strategic patch.
	// TODO: Move all structs in apply to use raw data. Can be done once
	// builder has a RawResult method which delivers raw data instead of
	// internal objects.
	versionedObject, _, err := p.decoder.Decode(current, nil, nil)
	if err != nil {
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf("converting encoded server-side object back to versioned struct:\n%v\nfor:", obj), source, err)
	}

	// Compute a three way strategic merge patch to send to server.
	patch, err := strategicpatch.CreateThreeWayMergePatch(original, modified, current, versionedObject, true)
	if err != nil {
		format := "creating patch with:\noriginal:\n%s\nmodified:\n%s\ncurrent:\n%s\nfor:"
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf(format, original, modified, current), source, err)
	}

	_, err = p.helper.Patch(namespace, name, api.StrategicMergePatchType, patch)
	return patch, err
}

func (p *patcher) patch(current runtime.Object, modified []byte, source, namespace, name string) ([]byte, error) {
	var getErr error
	patchBytes, err := p.patchSimple(current, modified, source, namespace, name)
	for i := 1; i <= maxPatchRetry && errors.IsConflict(err); i++ {
		if i > triesBeforeBackOff {
			p.backOff.Sleep(backOffPeriod)
		}
		current, getErr = p.helper.Get(namespace, name, false)
		if getErr != nil {
			return nil, getErr
		}
		patchBytes, err = p.patchSimple(current, modified, source, namespace, name)
	}

	return patchBytes, err
}
<commit_msg>kubectl apply add --overwrite flag<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"time"

	"github.com/jonboulle/clockwork"
	"github.com/renstrom/dedent"
	"github.com/spf13/cobra"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/kubectl"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
	"k8s.io/kubernetes/pkg/kubectl/resource"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/strategicpatch"
)

// ApplyOptions stores cmd.Flag values for apply.  As new fields are added,
// add them here instead of referencing the cmd.Flags()
type ApplyOptions struct {
	Filenames []string
	Recursive bool
}

const (
	// maxPatchRetry is the maximum number of conflicts retry for during a patch operation before returning failure
	maxPatchRetry = 5
	// backOffPeriod is the period to back off when apply patch resutls in error.
	backOffPeriod = 1 * time.Second
	// how many times we can retry before back off
	triesBeforeBackOff = 1
)

var (
	apply_long = dedent.Dedent(`
		Apply a configuration to a resource by filename or stdin.
		This resource will be created if it doesn't exist yet.
		To use 'apply', always create the resource initially with either 'apply' or 'create --save-config'.

		JSON and YAML formats are accepted.`)

	apply_example = dedent.Dedent(`
		# Apply the configuration in pod.json to a pod.
		kubectl apply -f ./pod.json

		# Apply the JSON passed into stdin to a pod.
		cat pod.json | kubectl apply -f -`)
)

func NewCmdApply(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	options := &ApplyOptions{}

	cmd := &cobra.Command{
		Use:     "apply -f FILENAME",
		Short:   "Apply a configuration to a resource by filename or stdin",
		Long:    apply_long,
		Example: apply_example,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(validateArgs(cmd, args))
			cmdutil.CheckErr(cmdutil.ValidateOutputArgs(cmd))
			cmdutil.CheckErr(RunApply(f, cmd, out, options))
		},
	}

	usage := "Filename, directory, or URL to file that contains the configuration to apply"
	kubectl.AddJsonFilenameFlag(cmd, &options.Filenames, usage)
	cmd.MarkFlagRequired("filename")
	cmd.Flags().Bool("overwrite", true, "Automatically resolve conflicts between the modified and live configuration by using values from the modified configuration")
	cmdutil.AddValidateFlags(cmd)
	cmdutil.AddRecursiveFlag(cmd, &options.Recursive)
	cmdutil.AddOutputFlagsForMutation(cmd)
	cmdutil.AddRecordFlag(cmd)
	cmdutil.AddInclude3rdPartyFlags(cmd)
	return cmd
}

func validateArgs(cmd *cobra.Command, args []string) error {
	if len(args) != 0 {
		return cmdutil.UsageError(cmd, "Unexpected args: %v", args)
	}

	return nil
}

func RunApply(f *cmdutil.Factory, cmd *cobra.Command, out io.Writer, options *ApplyOptions) error {
	shortOutput := cmdutil.GetFlagString(cmd, "output") == "name"
	schema, err := f.Validator(cmdutil.GetFlagBool(cmd, "validate"), cmdutil.GetFlagString(cmd, "schema-cache-dir"))
	if err != nil {
		return err
	}

	cmdNamespace, enforceNamespace, err := f.DefaultNamespace()
	if err != nil {
		return err
	}

	mapper, typer := f.Object(cmdutil.GetIncludeThirdPartyAPIs(cmd))
	r := resource.NewBuilder(mapper, typer, resource.ClientMapperFunc(f.ClientForMapping), f.Decoder(true)).
		Schema(schema).
		ContinueOnError().
		NamespaceParam(cmdNamespace).DefaultNamespace().
		FilenameParam(enforceNamespace, options.Recursive, options.Filenames...).
		Flatten().
		Do()
	err = r.Err()
	if err != nil {
		return err
	}

	encoder := f.JSONEncoder()
	decoder := f.Decoder(false)

	count := 0
	err = r.Visit(func(info *resource.Info, err error) error {
		// In this method, info.Object contains the object retrieved from the server
		// and info.VersionedObject contains the object decoded from the input source.
		if err != nil {
			return err
		}

		// Get the modified configuration of the object. Embed the result
		// as an annotation in the modified configuration, so that it will appear
		// in the patch sent to the server.
		modified, err := kubectl.GetModifiedConfiguration(info, true, encoder)
		if err != nil {
			return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving modified configuration from:\n%v\nfor:", info), info.Source, err)
		}

		if err := info.Get(); err != nil {
			if !errors.IsNotFound(err) {
				return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving current configuration of:\n%v\nfrom server for:", info), info.Source, err)
			}
			// Create the resource if it doesn't exist
			// First, update the annotation used by kubectl apply
			if err := kubectl.CreateApplyAnnotation(info, encoder); err != nil {
				return cmdutil.AddSourceToErr("creating", info.Source, err)
			}

			if cmdutil.ShouldRecord(cmd, info) {
				if err := cmdutil.RecordChangeCause(info.Object, f.Command()); err != nil {
					return cmdutil.AddSourceToErr("creating", info.Source, err)
				}
			}

			// Then create the resource and skip the three-way merge
			if err := createAndRefresh(info); err != nil {
				return cmdutil.AddSourceToErr("creating", info.Source, err)
			}
			count++
			cmdutil.PrintSuccess(mapper, shortOutput, out, info.Mapping.Resource, info.Name, "created")
			return nil
		}

		overwrite := cmdutil.GetFlagBool(cmd, "overwrite")
		helper := resource.NewHelper(info.Client, info.Mapping)
		patcher := NewPatcher(encoder, decoder, info.Mapping, helper, overwrite)

		patchBytes, err := patcher.patch(info.Object, modified, info.Source, info.Namespace, info.Name)
		if err != nil {
			return cmdutil.AddSourceToErr(fmt.Sprintf("applying patch:\n%s\nto:\n%v\nfor:", patchBytes, info), info.Source, err)
		}

		if cmdutil.ShouldRecord(cmd, info) {
			patch, err := cmdutil.ChangeResourcePatch(info, f.Command())
			if err != nil {
				return err
			}
			_, err = helper.Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch)
			if err != nil {
				return cmdutil.AddSourceToErr(fmt.Sprintf("applying patch:\n%s\nto:\n%v\nfor:", patch, info), info.Source, err)
			}
		}

		count++
		cmdutil.PrintSuccess(mapper, shortOutput, out, info.Mapping.Resource, info.Name, "configured")
		return nil
	})

	if err != nil {
		return err
	}

	if count == 0 {
		return fmt.Errorf("no objects passed to apply")
	}

	return nil
}

type patcher struct {
	encoder runtime.Encoder
	decoder runtime.Decoder

	mapping *meta.RESTMapping
	helper  *resource.Helper

	overwrite bool
	backOff   clockwork.Clock
}

func NewPatcher(encoder runtime.Encoder, decoder runtime.Decoder, mapping *meta.RESTMapping, helper *resource.Helper, overwrite bool) *patcher {
	return &patcher{
		encoder:   encoder,
		decoder:   decoder,
		mapping:   mapping,
		helper:    helper,
		overwrite: overwrite,
		backOff:   clockwork.NewRealClock(),
	}
}

func (p *patcher) patchSimple(obj runtime.Object, modified []byte, source, namespace, name string) ([]byte, error) {
	// Serialize the current configuration of the object from the server.
	current, err := runtime.Encode(p.encoder, obj)
	if err != nil {
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf("serializing current configuration from:\n%v\nfor:", obj), source, err)
	}

	// Retrieve the original configuration of the object from the annotation.
	original, err := kubectl.GetOriginalConfiguration(p.mapping, obj)
	if err != nil {
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf("retrieving original configuration from:\n%v\nfor:", obj), source, err)
	}

	// Create the versioned struct from the original from the server for
	// strategic patch.
	// TODO: Move all structs in apply to use raw data. Can be done once
	// builder has a RawResult method which delivers raw data instead of
	// internal objects.
	versionedObject, _, err := p.decoder.Decode(current, nil, nil)
	if err != nil {
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf("converting encoded server-side object back to versioned struct:\n%v\nfor:", obj), source, err)
	}

	// Compute a three way strategic merge patch to send to server.
	patch, err := strategicpatch.CreateThreeWayMergePatch(original, modified, current, versionedObject, p.overwrite)
	if err != nil {
		format := "creating patch with:\noriginal:\n%s\nmodified:\n%s\ncurrent:\n%s\nfor:"
		return nil, cmdutil.AddSourceToErr(fmt.Sprintf(format, original, modified, current), source, err)
	}

	_, err = p.helper.Patch(namespace, name, api.StrategicMergePatchType, patch)
	return patch, err
}

func (p *patcher) patch(current runtime.Object, modified []byte, source, namespace, name string) ([]byte, error) {
	var getErr error
	patchBytes, err := p.patchSimple(current, modified, source, namespace, name)
	for i := 1; i <= maxPatchRetry && errors.IsConflict(err); i++ {
		if i > triesBeforeBackOff {
			p.backOff.Sleep(backOffPeriod)
		}
		current, getErr = p.helper.Get(namespace, name, false)
		if getErr != nil {
			return nil, getErr
		}
		patchBytes, err = p.patchSimple(current, modified, source, namespace, name)
	}

	return patchBytes, err
}
<|endoftext|>


<commit_before>// +build windows

/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package lock

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"
	"unsafe"
)

var (
	modkernel32    = syscall.NewLazyDLL("kernel32.dll")
	procLockFileEx = modkernel32.NewProc("LockFileEx")
)

const (
	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
	lockFileExclusiveLock   = 2
	lockFileFailImmediately = 1

	// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx
	errLockViolation syscall.Errno = 0x21
)

// lockedOpenFile is an internal function.
func lockedOpenFile(path string, flag int, perm os.FileMode, lockType uint32) (*LockedFile, error) {
	f, err := Open(path, flag, perm)
	if err != nil {
		return nil, err
	}

	if err = lockFile(syscall.Handle(f.Fd()), lockType); err != nil {
		f.Close()
		return nil, err
	}

	st, err := os.Stat(path)
	if err != nil {
		f.Close()
		return nil, err
	}

	if st.IsDir() {
		f.Close()
		return nil, &os.PathError{
			Op:   "open",
			Path: path,
			Err:  syscall.EISDIR,
		}
	}

	return &LockedFile{File: f}, nil
}

// TryLockedOpenFile - tries a new write lock, functionality
// it is similar to LockedOpenFile with with syscall.LOCK_EX
// mode but along with syscall.LOCK_NB such that the function
// doesn't wait forever but instead returns if it cannot
// acquire a write lock.
func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
	return lockedOpenFile(path, flag, perm, lockFileFailImmediately)
}

// LockedOpenFile - initializes a new lock and protects
// the file from concurrent access.
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
	return lockedOpenFile(path, flag, perm, 0)
}

// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path when needed, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), or is short enough, fixLongPath returns
// path unmodified.
//
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
func fixLongPath(path string) string {
	// Do nothing (and don't allocate) if the path is "short".
	// Empirically (at least on the Windows Server 2013 builder),
	// the kernel is arbitrarily okay with < 248 bytes. That
	// matches what the docs above say:
	// "When using an API to create a directory, the specified
	// path cannot be so long that you cannot append an 8.3 file
	// name (that is, the directory name cannot exceed MAX_PATH
	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
	//
	// The MSDN docs appear to say that a normal path that is 248 bytes long
	// will work; empirically the path must be less then 248 bytes long.
	if len(path) < 248 {
		// Don't fix. (This is how Go 1.7 and earlier worked,
		// not automatically generating the \\?\ form)
		return path
	}

	// The extended form begins with \\?\, as in
	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
	// The extended form disables evaluation of . and .. path
	// elements and disables the interpretation of / as equivalent
	// to \. The conversion here rewrites / to \ and elides
	// . elements as well as trailing or duplicate separators. For
	// simplicity it avoids the conversion entirely for relative
	// paths or paths containing .. elements. For now,
	// \\server\share paths are not converted to
	// \\?\UNC\server\share paths because the rules for doing so
	// are less well-specified.
	if len(path) >= 2 && path[:2] == `\\` {
		// Don't canonicalize UNC paths.
		return path
	}
	if !filepath.IsAbs(path) {
		// Relative path
		return path
	}

	const prefix = `\\?`

	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
	copy(pathbuf, prefix)
	n := len(path)
	r, w := 0, len(prefix)
	for r < n {
		switch {
		case os.IsPathSeparator(path[r]):
			// empty block
			r++
		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
			// /./
			r++
		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
			// /../ is currently unhandled
			return path
		default:
			pathbuf[w] = '\\'
			w++
			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
				pathbuf[w] = path[r]
				w++
			}
		}
	}
	// A drive's root directory needs a trailing \
	if w == len(`\\?\c:`) {
		pathbuf[w] = '\\'
		w++
	}
	return string(pathbuf[:w])
}

// perm param is ignored, on windows file perms/NT acls
// are not octet combinations. Providing access to NT
// acls is out of scope here.
func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
	if path == "" {
		return nil, syscall.ERROR_FILE_NOT_FOUND
	}

	pathp, err := syscall.UTF16PtrFromString(fixLongPath(path))
	if err != nil {
		return nil, err
	}

	var access uint32
	switch flag {
	case syscall.O_RDONLY:
		access = syscall.GENERIC_READ
	case syscall.O_WRONLY:
		access = syscall.GENERIC_WRITE
	case syscall.O_RDWR:
		fallthrough
	case syscall.O_RDWR | syscall.O_CREAT:
		fallthrough
	case syscall.O_WRONLY | syscall.O_CREAT:
		access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
	case syscall.O_WRONLY | syscall.O_CREAT | syscall.O_APPEND:
		access = syscall.FILE_APPEND_DATA
	default:
		return nil, fmt.Errorf("Unsupported flag (%d)", flag)
	}

	var createflag uint32
	switch {
	case flag&syscall.O_CREAT == syscall.O_CREAT:
		createflag = syscall.OPEN_ALWAYS
	default:
		createflag = syscall.OPEN_EXISTING
	}

	shareflag := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
	accessAttr := uint32(syscall.FILE_ATTRIBUTE_NORMAL | 0x80000000)

	fd, err := syscall.CreateFile(pathp, access, shareflag, nil, createflag, accessAttr, 0)
	if err != nil {
		return nil, err
	}

	return os.NewFile(uintptr(fd), path), nil
}

func lockFile(fd syscall.Handle, flags uint32) error {
	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
	var flag uint32 = lockFileExclusiveLock // Lockfile exlusive.
	flag |= flags

	if fd == syscall.InvalidHandle {
		return nil
	}

	err := lockFileEx(fd, flag, 1, 0, &syscall.Overlapped{})
	if err == nil {
		return nil
	} else if err.Error() == "The process cannot access the file because another process has locked a portion of the file." {
		return ErrAlreadyLocked
	} else if err != errLockViolation {
		return err
	}

	return nil
}

func lockFileEx(h syscall.Handle, flags, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
	var reserved = uint32(0)
	r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags),
		uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
	if r1 == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}
<commit_msg>for O_RDONLY mode hold shared locks on windows (#6454)<commit_after>// +build windows

/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package lock

import (
	"fmt"
	"os"
	"path/filepath"
	"syscall"
	"unsafe"
)

var (
	modkernel32    = syscall.NewLazyDLL("kernel32.dll")
	procLockFileEx = modkernel32.NewProc("LockFileEx")
)

const (
	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
	lockFileExclusiveLock   = 2
	lockFileFailImmediately = 1

	// see https://msdn.microsoft.com/en-us/library/windows/desktop/ms681382(v=vs.85).aspx
	errLockViolation syscall.Errno = 0x21
)

// lockedOpenFile is an internal function.
func lockedOpenFile(path string, flag int, perm os.FileMode, lockType uint32) (*LockedFile, error) {
	f, err := Open(path, flag, perm)
	if err != nil {
		return nil, err
	}

	if err = lockFile(syscall.Handle(f.Fd()), lockType); err != nil {
		f.Close()
		return nil, err
	}

	st, err := os.Stat(path)
	if err != nil {
		f.Close()
		return nil, err
	}

	if st.IsDir() {
		f.Close()
		return nil, &os.PathError{
			Op:   "open",
			Path: path,
			Err:  syscall.EISDIR,
		}
	}

	return &LockedFile{File: f}, nil
}

// TryLockedOpenFile - tries a new write lock, functionality
// it is similar to LockedOpenFile with with syscall.LOCK_EX
// mode but along with syscall.LOCK_NB such that the function
// doesn't wait forever but instead returns if it cannot
// acquire a write lock.
func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
	var lockType uint32 = lockFileFailImmediately | lockFileExclusiveLock
	switch flag {
	case syscall.O_RDONLY:
		// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-lockfileex
		lockType = lockFileFailImmediately | 0 // Set this to enable shared lock and fail immediately.
	}
	return lockedOpenFile(path, flag, perm, lockType)
}

// LockedOpenFile - initializes a new lock and protects
// the file from concurrent access.
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
	var lockType uint32 = lockFileExclusiveLock
	switch flag {
	case syscall.O_RDONLY:
		// https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-lockfileex
		lockType = 0 // Set this to enable shared lock.
	}
	return lockedOpenFile(path, flag, perm, lockType)
}

// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path when needed, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), or is short enough, fixLongPath returns
// path unmodified.
//
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
func fixLongPath(path string) string {
	// Do nothing (and don't allocate) if the path is "short".
	// Empirically (at least on the Windows Server 2013 builder),
	// the kernel is arbitrarily okay with < 248 bytes. That
	// matches what the docs above say:
	// "When using an API to create a directory, the specified
	// path cannot be so long that you cannot append an 8.3 file
	// name (that is, the directory name cannot exceed MAX_PATH
	// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
	//
	// The MSDN docs appear to say that a normal path that is 248 bytes long
	// will work; empirically the path must be less then 248 bytes long.
	if len(path) < 248 {
		// Don't fix. (This is how Go 1.7 and earlier worked,
		// not automatically generating the \\?\ form)
		return path
	}

	// The extended form begins with \\?\, as in
	// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
	// The extended form disables evaluation of . and .. path
	// elements and disables the interpretation of / as equivalent
	// to \. The conversion here rewrites / to \ and elides
	// . elements as well as trailing or duplicate separators. For
	// simplicity it avoids the conversion entirely for relative
	// paths or paths containing .. elements. For now,
	// \\server\share paths are not converted to
	// \\?\UNC\server\share paths because the rules for doing so
	// are less well-specified.
	if len(path) >= 2 && path[:2] == `\\` {
		// Don't canonicalize UNC paths.
		return path
	}
	if !filepath.IsAbs(path) {
		// Relative path
		return path
	}

	const prefix = `\\?`

	pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
	copy(pathbuf, prefix)
	n := len(path)
	r, w := 0, len(prefix)
	for r < n {
		switch {
		case os.IsPathSeparator(path[r]):
			// empty block
			r++
		case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
			// /./
			r++
		case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
			// /../ is currently unhandled
			return path
		default:
			pathbuf[w] = '\\'
			w++
			for ; r < n && !os.IsPathSeparator(path[r]); r++ {
				pathbuf[w] = path[r]
				w++
			}
		}
	}
	// A drive's root directory needs a trailing \
	if w == len(`\\?\c:`) {
		pathbuf[w] = '\\'
		w++
	}
	return string(pathbuf[:w])
}

// Open - perm param is ignored, on windows file perms/NT acls
// are not octet combinations. Providing access to NT
// acls is out of scope here.
func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
	if path == "" {
		return nil, syscall.ERROR_FILE_NOT_FOUND
	}

	pathp, err := syscall.UTF16PtrFromString(fixLongPath(path))
	if err != nil {
		return nil, err
	}

	var access uint32
	switch flag {
	case syscall.O_RDONLY:
		access = syscall.GENERIC_READ
	case syscall.O_WRONLY:
		access = syscall.GENERIC_WRITE
	case syscall.O_RDWR:
		fallthrough
	case syscall.O_RDWR | syscall.O_CREAT:
		fallthrough
	case syscall.O_WRONLY | syscall.O_CREAT:
		access = syscall.GENERIC_READ | syscall.GENERIC_WRITE
	case syscall.O_WRONLY | syscall.O_CREAT | syscall.O_APPEND:
		access = syscall.FILE_APPEND_DATA
	default:
		return nil, fmt.Errorf("Unsupported flag (%d)", flag)
	}

	var createflag uint32
	switch {
	case flag&syscall.O_CREAT == syscall.O_CREAT:
		createflag = syscall.OPEN_ALWAYS
	default:
		createflag = syscall.OPEN_EXISTING
	}

	shareflag := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
	accessAttr := uint32(syscall.FILE_ATTRIBUTE_NORMAL | 0x80000000)

	fd, err := syscall.CreateFile(pathp, access, shareflag, nil, createflag, accessAttr, 0)
	if err != nil {
		return nil, err
	}

	return os.NewFile(uintptr(fd), path), nil
}

func lockFile(fd syscall.Handle, flags uint32) error {
	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx
	if fd == syscall.InvalidHandle {
		return nil
	}

	err := lockFileEx(fd, flags, 1, 0, &syscall.Overlapped{})
	if err == nil {
		return nil
	} else if err.Error() == "The process cannot access the file because another process has locked a portion of the file." {
		return ErrAlreadyLocked
	} else if err != errLockViolation {
		return err
	}

	return nil
}

func lockFileEx(h syscall.Handle, flags, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
	var reserved = uint32(0)
	r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags),
		uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
	if r1 == 0 {
		if e1 != 0 {
			err = error(e1)
		} else {
			err = syscall.EINVAL
		}
	}
	return
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package trace

import (
	k8sConst "github.com/cilium/cilium/pkg/k8s/apis/cilium.io"
	"github.com/cilium/cilium/pkg/labels"
)

const (
	// DefaultNamespace represents the default Kubernetes namespace.
	DefaultNamespace = "default"
)

func generateLabels(namespace string, labelsMap map[string]string) []string {
	var labelsArr []string
	temp := namespace
	if temp == "" {
		temp = DefaultNamespace
	}
	labelsArr = append(labelsArr, labels.GenerateK8sLabelString(k8sConst.PodNamespaceLabel, temp))
	for k, v := range labelsMap {
		labelsArr = append(labelsArr, labels.GenerateK8sLabelString(k, v))
	}
	return labelsArr
}
<commit_msg>policy/trace: Remove redundant files<commit_after><|endoftext|>


<commit_before>package reference

import (
	"github.com/Dalee/hitman/pkg/registry"
	"fmt"
	"strings"
)

type (
	// interface to hitman/pkg/registry
	registryInterface interface {
		GetImageDigestList(repo string) (*registry.RepositoryDigestList, error)
	}

	// GarbageDetectItem holds information about repository, deployed tags and garbage digests
	GarbageDetectItem struct {
		Repository        string
		DeployedTagList   []string
		GarbageDigestList []string
		GarbageTagList    []string
	}

	// GarbageDetectInfo holds whole list of GarbageDetectItem
	GarbageDetectInfo struct {
		Items []*GarbageDetectItem
	}
)

// StringInSlice checks is given string present in slice of strings
func StringInSlice(s string, sl []string) bool {
	for _, item := range sl {
		if strings.Compare(item, s) == 0 {
			return true
		}
	}
	return false
}

// RemoveDuplicates removes duplicated strings in slice (in-place)
func RemoveDuplicates(xs *[]string) {
	found := make(map[string]bool)
	j := 0
	for i, x := range *xs {
		if !found[x] {
			found[x] = true
			(*xs)[j] = (*xs)[i]
			j++
		}
	}
	*xs = (*xs)[:j]
}

// SliceHasItemsInSlice is checks that any string in src slice contains in dst slice
func SliceHasItemsInSlice(src, dst []string) bool {
	for _, left := range src {
		for _, right := range dst {
			if strings.Compare(left, right) == 0 {
				return true
			}
		}
	}
	return false
}

// DetectGarbage will detect garbage for a given set of deployed image references
func DetectGarbage(k8sImageList []string, api registryInterface, ignoreMissing bool) (*GarbageDetectInfo, error) {
	// remove duplicated entries
	RemoveDuplicates(&k8sImageList)

	// prepare k8s deployed list
	deployedImages := make(map[string][]string, 0)
	for _, imageRefSpec := range k8sImageList {
		u, err := DecodeReference(imageRefSpec)
		if err != nil {
			return nil, err
		}

		deployedImages[u.Repository] =
			append(deployedImages[u.Repository], u.Tag)
	}

	// prepare registry registered list
	registryImages := make(map[string][]*registry.RepositoryDigest)
	for repositoryPath := range deployedImages {

		imageInfo, err := api.GetImageDigestList(repositoryPath)
		if err != nil {
			// FIXME: either image is missed in repository or call failed
			// FIXME: make it more clear, right now threat it as missing image
			if ignoreMissing == false {
				return nil, fmt.Errorf(fmt.Sprintf("Unknown image: %s", repositoryPath))
			}
			continue
		}

		// part of code, which is way to allow simulate situation
		// when registry answers something, but this something is
		// not image requested (actually should never happen in real life)
		found := true
		for _, digest := range imageInfo.Children {
			if strings.Compare(digest.Path, repositoryPath) != 0 {
				found = false
				break
			}
		}

		if found {
			registryImages[repositoryPath] =
				append(registryImages[repositoryPath], imageInfo.Children...)
		}
	}

	// build repoPath: [digest, digest]
	//garbageImages := make(map[string][]string)
	detectInfo := new(GarbageDetectInfo)

	for repositoryPath, deployedTagList := range deployedImages {
		detectItem := &GarbageDetectItem{
			Repository: repositoryPath,
			DeployedTagList: deployedTagList,
			GarbageDigestList: []string{},
		}

		detectInfo.Items = append(detectInfo.Items, detectItem)

		imageDigestList, ok := registryImages[repositoryPath];
		if !ok {
			if ignoreMissing == false {
				return nil, fmt.Errorf(fmt.Sprintf("Unknown image: %s", repositoryPath))
			}
			continue
		}

		for _, digest := range imageDigestList {
			if SliceHasItemsInSlice(deployedTagList, digest.TagList) == false {
				detectItem.GarbageDigestList =
					append(detectItem.GarbageDigestList, digest.Name)

				detectItem.GarbageTagList =
					append(detectItem.GarbageTagList, digest.TagList...)
			}
		}
	}

	return detectInfo, nil
}
<commit_msg>maintain order (do not use map as ordered list)<commit_after>package reference

import (
	"github.com/Dalee/hitman/pkg/registry"
	"fmt"
	"strings"
)

type (
	// interface to hitman/pkg/registry
	registryInterface interface {
		GetImageDigestList(repo string) (*registry.RepositoryDigestList, error)
	}

	// GarbageDetectItem holds information about repository, deployed tags and garbage digests
	GarbageDetectItem struct {
		Repository        string
		DeployedTagList   []string
		GarbageDigestList []string
		GarbageTagList    []string
	}

	// GarbageDetectInfo holds whole list of GarbageDetectItem
	GarbageDetectInfo struct {
		Items []*GarbageDetectItem
	}
)

// StringInSlice checks is given string present in slice of strings
func StringInSlice(s string, sl []string) bool {
	for _, item := range sl {
		if strings.Compare(item, s) == 0 {
			return true
		}
	}
	return false
}

// RemoveDuplicates removes duplicated strings in slice (in-place)
func RemoveDuplicates(xs *[]string) {
	found := make(map[string]bool)
	j := 0
	for i, x := range *xs {
		if !found[x] {
			found[x] = true
			(*xs)[j] = (*xs)[i]
			j++
		}
	}
	*xs = (*xs)[:j]
}

// SliceHasItemsInSlice is checks that any string in src slice contains in dst slice
func SliceHasItemsInSlice(src, dst []string) bool {
	for _, left := range src {
		for _, right := range dst {
			if strings.Compare(left, right) == 0 {
				return true
			}
		}
	}
	return false
}

// DetectGarbage will detect garbage for a given set of deployed image references
func DetectGarbage(k8sImageList []string, api registryInterface, ignoreMissing bool) (*GarbageDetectInfo, error) {
	// remove duplicated entries
	RemoveDuplicates(&k8sImageList)

	// prepare k8s deployed list and deployed repository list (to keep order, map order is not defined)
	deployedImages := make(map[string][]string, 0)
	deployedImagesList := make([]string, 0)

	for _, imageRefSpec := range k8sImageList {
		u, err := DecodeReference(imageRefSpec)
		if err != nil {
			return nil, err
		}

		deployedImages[u.Repository] =
			append(deployedImages[u.Repository], u.Tag)

		// if repository is not registered in orderList, register it
		if StringInSlice(u.Repository, deployedImagesList) == false {
			deployedImagesList =
				append(deployedImagesList, u.Repository)
		}
	}

	// prepare registry registered list
	registryImages := make(map[string][]*registry.RepositoryDigest)
	for repositoryPath := range deployedImages {

		imageInfo, err := api.GetImageDigestList(repositoryPath)
		if err != nil {
			// FIXME: either image is missed in repository or call failed
			// FIXME: make it more clear, right now threat it as missing image
			if ignoreMissing == false {
				return nil, fmt.Errorf(fmt.Sprintf("Unknown image: %s", repositoryPath))
			}
			continue
		}

		// part of code, which is way to allow simulate situation
		// when registry answers something, but this something is
		// not image requested (actually should never happen in real life)
		found := true
		for _, digest := range imageInfo.Children {
			if strings.Compare(digest.Path, repositoryPath) != 0 {
				found = false
				break
			}
		}

		if found {
			registryImages[repositoryPath] =
				append(registryImages[repositoryPath], imageInfo.Children...)
		}
	}

	// build repoPath: [digest, digest]
	//garbageImages := make(map[string][]string)
	detectInfo := new(GarbageDetectInfo)

	for _, repositoryPath := range deployedImagesList {
		deployedTagList := deployedImages[repositoryPath]
		detectItem := &GarbageDetectItem{
			Repository: repositoryPath,
			DeployedTagList: deployedTagList,
			GarbageDigestList: []string{},
		}

		detectInfo.Items = append(detectInfo.Items, detectItem)

		imageDigestList, ok := registryImages[repositoryPath]
		if !ok {
			if ignoreMissing == false {
				return nil, fmt.Errorf(fmt.Sprintf("Unknown image: %s", repositoryPath))
			}
			continue
		}

		for _, digest := range imageDigestList {
			if SliceHasItemsInSlice(deployedTagList, digest.TagList) == false {
				detectItem.GarbageDigestList =
					append(detectItem.GarbageDigestList, digest.Name)

				detectItem.GarbageTagList =
					append(detectItem.GarbageTagList, digest.TagList...)
			}
		}
	}

	return detectInfo, nil
}
<|endoftext|>