<commit_before>package vm

import (
	"fmt"
	"net"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	bslcommon "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/common"
	bslcstem "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/stemcell"
	datatypes "github.com/maximilien/softlayer-go/data_types"
	sl "github.com/maximilien/softlayer-go/softlayer"

	"github.com/cloudfoundry/bosh-softlayer-cpi/common"
)

const SOFTLAYER_VM_CREATOR_LOG_TAG = "SoftLayerVMCreator"

type softLayerVirtualGuestCreator struct {
	softLayerClient        sl.Client
	agentEnvServiceFactory AgentEnvServiceFactory

	agentOptions AgentOptions
	logger       boshlog.Logger
	vmFinder     Finder
}

func NewSoftLayerCreator(vmFinder Finder, softLayerClient sl.Client, agentOptions AgentOptions, logger boshlog.Logger) VMCreator {
	bslcommon.TIMEOUT = 120 * time.Minute
	bslcommon.POLLING_INTERVAL = 5 * time.Second

	return &softLayerVirtualGuestCreator{
		vmFinder:        vmFinder,
		softLayerClient: softLayerClient,
		agentOptions:    agentOptions,
		logger:          logger,
	}
}

func (c *softLayerVirtualGuestCreator) Create(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	for _, network := range networks {
		switch network.Type {
		case "dynamic":
			if len(network.IP) == 0 {
				return c.createBySoftlayer(agentID, stemcell, cloudProps, networks, env)
			} else {
				return c.createByOSReload(agentID, stemcell, cloudProps, networks, env)
			}
		case "manual":
			continue
		case "vip":
			return nil, bosherr.Error("SoftLayer Not Support VIP netowrk")
		default:
			return nil, bosherr.Errorf("Softlayer Not Support This Kind Of Network: %s", network.Type)
		}
	}

	return nil, nil
}

// Private methods
func (c *softLayerVirtualGuestCreator) createBySoftlayer(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	virtualGuestTemplate, err := CreateVirtualGuestTemplate(stemcell, cloudProps)
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuest template")
	}

	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	virtualGuest, err := virtualGuestService.CreateObject(virtualGuestTemplate)
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuest from SoftLayer client")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, virtualGuest.Id, "Service Setup")
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", virtualGuest.Id)
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, virtualGuest.Id, cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return nil, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", virtualGuest.Id))
		}
	}

	vm, found, err := c.vmFinder.Find(virtualGuest.Id)
	if err != nil || !found {
		return nil, bosherr.WrapErrorf(err, "Cannot find VirtualGuest with id: %d.", virtualGuest.Id)
	}

	if len(cloudProps.BoshIp) == 0 {
		UpdateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", vm.GetPrimaryBackendIP(), vm.GetFullyQualifiedDomainName()))
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, vm.GetPrimaryBackendIP())
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus

		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			UpdateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	vm.ConfigureNetworks2(networks)

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)

	err = vm.UpdateAgentEnv(agentEnv)
	if err != nil {
		return nil, bosherr.WrapError(err, "Updating VM's agent env")
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = vm.SetVcapPassword(c.agentOptions.VcapPassword)
		if err != nil {
			return nil, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}

	return vm, nil
}

func (c *softLayerVirtualGuestCreator) createByOSReload(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	var virtualGuest datatypes.SoftLayer_Virtual_Guest

	if common.IsPrivateSubnet(net.ParseIP(networks.First().IP)) {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryBackendIpAddress(networks.First().IP)
		c.logger.Info(SOFTLAYER_VM_CREATOR_LOG_TAG, fmt.Sprintf("OS reload on the server id %d with stemcell %d", virtualGuest.Id, stemcell.ID()))
	} else {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryIpAddress(networks.First().IP)
	}

	if err != nil || virtualGuest.Id == 0 {
		return nil, bosherr.WrapErrorf(err, "Could not find VirtualGuest by ip address: %s", networks.First().IP)
	}

	c.logger.Info(SOFTLAYER_VM_CREATOR_LOG_TAG, fmt.Sprintf("OS reload on VirtualGuest %d using stemcell %d", virtualGuest.Id, stemcell.ID()))

	vm, found, err := c.vmFinder.Find(virtualGuest.Id)
	if err != nil || !found {
		return nil, bosherr.WrapErrorf(err, "Cannot find virtualGuest with id: %d", virtualGuest.Id)
	}

	bslcommon.TIMEOUT = 4 * time.Hour
	err = vm.ReloadOS(stemcell)
	if err != nil {
		return nil, bosherr.WrapError(err, "Failed to reload OS")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, vm.ID(), "Service Setup")
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", vm.ID())
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, vm.ID(), cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return nil, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", vm.ID()))
		}
	}

	if len(cloudProps.BoshIp) == 0 {
		UpdateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", vm.GetPrimaryBackendIP(), vm.GetFullyQualifiedDomainName()))
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, vm.GetPrimaryBackendIP())
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus

		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			UpdateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	vm, found, err = c.vmFinder.Find(virtualGuest.Id)
	if err != nil || !found {
		return nil, bosherr.WrapErrorf(err, "refresh VM with id: %d after os_reload", virtualGuest.Id)
	}

	vm.ConfigureNetworks2(networks)

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)
	if err != nil {
		return nil, bosherr.WrapErrorf(err, "Cannot create agent env for virtual guest with id: %d", vm.ID())
	}

	err = vm.UpdateAgentEnv(agentEnv)
	if err != nil {
		return nil, bosherr.WrapError(err, "Updating VM's agent env")
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = vm.SetVcapPassword(c.agentOptions.VcapPassword)
		if err != nil {
			return nil, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}
	return vm, nil
}
<commit_msg>Changed network type from manual to static<commit_after>package vm

import (
	"fmt"
	"net"
	"time"

	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"

	bslcommon "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/common"
	bslcstem "github.com/cloudfoundry/bosh-softlayer-cpi/softlayer/stemcell"
	datatypes "github.com/maximilien/softlayer-go/data_types"
	sl "github.com/maximilien/softlayer-go/softlayer"

	"github.com/cloudfoundry/bosh-softlayer-cpi/common"
)

const SOFTLAYER_VM_CREATOR_LOG_TAG = "SoftLayerVMCreator"

type softLayerVirtualGuestCreator struct {
	softLayerClient        sl.Client
	agentEnvServiceFactory AgentEnvServiceFactory

	agentOptions AgentOptions
	logger       boshlog.Logger
	vmFinder     Finder
}

func NewSoftLayerCreator(vmFinder Finder, softLayerClient sl.Client, agentOptions AgentOptions, logger boshlog.Logger) VMCreator {
	bslcommon.TIMEOUT = 120 * time.Minute
	bslcommon.POLLING_INTERVAL = 5 * time.Second

	return &softLayerVirtualGuestCreator{
		vmFinder:        vmFinder,
		softLayerClient: softLayerClient,
		agentOptions:    agentOptions,
		logger:          logger,
	}
}

func (c *softLayerVirtualGuestCreator) Create(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	for _, network := range networks {
		switch network.Type {
		case "dynamic":
			if len(network.IP) == 0 {
				return c.createBySoftlayer(agentID, stemcell, cloudProps, networks, env)
			} else {
				return c.createByOSReload(agentID, stemcell, cloudProps, networks, env)
			}
		case "static":
			continue
		case "vip":
			return nil, bosherr.Error("SoftLayer Not Support VIP netowrk")
		default:
			return nil, bosherr.Errorf("Softlayer Not Support This Kind Of Network: %s", network.Type)
		}
	}

	return nil, nil
}

// Private methods
func (c *softLayerVirtualGuestCreator) createBySoftlayer(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	virtualGuestTemplate, err := CreateVirtualGuestTemplate(stemcell, cloudProps)
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuest template")
	}

	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	virtualGuest, err := virtualGuestService.CreateObject(virtualGuestTemplate)
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuest from SoftLayer client")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, virtualGuest.Id, "Service Setup")
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", virtualGuest.Id)
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, virtualGuest.Id, cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return nil, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", virtualGuest.Id))
		}
	}

	vm, found, err := c.vmFinder.Find(virtualGuest.Id)
	if err != nil || !found {
		return nil, bosherr.WrapErrorf(err, "Cannot find VirtualGuest with id: %d.", virtualGuest.Id)
	}

	if len(cloudProps.BoshIp) == 0 {
		UpdateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", vm.GetPrimaryBackendIP(), vm.GetFullyQualifiedDomainName()))
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, vm.GetPrimaryBackendIP())
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus

		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			UpdateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	vm.ConfigureNetworks2(networks)

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)

	err = vm.UpdateAgentEnv(agentEnv)
	if err != nil {
		return nil, bosherr.WrapError(err, "Updating VM's agent env")
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = vm.SetVcapPassword(c.agentOptions.VcapPassword)
		if err != nil {
			return nil, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}

	return vm, nil
}

func (c *softLayerVirtualGuestCreator) createByOSReload(agentID string, stemcell bslcstem.Stemcell, cloudProps VMCloudProperties, networks Networks, env Environment) (VM, error) {
	virtualGuestService, err := c.softLayerClient.GetSoftLayer_Virtual_Guest_Service()
	if err != nil {
		return nil, bosherr.WrapError(err, "Creating VirtualGuestService from SoftLayer client")
	}

	var virtualGuest datatypes.SoftLayer_Virtual_Guest

	if common.IsPrivateSubnet(net.ParseIP(networks.First().IP)) {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryBackendIpAddress(networks.First().IP)
		c.logger.Info(SOFTLAYER_VM_CREATOR_LOG_TAG, fmt.Sprintf("OS reload on the server id %d with stemcell %d", virtualGuest.Id, stemcell.ID()))
	} else {
		virtualGuest, err = virtualGuestService.GetObjectByPrimaryIpAddress(networks.First().IP)
	}

	if err != nil || virtualGuest.Id == 0 {
		return nil, bosherr.WrapErrorf(err, "Could not find VirtualGuest by ip address: %s", networks.First().IP)
	}

	c.logger.Info(SOFTLAYER_VM_CREATOR_LOG_TAG, fmt.Sprintf("OS reload on VirtualGuest %d using stemcell %d", virtualGuest.Id, stemcell.ID()))

	vm, found, err := c.vmFinder.Find(virtualGuest.Id)
	if err != nil || !found {
		return nil, bosherr.WrapErrorf(err, "Cannot find virtualGuest with id: %d", virtualGuest.Id)
	}

	bslcommon.TIMEOUT = 4 * time.Hour
	err = vm.ReloadOS(stemcell)
	if err != nil {
		return nil, bosherr.WrapError(err, "Failed to reload OS")
	}

	if cloudProps.EphemeralDiskSize == 0 {
		err = bslcommon.WaitForVirtualGuestLastCompleteTransaction(c.softLayerClient, vm.ID(), "Service Setup")
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Waiting for VirtualGuest `%d` has Service Setup transaction complete", vm.ID())
		}
	} else {
		err = bslcommon.AttachEphemeralDiskToVirtualGuest(c.softLayerClient, vm.ID(), cloudProps.EphemeralDiskSize, c.logger)
		if err != nil {
			return nil, bosherr.WrapError(err, fmt.Sprintf("Attaching ephemeral disk to VirtualGuest `%d`", vm.ID()))
		}
	}

	if len(cloudProps.BoshIp) == 0 {
		UpdateEtcHostsOfBoshInit(fmt.Sprintf("%s  %s", vm.GetPrimaryBackendIP(), vm.GetFullyQualifiedDomainName()))
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, vm.GetPrimaryBackendIP())
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus
	} else {
		mbus, err := ParseMbusURL(c.agentOptions.Mbus, cloudProps.BoshIp)
		if err != nil {
			return nil, bosherr.WrapErrorf(err, "Cannot construct mbus url.")
		}
		c.agentOptions.Mbus = mbus

		switch c.agentOptions.Blobstore.Provider {
		case BlobstoreTypeDav:
			davConf := DavConfig(c.agentOptions.Blobstore.Options)
			UpdateDavConfig(&davConf, cloudProps.BoshIp)
		}
	}

	vm, found, err = c.vmFinder.Find(virtualGuest.Id)
	if err != nil || !found {
		return nil, bosherr.WrapErrorf(err, "refresh VM with id: %d after os_reload", virtualGuest.Id)
	}

	vm.ConfigureNetworks2(networks)

	agentEnv := CreateAgentUserData(agentID, cloudProps, networks, env, c.agentOptions)
	if err != nil {
		return nil, bosherr.WrapErrorf(err, "Cannot create agent env for virtual guest with id: %d", vm.ID())
	}

	err = vm.UpdateAgentEnv(agentEnv)
	if err != nil {
		return nil, bosherr.WrapError(err, "Updating VM's agent env")
	}

	if len(c.agentOptions.VcapPassword) > 0 {
		err = vm.SetVcapPassword(c.agentOptions.VcapPassword)
		if err != nil {
			return nil, bosherr.WrapError(err, "Updating VM's vcap password")
		}
	}
	return vm, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"time"

	"github.com/shurcooL/go/u/u4"
	"github.com/shurcooL/trayhost"
)

func debugMenuItems() []trayhost.MenuItem {
	return []trayhost.MenuItem{
		trayhost.MenuItem{
			Title: "Debug: Get Clipboard Content",
			Handler: func() {
				cc, err := trayhost.GetClipboardContent()
				log.Printf("GetClipboardContent() error: %v\n", err)
				log.Printf("Text: %q\n", cc.Text)
				log.Printf("Image: %v len(%v)\n", cc.Image.Kind, len(cc.Image.Bytes))
				log.Printf("Files: len(%v) %v\n", len(cc.Files), cc.Files)
			},
		},
		trayhost.MenuItem{
			Title: "Debug: Set Clipboard Text",
			Handler: func() {
				trayhost.SetClipboardText("http://www.example.com/image.png")
			},
		},
		trayhost.MenuItem{
			Title: "Debug: Notification",
			Handler: func() {
				handler := func() {
					u4.Open("http://www.example.com/image.png")
				}
				notification := trayhost.Notification{Title: "Success", Body: "http://www.example.com/image.png", Timeout: 3 * time.Second, Handler: handler}
				//trayhost.Notification{Title: "Upload Failed", Body: "error description goes here"}.Display()
				if cc, err := trayhost.GetClipboardContent(); err == nil && cc.Image.Kind != "" {
					notification.Image = cc.Image
				}
				notification.Display()
			},
		},
	}
}
<commit_msg>client: Apply gofmt -s to new file.<commit_after>package main

import (
	"log"
	"time"

	"github.com/shurcooL/go/u/u4"
	"github.com/shurcooL/trayhost"
)

func debugMenuItems() []trayhost.MenuItem {
	return []trayhost.MenuItem{
		{
			Title: "Debug: Get Clipboard Content",
			Handler: func() {
				cc, err := trayhost.GetClipboardContent()
				log.Printf("GetClipboardContent() error: %v\n", err)
				log.Printf("Text: %q\n", cc.Text)
				log.Printf("Image: %v len(%v)\n", cc.Image.Kind, len(cc.Image.Bytes))
				log.Printf("Files: len(%v) %v\n", len(cc.Files), cc.Files)
			},
		},
		{
			Title: "Debug: Set Clipboard Text",
			Handler: func() {
				trayhost.SetClipboardText("http://www.example.com/image.png")
			},
		},
		{
			Title: "Debug: Notification",
			Handler: func() {
				handler := func() {
					u4.Open("http://www.example.com/image.png")
				}
				notification := trayhost.Notification{Title: "Success", Body: "http://www.example.com/image.png", Timeout: 3 * time.Second, Handler: handler}
				//trayhost.Notification{Title: "Upload Failed", Body: "error description goes here"}.Display()
				if cc, err := trayhost.GetClipboardContent(); err == nil && cc.Image.Kind != "" {
					notification.Image = cc.Image
				}
				notification.Display()
			},
		},
	}
}
<|endoftext|>


<commit_before>// Package files is for storing files on the cozy, including binary ones like
// photos and movies. The range of possible operations with this endpoint goes
// from simple ones, like uploading a file, to more complex ones, like renaming
// a folder. It also ensure that an instance is not exceeding its quota, and
// keeps a trash to recover files recently deleted.
package files

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/gin-gonic/gin"
	"github.com/spf13/afero"
)

// DefaultContentType is used for files uploaded with no content-type
const DefaultContentType = "application/octet-stream"

type FileMetadata struct {
	DocType    string
	Name       string
	FolderID   string
	Executable bool
	Tags       []string
}

func (metadata *FileMetadata) Path() string {
	return escapeSlash(metadata.FolderID) + "/" + escapeSlash(metadata.Name)
}

func escapeSlash(str string) string {
	return strings.Replace(str, "/", "\\/", -1)
}

func extractTags(str string) []string {
	tags := make([]string, 0)
	for _, tag := range strings.Split(str, ",") {
		// @TODO: more sanitization maybe ?
		tag = strings.TrimSpace(tag)
		if tag != "" {
			tags = append(tags, tag)
		}
	}
	return tags
}

func checkParentFolderID(storage afero.Fs, folderID string) (bool, error) {
	if folderID == "" {
		return true, nil
	}

	exists, err := afero.DirExists(storage, folderID)
	if err != nil {
		return false, err
	}

	if !exists {
		return false, nil
	}

	return true, nil
}

// Upload is the method for uploading a file
//
// This will be used to upload a file
// @TODO(bruno): wip
func Upload(metadata *FileMetadata, storage afero.Fs, body io.ReadCloser) error {
	path := metadata.Path()

	defer body.Close()
	return afero.SafeWriteReader(storage, path, body)
}

// CreateDirectory is the method for creating a new directory
//
// @TODO(pierre): wip
func CreateDirectory(metadata *FileMetadata, storage afero.Fs) error {
	path := metadata.Path()

	exists, err := afero.DirExists(storage, path)
	if err != nil {
		return err
	}
	if exists {
		return fmt.Errorf("Directory already exists")
	}

	return storage.Mkdir(path, 0777)
}

// Handle all POST requests on /:folder-id and given the Type
// parameter of the request, it will either upload a new file or
// create a new directory.
//
// swagger:route POST /files/:folder-id files uploadFileOrCreateDir
func FolderPostHandler(c *gin.Context) {
	instanceInterface, exists := c.Get("instance")
	if !exists {
		err := fmt.Errorf("No instance found")
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	instance := instanceInterface.(*middlewares.Instance)
	storage, err := instance.GetStorageProvider()
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	tags := extractTags(c.Query("Tags"))

	metadata := &FileMetadata{
		Name:       c.Query("Name"),
		DocType:    c.Query("Type"),
		Executable: c.Query("Executable") == "true",
		FolderID:   c.Param("folder-id"),
		Tags:       tags,
	}

	contentType := c.ContentType()
	if contentType == "" {
		contentType = DefaultContentType
	}

	if metadata.Name == "" {
		err := fmt.Errorf("Missing Name in the query-string")
		c.AbortWithError(http.StatusUnprocessableEntity, err)
		return
	}

	exists, err = checkParentFolderID(storage, metadata.FolderID)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	if !exists {
		err := fmt.Errorf("Parent folder with given FolderID does not exist")
		c.AbortWithError(http.StatusNotFound, err)
		return
	}

	fmt.Printf("%s:\n\t- %+v\n\t- %v\n", metadata, contentType)

	switch metadata.DocType {
	case "io.cozy.files":
		err = Upload(metadata, storage, c.Request.Body)
	case "io.cozy.folders":
		err = CreateDirectory(metadata, storage)
	default:
		err = fmt.Errorf("Invalid Type in the query-string")
		c.AbortWithError(http.StatusUnprocessableEntity, err)
		return
	}

	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	data := []byte{'O', 'K'}
	c.Data(http.StatusCreated, jsonapi.ContentType, data)
}

// Routes sets the routing for the files service
func Routes(router *gin.RouterGroup) {
	router.POST("/:folder-id", FolderPostHandler)
}
<commit_msg>Linter a little happier<commit_after>// Package files is for storing files on the cozy, including binary ones like
// photos and movies. The range of possible operations with this endpoint goes
// from simple ones, like uploading a file, to more complex ones, like renaming
// a folder. It also ensure that an instance is not exceeding its quota, and
// keeps a trash to recover files recently deleted.
package files

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/cozy/cozy-stack/web/jsonapi"
	"github.com/cozy/cozy-stack/web/middlewares"
	"github.com/gin-gonic/gin"
	"github.com/spf13/afero"
)

// DefaultContentType is used for files uploaded with no content-type
const DefaultContentType = "application/octet-stream"

type fileMetadata struct {
	DocType    string
	Name       string
	FolderID   string
	Executable bool
	Tags       []string
}

func (metadata *fileMetadata) Path() string {
	return escapeSlash(metadata.FolderID) + "/" + escapeSlash(metadata.Name)
}

func escapeSlash(str string) string {
	return strings.Replace(str, "/", "\\/", -1)
}

func extractTags(str string) []string {
	var tags []string
	for _, tag := range strings.Split(str, ",") {
		// @TODO: more sanitization maybe ?
		tag = strings.TrimSpace(tag)
		if tag != "" {
			tags = append(tags, tag)
		}
	}
	return tags
}

func checkParentFolderID(storage afero.Fs, folderID string) (bool, error) {
	if folderID == "" {
		return true, nil
	}

	exists, err := afero.DirExists(storage, folderID)
	if err != nil {
		return false, err
	}

	if !exists {
		return false, nil
	}

	return true, nil
}

// Upload is the method for uploading a file
//
// This will be used to upload a file
// @TODO(bruno): wip
func Upload(metadata *fileMetadata, storage afero.Fs, body io.ReadCloser) error {
	path := metadata.Path()

	defer body.Close()
	return afero.SafeWriteReader(storage, path, body)
}

// CreateDirectory is the method for creating a new directory
//
// @TODO(pierre): wip
func CreateDirectory(metadata *fileMetadata, storage afero.Fs) error {
	path := metadata.Path()

	exists, err := afero.DirExists(storage, path)
	if err != nil {
		return err
	}
	if exists {
		return fmt.Errorf("Directory already exists")
	}

	return storage.Mkdir(path, 0777)
}

// FolderPostHandler Handle all POST requests on /:folder-id and given
// the Type parameter of the request, it will either upload a new file
// or create a new directory.
//
// swagger:route POST /files/:folder-id files uploadFileOrCreateDir
func FolderPostHandler(c *gin.Context) {
	instanceInterface, exists := c.Get("instance")
	if !exists {
		err := fmt.Errorf("No instance found")
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	instance := instanceInterface.(*middlewares.Instance)
	storage, err := instance.GetStorageProvider()
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	tags := extractTags(c.Query("Tags"))

	metadata := &fileMetadata{
		Name:       c.Query("Name"),
		DocType:    c.Query("Type"),
		Executable: c.Query("Executable") == "true",
		FolderID:   c.Param("folder-id"),
		Tags:       tags,
	}

	contentType := c.ContentType()
	if contentType == "" {
		contentType = DefaultContentType
	}

	if metadata.Name == "" {
		err = fmt.Errorf("Missing Name in the query-string")
		c.AbortWithError(http.StatusUnprocessableEntity, err)
		return
	}

	exists, err = checkParentFolderID(storage, metadata.FolderID)
	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}
	if !exists {
		err = fmt.Errorf("Parent folder with given FolderID does not exist")
		c.AbortWithError(http.StatusNotFound, err)
		return
	}

	fmt.Printf("%s:\n\t- %+v\n\t- %v\n", metadata.DocType, metadata, contentType)

	switch metadata.DocType {
	case "io.cozy.files":
		err = Upload(metadata, storage, c.Request.Body)
	case "io.cozy.folders":
		err = CreateDirectory(metadata, storage)
	default:
		err = fmt.Errorf("Invalid Type in the query-string")
		c.AbortWithError(http.StatusUnprocessableEntity, err)
		return
	}

	if err != nil {
		c.AbortWithError(http.StatusInternalServerError, err)
		return
	}

	data := []byte{'O', 'K'}
	c.Data(http.StatusCreated, jsonapi.ContentType, data)
}

// Routes sets the routing for the files service
func Routes(router *gin.RouterGroup) {
	router.POST("/:folder-id", FolderPostHandler)
}
<|endoftext|>


<commit_before>package join

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/etcd/store"
	"github.com/davecgh/go-spew/spew"
	"github.com/golang/glog"
	"github.com/sttts/elastic-etcd/node"
	"golang.org/x/net/context/ctxhttp"
)

// Strategy describes the member add strategy.
type Strategy string

const (
	livenessTimeout  = time.Second * 5
	etcdTimeout      = time.Second * 5
	discoveryTimeout = time.Second * 10

	// PreparedStrategy assumes that the admin prepares new member entries.
	PreparedStrategy = Strategy("prepared")

	// PruneStrategy aggressively removes dead members.
	PruneStrategy = Strategy("prune")

	// ReplaceStrategy defensively removes a dead member only when a cluster is full.
	ReplaceStrategy = Strategy("replace")

	// AddStrategy only adds a member until the cluster is full, never removes old members.
	AddStrategy = Strategy("add")

	maxUint = ^uint(0)
	maxInt  = int(maxUint >> 1)
)

// EtcdConfig is the result of the join algorithm, turned into etcd flags or env vars.
type EtcdConfig struct {
	InitialCluster      []string
	InitialClusterState string
	AdvertisePeerURLs   string
	Discovery           string
	Name                string
}

func alive(ctx context.Context, m client.Member) bool {
	ctx, _ = context.WithTimeout(ctx, livenessTimeout)
	glog.V(6).Infof("Testing liveness of %s=%v", m.Name, m.PeerURLs)
	for _, u := range m.PeerURLs {
		resp, err := ctxhttp.Get(ctx, http.DefaultClient, u+rafthttp.ProbingPrefix)
		if err == nil && resp.StatusCode == http.StatusOK {
			return true
		}
	}

	return false
}

func active(ctx context.Context, m client.Member) (bool, error) {
	ctx, _ = context.WithTimeout(ctx, etcdTimeout)

	c, err := client.New(client.Config{
		Endpoints:               m.ClientURLs,
		Transport:               client.DefaultTransport,
		HeaderTimeoutPerRequest: 5 * time.Second,
	})
	if err != nil {
		return false, err
	}
	mapi := client.NewMembersAPI(c)
	glog.V(6).Infof("Testing whether %s=%v knows the leader", m.Name, m.PeerURLs)
	leader, err := mapi.Leader(ctx)
	if err != nil {
		return false, err
	}
	return leader != nil, nil
}

func clusterExistingHeuristic(
	ctx context.Context,
	size int, nodes []node.DiscoveryNode,
) ([]node.DiscoveryNode, error) {
	quorum := size/2 + 1

	if nodes == nil {
		glog.V(4).Infof("No nodes found in discovery service. Assuming new cluster.")
		return nil, nil
	}

	wg := sync.WaitGroup{}
	wg.Add(len(nodes))
	activeNodes := make([]node.DiscoveryNode, 0, len(nodes))
	for _, n := range nodes {
		go func(n node.DiscoveryNode) {
			defer wg.Done()
			if !alive(ctx, n.Member) {
				glog.Infof("Node %s looks dead", n.NamedPeerURLs())
				return
			}
			if ok, err := active(ctx, n.Member); !ok {
				if err != nil {
					glog.Error(err)
				}
				glog.Infof("Node %s is not in a healthy cluster.", n.NamedPeerURLs())
				return
			}
			glog.Infof("Node %s looks alive and active in a cluster", n.NamedPeerURLs())
			activeNodes = append(activeNodes, n)
		}(n)
	}
	wg.Wait()

	if len(nodes) < quorum {
		glog.V(4).Infof(
			"Only %d nodes found in discovery service, less than a quorum of %d. Assuming new cluster.",
			len(nodes),
			quorum,
		)
		return nil, nil
	}

	if len(nodes) == size {
		glog.V(4).Infof("Cluster is full. Assuming existing cluster.")
		return activeNodes, nil
	}

	if len(activeNodes) > 0 {
		return activeNodes, nil
	}

	return nil, nil
}

func discoveryValue(ctx context.Context, baseURL, key string) (*store.Event, error) {
	ctx, _ = context.WithTimeout(ctx, discoveryTimeout)

	url := baseURL + key
	glog.V(6).Infof("Getting %s", url)
	resp, err := ctxhttp.Get(ctx, http.DefaultClient, url)
	if err != nil {
		return nil, err
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("status code %d from %q: %s", resp.StatusCode, url, body)
	}

	var res store.Event
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return nil, fmt.Errorf("invalid answer from %q: %v", url, err)
	}

	glog.V(9).Infof("Got: %s", spew.Sdump(res))

	return &res, nil
}

func deleteDiscoveryMachine(ctx context.Context, baseURL, id string) (bool, error) {
	ctx, _ = context.WithTimeout(ctx, discoveryTimeout)

	url := baseURL + "/" + strings.TrimLeft(id, "/")
	req, err := http.NewRequest("DELETE", url, strings.NewReader(""))
	if err != nil {
		return false, err
	}
	resp, err := ctxhttp.Do(ctx, http.DefaultClient, req)
	if err != nil {
		return false, err
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode == http.StatusNotFound {
		return false, nil
	}
	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		return false, fmt.Errorf("status code %d on DELETE for %q: %s", resp.StatusCode, url, body)
	}

	return true, nil
}

// Join adds a new member depending on the strategy and returns a matching etcd configuration.
func Join(
	discoveryURL, name, initialAdvertisePeerURLs string,
	fresh bool,
	clientPort, clusterSize int,
	strategy Strategy,
) (*EtcdConfig, error) {
	ctx := context.Background()

	res, err := discoveryValue(ctx, discoveryURL, "/")
	if err != nil {
		return nil, err
	}
	nodes := make([]node.DiscoveryNode, 0, len(res.Node.Nodes))
	for _, nn := range res.Node.Nodes {
		if nn.Value == nil {
			glog.V(5).Infof("Skipping %q because no value exists", nn.Key)
		}
		n, err := node.NewDiscoveryNode(*nn.Value, clientPort)
		if err != nil {
			glog.Warningf("invalid peer url %q in discovery service: %v", *nn.Value, err)
			continue
		}
		nodes = append(nodes, *n)
	}

	if clusterSize < 0 {
		res, err = discoveryValue(ctx, discoveryURL, "/_config/size")
		if err != nil {
			return nil, fmt.Errorf("cannot get discovery url cluster size: %v", err)
		}

		size, _ := strconv.ParseInt(*res.Node.Value, 10, 16)
		clusterSize = int(size)

		glog.V(2).Infof("Got a target cluster size of %d from the discovery url", clusterSize)
	} else if clusterSize == 0 {
		clusterSize = maxInt
	}

	activeNodes, err := clusterExistingHeuristic(ctx, clusterSize, nodes)
	if err != nil {
		return nil, err
	}
	if activeNodes != nil && len(activeNodes) == 0 {
		// cluster down. Restarting nodes with the same config.

		if fresh {
			return nil, errors.New("Cluster is down. A new node cannot join now.")
		}

		return &EtcdConfig{
			InitialClusterState: "existing",
			AdvertisePeerURLs:   initialAdvertisePeerURLs,
			Name:                name,
		}, nil
	} else if activeNodes != nil {
		activeNamedURLs := make([]string, 0, len(nodes))
		for _, n := range activeNodes {
			activeNamedURLs = append(activeNamedURLs, n.NamedPeerURLs()...)
		}

		advertisedURLs := strings.Split(initialAdvertisePeerURLs, ",")

		advertisedNamedURLs := make([]string, 0, len(initialAdvertisePeerURLs))
		for _, u := range advertisedURLs {
			advertisedNamedURLs = append(advertisedNamedURLs, fmt.Sprintf("%s=%s", name, u))
		}

		initialNamedURLs := []string{advertisedNamedURLs[0]}
		if strategy != PreparedStrategy && fresh {
			adder, err := newMemberAdder(
				activeNodes,
				strategy,
				clientPort,
				clusterSize,
				discoveryURL,
			)
			if err != nil {
				return nil, err
			}
			initialURLs, err := adder.Add(ctx, name, advertisedURLs)
			if err != nil {
				return nil, fmt.Errorf("unable to add node %q with peer urls %q to the cluster: %v", name, initialAdvertisePeerURLs, err)
			}

			initialNamedURLs = []string{}
			for _, u := range initialURLs {
				initialNamedURLs = append(initialNamedURLs, fmt.Sprintf("%s=%s", name, u))
			}
		}

		return &EtcdConfig{
			InitialCluster:      append(initialNamedURLs, activeNamedURLs...),
			InitialClusterState: "existing",
			AdvertisePeerURLs:   initialAdvertisePeerURLs,
			Name:                name,
		}, nil
	} else {
		return &EtcdConfig{
			InitialClusterState: "new",
			Discovery:           discoveryURL,
			AdvertisePeerURLs:   initialAdvertisePeerURLs,
			Name:                name,
		}, nil
	}
}
<commit_msg>Add locking<commit_after>package join

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/coreos/etcd/client"
	"github.com/coreos/etcd/rafthttp"
	"github.com/coreos/etcd/store"
	"github.com/davecgh/go-spew/spew"
	"github.com/golang/glog"
	"github.com/sttts/elastic-etcd/node"
	"golang.org/x/net/context/ctxhttp"
)

// Strategy describes the member add strategy.
type Strategy string

const (
	livenessTimeout  = time.Second * 5
	etcdTimeout      = time.Second * 5
	discoveryTimeout = time.Second * 10

	// PreparedStrategy assumes that the admin prepares new member entries.
	PreparedStrategy = Strategy("prepared")

	// PruneStrategy aggressively removes dead members.
	PruneStrategy = Strategy("prune")

	// ReplaceStrategy defensively removes a dead member only when a cluster is full.
	ReplaceStrategy = Strategy("replace")

	// AddStrategy only adds a member until the cluster is full, never removes old members.
	AddStrategy = Strategy("add")

	maxUint = ^uint(0)
	maxInt  = int(maxUint >> 1)
)

// EtcdConfig is the result of the join algorithm, turned into etcd flags or env vars.
type EtcdConfig struct {
	InitialCluster      []string
	InitialClusterState string
	AdvertisePeerURLs   string
	Discovery           string
	Name                string
}

func alive(ctx context.Context, m client.Member) bool {
	ctx, _ = context.WithTimeout(ctx, livenessTimeout)
	glog.V(6).Infof("Testing liveness of %s=%v", m.Name, m.PeerURLs)
	for _, u := range m.PeerURLs {
		resp, err := ctxhttp.Get(ctx, http.DefaultClient, u+rafthttp.ProbingPrefix)
		if err == nil && resp.StatusCode == http.StatusOK {
			return true
		}
	}

	return false
}

func active(ctx context.Context, m client.Member) (bool, error) {
	ctx, _ = context.WithTimeout(ctx, etcdTimeout)

	c, err := client.New(client.Config{
		Endpoints:               m.ClientURLs,
		Transport:               client.DefaultTransport,
		HeaderTimeoutPerRequest: 5 * time.Second,
	})
	if err != nil {
		return false, err
	}
	mapi := client.NewMembersAPI(c)
	glog.V(6).Infof("Testing whether %s=%v knows the leader", m.Name, m.PeerURLs)
	leader, err := mapi.Leader(ctx)
	if err != nil {
		return false, err
	}
	return leader != nil, nil
}

func clusterExistingHeuristic(
	ctx context.Context,
	size int, nodes []node.DiscoveryNode,
) ([]node.DiscoveryNode, error) {
	quorum := size/2 + 1

	if nodes == nil {
		glog.V(4).Infof("No nodes found in discovery service. Assuming new cluster.")
		return nil, nil
	}

	wg := sync.WaitGroup{}
	wg.Add(len(nodes))
	lock := sync.Mutex{}
	activeNodes := make([]node.DiscoveryNode, 0, len(nodes))
	for _, n := range nodes {
		go func(n node.DiscoveryNode) {
			defer wg.Done()
			if !alive(ctx, n.Member) {
				glog.Infof("Node %s looks dead", n.NamedPeerURLs())
				return
			}
			if ok, err := active(ctx, n.Member); !ok {
				if err != nil {
					glog.Error(err)
				}
				glog.Infof("Node %s is not in a healthy cluster.", n.NamedPeerURLs())
				return
			}
			glog.Infof("Node %s looks alive and active in a cluster", n.NamedPeerURLs())
			lock.Lock()
			defer lock.Unlock()
			activeNodes = append(activeNodes, n)
		}(n)
	}
	wg.Wait()

	if len(nodes) < quorum {
		glog.V(4).Infof(
			"Only %d nodes found in discovery service, less than a quorum of %d. Assuming new cluster.",
			len(nodes),
			quorum,
		)
		return nil, nil
	}

	if len(nodes) == size {
		glog.V(4).Infof("Cluster is full. Assuming existing cluster.")
		return activeNodes, nil
	}

	if len(activeNodes) > 0 {
		return activeNodes, nil
	}

	return nil, nil
}

func discoveryValue(ctx context.Context, baseURL, key string) (*store.Event, error) {
	ctx, _ = context.WithTimeout(ctx, discoveryTimeout)

	url := baseURL + key
	glog.V(6).Infof("Getting %s", url)
	resp, err := ctxhttp.Get(ctx, http.DefaultClient, url)
	if err != nil {
		return nil, err
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("status code %d from %q: %s", resp.StatusCode, url, body)
	}

	var res store.Event
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return nil, fmt.Errorf("invalid answer from %q: %v", url, err)
	}

	glog.V(9).Infof("Got: %s", spew.Sdump(res))

	return &res, nil
}

func deleteDiscoveryMachine(ctx context.Context, baseURL, id string) (bool, error) {
	ctx, _ = context.WithTimeout(ctx, discoveryTimeout)

	url := baseURL + "/" + strings.TrimLeft(id, "/")
	req, err := http.NewRequest("DELETE", url, strings.NewReader(""))
	if err != nil {
		return false, err
	}
	resp, err := ctxhttp.Do(ctx, http.DefaultClient, req)
	if err != nil {
		return false, err
	}
	defer func() { _ = resp.Body.Close() }()
	if resp.StatusCode == http.StatusNotFound {
		return false, nil
	}
	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		return false, fmt.Errorf("status code %d on DELETE for %q: %s", resp.StatusCode, url, body)
	}

	return true, nil
}

// Join adds a new member depending on the strategy and returns a matching etcd configuration.
func Join(
	discoveryURL, name, initialAdvertisePeerURLs string,
	fresh bool,
	clientPort, clusterSize int,
	strategy Strategy,
) (*EtcdConfig, error) {
	ctx := context.Background()

	res, err := discoveryValue(ctx, discoveryURL, "/")
	if err != nil {
		return nil, err
	}
	nodes := make([]node.DiscoveryNode, 0, len(res.Node.Nodes))
	for _, nn := range res.Node.Nodes {
		if nn.Value == nil {
			glog.V(5).Infof("Skipping %q because no value exists", nn.Key)
		}
		n, err := node.NewDiscoveryNode(*nn.Value, clientPort)
		if err != nil {
			glog.Warningf("invalid peer url %q in discovery service: %v", *nn.Value, err)
			continue
		}
		nodes = append(nodes, *n)
	}

	if clusterSize < 0 {
		res, err = discoveryValue(ctx, discoveryURL, "/_config/size")
		if err != nil {
			return nil, fmt.Errorf("cannot get discovery url cluster size: %v", err)
		}

		size, _ := strconv.ParseInt(*res.Node.Value, 10, 16)
		clusterSize = int(size)

		glog.V(2).Infof("Got a target cluster size of %d from the discovery url", clusterSize)
	} else if clusterSize == 0 {
		clusterSize = maxInt
	}

	activeNodes, err := clusterExistingHeuristic(ctx, clusterSize, nodes)
	if err != nil {
		return nil, err
	}
	if activeNodes != nil && len(activeNodes) == 0 {
		// cluster down. Restarting nodes with the same config.

		if fresh {
			return nil, errors.New("Cluster is down. A new node cannot join now.")
		}

		return &EtcdConfig{
			InitialClusterState: "existing",
			AdvertisePeerURLs:   initialAdvertisePeerURLs,
			Name:                name,
		}, nil
	} else if activeNodes != nil {
		activeNamedURLs := make([]string, 0, len(nodes))
		for _, n := range activeNodes {
			activeNamedURLs = append(activeNamedURLs, n.NamedPeerURLs()...)
		}

		advertisedURLs := strings.Split(initialAdvertisePeerURLs, ",")

		advertisedNamedURLs := make([]string, 0, len(initialAdvertisePeerURLs))
		for _, u := range advertisedURLs {
			advertisedNamedURLs = append(advertisedNamedURLs, fmt.Sprintf("%s=%s", name, u))
		}

		initialNamedURLs := []string{advertisedNamedURLs[0]}
		if strategy != PreparedStrategy && fresh {
			adder, err := newMemberAdder(
				activeNodes,
				strategy,
				clientPort,
				clusterSize,
				discoveryURL,
			)
			if err != nil {
				return nil, err
			}
			initialURLs, err := adder.Add(ctx, name, advertisedURLs)
			if err != nil {
				return nil, fmt.Errorf("unable to add node %q with peer urls %q to the cluster: %v", name, initialAdvertisePeerURLs, err)
			}

			initialNamedURLs = []string{}
			for _, u := range initialURLs {
				initialNamedURLs = append(initialNamedURLs, fmt.Sprintf("%s=%s", name, u))
			}
		}

		return &EtcdConfig{
			InitialCluster:      append(initialNamedURLs, activeNamedURLs...),
			InitialClusterState: "existing",
			AdvertisePeerURLs:   initialAdvertisePeerURLs,
			Name:                name,
		}, nil
	} else {
		return &EtcdConfig{
			InitialClusterState: "new",
			Discovery:           discoveryURL,
			AdvertisePeerURLs:   initialAdvertisePeerURLs,
			Name:                name,
		}, nil
	}
}
<|endoftext|>


<commit_before>package manager

import (
	"errors"
	"log"
	"sync"
	"time"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geolite2"
)

var (
	// ErrNilDataset is returned when CurrentAnnotator is nil.
	ErrNilDataset = errors.New("CurrentAnnotator is nil")

	// ErrPendingAnnotatorLoad is returned when a new annotator is requested, but not yet loaded.
	ErrPendingAnnotatorLoad = errors.New("annotator is loading")

	// ErrAnnotatorLoadFailed is returned when a requested annotator has failed to load.
	ErrAnnotatorLoadFailed = errors.New("unable to load annoator")

	// A mutex to make sure that we are not reading from the CurrentAnnotator
	// pointer while trying to update it
	currentDataMutex = &sync.RWMutex{}

	// CurrentAnnotator points to a GeoDataset struct containing the absolute
	// latest data for the annotator to search and reply with
	CurrentAnnotator api.Annotator
)

// AnnotatorMap manages all loading and fetching of Annotators.
// TODO - should we call this AnnotatorCache?
// TODO - should this be a generic cache of interface{}?
type AnnotatorMap struct {
	// Keys are date strings in YYYYMMDD format.
	annotators map[string]api.Annotator
	// Lock to be held when reading or writing the map.
	mutex  sync.RWMutex
	loader func(string) (api.Annotator, error)
}

// NewAnnotatorMap creates a new map that will use the provided loader for loading new Annotators.
func NewAnnotatorMap(loader func(string) (api.Annotator, error)) *AnnotatorMap {
	return &AnnotatorMap{annotators: make(map[string]api.Annotator, 10), loader: loader}
}

// NOTE: Should only be called by checkAndLoadAnnotator.
// Loads an annotator, and updates the pending map entry.
// On entry, the calling goroutine should "own" the
func (am *AnnotatorMap) loadAnnotator(dateString string) {
	// On entry, this goroutine has exclusive ownership of the
	// map entry, and the responsibility for loading the annotator.
	newAnn, err := am.loader(dateString)
	if err != nil {
		// TODO add a metric
		log.Println(err)
		return
	}

	am.mutex.Lock()
	defer am.mutex.Unlock()

	ann, ok := am.annotators[dateString]
	if !ok {
		// TODO add a metric
		// TODO handle error
	}
	if ann != nil {
		// TODO add a metric
		// TODO handle error
	}
	// TODO add a metric
	am.annotators[dateString] = newAnn
}

// This asynchronously attempts to set map entry to nil, and
// if successful, proceeds to asynchronously load the new dataset.
func (am *AnnotatorMap) checkAndLoadAnnotator(dateString string) {
	go func() {
		am.mutex.Lock()

		_, ok := am.annotators[dateString]
		if ok {
			// Another goroutine is already responsible for loading.
			am.mutex.Unlock()
			return
		}

		// Place marker so that other requesters know it is loading.
		am.annotators[dateString] = nil
		// Drop the lock before attempting to load the annotator.
		am.mutex.Unlock()
		am.loadAnnotator(dateString)
	}()
}

// GetAnnotator gets the named annotator, if already in the map.
// If not already loaded, this will trigger loading, and return ErrPendingAnnotatorLoad
func (am *AnnotatorMap) GetAnnotator(dateString string) (api.Annotator, error) {
	am.mutex.RLock()
	defer am.mutex.RUnlock()

	ann, ok := am.annotators[dateString]
	if !ok {
		am.checkAndLoadAnnotator(dateString)
		return nil, ErrPendingAnnotatorLoad
	}
	if ann == nil {
		return nil, ErrPendingAnnotatorLoad
	}
	return ann, nil
}

// GetAnnotator returns the correct annotator to use for a given timestamp.
func GetAnnotator(date time.Time) api.Annotator {
	// TODO - use the requested date
	// dateString := strconv.FormatInt(date.Unix(), encodingBase)
	currentDataMutex.RLock()
	ann := CurrentAnnotator
	currentDataMutex.RUnlock()
	return ann
}

// PopulateLatestData will search to the latest Geolite2 files
// available in GCS and will use them to create a new GeoDataset which
// it will place into the global scope as the latest version. It will
// do so safely with use of the currentDataMutex RW mutex. It it
// encounters an error, it will halt the program.
func PopulateLatestData() {
	data, err := geolite2.LoadLatestGeolite2File()
	if err != nil {
		log.Fatal(err)
	}
	currentDataMutex.Lock()
	CurrentAnnotator = data
	currentDataMutex.Unlock()
}
<commit_msg>various code improvements<commit_after>package manager

import (
	"errors"
	"log"
	"sync"
	"time"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geolite2"
)

var (
	// ErrNilDataset is returned when CurrentAnnotator is nil.
	ErrNilDataset = errors.New("CurrentAnnotator is nil")

	// ErrPendingAnnotatorLoad is returned when a new annotator is requested, but not yet loaded.
	ErrPendingAnnotatorLoad = errors.New("annotator is loading")

	// ErrAnnotatorLoadFailed is returned when a requested annotator has failed to load.
	ErrAnnotatorLoadFailed = errors.New("unable to load annoator")

	// These are UNEXPECTED errors!!
	ErrGoroutineNotOwner  = errors.New("Goroutine not owner")
	ErrMapEntryAlreadySet = errors.New("Map entry already set")

	// A mutex to make sure that we are not reading from the CurrentAnnotator
	// pointer while trying to update it
	currentDataMutex = &sync.RWMutex{}

	// CurrentAnnotator points to a GeoDataset struct containing the absolute
	// latest data for the annotator to search and reply with
	CurrentAnnotator api.Annotator
)

// AnnotatorMap manages all loading and fetching of Annotators.
// TODO - should we call this AnnotatorCache?
// TODO - should this be a generic cache of interface{}?
//
// Synchronization:
//  All accesses must hold the mutex.  If an element is not found, the
//  goroutine may attempt to take responsibility for loading it by obtaining
//  the write lock, and writing an entry with a nil pointer.
// TODO - still need a strategy for dealing with persistent errors.
type AnnotatorMap struct {
	// Keys are date strings in YYYYMMDD format.
	annotators map[string]api.Annotator
	// Lock to be held when reading or writing the map.
	mutex  sync.RWMutex
	loader func(string) (api.Annotator, error)
}

// NewAnnotatorMap creates a new map that will use the provided loader for loading new Annotators.
func NewAnnotatorMap(loader func(string) (api.Annotator, error)) *AnnotatorMap {
	return &AnnotatorMap{annotators: make(map[string]api.Annotator, 10), loader: loader}
}

// NOTE: Should only be called by checkAndLoadAnnotator.
// The calling goroutine should "own" the responsibility for
// setting the annotator.
func (am *AnnotatorMap) setAnnotatorIfNil(dateString string, ann api.Annotator) error {
	am.mutex.Lock()
	defer am.mutex.Unlock()

	old, ok := am.annotators[dateString]
	if !ok {
		return ErrGoroutineNotOwner
	}
	if old != nil {
		return ErrMapEntryAlreadySet
	}
	am.annotators[dateString] = ann
	return nil
}

func (am *AnnotatorMap) maybeSetNil(dateString string) bool {
	am.mutex.Lock()
	defer am.mutex.Unlock()
	_, ok := am.annotators[dateString]
	if ok {
		// Another goroutine is already responsible for loading.
		return false
	}

	// Place marker so that other requesters know it is loading.
	am.annotators[dateString] = nil
	return true
}

// This synchronously attempts to set map entry to nil, and
// if successful, proceeds to asynchronously load the new dataset.
func (am *AnnotatorMap) checkAndLoadAnnotator(dateString string) {
	reserved := am.maybeSetNil(dateString)
	if reserved {
		// This goroutine now has exclusive ownership of the
		// map entry, and the responsibility for loading the annotator.
		go func(dateString string) {
			newAnn, err := am.loader(dateString)
			if err != nil {
				// TODO add a metric
				log.Println(err)
				return
			}
			// Set the new annotator value.  Entry should be nil.
			err = am.setAnnotatorIfNil(dateString, newAnn)
			if err != nil {
				// TODO add a metric
				log.Println(err)
			}
		}(dateString)
	}
}

// GetAnnotator gets the named annotator, if already in the map.
// If not already loaded, this will trigger loading, and return ErrPendingAnnotatorLoad
func (am *AnnotatorMap) GetAnnotator(dateString string) (api.Annotator, error) {
	am.mutex.RLock()
	ann, ok := am.annotators[dateString]
	am.mutex.RUnlock()

	if !ok {
		// There is not yet any entry for this date.  Try to load it.
		am.checkAndLoadAnnotator(dateString)
		return nil, ErrPendingAnnotatorLoad
	}
	if ann == nil {
		// Another goroutine is already loading this entry.  Return error.
		return nil, ErrPendingAnnotatorLoad
	}
	return ann, nil
}

// GetAnnotator returns the correct annotator to use for a given timestamp.
func GetAnnotator(date time.Time) api.Annotator {
	// TODO - use the requested date
	// dateString := strconv.FormatInt(date.Unix(), encodingBase)
	currentDataMutex.RLock()
	ann := CurrentAnnotator
	currentDataMutex.RUnlock()
	return ann
}

// PopulateLatestData will search to the latest Geolite2 files
// available in GCS and will use them to create a new GeoDataset which
// it will place into the global scope as the latest version. It will
// do so safely with use of the currentDataMutex RW mutex. It it
// encounters an error, it will halt the program.
func PopulateLatestData() {
	data, err := geolite2.LoadLatestGeolite2File()
	if err != nil {
		log.Fatal(err)
	}
	currentDataMutex.Lock()
	CurrentAnnotator = data
	currentDataMutex.Unlock()
}
<|endoftext|>


<commit_before>package main // launchv.go

import (
	"flag"
	"fmt"
	"github.com/jonhadfield/findexec"
	"math/rand"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"
)

/*
REVISION HISTORY
======== =======
19 Jul 22 -- First version.  I'm writing this as I go along, pulling code from other pgms as I need them.
             I want this to take an input string on the command line.  This string will be a regexp used to match against a filename, like what rex.go does.
             From the resultant slice of matches of this regexp, I'll shuffle it and then feed them one at a time into vlc.
             So it looks like I'll need pieces of rex.go, shuffle code from bj.go, and then launching and external pgm code like I do in a few places now.
             The final launching loop will pause and exit if I want it to, like I did w/ the pid and windows title matching routines.  I'll let the import list auto-populate.
20 Jul 22 -- Added verboseFlag being set will have it output the filename w/ each loop iteration.  And I added 'x' to the exit key behavior.
21 Jul 22 -- Now called lauv, it will output n files on the command line to vlc.  This way I can use 'n' from within vlc.
22 Jul 22 -- I can't get this to work by putting several filenames on the command line and it reading them all in.  Maybe I'll try redirection.
23 Jul 22 -- I finally figured out how to work w/ variadic params, after searching online.  An answer in stack overflow helped me a lot.  Now it works.
24 Jul 22 -- I allow n, or numNames, to be zero.  That means that there is no limit to what's passed into vlc.
30 Jul 22 -- Decided to always have it print out number of matches and shuffling time.
             Now called launchv, because the old name of lauv just wasn't working for me.  It was hard to type.
19 Sep 22 -- Trying to get it so I don't need tcc.  I added the notccFlag to test it.  When that started working, I made the default of true, so it's always on.
               I had to add vlc to the path for it to work.
20 Sep 22 -- After writing and debugging findme, I now know that my issue all along was that actual quote characters were in the VLCPATH environment string.
               After removing those, the code now works as originally intended.
21 Sep 22 -- The code now has a default value for the location of C:\Program Files\VideoLAN\VLC.  The environment value VLCPATH can be used to change this.
               The code will exit if the find function returns a blank string.  This would be an opportunity to use the environment var to help the pgm find vlc.
*/

const lastModified = "Sep 21, 2022"

var includeRegex, excludeRegex *regexp.Regexp
var verboseFlag, veryverboseFlag, notccFlag, ok bool
var includeRexString, excludeRexString, searchPath, path, vPath string
var vlcPath = "C:\\Program Files\\VideoLAN\\VLC"
var numNames int

func main() {
	fmt.Printf(" launch vlc.go.  Last modified %s, compiled w/ %s\n\n", lastModified, runtime.Version())

	workingDir, _ := os.Getwd()
	execName, _ := os.Executable()
	ExecFI, _ := os.Stat(execName)
	LastLinkedTimeStamp := ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")

	path = os.Getenv("PATH")
	vPath, ok = os.LookupEnv("VLCPATH")
	if ok {
		vlcPath = strings.ReplaceAll(vPath, `"`, "") // Here I use back quotes to insert a literal quote.
	}
	searchPath = vlcPath + ";" + path

	flag.Usage = func() {
		fmt.Fprintf(flag.CommandLine.Output(), " This pgm will match an input regexp against all filenames in the current directory\n")
		fmt.Fprintf(flag.CommandLine.Output(), " shuffle them, and then output 'n' of them on the command line to vlc.\n")
		fmt.Fprintf(flag.CommandLine.Output(), " %s has timestamp of %s, working directory is %s, full name of executable is %s and vlcPath is %s.\n",
			ExecFI.Name(), LastLinkedTimeStamp, workingDir, execName, vlcPath)
		fmt.Fprintf(flag.CommandLine.Output(), " Usage: launchv <options> <input-regex> where <input-regex> cannot be empty. \n")
		fmt.Fprintln(flag.CommandLine.Output())
		flag.PrintDefaults()
	}

	flag.BoolVar(&verboseFlag, "v", false, " Verbose mode flag.")
	flag.BoolVar(&veryverboseFlag, "vv", false, " Very Verbose mode flag.")
	flag.StringVar(&excludeRexString, "x", "", " Exclude file regexp string, which is usually empty.")
	flag.IntVar(&numNames, "n", 20, " Number of file names to output on the commandline to vlc.")
	flag.BoolVar(&notccFlag, "not", true, " Not using tcc flag.") // Since the default is true, to make it false requires -not=false syntax.
	flag.Parse()
	//numNames += 2 // account for 2 extra items I have to add to the slice, ie, the -C and vlc add'l params.  Not needed anymore.
	if veryverboseFlag { // very verbose also turns on verbose flag.
		verboseFlag = true
	}

	if verboseFlag {
		fmt.Printf(" %s has timestamp of %s, working directory is %s, and full name of executable is %s.\n",
			ExecFI.Name(), LastLinkedTimeStamp, workingDir, execName)
	}
	if verboseFlag {
		fmt.Printf(" vlcPath = %s, searchPath is: \n", vlcPath)
		listPath(searchPath)
	}

	if flag.NArg() < 1 { // if there are more than 1 arguments, the extra ones are ignored.
		fmt.Printf(" Usage: launchv <options> <input-regex> where <input-regex> cannot be empty.  Exiting\n")
		os.Exit(0)
	}

	includeRexString = flag.Arg(0) // this is the first argument on the command line that is not the program name.
	var err error
	includeRegex, err = regexp.Compile(strings.ToLower(includeRexString))
	if err != nil {
		fmt.Printf(" Error from compiling the regexp input string is %v\n", err)
		os.Exit(1)
	}
	if excludeRexString != "" {
		excludeRegex, err = regexp.Compile(strings.ToLower(excludeRexString))
		if err != nil {
			fmt.Printf(" Error from compiling the exclude regexp is %v\n", err)
			os.Exit(1)
		}
	}

	fileNames := getFileNames(workingDir, includeRegex) // this slice of filenames matches the includeRegexp and does not match the excludeRegexp, if given.
	if verboseFlag {
		fmt.Printf(" There are %d filenames found using includeRexString = %q and %q, and excludeRexString = %q\n",
			len(fileNames), includeRexString, includeRegex.String(), excludeRexString)
	}

	if len(fileNames) == 0 {
		fmt.Printf(" No filenames matched the regexp of %q and were excluded by %q.  Exiting  \n", includeRexString, excludeRexString)
		os.Exit(0)
	}

	// Now to shuffle the file names slice.

	now := time.Now()
	rand.Seed(now.UnixNano())
	shuffleAmount := now.Nanosecond()/1e6 + now.Second() + now.Minute() + now.Day() + now.Hour() + now.Year()
	swapFnt := func(i, j int) {
		fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
	}
	for i := 0; i < shuffleAmount; i++ {
		rand.Shuffle(len(fileNames), swapFnt)
	}

	fmt.Printf(" Shuffled %d filenames %d times, which took %s.\n", len(fileNames), shuffleAmount, time.Since(now))

	// ready to start calling vlc

	// Turns out that the shell searches against the path on Windows, but just executing it here doesn't.  So I have to search the path myself.
	// Nope, I still have that wrong.  I need to start a command processor, too.  And vlc is not in the %PATH, but it does work when I just give it as a command without a path.

	var vlcStr, shellStr string
	if runtime.GOOS == "windows" {
		vlcStr = findexec.Find("vlc", searchPath) //Turns out that vlc was not in the path.  But it shows up when I use "which vlc".  So it seems that findexec doesn't find it on my win10 system.  So I added it to the path.
		//vlcStr = "vlc"
		//shellStr = os.Getenv("ComSpec") not needed anymore
	} else if runtime.GOOS == "linux" {
		vlcStr = findexec.Find("vlc", "")
		shellStr = "/bin/bash" // not needed as I found out by some experimentation on leox.
	}

	if vlcStr == "" {
		fmt.Printf(" vlcStr is null.  Exiting ")
		os.Exit(1)
	}

	// Time to run vlc.

	var execCmd *exec.Cmd

	variadicParam := []string{"-C", "vlc"} // This isn't really needed anymore.  I'll leave it here anyway, as a model in case I ever need to do this again.
	if notccFlag {
		variadicParam = []string{}
	}
	variadicParam = append(variadicParam, fileNames...)
	n := minInt(numNames, len(fileNames))
	if n > 0 {
		variadicParam = variadicParam[:n]
	}

	// For me to be able to pass a variadic param here, I must match the definition of the function, not pass some and then try the variadic syntax.
	// I got this answer from stack overflow.

	if runtime.GOOS == "windows" {
		if notccFlag {
			execCmd = exec.Command(vlcStr, variadicParam...)
		} else { // this isn't needed anymore.  I'll leave it here because it does work, in case I ever need to do this again.
			execCmd = exec.Command(shellStr, variadicParam...)
		}
		//switch n { // just to see if this works.  Once I figured out the variadic syntax I don't need a switch case statement here.
		//case 1:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0])
		//case 2:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1])
		//case 3:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1], fileNames[2])
		//case 4:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1], fileNames[2], fileNames[3])
		//case 5:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1], fileNames[2], fileNames[3], fileNames[4])
		//default:
		//	execCmd = exec.Command(shellStr, variadicParam...)
		//}
	} else if runtime.GOOS == "linux" { // I'm ignoring this for now.  I'll come back to it after I get the Windows code working.
		execCmd = exec.Command(vlcStr, fileNames...)
	}

	if veryverboseFlag {
		fmt.Printf(" vlcStr = %s, len of variadicParam = %d, and filenames in variadicParam are %v\n", vlcStr, len(variadicParam), variadicParam)
	}

	execCmd.Stdin = os.Stdin
	execCmd.Stdout = os.Stdout
	execCmd.Stderr = os.Stderr
	e := execCmd.Start()
	if e != nil {
		fmt.Printf(" Error returned by running vlc %s is %v\n", variadicParam, e)
	}
} // end main()

// ------------------------------------------------------------------------ getFileNames -------------------------------------------------------

func getFileNames(workingDir string, inputRegex *regexp.Regexp) []string {

	fileNames := myReadDir(workingDir, inputRegex) // excluding by regex, filesize or having an ext is done by MyReadDir.

	if veryverboseFlag {
		fmt.Printf(" Leaving getFileNames.  flag.Nargs=%d, len(flag.Args)=%d, len(fileNames)=%d\n", flag.NArg(), len(flag.Args()), len(fileNames))
	}

	return fileNames
}

// ------------------------------- myReadDir -----------------------------------

func myReadDir(dir string, inputRegex *regexp.Regexp) []string {

	dirEntries, err := os.ReadDir(dir)
	if err != nil {
		return nil
	}

	fileNames := make([]string, 0, len(dirEntries))
	for _, d := range dirEntries {
		lower := strings.ToLower(d.Name())
		if !inputRegex.MatchString(lower) { // skip dirEntries that do not match the input regex.
			continue
		}

		//quotedString := fmt.Sprintf("%q", d.Name())
		//fullPath, e := filepath.Abs(d.Name())
		//if e != nil {
		//	fmt.Fprintf(os.Stderr, " myReadDir error from filepath.Abs(%s) is %v\n", d.Name(), e)
		//}
		//fullPath = "file:///" + fullPath // I got this idea by reading the vlc help text
		if excludeRegex == nil {
			fileNames = append(fileNames, d.Name())
		} else if !excludeRegex.MatchString(lower) { // excludeRegex is not empty, so using it won't panic.
			fileNames = append(fileNames, d.Name())
		}
	}
	return fileNames
} // myReadDir

// ------------------------------ pause -----------------------------------------
/*
func pause() bool {
	fmt.Print(" Pausing the loop.  Hit <enter> to continue; 'n' or 'x' to exit  ")
	var ans string
	fmt.Scanln(&ans)
	ans = strings.ToLower(ans)
	if strings.HasPrefix(ans, "n") || strings.HasPrefix(ans, "x") {
		return true
	}
	return false
}
*/

// ------------------------------- minInt ----------------------------------------

func minInt(i, j int) int {
	if i <= j {
		return i
	}
	return j
}

// ------------------------------- listPath --------------------------------------

func listPath(path string) {
	splitEnv := strings.Split(path, ";")
	for _, s := range splitEnv {
		fmt.Printf(" %s\n", s)
	}
}

/* ------------------------------------------- MakeDateStr ---------------------------------------------------* */
/*
func MakeDateStr() string {

	const DateSepChar = "-"
	var dateStr string

	m, d, y := timlibg.TIME2MDY()
	timeNow := timlibg.GetDateTime()

	MSTR := strconv.Itoa(m)
	DSTR := strconv.Itoa(d)
	YSTR := strconv.Itoa(y)
	Hr := strconv.Itoa(timeNow.Hours)
	Min := strconv.Itoa(timeNow.Minutes)
	Sec := strconv.Itoa(timeNow.Seconds)

	dateStr = "_" + MSTR + DateSepChar + DSTR + DateSepChar + YSTR + "_" + Hr + DateSepChar + Min + DateSepChar + Sec + "__" + timeNow.DayOfWeekStr
	return dateStr
} // MakeDateStr
*/
<commit_msg>09/21/2022 16:43:47 launchv/launchv.go -- should work on linux too<commit_after>package main // launchv.go

import (
	"flag"
	"fmt"
	"github.com/jonhadfield/findexec"
	"math/rand"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
	"time"
)

/*
REVISION HISTORY
======== =======
19 Jul 22 -- First version.  I'm writing this as I go along, pulling code from other pgms as I need them.
             I want this to take an input string on the command line.  This string will be a regexp used to match against a filename, like what rex.go does.
             From the resultant slice of matches of this regexp, I'll shuffle it and then feed them one at a time into vlc.
             So it looks like I'll need pieces of rex.go, shuffle code from bj.go, and then launching and external pgm code like I do in a few places now.
             The final launching loop will pause and exit if I want it to, like I did w/ the pid and windows title matching routines.  I'll let the import list auto-populate.
20 Jul 22 -- Added verboseFlag being set will have it output the filename w/ each loop iteration.  And I added 'x' to the exit key behavior.
21 Jul 22 -- Now called lauv, it will output n files on the command line to vlc.  This way I can use 'n' from within vlc.
22 Jul 22 -- I can't get this to work by putting several filenames on the command line and it reading them all in.  Maybe I'll try redirection.
23 Jul 22 -- I finally figured out how to work w/ variadic params, after searching online.  An answer in stack overflow helped me a lot.  Now it works.
24 Jul 22 -- I allow n, or numNames, to be zero.  That means that there is no limit to what's passed into vlc.
30 Jul 22 -- Decided to always have it print out number of matches and shuffling time.
             Now called launchv, because the old name of lauv just wasn't working for me.  It was hard to type.
19 Sep 22 -- Trying to get it so I don't need tcc.  I added the notccFlag to test it.  When that started working, I made the default of true, so it's always on.
               I had to add vlc to the path for it to work.
20 Sep 22 -- After writing and debugging findme, I now know that my issue all along was that actual quote characters were in the VLCPATH environment string.
               After removing those, the code now works as originally intended.
21 Sep 22 -- The code now has a default value for the location of C:\Program Files\VideoLAN\VLC.  The environment value VLCPATH can be used to change this.
               The code will exit if the find function returns a blank string.  This would be an opportunity to use the environment var to help the pgm find vlc.
               And since linux and windows use different characters as subdir separators in the PATH, and the filesystem uses a different delimiter, I have to use a conditional
               based on runtime.GOOS.
*/

const lastModified = "Sep 21, 2022"

var includeRegex, excludeRegex *regexp.Regexp
var verboseFlag, veryverboseFlag, notccFlag, ok bool
var includeRexString, excludeRexString, searchPath, path, vPath string
var vlcPath = "C:\\Program Files\\VideoLAN\\VLC"
var numNames int

func main() {
	fmt.Printf(" launch vlc.go.  Last modified %s, compiled w/ %s\n\n", lastModified, runtime.Version())

	workingDir, _ := os.Getwd()
	execName, _ := os.Executable()
	ExecFI, _ := os.Stat(execName)
	LastLinkedTimeStamp := ExecFI.ModTime().Format("Mon Jan 2 2006 15:04:05 MST")

	path = os.Getenv("PATH")
	vPath, ok = os.LookupEnv("VLCPATH")
	if ok {
		vlcPath = strings.ReplaceAll(vPath, `"`, "") // Here I use back quotes to insert a literal quote.
	}
	if runtime.GOOS == "windows" {
		searchPath = vlcPath + ";" + path
	} else if runtime.GOOS == "linux" && ok {
		searchPath = vlcPath + ":" + path
	} else { // on linux and not ok, meaning environment variable VLCPATH is empty.
		searchPath = path
	}

	flag.Usage = func() {
		fmt.Fprintf(flag.CommandLine.Output(), " This pgm will match an input regexp against all filenames in the current directory\n")
		fmt.Fprintf(flag.CommandLine.Output(), " shuffle them, and then output 'n' of them on the command line to vlc.\n")
		fmt.Fprintf(flag.CommandLine.Output(), " %s has timestamp of %s, working directory is %s, full name of executable is %s and vlcPath is %s.\n",
			ExecFI.Name(), LastLinkedTimeStamp, workingDir, execName, vlcPath)
		fmt.Fprintf(flag.CommandLine.Output(), " Usage: launchv <options> <input-regex> where <input-regex> cannot be empty. \n")
		fmt.Fprintln(flag.CommandLine.Output())
		flag.PrintDefaults()
	}

	flag.BoolVar(&verboseFlag, "v", false, " Verbose mode flag.")
	flag.BoolVar(&veryverboseFlag, "vv", false, " Very Verbose mode flag.")
	flag.StringVar(&excludeRexString, "x", "", " Exclude file regexp string, which is usually empty.")
	flag.IntVar(&numNames, "n", 20, " Number of file names to output on the commandline to vlc.")
	flag.BoolVar(&notccFlag, "not", true, " Not using tcc flag.") // Since the default is true, to make it false requires -not=false syntax.
	flag.Parse()
	//numNames += 2 // account for 2 extra items I have to add to the slice, ie, the -C and vlc add'l params.  Not needed anymore.
	if veryverboseFlag { // very verbose also turns on verbose flag.
		verboseFlag = true
	}

	if verboseFlag {
		fmt.Printf(" %s has timestamp of %s, working directory is %s, and full name of executable is %s.\n",
			ExecFI.Name(), LastLinkedTimeStamp, workingDir, execName)
	}
	if verboseFlag {
		fmt.Printf(" vlcPath = %s, searchPath is: \n", vlcPath)
		listPath(searchPath)
	}

	if flag.NArg() < 1 { // if there are more than 1 arguments, the extra ones are ignored.
		fmt.Printf(" Usage: launchv <options> <input-regex> where <input-regex> cannot be empty.  Exiting\n")
		os.Exit(0)
	}

	includeRexString = flag.Arg(0) // this is the first argument on the command line that is not the program name.
	var err error
	includeRegex, err = regexp.Compile(strings.ToLower(includeRexString))
	if err != nil {
		fmt.Printf(" Error from compiling the regexp input string is %v\n", err)
		os.Exit(1)
	}
	if excludeRexString != "" {
		excludeRegex, err = regexp.Compile(strings.ToLower(excludeRexString))
		if err != nil {
			fmt.Printf(" Error from compiling the exclude regexp is %v\n", err)
			os.Exit(1)
		}
	}

	fileNames := getFileNames(workingDir, includeRegex) // this slice of filenames matches the includeRegexp and does not match the excludeRegexp, if given.
	if verboseFlag {
		fmt.Printf(" There are %d filenames found using includeRexString = %q and %q, and excludeRexString = %q\n",
			len(fileNames), includeRexString, includeRegex.String(), excludeRexString)
	}

	if len(fileNames) == 0 {
		fmt.Printf(" No filenames matched the regexp of %q and were excluded by %q.  Exiting  \n", includeRexString, excludeRexString)
		os.Exit(0)
	}

	// Now to shuffle the file names slice.

	now := time.Now()
	rand.Seed(now.UnixNano())
	shuffleAmount := now.Nanosecond()/1e6 + now.Second() + now.Minute() + now.Day() + now.Hour() + now.Year()
	swapFnt := func(i, j int) {
		fileNames[i], fileNames[j] = fileNames[j], fileNames[i]
	}
	for i := 0; i < shuffleAmount; i++ {
		rand.Shuffle(len(fileNames), swapFnt)
	}

	fmt.Printf(" Shuffled %d filenames %d times, which took %s.\n", len(fileNames), shuffleAmount, time.Since(now))

	// ready to start calling vlc

	// Turns out that the shell searches against the path on Windows, but just executing it here doesn't.  So I have to search the path myself.
	// Nope, I still have that wrong.  I need to start a command processor, too.  And vlc is not in the %PATH, but it does work when I just give it as a command without a path.

	var vlcStr, shellStr string
	if runtime.GOOS == "windows" {
		vlcStr = findexec.Find("vlc", searchPath) //Turns out that vlc was not in the path.  But it shows up when I use "which vlc".  So it seems that findexec doesn't find it on my win10 system.  So I added it to the path.
		//vlcStr = "vlc"
		//shellStr = os.Getenv("ComSpec") not needed anymore
	} else if runtime.GOOS == "linux" {
		vlcStr = findexec.Find("vlc", "")
		shellStr = "/bin/bash" // not needed as I found out by some experimentation on leox.
	}

	if vlcStr == "" {
		fmt.Printf(" vlcStr is null.  Exiting ")
		os.Exit(1)
	}

	// Time to run vlc.

	var execCmd *exec.Cmd

	variadicParam := []string{"-C", "vlc"} // This isn't really needed anymore.  I'll leave it here anyway, as a model in case I ever need to do this again.
	if notccFlag {
		variadicParam = []string{}
	}
	variadicParam = append(variadicParam, fileNames...)
	n := minInt(numNames, len(fileNames))
	if n > 0 {
		variadicParam = variadicParam[:n]
	}

	// For me to be able to pass a variadic param here, I must match the definition of the function, not pass some and then try the variadic syntax.
	// I got this answer from stack overflow.

	if runtime.GOOS == "windows" {
		if notccFlag {
			execCmd = exec.Command(vlcStr, variadicParam...)
		} else { // this isn't needed anymore.  I'll leave it here because it does work, in case I ever need to do this again.
			execCmd = exec.Command(shellStr, variadicParam...)
		}
		//switch n { // just to see if this works.  Once I figured out the variadic syntax I don't need a switch case statement here.
		//case 1:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0])
		//case 2:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1])
		//case 3:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1], fileNames[2])
		//case 4:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1], fileNames[2], fileNames[3])
		//case 5:
		//	execCmd = exec.Command(shellStr, "-C", vlcStr, fileNames[0], fileNames[1], fileNames[2], fileNames[3], fileNames[4])
		//default:
		//	execCmd = exec.Command(shellStr, variadicParam...)
		//}
	} else if runtime.GOOS == "linux" { // I'm ignoring this for now.  I'll come back to it after I get the Windows code working.
		execCmd = exec.Command(vlcStr, fileNames...)
	}

	if veryverboseFlag {
		fmt.Printf(" vlcStr = %s, len of variadicParam = %d, and filenames in variadicParam are %v\n", vlcStr, len(variadicParam), variadicParam)
	}

	execCmd.Stdin = os.Stdin
	execCmd.Stdout = os.Stdout
	execCmd.Stderr = os.Stderr
	e := execCmd.Start()
	if e != nil {
		fmt.Printf(" Error returned by running vlc %s is %v\n", variadicParam, e)
	}
} // end main()

// ------------------------------------------------------------------------ getFileNames -------------------------------------------------------

func getFileNames(workingDir string, inputRegex *regexp.Regexp) []string {

	fileNames := myReadDir(workingDir, inputRegex) // excluding by regex, filesize or having an ext is done by MyReadDir.

	if veryverboseFlag {
		fmt.Printf(" Leaving getFileNames.  flag.Nargs=%d, len(flag.Args)=%d, len(fileNames)=%d\n", flag.NArg(), len(flag.Args()), len(fileNames))
	}

	return fileNames
}

// ------------------------------- myReadDir -----------------------------------

func myReadDir(dir string, inputRegex *regexp.Regexp) []string {

	dirEntries, err := os.ReadDir(dir)
	if err != nil {
		return nil
	}

	fileNames := make([]string, 0, len(dirEntries))
	for _, d := range dirEntries {
		lower := strings.ToLower(d.Name())
		if !inputRegex.MatchString(lower) { // skip dirEntries that do not match the input regex.
			continue
		}

		//quotedString := fmt.Sprintf("%q", d.Name())
		//fullPath, e := filepath.Abs(d.Name())
		//if e != nil {
		//	fmt.Fprintf(os.Stderr, " myReadDir error from filepath.Abs(%s) is %v\n", d.Name(), e)
		//}
		//fullPath = "file:///" + fullPath // I got this idea by reading the vlc help text
		if excludeRegex == nil {
			fileNames = append(fileNames, d.Name())
		} else if !excludeRegex.MatchString(lower) { // excludeRegex is not empty, so using it won't panic.
			fileNames = append(fileNames, d.Name())
		}
	}
	return fileNames
} // myReadDir

// ------------------------------ pause -----------------------------------------
/*
func pause() bool {
	fmt.Print(" Pausing the loop.  Hit <enter> to continue; 'n' or 'x' to exit  ")
	var ans string
	fmt.Scanln(&ans)
	ans = strings.ToLower(ans)
	if strings.HasPrefix(ans, "n") || strings.HasPrefix(ans, "x") {
		return true
	}
	return false
}
*/

// ------------------------------- minInt ----------------------------------------

func minInt(i, j int) int {
	if i <= j {
		return i
	}
	return j
}

// ------------------------------- listPath --------------------------------------

func listPath(path string) {
	splitEnv := strings.Split(path, ";")
	for _, s := range splitEnv {
		fmt.Printf(" %s\n", s)
	}
}

/* ------------------------------------------- MakeDateStr ---------------------------------------------------* */
/*
func MakeDateStr() string {

	const DateSepChar = "-"
	var dateStr string

	m, d, y := timlibg.TIME2MDY()
	timeNow := timlibg.GetDateTime()

	MSTR := strconv.Itoa(m)
	DSTR := strconv.Itoa(d)
	YSTR := strconv.Itoa(y)
	Hr := strconv.Itoa(timeNow.Hours)
	Min := strconv.Itoa(timeNow.Minutes)
	Sec := strconv.Itoa(timeNow.Seconds)

	dateStr = "_" + MSTR + DateSepChar + DSTR + DateSepChar + YSTR + "_" + Hr + DateSepChar + Min + DateSepChar + Sec + "__" + timeNow.DayOfWeekStr
	return dateStr
} // MakeDateStr
*/
<|endoftext|>


<commit_before><commit_msg>allow admins to impersonate other users<commit_after><|endoftext|>


<commit_before>package cachecheck

import (
	"testing"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/registry/storage/cache"
)

// CheckBlobDescriptorCache takes a cache implementation through a common set
// of operations. If adding new tests, please add them here so new
// implementations get the benefit. This should be used for unit tests.
func CheckBlobDescriptorCache(t *testing.T, provider cache.BlobDescriptorCacheProvider) {
	ctx := context.Background()

	checkBlobDescriptorCacheEmptyRepository(t, ctx, provider)
	checkBlobDescriptorCacheSetAndRead(t, ctx, provider)
}

func checkBlobDescriptorCacheEmptyRepository(t *testing.T, ctx context.Context, provider cache.BlobDescriptorCacheProvider) {
	if _, err := provider.Stat(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"); err != distribution.ErrBlobUnknown {
		t.Fatalf("expected unknown blob error with empty store: %v", err)
	}

	cache, err := provider.RepositoryScoped("")
	if err == nil {
		t.Fatalf("expected an error when asking for invalid repo")
	}

	cache, err = provider.RepositoryScoped("foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting repository: %v", err)
	}

	if err := cache.SetDescriptor(ctx, "", distribution.Descriptor{
		Digest:    "sha384:abc",
		Size:      10,
		MediaType: "application/octet-stream"}); err != digest.ErrDigestInvalidFormat {
		t.Fatalf("expected error with invalid digest: %v", err)
	}

	if err := cache.SetDescriptor(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", distribution.Descriptor{
		Digest:    "",
		Size:      10,
		MediaType: "application/octet-stream"}); err == nil {
		t.Fatalf("expected error setting value on invalid descriptor")
	}

	if _, err := cache.Stat(ctx, ""); err != digest.ErrDigestInvalidFormat {
		t.Fatalf("expected error checking for cache item with empty digest: %v", err)
	}

	if _, err := cache.Stat(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"); err != distribution.ErrBlobUnknown {
		t.Fatalf("expected unknown blob error with empty repo: %v", err)
	}
}

func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provider cache.BlobDescriptorCacheProvider) {
	localDigest := digest.Digest("sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")
	expected := distribution.Descriptor{
		Digest:    "sha256:abc1111111111111111111111111111111111111111111111111111111111111",
		Size:      10,
		MediaType: "application/octet-stream"}

	cache, err := provider.RepositoryScoped("foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting scoped cache: %v", err)
	}

	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
		t.Fatalf("error setting descriptor: %v", err)
	}

	desc, err := cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error statting fake2:abc: %v", err)
	}

	if expected != desc {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// also check that we set the canonical key ("fake:abc")
	desc, err = cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("descriptor not returned for canonical key: %v", err)
	}

	if expected != desc {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// ensure that global gets extra descriptor mapping
	desc, err = provider.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc)
	}

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// get at it through canonical descriptor
	desc, err = provider.Stat(ctx, expected.Digest)
	if err != nil {
		t.Fatalf("unexpected error checking glboal descriptor: %v", err)
	}

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// now, we set the repo local mediatype to something else and ensure it
	// doesn't get changed in the provider cache.
	expected.MediaType = "application/json"

	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
		t.Fatalf("unexpected error setting descriptor: %v", err)
	}

	desc, err = cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error getting descriptor: %v", err)
	}

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
	}

	desc, err = provider.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error getting global descriptor: %v", err)
	}

	expected.MediaType = "application/octet-stream" // expect original mediatype in global

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
	}
}

func checkBlobDescriptorClear(t *testing.T, ctx context.Context, provider cache.BlobDescriptorCacheProvider) {
	localDigest := digest.Digest("sha384:abc")
	expected := distribution.Descriptor{
		Digest:    "sha256:abc",
		Size:      10,
		MediaType: "application/octet-stream"}

	cache, err := provider.RepositoryScoped("foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting scoped cache: %v", err)
	}

	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
		t.Fatalf("error setting descriptor: %v", err)
	}

	desc, err := cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error statting fake2:abc: %v", err)
	}

	if expected != desc {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	err = cache.Clear(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error deleting descriptor")
	}

	nonExistantDigest := digest.Digest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
	err = cache.Clear(ctx, nonExistantDigest)
	if err == nil {
		t.Fatalf("expected error deleting unknown descriptor")
	}
}
<commit_msg>Correct test digest lengths and enable all unit tests<commit_after>package cachecheck

import (
	"testing"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/registry/storage/cache"
)

// CheckBlobDescriptorCache takes a cache implementation through a common set
// of operations. If adding new tests, please add them here so new
// implementations get the benefit. This should be used for unit tests.
func CheckBlobDescriptorCache(t *testing.T, provider cache.BlobDescriptorCacheProvider) {
	ctx := context.Background()

	checkBlobDescriptorCacheEmptyRepository(t, ctx, provider)
	checkBlobDescriptorCacheSetAndRead(t, ctx, provider)
	checkBlobDescriptorCacheClear(t, ctx, provider)
}

func checkBlobDescriptorCacheEmptyRepository(t *testing.T, ctx context.Context, provider cache.BlobDescriptorCacheProvider) {
	if _, err := provider.Stat(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"); err != distribution.ErrBlobUnknown {
		t.Fatalf("expected unknown blob error with empty store: %v", err)
	}

	cache, err := provider.RepositoryScoped("")
	if err == nil {
		t.Fatalf("expected an error when asking for invalid repo")
	}

	cache, err = provider.RepositoryScoped("foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting repository: %v", err)
	}

	if err := cache.SetDescriptor(ctx, "", distribution.Descriptor{
		Digest:    "sha384:abc",
		Size:      10,
		MediaType: "application/octet-stream"}); err != digest.ErrDigestInvalidFormat {
		t.Fatalf("expected error with invalid digest: %v", err)
	}

	if err := cache.SetDescriptor(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", distribution.Descriptor{
		Digest:    "",
		Size:      10,
		MediaType: "application/octet-stream"}); err == nil {
		t.Fatalf("expected error setting value on invalid descriptor")
	}

	if _, err := cache.Stat(ctx, ""); err != digest.ErrDigestInvalidFormat {
		t.Fatalf("expected error checking for cache item with empty digest: %v", err)
	}

	if _, err := cache.Stat(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"); err != distribution.ErrBlobUnknown {
		t.Fatalf("expected unknown blob error with empty repo: %v", err)
	}
}

func checkBlobDescriptorCacheSetAndRead(t *testing.T, ctx context.Context, provider cache.BlobDescriptorCacheProvider) {
	localDigest := digest.Digest("sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")
	expected := distribution.Descriptor{
		Digest:    "sha256:abc1111111111111111111111111111111111111111111111111111111111111",
		Size:      10,
		MediaType: "application/octet-stream"}

	cache, err := provider.RepositoryScoped("foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting scoped cache: %v", err)
	}

	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
		t.Fatalf("error setting descriptor: %v", err)
	}

	desc, err := cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error statting fake2:abc: %v", err)
	}

	if expected != desc {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// also check that we set the canonical key ("fake:abc")
	desc, err = cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("descriptor not returned for canonical key: %v", err)
	}

	if expected != desc {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// ensure that global gets extra descriptor mapping
	desc, err = provider.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc)
	}

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// get at it through canonical descriptor
	desc, err = provider.Stat(ctx, expected.Digest)
	if err != nil {
		t.Fatalf("unexpected error checking glboal descriptor: %v", err)
	}

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	// now, we set the repo local mediatype to something else and ensure it
	// doesn't get changed in the provider cache.
	expected.MediaType = "application/json"

	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
		t.Fatalf("unexpected error setting descriptor: %v", err)
	}

	desc, err = cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error getting descriptor: %v", err)
	}

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
	}

	desc, err = provider.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error getting global descriptor: %v", err)
	}

	expected.MediaType = "application/octet-stream" // expect original mediatype in global

	if desc != expected {
		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
	}
}

func checkBlobDescriptorCacheClear(t *testing.T, ctx context.Context, provider cache.BlobDescriptorCacheProvider) {
	localDigest := digest.Digest("sha384:def111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")
	expected := distribution.Descriptor{
		Digest:    "sha256:def1111111111111111111111111111111111111111111111111111111111111",
		Size:      10,
		MediaType: "application/octet-stream"}

	cache, err := provider.RepositoryScoped("foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting scoped cache: %v", err)
	}

	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
		t.Fatalf("error setting descriptor: %v", err)
	}

	desc, err := cache.Stat(ctx, localDigest)
	if err != nil {
		t.Fatalf("unexpected error statting fake2:abc: %v", err)
	}

	if expected != desc {
		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
	}

	err = cache.Clear(ctx, localDigest)
	if err != nil {
		t.Error(err)
	}

	desc, err = cache.Stat(ctx, localDigest)
	if err == nil {
		t.Fatalf("expected error statting deleted blob: %v", err)
	}
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"time"
)

type Token struct {
	AccessToken  string `json:"access_token"`
	TokenType    string `json:"token_type,omitempty"`
	ExpiresIn    int64  `json:"expires_in"`
	RefreshToken string `json:"refresh_token,omitempty"`
}

type TokenHolder struct {
	currToken    *Token
	base         string
	method       string
	apiAuths     string
	clientId     string
	clientSecret string
	expiresIn    int64
}

var (
	errEmptyAuths = errors.New("empty auth string or clientId and clientSecret")
)

func NewTokenHolder(baseUrl string) *TokenHolder {
	return &TokenHolder{
		base:   baseUrl,
		method: "POST",
	}
}

func (th *TokenHolder) SetAuth(auths string) {
	th.apiAuths = auths
}

func (th *TokenHolder) SetClient(id, secret string) {
	th.clientId = id
	th.clientSecret = secret
}

func (th *TokenHolder) Expired() bool {
	return th.expiresIn < time.Now().Unix()
}

func (th *TokenHolder) GetAuthToken() (token string, err error) {
	if th.currToken == nil || th.Expired() {
		th.currToken, err = th.requestToken()
		if err != nil {
			return "", err
		}
		th.expiresIn = th.currToken.ExpiresIn
	}
	token = th.currToken.AccessToken
	return
}

func (th *TokenHolder) requestToken() (token *Token, err error) {
	var resp []byte
	if th.apiAuths != "" { // for ExMail
		body_str := "grant_type=client_credentials"
		resp, err = DoHTTP("POST", th.base, th.apiAuths, bytes.NewBufferString(body_str))

	} else if th.clientId != "" && th.clientSecret != "" { // for ExWechat
		uri := fmt.Sprintf("%s?corpid=%s&corpsecret=%s", th.base, th.clientId, th.clientSecret)
		resp, err = DoHTTP("GET", uri, "", nil)
	} else {
		err = errEmptyAuths
	}

	if err != nil {
		log.Printf(" err %s", err)
		return
	}

	obj := &Token{}
	err = json.Unmarshal(resp, obj)
	if err != nil {
		log.Printf("unmarshal err %s", err)
		return
	}
	token = obj

	return
}
<commit_msg>fixes token expire<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"time"
)

type Token struct {
	AccessToken  string `json:"access_token"`
	TokenType    string `json:"token_type,omitempty"`
	ExpiresIn    int64  `json:"expires_in"`
	RefreshToken string `json:"refresh_token,omitempty"`
}

type TokenHolder struct {
	currToken    *Token
	base         string
	method       string
	apiAuths     string
	clientId     string
	clientSecret string
	expiresAt    int64
}

var (
	errEmptyAuths = errors.New("empty auth string or clientId and clientSecret")
)

func NewTokenHolder(baseUrl string) *TokenHolder {
	return &TokenHolder{
		base:   baseUrl,
		method: "POST",
	}
}

func (th *TokenHolder) SetAuth(auths string) {
	th.apiAuths = auths
}

func (th *TokenHolder) SetClient(id, secret string) {
	th.clientId = id
	th.clientSecret = secret
}

func (th *TokenHolder) Expired() bool {
	return th.expiresAt < time.Now().Unix()
}

func (th *TokenHolder) GetAuthToken() (token string, err error) {
	if th.currToken == nil || th.Expired() {
		log.Print("token is nil or expired, refreshing it")
		th.currToken, err = th.requestToken()
		if err != nil {
			return "", err
		}
		log.Print("got token", th.currToken)
		th.expiresAt = time.Now().Unix() + th.currToken.ExpiresIn
	}
	token = th.currToken.AccessToken
	return
}

func (th *TokenHolder) requestToken() (token *Token, err error) {
	var resp []byte
	if th.apiAuths != "" { // for ExMail
		body_str := "grant_type=client_credentials"
		resp, err = DoHTTP("POST", th.base, th.apiAuths, bytes.NewBufferString(body_str))

	} else if th.clientId != "" && th.clientSecret != "" { // for ExWechat
		uri := fmt.Sprintf("%s?corpid=%s&corpsecret=%s", th.base, th.clientId, th.clientSecret)
		resp, err = DoHTTP("GET", uri, "", nil)
	} else {
		err = errEmptyAuths
	}

	if err != nil {
		log.Printf(" err %s", err)
		return
	}

	obj := &Token{}
	err = json.Unmarshal(resp, obj)
	if err != nil {
		log.Printf("unmarshal err %s", err)
		return
	}
	token = obj

	return
}
<|endoftext|>