<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package complete_chassis_reboot_test

import (
	"context"
	"sort"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/featureprofiles/internal/fptest"
	spb "github.com/openconfig/gnoi/system"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/testt"
)

const (
	oneMinuteInNanoSecond = 6e10
	oneSecondInNanoSecond = 1e9
	rebootDelay           = 120
	// Maximum reboot time is 900 seconds (15 minutes).
	maxRebootTime = 900
)

func TestMain(m *testing.M) {
	fptest.RunTests(m)
}

// Test cases:
//  1) Send gNOI reboot request using the method COLD with the delay of N seconds.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: In nanoseconds before issuing reboot.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT remains reachable for N seconds by checking DUT current time is updated.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//  2) Send gNOI reboot request using the method COLD without delay.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: 0 - no delay.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//
// Topology:
//   dut:port1 <--> ate:port1
//
// Test notes:
//  - A RebootRequest requests the specified target be rebooted using the specified
//    method after the specified delay.  Only the DEFAULT method with a delay of 0
//    is guaranteed to be accepted for all target types.
//  - A RebootMethod determines what should be done with a target when a Reboot is
//    requested.  Only the COLD method is required to be supported by all
//    targets.  Methods the target does not support should result in failure.
//
//  - gnoi operation commands can be sent and tested using CLI command grpcurl.
//    https://github.com/fullstorydev/grpcurl
//

func TestChassisReboot(t *testing.T) {
	dut := ondatra.DUT(t, "dut")

	cases := []struct {
		desc          string
		rebootRequest *spb.RebootRequest
	}{
		{
			desc: "with delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   rebootDelay * oneSecondInNanoSecond,
				Message: "Reboot chassis with delay",
				Force:   true,
			}},
		{
			desc: "without delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   0,
				Message: "Reboot chassis without delay",
				Force:   true,
			}},
	}

	expectedVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
	sort.Strings(expectedVersion)
	t.Logf("DUT software version: %v", expectedVersion)
	gnoiClient := dut.RawAPIs().GNOI().Default(t)

	for _, tc := range cases {
		t.Run(tc.desc, func(t *testing.T) {
			// TODO: Remove t.Skipf() after reboot with no delay issue is supported.
			if tc.rebootRequest.GetDelay() == 0 {
				t.Skipf("delay 0 option is not working due to known bug.")
			}

			bootTimeBeforeReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time before reboot: %v", bootTimeBeforeReboot)
			PrevTime := dut.Telemetry().System().CurrentDatetime().Get(t)
			start := time.Now()

			t.Logf("Send reboot request: %v", tc.rebootRequest)
			rebootResponse, err := gnoiClient.System().Reboot(context.Background(), tc.rebootRequest)
			t.Logf("Got reboot response: %v, err: %v", rebootResponse, err)
			if err != nil {
				t.Fatalf("Failed to reboot chassis with unexpected err: %v", err)
			}

			if tc.rebootRequest.GetDelay() > 1 {
				// DUT remains reachable for N seconds of delay by checking DUT time is updated.
				t.Logf("DUT remains reachable with the delay of %v seconds", rebootDelay)
				for {
					t.Logf("Time elapsed %.2f seconds since reboot was requested.", time.Since(start).Seconds())
					if uint64(time.Since(start).Seconds()) > rebootDelay {
						t.Logf("Time elapsed %v seconds > %v reboot delay", time.Since(start), rebootDelay)
						break
					}
					latestTime := dut.Telemetry().System().CurrentDatetime().Get(t)
					if latestTime == PrevTime {
						t.Errorf("Get latest system time: got %v, want newer time than %v", latestTime, PrevTime)
					}
					PrevTime = latestTime
					time.Sleep(10 * time.Second)
				}
			}

			startReboot := time.Now()
			t.Logf("Wait for DUT to boot up by polling the telemetry output.")
			for {
				var currentTime string
				t.Logf("Time elapsed %.2f seconds since reboot started.", time.Since(startReboot).Seconds())
				time.Sleep(30 * time.Second)
				if errMsg := testt.CaptureFatal(t, func(t testing.TB) {
					currentTime = dut.Telemetry().System().CurrentDatetime().Get(t)
				}); errMsg != nil {
					t.Logf("Got testt.CaptureFatal errMsg: %s, keep polling ...", *errMsg)
				} else {
					t.Logf("Device rebooted successfully with received time: %v", currentTime)
					break
				}

				if uint64(time.Since(startReboot).Seconds()) > maxRebootTime {
					t.Errorf("Check boot time: got %v, want < %v", time.Since(startReboot), maxRebootTime)
				}
			}
			t.Logf("Device boot time: %.2f seconds", time.Since(startReboot).Seconds())

			bootTimeAfterReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time after reboot: %v", bootTimeAfterReboot)
			if bootTimeAfterReboot <= bootTimeBeforeReboot {
				t.Errorf("Get boot time: got %v, want > %v", bootTimeAfterReboot, bootTimeBeforeReboot)
			}

			swVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
			sort.Strings(swVersion)
			t.Logf("DUT software version after reboot: %v", swVersion)
			if diff := cmp.Diff(expectedVersion, swVersion); diff != "" {
				t.Errorf("Software version differed (-want +got):\n%v", diff)
			}
		})
	}
}
<commit_msg>Chassis Reboot Test: Gather DUT datetime after sleep (#193)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package complete_chassis_reboot_test

import (
	"context"
	"sort"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/openconfig/featureprofiles/internal/fptest"
	spb "github.com/openconfig/gnoi/system"
	"github.com/openconfig/ondatra"
	"github.com/openconfig/testt"
)

const (
	oneMinuteInNanoSecond = 6e10
	oneSecondInNanoSecond = 1e9
	rebootDelay           = 120
	// Maximum reboot time is 900 seconds (15 minutes).
	maxRebootTime = 900
)

func TestMain(m *testing.M) {
	fptest.RunTests(m)
}

// Test cases:
//  1) Send gNOI reboot request using the method COLD with the delay of N seconds.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: In nanoseconds before issuing reboot.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT remains reachable for N seconds by checking DUT current time is updated.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//  2) Send gNOI reboot request using the method COLD without delay.
//     - method: Only the COLD method is required to be supported by all targets.
//     - Delay: 0 - no delay.
//     - message: Informational reason for the reboot.
//     - force: Force reboot if basic checks fail. (ex. uncommitted configuration).
//   - Verify the following items.
//     - DUT boot time is updated after reboot.
//     - DUT software version is the same after the reboot.
//
// Topology:
//   dut:port1 <--> ate:port1
//
// Test notes:
//  - A RebootRequest requests the specified target be rebooted using the specified
//    method after the specified delay.  Only the DEFAULT method with a delay of 0
//    is guaranteed to be accepted for all target types.
//  - A RebootMethod determines what should be done with a target when a Reboot is
//    requested.  Only the COLD method is required to be supported by all
//    targets.  Methods the target does not support should result in failure.
//
//  - gnoi operation commands can be sent and tested using CLI command grpcurl.
//    https://github.com/fullstorydev/grpcurl
//

func TestChassisReboot(t *testing.T) {
	dut := ondatra.DUT(t, "dut")

	cases := []struct {
		desc          string
		rebootRequest *spb.RebootRequest
	}{
		{
			desc: "with delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   rebootDelay * oneSecondInNanoSecond,
				Message: "Reboot chassis with delay",
				Force:   true,
			}},
		{
			desc: "without delay",
			rebootRequest: &spb.RebootRequest{
				Method:  spb.RebootMethod_COLD,
				Delay:   0,
				Message: "Reboot chassis without delay",
				Force:   true,
			}},
	}

	expectedVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
	sort.Strings(expectedVersion)
	t.Logf("DUT software version: %v", expectedVersion)
	gnoiClient := dut.RawAPIs().GNOI().Default(t)

	for _, tc := range cases {
		t.Run(tc.desc, func(t *testing.T) {
			// TODO: Remove t.Skipf() after reboot with no delay issue is supported.
			if tc.rebootRequest.GetDelay() == 0 {
				t.Skipf("delay 0 option is not working due to known bug.")
			}

			bootTimeBeforeReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time before reboot: %v", bootTimeBeforeReboot)
			prevTime, err := time.Parse(time.RFC3339, dut.Telemetry().System().CurrentDatetime().Get(t))
			if err != nil {
				t.Fatalf("Failed parsing current-datetime: %s", err)
			}
			start := time.Now()

			t.Logf("Send reboot request: %v", tc.rebootRequest)
			rebootResponse, err := gnoiClient.System().Reboot(context.Background(), tc.rebootRequest)
			t.Logf("Got reboot response: %v, err: %v", rebootResponse, err)
			if err != nil {
				t.Fatalf("Failed to reboot chassis with unexpected err: %v", err)
			}

			if tc.rebootRequest.GetDelay() > 1 {
				t.Logf("Validating DUT remains reachable for at least %d seconds", rebootDelay)
				for {
					time.Sleep(10 * time.Second)
					t.Logf("Time elapsed %.2f seconds since reboot was requested.", time.Since(start).Seconds())
					if uint64(time.Since(start).Seconds()) > rebootDelay {
						t.Logf("Time elapsed %.2f seconds > %d reboot delay", time.Since(start).Seconds(), rebootDelay)
						break
					}
					latestTime, err := time.Parse(time.RFC3339, dut.Telemetry().System().CurrentDatetime().Get(t))
					if err != nil {
						t.Fatalf("Failed parsing current-datetime: %s", err)
					}
					if latestTime.Before(prevTime) || latestTime.Equal(prevTime) {
						t.Errorf("Get latest system time: got %v, want newer time than %v", latestTime, prevTime)
					}
					prevTime = latestTime
				}
			}

			startReboot := time.Now()
			t.Logf("Wait for DUT to boot up by polling the telemetry output.")
			for {
				var currentTime string
				t.Logf("Time elapsed %.2f seconds since reboot started.", time.Since(startReboot).Seconds())
				time.Sleep(30 * time.Second)
				if errMsg := testt.CaptureFatal(t, func(t testing.TB) {
					currentTime = dut.Telemetry().System().CurrentDatetime().Get(t)
				}); errMsg != nil {
					t.Logf("Got testt.CaptureFatal errMsg: %s, keep polling ...", *errMsg)
				} else {
					t.Logf("Device rebooted successfully with received time: %v", currentTime)
					break
				}

				if uint64(time.Since(startReboot).Seconds()) > maxRebootTime {
					t.Errorf("Check boot time: got %v, want < %v", time.Since(startReboot), maxRebootTime)
				}
			}
			t.Logf("Device boot time: %.2f seconds", time.Since(startReboot).Seconds())

			bootTimeAfterReboot := dut.Telemetry().System().BootTime().Get(t)
			t.Logf("DUT boot time after reboot: %v", bootTimeAfterReboot)
			if bootTimeAfterReboot <= bootTimeBeforeReboot {
				t.Errorf("Get boot time: got %v, want > %v", bootTimeAfterReboot, bootTimeBeforeReboot)
			}

			swVersion := dut.Telemetry().ComponentAny().SoftwareVersion().Get(t)
			sort.Strings(swVersion)
			t.Logf("DUT software version after reboot: %v", swVersion)
			if diff := cmp.Diff(expectedVersion, swVersion); diff != "" {
				t.Errorf("Software version differed (-want +got):\n%v", diff)
			}
		})
	}
}
<|endoftext|>


<commit_before>package depot

import (
	"io"
	"sync"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot/event"
	"github.com/pivotal-golang/lager"
)

const ContainerInitializationFailedMessage = "failed to initialize container"

type client struct {
	*clientProvider
	logger lager.Logger
}

type clientProvider struct {
	totalCapacity   executor.ExecutorResources
	gardenStore     GardenStore
	allocationStore AllocationStore
	eventHub        event.Hub
	lockMap         map[string]*sync.Mutex
	lockForLockMap  *sync.Mutex
	resourcesLock   *sync.Mutex
}

type AllocationStore interface {
	List() []executor.Container
	Lookup(guid string) (executor.Container, error)
	Allocate(logger lager.Logger, container executor.Container) error
	Initialize(logger lager.Logger, guid string) error
	Fail(logger lager.Logger, guid string, reason string) error
	Deallocate(logger lager.Logger, guid string) error
}

type GardenStore interface {
	Create(logger lager.Logger, container executor.Container) (executor.Container, error)
	Lookup(guid string) (executor.Container, error)
	List(tags executor.Tags) ([]executor.Container, error)
	Destroy(logger lager.Logger, guid string) error
	Ping() error
	Run(logger lager.Logger, container executor.Container) error
	Stop(logger lager.Logger, guid string) error
	GetFiles(guid, sourcePath string) (io.ReadCloser, error)
}

func NewClientProvider(
	totalCapacity executor.ExecutorResources,
	allocationStore AllocationStore,
	gardenStore GardenStore,
	eventHub event.Hub,
) executor.ClientProvider {
	return &clientProvider{
		totalCapacity:   totalCapacity,
		allocationStore: allocationStore,
		gardenStore:     gardenStore,
		eventHub:        eventHub,
		lockMap:         map[string]*sync.Mutex{},
		lockForLockMap:  new(sync.Mutex),
		resourcesLock:   new(sync.Mutex),
	}
}

func (provider *clientProvider) WithLogger(logger lager.Logger) executor.Client {
	return &client{
		provider,
		logger.Session("depot-client"),
	}
}

func (c *client) getLock(guid string) *sync.Mutex {
	var lock *sync.Mutex
	var found bool

	if lock, found = c.lockMap[guid]; !found {
		c.lockForLockMap.Lock()
		if lock, found = c.lockMap[guid]; !found {
			lock = new(sync.Mutex)
			c.lockMap[guid] = lock
		}
		c.lockForLockMap.Unlock()
	}

	return lock
}

func (c *client) removeLockFromMap(guid string) {
	c.lockForLockMap.Lock()
	defer c.lockForLockMap.Unlock()
	delete(c.lockMap, guid)
}

func (c *client) AllocateContainers(executorContainers []executor.Container) (map[string]string, error) {
	logger := c.logger.Session("allocate")

	errMessageMap := map[string]string{}
	eligibleContainers := make([]executor.Container, 0, len(executorContainers))

	for _, executorContainer := range executorContainers {
		if executorContainer.CPUWeight > 100 || executorContainer.CPUWeight < 0 {
			logger.Debug("invalid-cpu-weight", lager.Data{
				"guid":      executorContainer.Guid,
				"cpuweight": executorContainer.CPUWeight,
			})
			errMessageMap[executorContainer.Guid] = executor.ErrLimitsInvalid.Error()
			continue
		} else if executorContainer.CPUWeight == 0 {
			executorContainer.CPUWeight = 100
		}

		if executorContainer.Guid == "" {
			logger.Debug("empty-guid")
			errMessageMap[executorContainer.Guid] = executor.ErrGuidNotSpecified.Error()
			continue
		}

		eligibleContainers = append(eligibleContainers, executorContainer)
	}

	c.resourcesLock.Lock()
	defer c.resourcesLock.Unlock()

	allocatableContainers, unallocatableContainers, err := c.checkSpace(eligibleContainers)
	if err != nil {
		logger.Error("failed-to-allocate-containers", err)
		return nil, executor.ErrFailureToCheckSpace
	}

	for _, allocatableContainer := range allocatableContainers {
		if err = c.allocationStore.Allocate(logger, allocatableContainer); err != nil {
			logger.Debug(
				"failed-to-allocate-container",
				lager.Data{
					"guid":         allocatableContainer.Guid,
					"errormessage": err.Error(),
				},
			)
			errMessageMap[allocatableContainer.Guid] = err.Error()
		} else {
			// TODO
			//c.eventHub.EmitEvent(executor.NewContainerReservedEvent(allocatableContainer))
		}
	}

	for _, unallocatableContainer := range unallocatableContainers {
		logger.Debug(
			"failed-to-allocate-container",
			lager.Data{
				"guid":         unallocatableContainer.Guid,
				"errormessage": executor.ErrInsufficientResourcesAvailable.Error(),
			},
		)
		errMessageMap[unallocatableContainer.Guid] = executor.ErrInsufficientResourcesAvailable.Error()
	}

	return errMessageMap, nil
}

func (c *client) GetContainer(guid string) (executor.Container, error) {
	logger := c.logger.Session("get", lager.Data{
		"guid": guid,
	})

	lock := c.getLock(guid)
	lock.Lock()
	defer lock.Unlock()

	container, err := c.allocationStore.Lookup(guid)
	if err != nil {
		container, err = c.gardenStore.Lookup(guid)
		if err != nil {
			logger.Error("container-not-found", err)
			return executor.Container{}, executor.ErrContainerNotFound
		}
	}

	return container, nil
}

func (c *client) RunContainer(guid string) error {
	logger := c.logger.Session("run", lager.Data{
		"guid": guid,
	})

	err := c.allocationStore.Initialize(logger, guid)
	if err != nil {
		logger.Error("failed-to-initialize-container", err)
		return err
	}

	go func() {
		lock := c.getLock(guid)
		lock.Lock()
		defer lock.Unlock()

		container, err := c.allocationStore.Lookup(guid)
		if err != nil {
			logger.Error("failed-to-find-container", err)
			return
		}

		container, err = c.gardenStore.Create(logger, container)
		if err != nil {
			logger.Error("failed-to-create-container", err)

			c.allocationStore.Fail(logger, guid, ContainerInitializationFailedMessage)
			//c.eventHub.EmitEvent(executor.NewContainerCompleteEvent(container))

			return
		}

		err = c.allocationStore.Deallocate(logger, guid)
		if err == executor.ErrContainerNotFound {
			logger.Debug("container-deleted-while-initilizing")

			err := c.gardenStore.Destroy(logger, guid)
			if err != nil {
				logger.Error("failed-to-destroy-newly-initialized-container", err)
			} else {
				logger.Info("destroyed-newly-initialized-container")
			}

			return
		} else if err != nil {
			logger.Error("failed-to-remove-allocated-container-for-unknown-reason", err)
			return
		}

		err = c.gardenStore.Run(logger, container)
		if err != nil {
			logger.Error("failed-to-run-newly-created-container", err)
		}
	}()

	return nil
}

func tagsMatch(needles, haystack executor.Tags) bool {
	for k, v := range needles {
		if haystack[k] != v {
			return false
		}
	}

	return true
}

func (c *client) ListContainers(tags executor.Tags) ([]executor.Container, error) {
	// Order here is important; listing containers from garden takes time, and in
	// that time a container may transition from allocation store to garden
	// store.
	//
	// In this case, if the garden store were fetched first, the container would
	// not be listed anywhere. This ordering guarantees that it will at least
	// show up allocated.
	allAllocatedContainers := c.allocationStore.List()
	allocatedContainers := make([]executor.Container, 0, len(allAllocatedContainers))
	for _, container := range allAllocatedContainers {
		if tagsMatch(tags, container.Tags) {
			allocatedContainers = append(allocatedContainers, container)
		}
	}
	gardenContainers, err := c.gardenStore.List(tags)
	if err != nil {
		return nil, err
	}

	// Order here is important; we want allocated containers to override garden
	// containers as a member of latter is uninitialized until its counterpart
	// in the former is gone and we don't want to publicize containers that are
	// in a half-baked state.
	containersByGuid := map[string]executor.Container{}
	for _, gardenContainer := range gardenContainers {
		containersByGuid[gardenContainer.Guid] = gardenContainer
	}
	for _, allocatedContainer := range allocatedContainers {
		containersByGuid[allocatedContainer.Guid] = allocatedContainer
	}

	containers := make([]executor.Container, 0, len(containersByGuid))
	for _, container := range containersByGuid {
		containers = append(containers, container)
	}

	return containers, nil
}

func (c *client) StopContainer(guid string) error {
	logger := c.logger.Session("stop", lager.Data{
		"guid": guid,
	})

	logger.Info("stopping-container")
	return c.gardenStore.Stop(logger, guid)
}

func (c *client) DeleteContainer(guid string) error {
	logger := c.logger.Session("delete", lager.Data{
		"guid": guid,
	})

	lock := c.getLock(guid)
	lock.Lock()
	defer lock.Unlock()

	allocationStoreErr := c.allocationStore.Deallocate(logger, guid)
	if allocationStoreErr != nil {
		logger.Debug("did-not-delete-allocation-container", lager.Data{"message": allocationStoreErr.Error()})
	}

	if _, err := c.gardenStore.Lookup(guid); err != nil {
		logger.Debug("garden-container-not-found", lager.Data{"message": err.Error()})
		c.removeLockFromMap(guid)
		return allocationStoreErr
	} else if err := c.gardenStore.Destroy(logger, guid); err != nil {
		logger.Error("failed-to-delete-garden-container", err)
		return err
	}

	c.removeLockFromMap(guid)
	return nil
}

func (c *client) remainingResources() (executor.ExecutorResources, error) {
	remainingResources, err := c.TotalResources()
	if err != nil {
		return executor.ExecutorResources{}, err
	}

	allocatedContainers := c.allocationStore.List()
	for _, allocation := range allocatedContainers {
		remainingResources.Containers--
		remainingResources.DiskMB -= allocation.DiskMB
		remainingResources.MemoryMB -= allocation.MemoryMB
	}

	gardenContainers, err := c.gardenStore.List(nil)
	if err != nil {
		return executor.ExecutorResources{}, err
	}

	for _, gardenContainer := range gardenContainers {
		remainingResources.Containers--
		remainingResources.DiskMB -= gardenContainer.DiskMB
		remainingResources.MemoryMB -= gardenContainer.MemoryMB
	}

	return remainingResources, nil
}

func (c *client) RemainingResources() (executor.ExecutorResources, error) {
	c.resourcesLock.Lock()
	defer c.resourcesLock.Unlock()

	return c.remainingResources()
}

func (c *client) Ping() error {
	return c.gardenStore.Ping()
}

func (c *client) TotalResources() (executor.ExecutorResources, error) {
	totalCapacity := c.totalCapacity

	return executor.ExecutorResources{
		MemoryMB:   totalCapacity.MemoryMB,
		DiskMB:     totalCapacity.DiskMB,
		Containers: totalCapacity.Containers,
	}, nil
}

func (c *client) GetFiles(guid, sourcePath string) (io.ReadCloser, error) {
	return c.gardenStore.GetFiles(guid, sourcePath)
}

func (c *client) SubscribeToEvents() (<-chan executor.Event, error) {
	return c.eventHub.Subscribe(), nil
}

func (c *client) checkSpace(containers []executor.Container) ([]executor.Container, []executor.Container, error) {
	remainingResources, err := c.remainingResources()
	if err != nil {
		return nil, nil, err
	}

	allocatableContainers := make([]executor.Container, 0, len(containers))
	unallocatableContainers := make([]executor.Container, 0, len(containers))

	// Can be optimized to select containers such that maximum number of containers can be allocated
	for _, container := range containers {
		if remainingResources.MemoryMB < container.MemoryMB {
			unallocatableContainers = append(unallocatableContainers, container)
			continue
		}

		if remainingResources.DiskMB < container.DiskMB {
			unallocatableContainers = append(unallocatableContainers, container)
			continue
		}

		if remainingResources.Containers < 1 {
			unallocatableContainers = append(unallocatableContainers, container)
			continue
		}

		remainingResources.MemoryMB -= container.MemoryMB
		remainingResources.DiskMB -= container.DiskMB
		remainingResources.Containers--
		allocatableContainers = append(allocatableContainers, container)
	}

	return allocatableContainers, unallocatableContainers, nil
}
<commit_msg>Replaced double checked locking by one global lock<commit_after>package depot

import (
	"io"
	"sync"

	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/executor/depot/event"
	"github.com/pivotal-golang/lager"
)

const ContainerInitializationFailedMessage = "failed to initialize container"

type client struct {
	*clientProvider
	logger lager.Logger
}

type clientProvider struct {
	totalCapacity   executor.ExecutorResources
	gardenStore     GardenStore
	allocationStore AllocationStore
	eventHub        event.Hub
	lockMap         map[string]*sync.Mutex
	lockForLockMap  *sync.Mutex
	resourcesLock   *sync.Mutex
}

type AllocationStore interface {
	List() []executor.Container
	Lookup(guid string) (executor.Container, error)
	Allocate(logger lager.Logger, container executor.Container) error
	Initialize(logger lager.Logger, guid string) error
	Fail(logger lager.Logger, guid string, reason string) error
	Deallocate(logger lager.Logger, guid string) error
}

type GardenStore interface {
	Create(logger lager.Logger, container executor.Container) (executor.Container, error)
	Lookup(guid string) (executor.Container, error)
	List(tags executor.Tags) ([]executor.Container, error)
	Destroy(logger lager.Logger, guid string) error
	Ping() error
	Run(logger lager.Logger, container executor.Container) error
	Stop(logger lager.Logger, guid string) error
	GetFiles(guid, sourcePath string) (io.ReadCloser, error)
}

func NewClientProvider(
	totalCapacity executor.ExecutorResources,
	allocationStore AllocationStore,
	gardenStore GardenStore,
	eventHub event.Hub,
) executor.ClientProvider {
	return &clientProvider{
		totalCapacity:   totalCapacity,
		allocationStore: allocationStore,
		gardenStore:     gardenStore,
		eventHub:        eventHub,
		lockMap:         map[string]*sync.Mutex{},
		lockForLockMap:  new(sync.Mutex),
		resourcesLock:   new(sync.Mutex),
	}
}

func (provider *clientProvider) WithLogger(logger lager.Logger) executor.Client {
	return &client{
		provider,
		logger.Session("depot-client"),
	}
}

func (c *client) getLock(guid string) *sync.Mutex {
	var lock *sync.Mutex
	var found bool

	c.lockForLockMap.Lock()
	defer c.lockForLockMap.Unlock()

	if lock, found = c.lockMap[guid]; !found {
		lock = new(sync.Mutex)
		c.lockMap[guid] = lock
	}

	return lock
}

func (c *client) removeLockFromMap(guid string) {
	c.lockForLockMap.Lock()
	defer c.lockForLockMap.Unlock()
	delete(c.lockMap, guid)
}

func (c *client) AllocateContainers(executorContainers []executor.Container) (map[string]string, error) {
	logger := c.logger.Session("allocate")

	errMessageMap := map[string]string{}
	eligibleContainers := make([]executor.Container, 0, len(executorContainers))

	for _, executorContainer := range executorContainers {
		if executorContainer.CPUWeight > 100 || executorContainer.CPUWeight < 0 {
			logger.Debug("invalid-cpu-weight", lager.Data{
				"guid":      executorContainer.Guid,
				"cpuweight": executorContainer.CPUWeight,
			})
			errMessageMap[executorContainer.Guid] = executor.ErrLimitsInvalid.Error()
			continue
		} else if executorContainer.CPUWeight == 0 {
			executorContainer.CPUWeight = 100
		}

		if executorContainer.Guid == "" {
			logger.Debug("empty-guid")
			errMessageMap[executorContainer.Guid] = executor.ErrGuidNotSpecified.Error()
			continue
		}

		eligibleContainers = append(eligibleContainers, executorContainer)
	}

	c.resourcesLock.Lock()
	defer c.resourcesLock.Unlock()

	allocatableContainers, unallocatableContainers, err := c.checkSpace(eligibleContainers)
	if err != nil {
		logger.Error("failed-to-allocate-containers", err)
		return nil, executor.ErrFailureToCheckSpace
	}

	for _, allocatableContainer := range allocatableContainers {
		if err = c.allocationStore.Allocate(logger, allocatableContainer); err != nil {
			logger.Debug(
				"failed-to-allocate-container",
				lager.Data{
					"guid":         allocatableContainer.Guid,
					"errormessage": err.Error(),
				},
			)
			errMessageMap[allocatableContainer.Guid] = err.Error()
		} else {
			// TODO
			//c.eventHub.EmitEvent(executor.NewContainerReservedEvent(allocatableContainer))
		}
	}

	for _, unallocatableContainer := range unallocatableContainers {
		logger.Debug(
			"failed-to-allocate-container",
			lager.Data{
				"guid":         unallocatableContainer.Guid,
				"errormessage": executor.ErrInsufficientResourcesAvailable.Error(),
			},
		)
		errMessageMap[unallocatableContainer.Guid] = executor.ErrInsufficientResourcesAvailable.Error()
	}

	return errMessageMap, nil
}

func (c *client) GetContainer(guid string) (executor.Container, error) {
	logger := c.logger.Session("get", lager.Data{
		"guid": guid,
	})

	lock := c.getLock(guid)
	lock.Lock()
	defer lock.Unlock()

	container, err := c.allocationStore.Lookup(guid)
	if err != nil {
		container, err = c.gardenStore.Lookup(guid)
		if err != nil {
			logger.Error("container-not-found", err)
			return executor.Container{}, executor.ErrContainerNotFound
		}
	}

	return container, nil
}

func (c *client) RunContainer(guid string) error {
	logger := c.logger.Session("run", lager.Data{
		"guid": guid,
	})

	err := c.allocationStore.Initialize(logger, guid)
	if err != nil {
		logger.Error("failed-to-initialize-container", err)
		return err
	}

	go func() {
		lock := c.getLock(guid)
		lock.Lock()
		defer lock.Unlock()

		container, err := c.allocationStore.Lookup(guid)
		if err != nil {
			logger.Error("failed-to-find-container", err)
			return
		}

		container, err = c.gardenStore.Create(logger, container)
		if err != nil {
			logger.Error("failed-to-create-container", err)

			c.allocationStore.Fail(logger, guid, ContainerInitializationFailedMessage)
			//c.eventHub.EmitEvent(executor.NewContainerCompleteEvent(container))

			return
		}

		err = c.allocationStore.Deallocate(logger, guid)
		if err == executor.ErrContainerNotFound {
			logger.Debug("container-deleted-while-initilizing")

			err := c.gardenStore.Destroy(logger, guid)
			if err != nil {
				logger.Error("failed-to-destroy-newly-initialized-container", err)
			} else {
				logger.Info("destroyed-newly-initialized-container")
			}

			return
		} else if err != nil {
			logger.Error("failed-to-remove-allocated-container-for-unknown-reason", err)
			return
		}

		err = c.gardenStore.Run(logger, container)
		if err != nil {
			logger.Error("failed-to-run-newly-created-container", err)
		}
	}()

	return nil
}

func tagsMatch(needles, haystack executor.Tags) bool {
	for k, v := range needles {
		if haystack[k] != v {
			return false
		}
	}

	return true
}

func (c *client) ListContainers(tags executor.Tags) ([]executor.Container, error) {
	// Order here is important; listing containers from garden takes time, and in
	// that time a container may transition from allocation store to garden
	// store.
	//
	// In this case, if the garden store were fetched first, the container would
	// not be listed anywhere. This ordering guarantees that it will at least
	// show up allocated.
	allAllocatedContainers := c.allocationStore.List()
	allocatedContainers := make([]executor.Container, 0, len(allAllocatedContainers))
	for _, container := range allAllocatedContainers {
		if tagsMatch(tags, container.Tags) {
			allocatedContainers = append(allocatedContainers, container)
		}
	}
	gardenContainers, err := c.gardenStore.List(tags)
	if err != nil {
		return nil, err
	}

	// Order here is important; we want allocated containers to override garden
	// containers as a member of latter is uninitialized until its counterpart
	// in the former is gone and we don't want to publicize containers that are
	// in a half-baked state.
	containersByGuid := map[string]executor.Container{}
	for _, gardenContainer := range gardenContainers {
		containersByGuid[gardenContainer.Guid] = gardenContainer
	}
	for _, allocatedContainer := range allocatedContainers {
		containersByGuid[allocatedContainer.Guid] = allocatedContainer
	}

	containers := make([]executor.Container, 0, len(containersByGuid))
	for _, container := range containersByGuid {
		containers = append(containers, container)
	}

	return containers, nil
}

func (c *client) StopContainer(guid string) error {
	logger := c.logger.Session("stop", lager.Data{
		"guid": guid,
	})

	logger.Info("stopping-container")
	return c.gardenStore.Stop(logger, guid)
}

func (c *client) DeleteContainer(guid string) error {
	logger := c.logger.Session("delete", lager.Data{
		"guid": guid,
	})

	lock := c.getLock(guid)
	lock.Lock()
	defer lock.Unlock()

	allocationStoreErr := c.allocationStore.Deallocate(logger, guid)
	if allocationStoreErr != nil {
		logger.Debug("did-not-delete-allocation-container", lager.Data{"message": allocationStoreErr.Error()})
	}

	if _, err := c.gardenStore.Lookup(guid); err != nil {
		logger.Debug("garden-container-not-found", lager.Data{"message": err.Error()})
		c.removeLockFromMap(guid)
		return allocationStoreErr
	} else if err := c.gardenStore.Destroy(logger, guid); err != nil {
		logger.Error("failed-to-delete-garden-container", err)
		return err
	}

	c.removeLockFromMap(guid)
	return nil
}

func (c *client) remainingResources() (executor.ExecutorResources, error) {
	remainingResources, err := c.TotalResources()
	if err != nil {
		return executor.ExecutorResources{}, err
	}

	allocatedContainers := c.allocationStore.List()
	for _, allocation := range allocatedContainers {
		remainingResources.Containers--
		remainingResources.DiskMB -= allocation.DiskMB
		remainingResources.MemoryMB -= allocation.MemoryMB
	}

	gardenContainers, err := c.gardenStore.List(nil)
	if err != nil {
		return executor.ExecutorResources{}, err
	}

	for _, gardenContainer := range gardenContainers {
		remainingResources.Containers--
		remainingResources.DiskMB -= gardenContainer.DiskMB
		remainingResources.MemoryMB -= gardenContainer.MemoryMB
	}

	return remainingResources, nil
}

func (c *client) RemainingResources() (executor.ExecutorResources, error) {
	c.resourcesLock.Lock()
	defer c.resourcesLock.Unlock()

	return c.remainingResources()
}

func (c *client) Ping() error {
	return c.gardenStore.Ping()
}

func (c *client) TotalResources() (executor.ExecutorResources, error) {
	totalCapacity := c.totalCapacity

	return executor.ExecutorResources{
		MemoryMB:   totalCapacity.MemoryMB,
		DiskMB:     totalCapacity.DiskMB,
		Containers: totalCapacity.Containers,
	}, nil
}

func (c *client) GetFiles(guid, sourcePath string) (io.ReadCloser, error) {
	return c.gardenStore.GetFiles(guid, sourcePath)
}

func (c *client) SubscribeToEvents() (<-chan executor.Event, error) {
	return c.eventHub.Subscribe(), nil
}

func (c *client) checkSpace(containers []executor.Container) ([]executor.Container, []executor.Container, error) {
	remainingResources, err := c.remainingResources()
	if err != nil {
		return nil, nil, err
	}

	allocatableContainers := make([]executor.Container, 0, len(containers))
	unallocatableContainers := make([]executor.Container, 0, len(containers))

	// Can be optimized to select containers such that maximum number of containers can be allocated
	for _, container := range containers {
		if remainingResources.MemoryMB < container.MemoryMB {
			unallocatableContainers = append(unallocatableContainers, container)
			continue
		}

		if remainingResources.DiskMB < container.DiskMB {
			unallocatableContainers = append(unallocatableContainers, container)
			continue
		}

		if remainingResources.Containers < 1 {
			unallocatableContainers = append(unallocatableContainers, container)
			continue
		}

		remainingResources.MemoryMB -= container.MemoryMB
		remainingResources.DiskMB -= container.DiskMB
		remainingResources.Containers--
		allocatableContainers = append(allocatableContainers, container)
	}

	return allocatableContainers, unallocatableContainers, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dbdaemon

import (
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"cloud.google.com/go/storage"
	"k8s.io/client-go/util/retry"
	"k8s.io/klog/v2"
)

const gsPrefix = "gs://"

// Override library functions for the benefit of unit tests.
var (
	lsnrctl = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "lsnrctl")
	}
	rman = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "rman")
	}
	orapwd = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "orapwd")
	}
	impdp = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "impdp")
	}
	expdp = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "expdp")
	}
)

const (
	contentTypePlainText = "plain/text"
	contentTypeGZ        = "application/gzip"
)

// osUtil was defined for tests.
type osUtil interface {
	runCommand(bin string, params []string) error
	isReturnCodeEqual(err error, code int) bool
	createFile(file string, content io.Reader) error
	removeFile(file string) error
}

type osUtilImpl struct {
}

func (o *osUtilImpl) runCommand(bin string, params []string) error {
	ohome := os.Getenv("ORACLE_HOME")
	klog.InfoS("executing command with args", "cmd", bin, "params", params, "ORACLE_SID", os.Getenv("ORACLE_SID"), "ORACLE_HOME", ohome, "TNS_ADMIN", os.Getenv("TNS_ADMIN"))
	switch bin {
	case lsnrctl(ohome), rman(ohome), orapwd(ohome), impdp(ohome), expdp(ohome):
	default:
		klog.InfoS("command not supported", "bin", bin)
		return fmt.Errorf("command %q is not supported", bin)
	}
	cmd := exec.Command(bin)
	cmd.Args = append(cmd.Args, params...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (o *osUtilImpl) isReturnCodeEqual(err error, code int) bool {
	if exitError, ok := err.(*exec.ExitError); ok {
		return exitError.ExitCode() == code
	}
	return false
}

func (o *osUtilImpl) createFile(file string, content io.Reader) error {
	dir := filepath.Dir(file)
	if err := os.MkdirAll(dir, 0750); err != nil {
		return fmt.Errorf("couldn't create dir err: %v", err)
	}
	f, err := os.Create(file) // truncates if file exists.
	if err != nil {
		return fmt.Errorf("couldn't create file err: %v", err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			klog.Warningf("failed to close %v: %v", f, err)
		}
	}()
	if _, err := io.Copy(f, content); err != nil {
		return fmt.Errorf("copying contents failed: %v", err)
	}
	return nil
}

func (o *osUtilImpl) removeFile(file string) error {
	return os.Remove(file)
}

// GCSUtil contains helper methods for reading/writing GCS objects.
type GCSUtil interface {
	// Download returns an io.ReadCloser for GCS object at given gcsPath.
	Download(ctx context.Context, gcsPath string) (io.ReadCloser, error)
	// UploadFile uploads contents of a file at filepath to gcsPath location in
	// GCS and sets object's contentType.
	// If gcsPath ends with .gz it also compresses the uploaded contents
	// and sets object's content type to application/gzip.
	UploadFile(ctx context.Context, gcsPath, filepath, contentType string) error
	// SplitURI takes a GCS URI and splits it into bucket and object names. If the URI does not have
	// the gs:// scheme, or the URI doesn't specify both a bucket and an object name, returns an error.
	SplitURI(url string) (bucket, name string, err error)
}

type gcsUtilImpl struct{}

func (g *gcsUtilImpl) Download(ctx context.Context, gcsPath string) (io.ReadCloser, error) {
	bucket, name, err := g.SplitURI(gcsPath)
	if err != nil {
		return nil, err
	}

	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to init GCS client: %v", err)
	}
	defer client.Close()

	reader, err := client.Bucket(bucket).Object(name).NewReader(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to read URL %s: %v", gcsPath, err)
	}

	return reader, nil
}

func (g *gcsUtilImpl) UploadFile(ctx context.Context, gcsPath, filePath, contentType string) error {
	return retry.OnError(retry.DefaultBackoff, func(err error) bool {
		klog.ErrorS(err, "failed to upload a file")
		// tried to cast err to *googleapi.Error with errors.As and wrap the error
		// in uploadFile. returned err is not a *googleapi.Error.
		return err != nil && strings.Contains(err.Error(), "compute: Received 500 ")
	}, func() error {
		return g.uploadFile(ctx, gcsPath, filePath, contentType)
	})

}

// uploadFile is the implementation of UploadFile to be wrapped with retry logic.
func (g *gcsUtilImpl) uploadFile(ctx context.Context, gcsPath, filePath, contentType string) error {
	bucket, name, err := g.SplitURI(gcsPath)
	if err != nil {
		return err
	}

	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			klog.Warningf("failed to close %v: %v", f, err)
		}
	}()

	client, err := storage.NewClient(ctx)
	if err != nil {
		return fmt.Errorf("failed to init GCS client: %v", err)
	}
	defer client.Close()

	b := client.Bucket(bucket)
	// check if bucket exists and it is accessible
	if _, err := b.Attrs(ctx); err != nil {
		return err
	}

	gcsWriter := b.Object(name).NewWriter(ctx)
	gcsWriter.ContentType = contentType
	defer gcsWriter.Close()

	var writer io.WriteCloser = gcsWriter
	if strings.HasSuffix(gcsPath, ".gz") {
		gcsWriter.ContentType = contentTypeGZ
		writer = gzip.NewWriter(gcsWriter)
		defer writer.Close()
	}

	_, err = io.Copy(writer, f)
	if err != nil {
		return fmt.Errorf("failed to write file %s to %s: %v", filePath, gcsPath, err)
	}

	return nil
}

func (g *gcsUtilImpl) SplitURI(url string) (bucket, name string, err error) {
	u := strings.TrimPrefix(url, gsPrefix)
	if u == url {
		return "", "", fmt.Errorf("URL %q is missing the %q prefix", url, gsPrefix)
	}
	if i := strings.Index(u, "/"); i >= 2 {
		return u[:i], u[i+1:], nil
	}
	return "", "", fmt.Errorf("URL %q does not specify a bucket and a name", url)
}
<commit_msg>Improve error handling for GCS uploads.<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dbdaemon

import (
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"cloud.google.com/go/storage"
	"k8s.io/client-go/util/retry"
	"k8s.io/klog/v2"
)

const gsPrefix = "gs://"

// Override library functions for the benefit of unit tests.
var (
	lsnrctl = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "lsnrctl")
	}
	rman = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "rman")
	}
	orapwd = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "orapwd")
	}
	impdp = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "impdp")
	}
	expdp = func(databaseHome string) string {
		return filepath.Join(databaseHome, "bin", "expdp")
	}
)

const (
	contentTypePlainText = "plain/text"
	contentTypeGZ        = "application/gzip"
)

// osUtil was defined for tests.
type osUtil interface {
	runCommand(bin string, params []string) error
	isReturnCodeEqual(err error, code int) bool
	createFile(file string, content io.Reader) error
	removeFile(file string) error
}

type osUtilImpl struct {
}

func (o *osUtilImpl) runCommand(bin string, params []string) error {
	ohome := os.Getenv("ORACLE_HOME")
	klog.InfoS("executing command with args", "cmd", bin, "params", params, "ORACLE_SID", os.Getenv("ORACLE_SID"), "ORACLE_HOME", ohome, "TNS_ADMIN", os.Getenv("TNS_ADMIN"))
	switch bin {
	case lsnrctl(ohome), rman(ohome), orapwd(ohome), impdp(ohome), expdp(ohome):
	default:
		klog.InfoS("command not supported", "bin", bin)
		return fmt.Errorf("command %q is not supported", bin)
	}
	cmd := exec.Command(bin)
	cmd.Args = append(cmd.Args, params...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (o *osUtilImpl) isReturnCodeEqual(err error, code int) bool {
	if exitError, ok := err.(*exec.ExitError); ok {
		return exitError.ExitCode() == code
	}
	return false
}

func (o *osUtilImpl) createFile(file string, content io.Reader) error {
	dir := filepath.Dir(file)
	if err := os.MkdirAll(dir, 0750); err != nil {
		return fmt.Errorf("couldn't create dir err: %v", err)
	}
	f, err := os.Create(file) // truncates if file exists.
	if err != nil {
		return fmt.Errorf("couldn't create file err: %v", err)
	}
	defer func() {
		if err := f.Close(); err != nil {
			klog.Warningf("failed to close %v: %v", f, err)
		}
	}()
	if _, err := io.Copy(f, content); err != nil {
		return fmt.Errorf("copying contents failed: %v", err)
	}
	return nil
}

func (o *osUtilImpl) removeFile(file string) error {
	return os.Remove(file)
}

// GCSUtil contains helper methods for reading/writing GCS objects.
type GCSUtil interface {
	// Download returns an io.ReadCloser for GCS object at given gcsPath.
	Download(ctx context.Context, gcsPath string) (io.ReadCloser, error)
	// UploadFile uploads contents of a file at filepath to gcsPath location in
	// GCS and sets object's contentType.
	// If gcsPath ends with .gz it also compresses the uploaded contents
	// and sets object's content type to application/gzip.
	UploadFile(ctx context.Context, gcsPath, filepath, contentType string) error
	// SplitURI takes a GCS URI and splits it into bucket and object names. If the URI does not have
	// the gs:// scheme, or the URI doesn't specify both a bucket and an object name, returns an error.
	SplitURI(url string) (bucket, name string, err error)
}

type gcsUtilImpl struct{}

func (g *gcsUtilImpl) Download(ctx context.Context, gcsPath string) (io.ReadCloser, error) {
	bucket, name, err := g.SplitURI(gcsPath)
	if err != nil {
		return nil, err
	}

	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to init GCS client: %v", err)
	}
	defer client.Close()

	reader, err := client.Bucket(bucket).Object(name).NewReader(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to read URL %s: %v", gcsPath, err)
	}

	return reader, nil
}

func (g *gcsUtilImpl) UploadFile(ctx context.Context, gcsPath, filePath, contentType string) error {
	return retry.OnError(retry.DefaultBackoff, func(err error) bool {
		klog.ErrorS(err, "failed to upload a file")
		// tried to cast err to *googleapi.Error with errors.As and wrap the error
		// in uploadFile. returned err is not a *googleapi.Error.
		return err != nil && strings.Contains(err.Error(), "compute: Received 500 ")
	}, func() error {
		return g.uploadFile(ctx, gcsPath, filePath, contentType)
	})

}

// uploadFile is the implementation of UploadFile to be wrapped with retry logic.
func (g *gcsUtilImpl) uploadFile(ctx context.Context, gcsPath, filePath, contentType string) error {
	bucket, name, err := g.SplitURI(gcsPath)
	if err != nil {
		return err
	}

	f, err := os.Open(filePath)
	if err != nil {
		return err
	}
	defer func() {
		if err := f.Close(); err != nil {
			klog.Warningf("failed to close %v: %v", f, err)
		}
	}()

	client, err := storage.NewClient(ctx)
	if err != nil {
		return fmt.Errorf("failed to init GCS client: %v", err)
	}
	defer client.Close()

	b := client.Bucket(bucket)
	// check if bucket exists and it is accessible
	if _, err := b.Attrs(ctx); err != nil {
		return err
	}

	gcsWriter := b.Object(name).NewWriter(ctx)
	gcsWriter.ContentType = contentType

	var writer io.WriteCloser = gcsWriter
	if strings.HasSuffix(gcsPath, ".gz") {
		gcsWriter.ContentType = contentTypeGZ
		writer = gzip.NewWriter(gcsWriter)
	}

	_, err = io.Copy(writer, f)
	if err != nil {
		return fmt.Errorf("failed to write file %s to %s: %v", filePath, gcsPath, err)
	}
	if err = writer.Close(); err != nil {
		return fmt.Errorf("failed to complete writing file %s to %s: %v", filePath, gcsPath, err)
	}
	if err = gcsWriter.Close(); err != nil {
		return fmt.Errorf("failed to complete writing file %s to %s: %v", filePath, gcsPath, err)
	}

	return nil
}

func (g *gcsUtilImpl) SplitURI(url string) (bucket, name string, err error) {
	u := strings.TrimPrefix(url, gsPrefix)
	if u == url {
		return "", "", fmt.Errorf("URL %q is missing the %q prefix", url, gsPrefix)
	}
	if i := strings.Index(u, "/"); i >= 2 {
		return u[:i], u[i+1:], nil
	}
	return "", "", fmt.Errorf("URL %q does not specify a bucket and a name", url)
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

                 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package broadcast

import (
	"github.com/hyperledger/fabric/orderer/common/filter"
	cb "github.com/hyperledger/fabric/protos/common"
	ab "github.com/hyperledger/fabric/protos/orderer"
	"github.com/op/go-logging"

	"github.com/golang/protobuf/proto"
)

var logger = logging.MustGetLogger("orderer/common/broadcast")

func init() {
	logging.SetLevel(logging.DEBUG, "")
}

// Handler defines an interface which handles broadcasts
type Handler interface {
	// Handle starts a service thread for a given gRPC connection and services the broadcast connection
	Handle(srv ab.AtomicBroadcast_BroadcastServer) error
}

// SupportManager provides a way for the Handler to look up the Support for a chain
type SupportManager interface {
	// GetChain gets the chain support for a given ChainID
	GetChain(chainID string) (Support, bool)

	// ProposeChain accepts a configuration transaction for a chain which does not already exists
	// The status returned is whether the proposal is accepted for consideration, only after consensus
	// occurs will the proposal be committed or rejected
	ProposeChain(env *cb.Envelope) cb.Status
}

// Support provides the backing resources needed to support broadcast on a chain
type Support interface {
	// Enqueue accepts a message and returns true on acceptance, or false on shutdown
	Enqueue(env *cb.Envelope) bool

	// Filters returns the set of broadcast filters for this chain
	Filters() *filter.RuleSet
}

type handlerImpl struct {
	queueSize int
	sm        SupportManager
}

// NewHandlerImpl constructs a new implementation of the Handler interface
func NewHandlerImpl(sm SupportManager, queueSize int) Handler {
	return &handlerImpl{
		queueSize: queueSize,
		sm:        sm,
	}
}

// Handle starts a service thread for a given gRPC connection and services the broadcast connection
func (bh *handlerImpl) Handle(srv ab.AtomicBroadcast_BroadcastServer) error {
	b := newBroadcaster(bh)
	defer close(b.queue)
	go b.drainQueue()
	return b.queueEnvelopes(srv)
}

type msgAndSupport struct {
	msg     *cb.Envelope
	support Support
}

type broadcaster struct {
	bs       *handlerImpl
	queue    chan *msgAndSupport
	exitChan chan struct{}
}

func newBroadcaster(bs *handlerImpl) *broadcaster {
	b := &broadcaster{
		bs:       bs,
		queue:    make(chan *msgAndSupport, bs.queueSize),
		exitChan: make(chan struct{}),
	}
	return b
}

func (b *broadcaster) drainQueue() {
	defer close(b.exitChan)
	for msgAndSupport := range b.queue {
		if !msgAndSupport.support.Enqueue(msgAndSupport.msg) {
			logger.Debugf("Consenter instructed us to shut down")
			return
		}
	}
	logger.Debugf("Exiting because the queue channel closed")
}

func (b *broadcaster) queueEnvelopes(srv ab.AtomicBroadcast_BroadcastServer) error {

	for {
		select {
		case <-b.exitChan:
			return nil
		default:
		}
		msg, err := srv.Recv()
		if err != nil {
			return err
		}

		payload := &cb.Payload{}
		err = proto.Unmarshal(msg.Payload, payload)
		if payload.Header == nil || payload.Header.ChainHeader == nil || payload.Header.ChainHeader.ChainID == "" {
			logger.Debugf("Received malformed message, dropping connection")
			return srv.Send(&ab.BroadcastResponse{Status: cb.Status_BAD_REQUEST})
		}

		support, ok := b.bs.sm.GetChain(payload.Header.ChainHeader.ChainID)
		if !ok {
			// Chain not found, maybe create one?
			if payload.Header.ChainHeader.Type != int32(cb.HeaderType_CONFIGURATION_TRANSACTION) {
				err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_NOT_FOUND})
			} else {
				logger.Debugf("Proposing new chain")
				err = srv.Send(&ab.BroadcastResponse{Status: b.bs.sm.ProposeChain(msg)})
			}
		} else {
			// Normal transaction for existing chain
			_, filterErr := support.Filters().Apply(msg)

			if filterErr != nil {
				logger.Debugf("Rejecting broadcast message")
				err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_BAD_REQUEST})
			} else {
				select {
				case b.queue <- &msgAndSupport{msg: msg, support: support}:
					err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_SUCCESS})
				default:
					err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_SERVICE_UNAVAILABLE})
				}
			}
		}

		if err != nil {
			return err
		}
	}
}
<commit_msg>[FAB-1447] Broadcaster exits normally at EOF<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

                 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package broadcast

import (
	"github.com/hyperledger/fabric/orderer/common/filter"
	cb "github.com/hyperledger/fabric/protos/common"
	ab "github.com/hyperledger/fabric/protos/orderer"
	"github.com/op/go-logging"

	"io"

	"github.com/golang/protobuf/proto"
)

var logger = logging.MustGetLogger("orderer/common/broadcast")

func init() {
	logging.SetLevel(logging.DEBUG, "")
}

// Handler defines an interface which handles broadcasts
type Handler interface {
	// Handle starts a service thread for a given gRPC connection and services the broadcast connection
	Handle(srv ab.AtomicBroadcast_BroadcastServer) error
}

// SupportManager provides a way for the Handler to look up the Support for a chain
type SupportManager interface {
	// GetChain gets the chain support for a given ChainID
	GetChain(chainID string) (Support, bool)

	// ProposeChain accepts a configuration transaction for a chain which does not already exists
	// The status returned is whether the proposal is accepted for consideration, only after consensus
	// occurs will the proposal be committed or rejected
	ProposeChain(env *cb.Envelope) cb.Status
}

// Support provides the backing resources needed to support broadcast on a chain
type Support interface {
	// Enqueue accepts a message and returns true on acceptance, or false on shutdown
	Enqueue(env *cb.Envelope) bool

	// Filters returns the set of broadcast filters for this chain
	Filters() *filter.RuleSet
}

type handlerImpl struct {
	queueSize int
	sm        SupportManager
}

// NewHandlerImpl constructs a new implementation of the Handler interface
func NewHandlerImpl(sm SupportManager, queueSize int) Handler {
	return &handlerImpl{
		queueSize: queueSize,
		sm:        sm,
	}
}

// Handle starts a service thread for a given gRPC connection and services the broadcast connection
func (bh *handlerImpl) Handle(srv ab.AtomicBroadcast_BroadcastServer) error {
	b := newBroadcaster(bh)
	defer close(b.queue)
	go b.drainQueue()
	return b.queueEnvelopes(srv)
}

type msgAndSupport struct {
	msg     *cb.Envelope
	support Support
}

type broadcaster struct {
	bs       *handlerImpl
	queue    chan *msgAndSupport
	exitChan chan struct{}
}

func newBroadcaster(bs *handlerImpl) *broadcaster {
	b := &broadcaster{
		bs:       bs,
		queue:    make(chan *msgAndSupport, bs.queueSize),
		exitChan: make(chan struct{}),
	}
	return b
}

func (b *broadcaster) drainQueue() {
	defer close(b.exitChan)
	for msgAndSupport := range b.queue {
		if !msgAndSupport.support.Enqueue(msgAndSupport.msg) {
			logger.Debugf("Consenter instructed us to shut down")
			return
		}
	}
	logger.Debugf("Exiting because the queue channel closed")
}

func (b *broadcaster) queueEnvelopes(srv ab.AtomicBroadcast_BroadcastServer) error {

	for {
		select {
		case <-b.exitChan:
			return nil
		default:
		}
		msg, err := srv.Recv()
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}

		payload := &cb.Payload{}
		err = proto.Unmarshal(msg.Payload, payload)
		if payload.Header == nil || payload.Header.ChainHeader == nil || payload.Header.ChainHeader.ChainID == "" {
			logger.Debugf("Received malformed message, dropping connection")
			return srv.Send(&ab.BroadcastResponse{Status: cb.Status_BAD_REQUEST})
		}

		support, ok := b.bs.sm.GetChain(payload.Header.ChainHeader.ChainID)
		if !ok {
			// Chain not found, maybe create one?
			if payload.Header.ChainHeader.Type != int32(cb.HeaderType_CONFIGURATION_TRANSACTION) {
				err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_NOT_FOUND})
			} else {
				logger.Debugf("Proposing new chain")
				err = srv.Send(&ab.BroadcastResponse{Status: b.bs.sm.ProposeChain(msg)})
			}
		} else {
			// Normal transaction for existing chain
			_, filterErr := support.Filters().Apply(msg)

			if filterErr != nil {
				logger.Debugf("Rejecting broadcast message")
				err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_BAD_REQUEST})
			} else {
				select {
				case b.queue <- &msgAndSupport{msg: msg, support: support}:
					err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_SUCCESS})
				default:
					err = srv.Send(&ab.BroadcastResponse{Status: cb.Status_SERVICE_UNAVAILABLE})
				}
			}
		}

		if err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>package packet

import (
	"bytes"
	"encoding/binary"

	"github.com/jsimonetti/go-artnet/packet/code"
	"github.com/jsimonetti/go-artnet/version"
)

var _ ArtNetPacket = &ArtCommandPacket{}

// ArtCommandPacket contains an ArtCommand Packet.
//
// The ArtCommand packet is used to send property set style commands. The packet can be
// unicast or broadcast, the decision being application specific.
//
// Packet Strategy:
//  Controller -  Receive:            Application Specific
//                Unicast Transmit:   Application Specific
//                Broadcast Transmit: Application Specific
//  Node -        Receive:            Application Specific
//                Unicast Transmit:   Application Specific
//                Broadcast Transmit: Application Specific
//  MediaServer - Receive:            Application Specific
//                Unicast Transmit:   Application Specific
//                Broadcast Transmit: Application Specific
type ArtCommandPacket struct {
	// Inherit the Header header
	Header

	// this packet type contains a version
	version [2]byte

	// ESTAmanufacturer contains a code used to represent equipment manufacturer.
	// They are assigned by ESTA. This field can be interpreted as two ASCII bytes
	// representing the manufacturer initials.
	ESTAmanufacturer [2]byte

	// Length indicates the length of the data
	Length uint16

	// Data is an ASCII string, null terminated. Max length is 512 bytes including the null terminator
	Data string
}

// NewArtCommandPacket returns an ArtNetPacket with the correct OpCode
func NewArtCommandPacket() *ArtCommandPacket {
	return &ArtCommandPacket{
		Header: Header{
			OpCode: code.OpCommand,
			id:     ArtNet,
		},
		version: version.Bytes(),
	}
}

// MarshalBinary marshals an ArtCommandPacket into a byte slice.
func (p *ArtCommandPacket) MarshalBinary() ([]byte, error) {
	var buf bytes.Buffer
	if err := binary.Write(&buf, binary.BigEndian, p); err != nil {
		return nil, err
	}
	return buf.Bytes(), p.validate()
}

// UnmarshalBinary unmarshals the contents of a byte slice into an ArtCommandPacket.
//TODO
func (p *ArtCommandPacket) UnmarshalBinary(b []byte) error {
	return p.validate()
}

// artPacket is an empty method to sattisfy the ArtNetPacket interface.
func (p *ArtCommandPacket) validate() error {
	if p.OpCode != code.OpCommand {
		return errInvalidOpCode
	}
	return nil
}
<commit_msg>Explain commands and add fixed esta code<commit_after>package packet

import (
	"bytes"
	"encoding/binary"

	"github.com/jsimonetti/go-artnet/packet/code"
	"github.com/jsimonetti/go-artnet/version"
)

var _ ArtNetPacket = &ArtCommandPacket{}

// ArtCommandPacket contains an ArtCommand Packet.
//
// The ArtCommand packet is used to send property set style commands. The packet can be
// unicast or broadcast, the decision being application specific.
//
// The Data field contains the command text. The text is ASCII encoded and is null terminated
// and is case insensitive. It is legal, although inefficient, to set the Data array size to
// the maximum of 512 and null pad unused entries.
// The command text may contain multiple commands and adheres to the following syntax:
//
//   Command=Data&
//
// The ampersand is a break between commands. Also note that the text is capitalised for
// readability; it is case insensitive. Thus far, two commands are defined by Art-Net. It is
// anticipated that additional commands will be added as other manufacturers register commands
// which have industry wide relevance. These commands shall be transmitted with EstaMan = 0xFFFF.
//
// Packet Strategy:
//  Controller -  Receive:            Application Specific
//                Unicast Transmit:   Application Specific
//                Broadcast Transmit: Application Specific
//  Node -        Receive:            Application Specific
//                Unicast Transmit:   Application Specific
//                Broadcast Transmit: Application Specific
//  MediaServer - Receive:            Application Specific
//                Unicast Transmit:   Application Specific
//                Broadcast Transmit: Application Specific
type ArtCommandPacket struct {
	// Inherit the Header header
	Header

	// this packet type contains a version
	version [2]byte

	// estamanufacturer contains a code used to represent equipment manufacturer.
	estamanufacturer [2]byte

	// Length indicates the length of the data
	Length uint16

	// Data is an ASCII string, null terminated. Max length is 512 bytes including the null terminator
	Data string
}

// NewArtCommandPacket returns an ArtNetPacket with the correct OpCode
func NewArtCommandPacket() *ArtCommandPacket {
	return &ArtCommandPacket{
		Header: Header{
			OpCode: code.OpCommand,
			id:     ArtNet,
		},
		version:          version.Bytes(),
		estamanufacturer: [2]byte{0xff, 0xff},
	}
}

// MarshalBinary marshals an ArtCommandPacket into a byte slice.
func (p *ArtCommandPacket) MarshalBinary() ([]byte, error) {
	var buf bytes.Buffer
	if err := binary.Write(&buf, binary.BigEndian, p); err != nil {
		return nil, err
	}
	return buf.Bytes(), p.validate()
}

// UnmarshalBinary unmarshals the contents of a byte slice into an ArtCommandPacket.
//TODO
func (p *ArtCommandPacket) UnmarshalBinary(b []byte) error {
	return p.validate()
}

// artPacket is an empty method to sattisfy the ArtNetPacket interface.
func (p *ArtCommandPacket) validate() error {
	if p.OpCode != code.OpCommand {
		return errInvalidOpCode
	}
	return nil
}
<|endoftext|>


<commit_before>package css

import (
	"fmt"
	"strings"
)

type RuleSet struct {
	selectors  []string
	properties []Property
}

func Rule() RuleSet {
	return RuleSet{}
}

func (rule RuleSet) For(selector Selector) RuleSet {
	rule.selectors = append(rule.selectors, selector.Selector())
	return rule
}

func (rule RuleSet) Set(properties ...Property) RuleSet {
	rule.properties = append(rule.properties, properties...)
	return rule
}

func (rule RuleSet) String() string {

	selectors := strings.Join(selectorsToStrings(rule.selectors), ",")
	properties := make([]string, 0, 1)
	for _, property := range rule.properties {
		properties = append(properties, property.String())
	}
	propertiesString := strings.Join(properties, "\n")

	return fmt.Sprintf("%v {\n%v\n}\n", selectors, propertiesString)
}
<commit_msg>Changed storage type from []string to []Selector and merged Rule into For()<commit_after>package css

import (
	"fmt"
	"strings"
)

type RuleSet struct {
	selectors  []Selector
	properties []Property
}

func For(selectors ...Selector) RuleSet {
	ruleset := RuleSet{}
	ruleset.selectors = append(ruleset.selectors, selectors...)
	return ruleset
}

func (rule RuleSet) Set(properties ...Property) RuleSet {
	rule.properties = append(rule.properties, properties...)
	return rule
}

func (rule RuleSet) String() string {

	selectors := strings.Join(selectorsToStrings(rule.selectors), ",")
	properties := make([]string, 0, 1)
	for _, property := range rule.properties {
		properties = append(properties, property.String())
	}
	propertiesString := strings.Join(properties, "\n")

	return fmt.Sprintf("%v {\n%v\n}\n", selectors, propertiesString)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
	"strconv"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"
const _OUTPUT_FILENAME = "output.csv"
const QUERY_LIMIT = 100
const _PENALTY_PERCENTAGE_CUTOFF = 0.10

//How many solves a user must have to have their relative scale included.
//A low value gives you far more very low or very high scores than you shoul get.
const _MINIMUM_SOLVES = 10

var noLimitFlag bool
var printPuzzleDataFlag bool
var cullCheaterPercentageFlag float64
var minimumSolvesFlag int
var useMockData bool

func init() {
	flag.BoolVar(&noLimitFlag, "a", false, "Specify to execute the solves query with no limit.")
	flag.BoolVar(&printPuzzleDataFlag, "p", false, "Specify that you want puzzle data printed out in the output.")
	flag.Float64Var(&cullCheaterPercentageFlag, "c", _PENALTY_PERCENTAGE_CUTOFF, "What percentage of solve time must be penalty for someone to be considered a cheater.")
	flag.IntVar(&minimumSolvesFlag, "n", _MINIMUM_SOLVES, "How many solves a user must have their scores considered.")
	flag.BoolVar(&useMockData, "m", false, "Use mock data (useful if you don't have a real database to test with).")
}

type dbConfig struct {
	Url               string
	Username          string
	Password          string
	DbName            string
	SolvesTable       string
	SolvesID          string
	SolvesPuzzleID    string
	SolvesTotalTime   string
	SolvesPenaltyTime string
	SolvesUser        string
	PuzzlesTable      string
	PuzzlesID         string
	PuzzlesDifficulty string
	PuzzlesName       string
	PuzzlesPuzzle     string
}

type solve struct {
	puzzleID    int
	totalTime   int
	penaltyTime int
}

type userSolvesCollection struct {
	solves []solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
	difficultyRating       int
	name                   string
	puzzle                 string
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve solve) bool {
	//Cull obviously incorrect solves.
	if solve.totalTime == 0 {
		return false
	}

	//Cull solves that leaned too heavily on hints.
	if float64(solve.penaltyTime)/float64(solve.totalTime) > cullCheaterPercentageFlag {
		return false
	}

	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
	return true
}

//Whehter or not this should be included in calculation.
//Basically, whether the reltaiveDifficulties will all be valid.
//Normally this returns false if there is only one solve by the user, but could also
//happen when there are multiple solves but (crazily enough) they all have exactly the same solveTime.
//This DOES happen in the production dataset.
func (self *userSolvesCollection) valid() bool {
	if self.max == self.min {
		return false
	}

	if len(self.solves) < minimumSolvesFlag {
		return false
	}

	return true
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {

	flag.Parse()

	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	difficutlyRatingsChan := make(chan map[int]puzzle)

	go getPuzzleDifficultyRatings(&config, difficutlyRatingsChan)

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	var solvesQuery string

	if noLimitFlag {
		log.Println("Running without a limit for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s, %s from %s"
	} else {
		log.Println("Running with a limit of ", QUERY_LIMIT, " for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s, %s from %s limit " + strconv.Itoa(QUERY_LIMIT)
	}

	res, err := db.Start(solvesQuery, config.SolvesUser, config.SolvesPuzzleID, config.SolvesTotalTime, config.SolvesPenaltyTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int
	var skippedSolves int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		if !userSolves.addSolve(solve{row.Int(1), row.Int(2), row.Int(3)}) {
			skippedSolves++
		}
		i++
	}

	log.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")
	log.Println("Skipped ", skippedSolves, " solves that cheated too much.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if !collection.valid() {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	log.Println("Skipped ", skippedUsers, " users because they did not have enough solve times.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{id: puzzleID, userRelativeDifficulty: sum / float32(len(difficulties)), difficultyRating: -1}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})

	//Merge in the difficulty ratings from the server.
	difficultyRatings := <-difficutlyRatingsChan

	for i, puzzle := range puzzles {
		info, ok := difficultyRatings[puzzle.id]
		if ok {
			puzzle.difficultyRating = info.difficultyRating
			puzzle.name = info.name
			puzzle.puzzle = info.puzzle
		}
		//It's not a pointer so we have to copy it back.
		puzzles[i] = puzzle
	}

	//Now print the results to stdout.

	csvOut := csv.NewWriter(os.Stdout)

	for _, puzzle := range puzzles {
		temp := []string{strconv.Itoa(puzzle.id), strconv.Itoa(puzzle.difficultyRating), fmt.Sprintf("%g", puzzle.userRelativeDifficulty), puzzle.name}
		if printPuzzleDataFlag {
			temp = append(temp, puzzle.puzzle)
		}
		csvOut.Write(temp)
	}

	csvOut.Flush()

}

func getPuzzleDifficultyRatings(config *dbConfig, result chan map[int]puzzle) {

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s, %s from %s", config.PuzzlesID, config.PuzzlesDifficulty, config.PuzzlesName, config.PuzzlesPuzzle, config.PuzzlesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	puzzles := make(map[int]puzzle)

	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		puzzles[row.Int(0)] = puzzle{id: row.Int(0), difficultyRating: row.Int(1), name: row.Str(2), puzzle: row.Str(3)}
	}

	result <- puzzles

}
<commit_msg>Made it so we will use the mock connection if told to (but then all hell breaks loose, given that it's not implemented).<commit_after>package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/ziutek/mymysql/mysql"
	_ "github.com/ziutek/mymysql/native"
	"log"
	"os"
	"sort"
	"strconv"
)

const _DB_CONFIG_FILENAME = "db_config.SECRET.json"
const _OUTPUT_FILENAME = "output.csv"
const QUERY_LIMIT = 100
const _PENALTY_PERCENTAGE_CUTOFF = 0.10

//How many solves a user must have to have their relative scale included.
//A low value gives you far more very low or very high scores than you shoul get.
const _MINIMUM_SOLVES = 10

var noLimitFlag bool
var printPuzzleDataFlag bool
var cullCheaterPercentageFlag float64
var minimumSolvesFlag int
var useMockData bool

func init() {
	flag.BoolVar(&noLimitFlag, "a", false, "Specify to execute the solves query with no limit.")
	flag.BoolVar(&printPuzzleDataFlag, "p", false, "Specify that you want puzzle data printed out in the output.")
	flag.Float64Var(&cullCheaterPercentageFlag, "c", _PENALTY_PERCENTAGE_CUTOFF, "What percentage of solve time must be penalty for someone to be considered a cheater.")
	flag.IntVar(&minimumSolvesFlag, "n", _MINIMUM_SOLVES, "How many solves a user must have their scores considered.")
	flag.BoolVar(&useMockData, "m", false, "Use mock data (useful if you don't have a real database to test with).")
}

type dbConfig struct {
	Url               string
	Username          string
	Password          string
	DbName            string
	SolvesTable       string
	SolvesID          string
	SolvesPuzzleID    string
	SolvesTotalTime   string
	SolvesPenaltyTime string
	SolvesUser        string
	PuzzlesTable      string
	PuzzlesID         string
	PuzzlesDifficulty string
	PuzzlesName       string
	PuzzlesPuzzle     string
}

type solve struct {
	puzzleID    int
	totalTime   int
	penaltyTime int
}

type userSolvesCollection struct {
	solves []solve
	max    int
	min    int
}

type puzzle struct {
	id                     int
	userRelativeDifficulty float32
	difficultyRating       int
	name                   string
	puzzle                 string
}

type puzzles []puzzle

type byUserRelativeDifficulty struct {
	puzzles
}

func (self puzzles) Len() int {
	return len(self)
}

func (self puzzles) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byUserRelativeDifficulty) Less(i, j int) bool {
	return self.puzzles[i].userRelativeDifficulty < self.puzzles[j].userRelativeDifficulty
}

func (self *userSolvesCollection) addSolve(solve solve) bool {
	//Cull obviously incorrect solves.
	if solve.totalTime == 0 {
		return false
	}

	//Cull solves that leaned too heavily on hints.
	if float64(solve.penaltyTime)/float64(solve.totalTime) > cullCheaterPercentageFlag {
		return false
	}

	self.solves = append(self.solves, solve)
	if len(self.solves) == 1 {
		self.max = solve.totalTime
		self.min = solve.totalTime
	} else {
		if self.max < solve.totalTime {
			self.max = solve.totalTime
		}
		if self.min > solve.totalTime {
			self.min = solve.totalTime
		}
	}
	return true
}

//Whehter or not this should be included in calculation.
//Basically, whether the reltaiveDifficulties will all be valid.
//Normally this returns false if there is only one solve by the user, but could also
//happen when there are multiple solves but (crazily enough) they all have exactly the same solveTime.
//This DOES happen in the production dataset.
func (self *userSolvesCollection) valid() bool {
	if self.max == self.min {
		return false
	}

	if len(self.solves) < minimumSolvesFlag {
		return false
	}

	return true
}

func (self *userSolvesCollection) relativeDifficulties() map[int]float32 {
	//Returns a map of puzzle id to relative difficulty, normalized by our max and min.
	avgSolveTimes := make(map[int]float32)
	//Keep track of how many times we've seen each puzzle solved by this user so we can do correct averaging.
	avgSolveTimesCount := make(map[int]int)

	//First, collect the average solve time (in case the same user has solved more than once the same puzzle)

	for _, solve := range self.solves {
		currentAvgSolveTime := avgSolveTimes[solve.puzzleID]

		avgSolveTimes[solve.puzzleID] = (currentAvgSolveTime*float32(avgSolveTimesCount[solve.puzzleID]) + float32(solve.totalTime)) / float32(avgSolveTimesCount[solve.puzzleID]+1)

		avgSolveTimesCount[solve.puzzleID]++
	}

	//Now, relativize all of the scores.

	result := make(map[int]float32)

	for puzzleID, avgSolveTime := range avgSolveTimes {
		result[puzzleID] = (avgSolveTime - float32(self.min)) / float32(self.max-self.min)
	}

	return result
}

func main() {

	flag.Parse()

	file, err := os.Open(_DB_CONFIG_FILENAME)
	if err != nil {
		log.Fatal("Could not find the config file at ", _DB_CONFIG_FILENAME, ". You should copy the SAMPLE one to that filename and configure.")
		os.Exit(1)
	}
	defer file.Close()
	decoder := json.NewDecoder(file)
	var config dbConfig
	if err := decoder.Decode(&config); err != nil {
		log.Fatal("There was an error parsing JSON from the config file: ", err)
		os.Exit(1)
	}

	difficutlyRatingsChan := make(chan map[int]puzzle)

	go getPuzzleDifficultyRatings(&config, difficutlyRatingsChan)

	var db mysql.Conn

	if useMockData {
		db = &mockConnection{}
	} else {
		db = mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)
	}

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	var solvesQuery string

	if noLimitFlag {
		log.Println("Running without a limit for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s, %s from %s"
	} else {
		log.Println("Running with a limit of ", QUERY_LIMIT, " for number of solves to retrieve.")
		solvesQuery = "select %s, %s, %s, %s from %s limit " + strconv.Itoa(QUERY_LIMIT)
	}

	res, err := db.Start(solvesQuery, config.SolvesUser, config.SolvesPuzzleID, config.SolvesTotalTime, config.SolvesPenaltyTime, config.SolvesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	solvesByUser := make(map[string]*userSolvesCollection)

	var userSolves *userSolvesCollection
	var ok bool
	var i int
	var skippedSolves int

	//First, process all user records in the DB to collect all solves by userName.
	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		userSolves, ok = solvesByUser[row.Str(0)]

		if !ok {
			userSolves = new(userSolvesCollection)
			solvesByUser[row.Str(0)] = userSolves
		}

		if !userSolves.addSolve(solve{row.Int(1), row.Int(2), row.Int(3)}) {
			skippedSolves++
		}
		i++
	}

	log.Println("Processed ", i, " solves by ", len(solvesByUser), " users.")
	log.Println("Skipped ", skippedSolves, " solves that cheated too much.")

	//Now get the relative difficulty for each user's puzzles, and collect them.

	relativeDifficultiesByPuzzle := make(map[int][]float32)

	var skippedUsers int

	for _, collection := range solvesByUser {

		if !collection.valid() {
			skippedUsers++
			continue
		}

		for puzzleID, relativeDifficulty := range collection.relativeDifficulties() {
			relativeDifficultiesByPuzzle[puzzleID] = append(relativeDifficultiesByPuzzle[puzzleID], relativeDifficulty)
		}

	}

	log.Println("Skipped ", skippedUsers, " users because they did not have enough solve times.")

	puzzles := make([]puzzle, len(relativeDifficultiesByPuzzle))

	var index int

	for puzzleID, difficulties := range relativeDifficultiesByPuzzle {
		var sum float32
		for _, difficulty := range difficulties {
			sum += difficulty
		}
		puzzles[index] = puzzle{id: puzzleID, userRelativeDifficulty: sum / float32(len(difficulties)), difficultyRating: -1}
		index++
	}

	//Sort the puzzles by relative user difficulty
	//We actually don't need the wrapper, since it will modify the underlying slice.
	sort.Sort(byUserRelativeDifficulty{puzzles})

	//Merge in the difficulty ratings from the server.
	difficultyRatings := <-difficutlyRatingsChan

	for i, puzzle := range puzzles {
		info, ok := difficultyRatings[puzzle.id]
		if ok {
			puzzle.difficultyRating = info.difficultyRating
			puzzle.name = info.name
			puzzle.puzzle = info.puzzle
		}
		//It's not a pointer so we have to copy it back.
		puzzles[i] = puzzle
	}

	//Now print the results to stdout.

	csvOut := csv.NewWriter(os.Stdout)

	for _, puzzle := range puzzles {
		temp := []string{strconv.Itoa(puzzle.id), strconv.Itoa(puzzle.difficultyRating), fmt.Sprintf("%g", puzzle.userRelativeDifficulty), puzzle.name}
		if printPuzzleDataFlag {
			temp = append(temp, puzzle.puzzle)
		}
		csvOut.Write(temp)
	}

	csvOut.Flush()

}

func getPuzzleDifficultyRatings(config *dbConfig, result chan map[int]puzzle) {

	db := mysql.New("tcp", "", config.Url, config.Username, config.Password, config.DbName)

	if err := db.Connect(); err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	res, err := db.Start("select %s, %s, %s, %s from %s", config.PuzzlesID, config.PuzzlesDifficulty, config.PuzzlesName, config.PuzzlesPuzzle, config.PuzzlesTable)

	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}

	puzzles := make(map[int]puzzle)

	for {

		row, _ := res.GetRow()

		if row == nil {
			break
		}

		puzzles[row.Int(0)] = puzzle{id: row.Int(0), difficultyRating: row.Int(1), name: row.Str(2), puzzle: row.Str(3)}
	}

	result <- puzzles

}
<|endoftext|>


<commit_before>package bolt

import (
	"github.com/ansible-semaphore/semaphore/db"
	"go.etcd.io/bbolt"
	"time"
)

func (d *BoltDb) CreateTask(task db.Task) (newTask db.Task, err error) {
	task.Created = time.Now()
	res, err := d.createObject(0, db.TaskProps, task)
	if err != nil {
		return
	}
	newTask = res.(db.Task)
	return
}

func (d *BoltDb) UpdateTask(task db.Task) error {
	return d.updateObject(0, db.TaskProps, task)
}

func (d *BoltDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) {
	newOutput, err := d.createObject(output.TaskID, db.TaskOutputProps, output)
	if err != nil {
		return db.TaskOutput{}, err
	}
	return newOutput.(db.TaskOutput), nil
}

func (d *BoltDb) getTasks(projectID int, templateID *int, params db.RetrieveQueryParams) (tasksWithTpl []db.TaskWithTpl, err error) {
	var tasks []db.Task

	err = d.getObjects(0, db.TaskProps, params, func(tsk interface{}) bool {
		task := tsk.(db.Task)

		if task.ProjectID != projectID {
			return false
		}

		if templateID != nil && task.TemplateID != *templateID {
			return false
		}

		return true
	}, &tasks)

	if err != nil {
		return
	}

	var templates = make(map[int]db.Template)
	var users = make(map[int]db.User)

	tasksWithTpl = make([]db.TaskWithTpl, len(tasks))
	for i, task := range tasks {
		tpl, ok := templates[task.TemplateID]
		if !ok {
			if templateID == nil {
				tpl, _ = d.getRawTemplate(task.ProjectID, task.TemplateID)
			} else {
				tpl, _ = d.getRawTemplate(task.ProjectID, *templateID)
			}
			templates[task.TemplateID] = tpl
		}
		tasksWithTpl[i] = db.TaskWithTpl{Task: task}
		tasksWithTpl[i].TemplatePlaybook = tpl.Playbook
		tasksWithTpl[i].TemplateAlias = tpl.Name
		tasksWithTpl[i].TemplateType = tpl.Type
		if task.UserID != nil {
			usr, ok := users[*task.UserID]
			if !ok {
				// trying to get user , but ignore error, because
				// user can be deleted, and it is ok
				usr, _ = d.GetUser(*task.UserID)
				users[*task.UserID] = usr
			}
			tasksWithTpl[i].UserName = &usr.Name
		}

		err = tasksWithTpl[i].Fill(d)
		if err != nil {
			return
		}
	}

	return
}

func (d *BoltDb) GetTask(projectID int, taskID int) (task db.Task, err error) {
	err = d.getObject(0, db.TaskProps, intObjectID(taskID), &task)
	if err != nil {
		return
	}

	if task.ProjectID != projectID {
		task = db.Task{}
		err = db.ErrNotFound
		return
	}

	return
}

func (d *BoltDb) GetTemplateTasks(projectID int, templateID int, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
	return d.getTasks(projectID, &templateID, params)
}

func (d *BoltDb) GetProjectTasks(projectID int, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
	return d.getTasks(projectID, nil, params)
}

func (d *BoltDb) deleteTaskWithOutputs(projectID int, taskID int, tx *bbolt.Tx) (err error) {
	// check if task exists in the project
	_, err = d.GetTask(projectID, taskID)
	if err != nil {
		return
	}

	err = d.deleteObject(0, db.TaskProps, intObjectID(taskID), tx)
	if err != nil {
		return
	}

	return tx.DeleteBucket(makeBucketId(db.TaskOutputProps, taskID))
}

func (d *BoltDb) DeleteTaskWithOutputs(projectID int, taskID int) error {
	return d.db.Update(func(tx *bbolt.Tx) error {
		return d.deleteTaskWithOutputs(projectID, taskID, tx)
	})
}

func (d *BoltDb) GetTaskOutputs(projectID int, taskID int) (outputs []db.TaskOutput, err error) {
	// check if task exists in the project
	_, err = d.GetTask(projectID, taskID)

	if err != nil {
		return
	}

	err = d.getObjects(taskID, db.TaskOutputProps, db.RetrieveQueryParams{}, nil, &outputs)

	return
}
<commit_msg>fix(be): ignore not found error when deleting task output bucket<commit_after>package bolt

import (
	"github.com/ansible-semaphore/semaphore/db"
	"go.etcd.io/bbolt"
	"time"
)

func (d *BoltDb) CreateTask(task db.Task) (newTask db.Task, err error) {
	task.Created = time.Now()
	res, err := d.createObject(0, db.TaskProps, task)
	if err != nil {
		return
	}
	newTask = res.(db.Task)
	return
}

func (d *BoltDb) UpdateTask(task db.Task) error {
	return d.updateObject(0, db.TaskProps, task)
}

func (d *BoltDb) CreateTaskOutput(output db.TaskOutput) (db.TaskOutput, error) {
	newOutput, err := d.createObject(output.TaskID, db.TaskOutputProps, output)
	if err != nil {
		return db.TaskOutput{}, err
	}
	return newOutput.(db.TaskOutput), nil
}

func (d *BoltDb) getTasks(projectID int, templateID *int, params db.RetrieveQueryParams) (tasksWithTpl []db.TaskWithTpl, err error) {
	var tasks []db.Task

	err = d.getObjects(0, db.TaskProps, params, func(tsk interface{}) bool {
		task := tsk.(db.Task)

		if task.ProjectID != projectID {
			return false
		}

		if templateID != nil && task.TemplateID != *templateID {
			return false
		}

		return true
	}, &tasks)

	if err != nil {
		return
	}

	var templates = make(map[int]db.Template)
	var users = make(map[int]db.User)

	tasksWithTpl = make([]db.TaskWithTpl, len(tasks))
	for i, task := range tasks {
		tpl, ok := templates[task.TemplateID]
		if !ok {
			if templateID == nil {
				tpl, _ = d.getRawTemplate(task.ProjectID, task.TemplateID)
			} else {
				tpl, _ = d.getRawTemplate(task.ProjectID, *templateID)
			}
			templates[task.TemplateID] = tpl
		}
		tasksWithTpl[i] = db.TaskWithTpl{Task: task}
		tasksWithTpl[i].TemplatePlaybook = tpl.Playbook
		tasksWithTpl[i].TemplateAlias = tpl.Name
		tasksWithTpl[i].TemplateType = tpl.Type
		if task.UserID != nil {
			usr, ok := users[*task.UserID]
			if !ok {
				// trying to get user , but ignore error, because
				// user can be deleted, and it is ok
				usr, _ = d.GetUser(*task.UserID)
				users[*task.UserID] = usr
			}
			tasksWithTpl[i].UserName = &usr.Name
		}

		err = tasksWithTpl[i].Fill(d)
		if err != nil {
			return
		}
	}

	return
}

func (d *BoltDb) GetTask(projectID int, taskID int) (task db.Task, err error) {
	err = d.getObject(0, db.TaskProps, intObjectID(taskID), &task)
	if err != nil {
		return
	}

	if task.ProjectID != projectID {
		task = db.Task{}
		err = db.ErrNotFound
		return
	}

	return
}

func (d *BoltDb) GetTemplateTasks(projectID int, templateID int, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
	return d.getTasks(projectID, &templateID, params)
}

func (d *BoltDb) GetProjectTasks(projectID int, params db.RetrieveQueryParams) ([]db.TaskWithTpl, error) {
	return d.getTasks(projectID, nil, params)
}

func (d *BoltDb) deleteTaskWithOutputs(projectID int, taskID int, tx *bbolt.Tx) (err error) {
	// check if task exists in the project
	_, err = d.GetTask(projectID, taskID)
	if err != nil {
		return
	}

	err = d.deleteObject(0, db.TaskProps, intObjectID(taskID), tx)
	if err != nil {
		return
	}

	err = tx.DeleteBucket(makeBucketId(db.TaskOutputProps, taskID))
	if err == bbolt.ErrBucketNotFound {
		err = nil
	}

	return
}

func (d *BoltDb) DeleteTaskWithOutputs(projectID int, taskID int) error {
	return d.db.Update(func(tx *bbolt.Tx) error {
		return d.deleteTaskWithOutputs(projectID, taskID, tx)
	})
}

func (d *BoltDb) GetTaskOutputs(projectID int, taskID int) (outputs []db.TaskOutput, err error) {
	// check if task exists in the project
	_, err = d.GetTask(projectID, taskID)

	if err != nil {
		return
	}

	err = d.getObjects(taskID, db.TaskOutputProps, db.RetrieveQueryParams{}, nil, &outputs)

	return
}
<|endoftext|>