<commit_before>package state

import (
	"fmt"
	"strings"
	"sync"
	"time"

	swanevent "github.com/Dataman-Cloud/swan/src/manager/event"
	"github.com/Dataman-Cloud/swan/src/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/src/types"

	"github.com/Sirupsen/logrus"
)

//  TASK_STAGING = 6;  // Initial state. Framework status updates should not use.
//  TASK_STARTING = 0; // The task is being launched by the executor.
//  TASK_RUNNING = 1;
//  TASK_KILLING = 8;  // The task is being killed by the executor.
//  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
//  TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
//  TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
//  TASK_ERROR = 7;    // TERMINAL: The task description contains an error.
//  TASK_LOST = 5;     // TERMINAL: The task failed but can be rescheduled.
//  TASK_DROPPED = 9;  // TERMINAL.
//  TASK_UNREACHABLE = 10;
//  TASK_GONE = 11;    // TERMINAL.
//  TASK_GONE_BY_OPERATOR = 12;
//  TASK_UNKNOWN = 13;

const (
	SLOT_STATE_PENDING_OFFER = "slot_task_pending_offer"
	SLOT_STATE_PENDING_KILL  = "slot_task_pending_killed"

	SLOT_STATE_TASK_STAGING          = "slot_task_staging"
	SLOT_STATE_TASK_STARTING         = "slot_task_starting"
	SLOT_STATE_TASK_RUNNING          = "slot_task_running"
	SLOT_STATE_TASK_KILLING          = "slot_task_killing"
	SLOT_STATE_TASK_FINISHED         = "slot_task_finished"
	SLOT_STATE_TASK_FAILED           = "slot_task_failed"
	SLOT_STATE_TASK_KILLED           = "slot_task_killed"
	SLOT_STATE_TASK_ERROR            = "slot_task_error"
	SLOT_STATE_TASK_LOST             = "slot_task_lost"
	SLOT_STATE_TASK_DROPPED          = "slot_task_dropped"
	SLOT_STATE_TASK_UNREACHABLE      = "slot_task_unreachable"
	SLOT_STATE_TASK_GONE             = "slot_task_gone"
	SLOT_STATE_TASK_GONE_BY_OPERATOR = "slot_task_gone_by_operator"
	SLOT_STATE_TASK_UNKNOWN          = "slot_task_unknown"
)

type SlotStateCallbackFuncs func(slot *Slot)

type Slot struct {
	Index           int
	Id              string
	App             *App
	Version         *types.Version
	State           string
	StatesCallbacks map[string][]SlotStateCallbackFuncs

	CurrentTask *Task
	TaskHistory []*Task

	OfferId       string
	AgentId       string
	Ip            string
	AgentHostName string

	resourceReservationLock sync.Mutex

	MarkForDeletion      bool
	MarkForRollingUpdate bool

	restartPolicy *RestartPolicy
}

func NewSlot(app *App, version *types.Version, index int) *Slot {
	slot := &Slot{
		Index:       index,
		App:         app,
		Version:     version,
		TaskHistory: make([]*Task, 0),
		Id:          fmt.Sprintf("%d-%s-%s-%s", index, version.AppId, version.RunAs, app.MesosConnector.ClusterId), // should be app.AppId

		resourceReservationLock: sync.Mutex{},

		MarkForRollingUpdate: false,
		MarkForDeletion:      false,
		StatesCallbacks:      make(map[string][]SlotStateCallbackFuncs),
	}

	slot.DispatchNewTask(slot.Version)

	// initialize restart policy
	testAndRestartFunc := func(s *Slot) bool {
		if s.StateIs(SLOT_STATE_TASK_RUNNING) {
			return true
		} else {
			s.Archive()
			s.DispatchNewTask(slot.Version)
		}

		return false
	}

	//slot.restartPolicy = NewRestartPolicy(slot, slot.Version.BackoffSeconds,
	//slot.Version.BackoffFactor, slot.Version.MaxLaunchDelaySeconds, testAndRestartFunc)
	slot.restartPolicy = NewRestartPolicy(slot, time.Second*30, 1.2, time.Second*300, testAndRestartFunc)

	return slot
}

// kill task doesn't need cleanup slot from app.Slots
func (slot *Slot) KillTask() {
	slot.StopRestartPolicy()
	slot.SetState(SLOT_STATE_PENDING_KILL)
	slot.CurrentTask.Kill()
}

// kill task and make slot sweeped after successfully kill task
func (slot *Slot) Kill() {
	slot.StopRestartPolicy()

	slot.MarkForDeletion = true
	slot.SetState(SLOT_STATE_PENDING_KILL)

	slot.CurrentTask.Kill()
}

func (slot *Slot) Archive() {
	slot.TaskHistory = append(slot.TaskHistory, slot.CurrentTask)
}

func (slot *Slot) DispatchNewTask(version *types.Version) {
	slot.Version = version
	slot.CurrentTask = NewTask(slot.App, slot.Version, slot)
	slot.SetState(SLOT_STATE_PENDING_OFFER)

	slot.App.OfferAllocatorRef.PutSlotBackToPendingQueue(slot)

}

func (slot *Slot) Update(version *types.Version) {
	logrus.Infof("update slot %s with version ID %s", slot.Id, version.ID)

	slot.MarkForRollingUpdate = true // mark as in progress of rolling update

	onSlotFinished := func(slot *Slot) {
		logrus.Infof("onSlotFinished func")
		slot.Archive()
		slot.DispatchNewTask(version)
	}
	slot.RegisterStateCallbacks(SLOT_STATE_TASK_FINISHED, onSlotFinished)

	slot.KillTask() // kill task but doesn't clean slot
}

func (slot *Slot) TestOfferMatch(ow *OfferWrapper) bool {
	return ow.CpuRemain() > slot.Version.Cpus &&
		ow.MemRemain() > slot.Version.Mem &&
		ow.DiskRemain() > slot.Version.Disk
}

func (slot *Slot) ReserveOfferAndPrepareTaskInfo(ow *OfferWrapper) (*OfferWrapper, *mesos.TaskInfo) {
	slot.resourceReservationLock.Lock()
	defer slot.resourceReservationLock.Unlock()

	ow.CpusUsed += slot.Version.Cpus
	ow.MemUsed += slot.Version.Mem
	ow.DiskUsed += slot.Version.Disk

	taskInfo := slot.CurrentTask.PrepareTaskInfo(ow)

	if slot.App.IsReplicates() { // reserve port only for replicates application
		ow.PortUsedSize += len(slot.Version.Container.Docker.PortMappings)
	}

	return ow, taskInfo
}

func (slot *Slot) Resources() []*mesos.Resource {
	var resources = []*mesos.Resource{}

	if slot.Version.Cpus > 0 {
		resources = append(resources, createScalarResource("cpus", slot.Version.Cpus))
	}

	if slot.Version.Mem > 0 {
		resources = append(resources, createScalarResource("mem", slot.Version.Cpus))
	}

	if slot.Version.Disk > 0 {
		resources = append(resources, createScalarResource("disk", slot.Version.Disk))
	}

	return resources
}

func (slot *Slot) StateIs(state string) bool {
	return slot.State == state
}

func (slot *Slot) SetState(state string) error {
	slot.State = state

	// handle callback
	if len(slot.StatesCallbacks[slot.State]) > 0 {
		for _, cb := range slot.StatesCallbacks[slot.State] {
			cb(slot)
		}
	}
	slot.RemoveStateCallbacks(slot.State)

	logrus.Infof("setting state for slot %s to %s", slot.Id, slot.State)

	switch slot.State {
	case SLOT_STATE_PENDING_KILL:
		slot.EmitTaskEvent(swanevent.EventTypeTaskRm)

	case SLOT_STATE_TASK_KILLED:
		slot.StopRestartPolicy()
	case SLOT_STATE_TASK_FINISHED:
		slot.StopRestartPolicy()
	case SLOT_STATE_TASK_RUNNING:
		slot.EmitTaskEvent(swanevent.EventTypeTaskAdd)
	case SLOT_STATE_TASK_LOST:

	case SLOT_STATE_TASK_FAILED:
		slot.EmitTaskEvent(swanevent.EventTypeTaskRm)
		// restart if needed
	default:
	}

	// skip app invalidation if slot state is not mesos driven
	if (slot.State != SLOT_STATE_PENDING_OFFER) ||
		(slot.State != SLOT_STATE_PENDING_KILL) {
		slot.App.InvalidateSlots()
	}
	return nil
}

func (slot *Slot) AppendStateCallbacks(state string, callback SlotStateCallbackFuncs) {
	slot.StatesCallbacks[state] = append(slot.StatesCallbacks[state], callback)
}

// empty callback queue first
func (slot *Slot) RegisterStateCallbacks(state string, callback SlotStateCallbackFuncs) {
	slot.RemoveStateCallbacks(state)
	slot.StatesCallbacks[state] = append(slot.StatesCallbacks[state], callback)
}

func (slot *Slot) RemoveStateCallbacks(state string) {
	slot.StatesCallbacks[state] = make([]SlotStateCallbackFuncs, 0)
}

func (slot *Slot) StopRestartPolicy() {
	if slot.restartPolicy != nil {
		slot.restartPolicy.Stop()
		slot.restartPolicy = nil
	}
}

func (slot *Slot) EmitTaskEvent(t string) {
	for _, port := range slot.CurrentTask.HostPorts {
		e := &swanevent.Event{Type: t}
		e.Payload = &swanevent.TaskInfo{
			Ip:     slot.AgentHostName,
			Port:   fmt.Sprintf("%d", port),
			TaskId: "task" + strings.ToLower(strings.Replace(slot.Id, "-", ".", -1)),
			Type:   "srv",
		}

		slot.App.EmitEvent(e)
	}
}
<commit_msg>add `constraints` for new framework<commit_after>package state

import (
	"fmt"
	"strings"
	"sync"
	"time"

	swanevent "github.com/Dataman-Cloud/swan/src/manager/event"
	"github.com/Dataman-Cloud/swan/src/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/src/types"

	"github.com/Sirupsen/logrus"
)

//  TASK_STAGING = 6;  // Initial state. Framework status updates should not use.
//  TASK_STARTING = 0; // The task is being launched by the executor.
//  TASK_RUNNING = 1;
//  TASK_KILLING = 8;  // The task is being killed by the executor.
//  TASK_FINISHED = 2; // TERMINAL: The task finished successfully.
//  TASK_FAILED = 3;   // TERMINAL: The task failed to finish successfully.
//  TASK_KILLED = 4;   // TERMINAL: The task was killed by the executor.
//  TASK_ERROR = 7;    // TERMINAL: The task description contains an error.
//  TASK_LOST = 5;     // TERMINAL: The task failed but can be rescheduled.
//  TASK_DROPPED = 9;  // TERMINAL.
//  TASK_UNREACHABLE = 10;
//  TASK_GONE = 11;    // TERMINAL.
//  TASK_GONE_BY_OPERATOR = 12;
//  TASK_UNKNOWN = 13;

const (
	SLOT_STATE_PENDING_OFFER = "slot_task_pending_offer"
	SLOT_STATE_PENDING_KILL  = "slot_task_pending_killed"

	SLOT_STATE_TASK_STAGING          = "slot_task_staging"
	SLOT_STATE_TASK_STARTING         = "slot_task_starting"
	SLOT_STATE_TASK_RUNNING          = "slot_task_running"
	SLOT_STATE_TASK_KILLING          = "slot_task_killing"
	SLOT_STATE_TASK_FINISHED         = "slot_task_finished"
	SLOT_STATE_TASK_FAILED           = "slot_task_failed"
	SLOT_STATE_TASK_KILLED           = "slot_task_killed"
	SLOT_STATE_TASK_ERROR            = "slot_task_error"
	SLOT_STATE_TASK_LOST             = "slot_task_lost"
	SLOT_STATE_TASK_DROPPED          = "slot_task_dropped"
	SLOT_STATE_TASK_UNREACHABLE      = "slot_task_unreachable"
	SLOT_STATE_TASK_GONE             = "slot_task_gone"
	SLOT_STATE_TASK_GONE_BY_OPERATOR = "slot_task_gone_by_operator"
	SLOT_STATE_TASK_UNKNOWN          = "slot_task_unknown"
)

type SlotStateCallbackFuncs func(slot *Slot)

type Slot struct {
	Index           int
	Id              string
	App             *App
	Version         *types.Version
	State           string
	StatesCallbacks map[string][]SlotStateCallbackFuncs

	CurrentTask *Task
	TaskHistory []*Task

	OfferId       string
	AgentId       string
	Ip            string
	AgentHostName string

	resourceReservationLock sync.Mutex

	MarkForDeletion      bool
	MarkForRollingUpdate bool

	restartPolicy *RestartPolicy
}

func NewSlot(app *App, version *types.Version, index int) *Slot {
	slot := &Slot{
		Index:       index,
		App:         app,
		Version:     version,
		TaskHistory: make([]*Task, 0),
		Id:          fmt.Sprintf("%d-%s-%s-%s", index, version.AppId, version.RunAs, app.MesosConnector.ClusterId), // should be app.AppId

		resourceReservationLock: sync.Mutex{},

		MarkForRollingUpdate: false,
		MarkForDeletion:      false,
		StatesCallbacks:      make(map[string][]SlotStateCallbackFuncs),
	}

	slot.DispatchNewTask(slot.Version)

	// initialize restart policy
	testAndRestartFunc := func(s *Slot) bool {
		if s.StateIs(SLOT_STATE_TASK_RUNNING) {
			return true
		} else {
			s.Archive()
			s.DispatchNewTask(slot.Version)
		}

		return false
	}

	//slot.restartPolicy = NewRestartPolicy(slot, slot.Version.BackoffSeconds,
	//slot.Version.BackoffFactor, slot.Version.MaxLaunchDelaySeconds, testAndRestartFunc)
	slot.restartPolicy = NewRestartPolicy(slot, time.Second*30, 1.2, time.Second*300, testAndRestartFunc)

	return slot
}

// kill task doesn't need cleanup slot from app.Slots
func (slot *Slot) KillTask() {
	slot.StopRestartPolicy()
	slot.SetState(SLOT_STATE_PENDING_KILL)
	slot.CurrentTask.Kill()
}

// kill task and make slot sweeped after successfully kill task
func (slot *Slot) Kill() {
	slot.StopRestartPolicy()

	slot.MarkForDeletion = true
	slot.SetState(SLOT_STATE_PENDING_KILL)

	slot.CurrentTask.Kill()
}

func (slot *Slot) Archive() {
	slot.TaskHistory = append(slot.TaskHistory, slot.CurrentTask)
}

func (slot *Slot) DispatchNewTask(version *types.Version) {
	slot.Version = version
	slot.CurrentTask = NewTask(slot.App, slot.Version, slot)
	slot.SetState(SLOT_STATE_PENDING_OFFER)

	slot.App.OfferAllocatorRef.PutSlotBackToPendingQueue(slot)

}

func (slot *Slot) Update(version *types.Version) {
	logrus.Infof("update slot %s with version ID %s", slot.Id, version.ID)

	slot.MarkForRollingUpdate = true // mark as in progress of rolling update

	onSlotFinished := func(slot *Slot) {
		logrus.Infof("onSlotFinished func")
		slot.Archive()
		slot.DispatchNewTask(version)
	}
	slot.RegisterStateCallbacks(SLOT_STATE_TASK_FINISHED, onSlotFinished)

	slot.KillTask() // kill task but doesn't clean slot
}

func (slot *Slot) TestOfferMatch(ow *OfferWrapper) bool {
	if slot.Version.Constraints != nil && len(slot.Version.Constraints) > 0 {
		constraints := slot.filterConstraints(slot.Version.Constraints)
		for _, constraint := range constraints {
			cons := strings.Split(constraint, ":")
			if cons[1] == "LIKE" {
				for _, attr := range ow.Offer.Attributes {
					var value string
					name := attr.GetName()
					switch attr.GetType() {
					case mesos.Value_SCALAR:
						value = fmt.Sprintf("%d", *attr.GetScalar().Value)
					case mesos.Value_TEXT:
						value = fmt.Sprintf("%s", *attr.GetText().Value)
					default:
						logrus.Errorf("Unsupported attribute value: %s", attr.GetType())
					}

					if name == cons[0] &&
						strings.Contains(value, cons[2]) &&
						ow.CpuRemain() > slot.Version.Cpus &&
						ow.MemRemain() > slot.Version.Mem &&
						ow.DiskRemain() > slot.Version.Disk {
						return true
					}
				}
				return false
			}
		}
	}

	return ow.CpuRemain() > slot.Version.Cpus &&
		ow.MemRemain() > slot.Version.Mem &&
		ow.DiskRemain() > slot.Version.Disk
}

func (slot *Slot) filterConstraints(constraints []string) []string {
	filteredConstraints := make([]string, 0)
	for _, constraint := range constraints {
		cons := strings.Split(constraint, ":")
		if len(cons) > 3 || len(cons) < 2 {
			logrus.Errorf("Malformed Constraints")
			continue
		}

		if cons[1] != "UNIQUE" && cons[1] != "LIKE" {
			logrus.Errorf("Constraints operator %s not supported", cons[1])
			continue
		}

		if cons[1] == "UNIQUE" && strings.ToLower(cons[0]) != "hostname" {
			logrus.Errorf("Constraints operator UNIQUE only support 'hostname': %s", cons[0])
			continue
		}

		if cons[1] == "LIKE" && len(cons) < 3 {
			logrus.Errorf("Constraints operator LIKE required two operands")
			continue
		}

		filteredConstraints = append(filteredConstraints, constraint)
	}

	return filteredConstraints
}

func (slot *Slot) ReserveOfferAndPrepareTaskInfo(ow *OfferWrapper) (*OfferWrapper, *mesos.TaskInfo) {
	slot.resourceReservationLock.Lock()
	defer slot.resourceReservationLock.Unlock()

	ow.CpusUsed += slot.Version.Cpus
	ow.MemUsed += slot.Version.Mem
	ow.DiskUsed += slot.Version.Disk

	taskInfo := slot.CurrentTask.PrepareTaskInfo(ow)

	if slot.App.IsReplicates() { // reserve port only for replicates application
		ow.PortUsedSize += len(slot.Version.Container.Docker.PortMappings)
	}

	return ow, taskInfo
}

func (slot *Slot) Resources() []*mesos.Resource {
	var resources = []*mesos.Resource{}

	if slot.Version.Cpus > 0 {
		resources = append(resources, createScalarResource("cpus", slot.Version.Cpus))
	}

	if slot.Version.Mem > 0 {
		resources = append(resources, createScalarResource("mem", slot.Version.Cpus))
	}

	if slot.Version.Disk > 0 {
		resources = append(resources, createScalarResource("disk", slot.Version.Disk))
	}

	return resources
}

func (slot *Slot) StateIs(state string) bool {
	return slot.State == state
}

func (slot *Slot) SetState(state string) error {
	slot.State = state

	// handle callback
	if len(slot.StatesCallbacks[slot.State]) > 0 {
		for _, cb := range slot.StatesCallbacks[slot.State] {
			cb(slot)
		}
	}
	slot.RemoveStateCallbacks(slot.State)

	logrus.Infof("setting state for slot %s to %s", slot.Id, slot.State)

	switch slot.State {
	case SLOT_STATE_PENDING_KILL:
		slot.EmitTaskEvent(swanevent.EventTypeTaskRm)

	case SLOT_STATE_TASK_KILLED:
		slot.StopRestartPolicy()
	case SLOT_STATE_TASK_FINISHED:
		slot.StopRestartPolicy()
	case SLOT_STATE_TASK_RUNNING:
		slot.EmitTaskEvent(swanevent.EventTypeTaskAdd)
	case SLOT_STATE_TASK_LOST:

	case SLOT_STATE_TASK_FAILED:
		slot.EmitTaskEvent(swanevent.EventTypeTaskRm)
		// restart if needed
	default:
	}

	// skip app invalidation if slot state is not mesos driven
	if (slot.State != SLOT_STATE_PENDING_OFFER) ||
		(slot.State != SLOT_STATE_PENDING_KILL) {
		slot.App.InvalidateSlots()
	}
	return nil
}

func (slot *Slot) AppendStateCallbacks(state string, callback SlotStateCallbackFuncs) {
	slot.StatesCallbacks[state] = append(slot.StatesCallbacks[state], callback)
}

// empty callback queue first
func (slot *Slot) RegisterStateCallbacks(state string, callback SlotStateCallbackFuncs) {
	slot.RemoveStateCallbacks(state)
	slot.StatesCallbacks[state] = append(slot.StatesCallbacks[state], callback)
}

func (slot *Slot) RemoveStateCallbacks(state string) {
	slot.StatesCallbacks[state] = make([]SlotStateCallbackFuncs, 0)
}

func (slot *Slot) StopRestartPolicy() {
	if slot.restartPolicy != nil {
		slot.restartPolicy.Stop()
		slot.restartPolicy = nil
	}
}

func (slot *Slot) EmitTaskEvent(t string) {
	for _, port := range slot.CurrentTask.HostPorts {
		e := &swanevent.Event{Type: t}
		e.Payload = &swanevent.TaskInfo{
			Ip:     slot.AgentHostName,
			Port:   fmt.Sprintf("%d", port),
			TaskId: "task" + strings.ToLower(strings.Replace(slot.Id, "-", ".", -1)),
			Type:   "srv",
		}

		slot.App.EmitEvent(e)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
	"fmt"
	"io/ioutil"

	"k8s.io/apimachinery/pkg/runtime/schema"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
	auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
	"k8s.io/apiserver/pkg/apis/audit/validation"
	"k8s.io/apiserver/pkg/audit"

	"github.com/golang/glog"
)

var (
	apiGroupVersions = []schema.GroupVersion{
		auditv1beta1.SchemeGroupVersion,
		auditv1alpha1.SchemeGroupVersion,
		auditv1.SchemeGroupVersion,
	}
	apiGroupVersionSet = map[schema.GroupVersion]bool{}
)

func init() {
	for _, gv := range apiGroupVersions {
		apiGroupVersionSet[gv] = true
	}
}

func LoadPolicyFromFile(filePath string) (*auditinternal.Policy, error) {
	if filePath == "" {
		return nil, fmt.Errorf("file path not specified")
	}
	policyDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file path %q: %+v", filePath, err)
	}

	policy := &auditinternal.Policy{}
	decoder := audit.Codecs.UniversalDecoder(apiGroupVersions...)

	_, gvk, err := decoder.Decode(policyDef, nil, policy)
	if err != nil {
		return nil, fmt.Errorf("failed decoding file %q: %v", filePath, err)
	}

	// Ensure the policy file contained an apiVersion and kind.
	if !apiGroupVersionSet[schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}] {
		return nil, fmt.Errorf("unknown group version field %v in policy file %s", gvk, filePath)
	}

	if err := validation.ValidatePolicy(policy); err != nil {
		return nil, err.ToAggregate()
	}

	policyCnt := len(policy.Rules)
	if policyCnt == 0 {
		return nil, fmt.Errorf("loaded illegal policy with 0 rules from file %s", filePath)
	}
	glog.V(4).Infof("Loaded %d audit policy rules from file %s", policyCnt, filePath)
	return policy, nil
}
<commit_msg>allow audit policy to be loaded from any byte source<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package policy

import (
	"fmt"
	"io/ioutil"

	"k8s.io/apimachinery/pkg/runtime/schema"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
	auditv1alpha1 "k8s.io/apiserver/pkg/apis/audit/v1alpha1"
	auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
	"k8s.io/apiserver/pkg/apis/audit/validation"
	"k8s.io/apiserver/pkg/audit"

	"github.com/golang/glog"
)

var (
	apiGroupVersions = []schema.GroupVersion{
		auditv1beta1.SchemeGroupVersion,
		auditv1alpha1.SchemeGroupVersion,
		auditv1.SchemeGroupVersion,
	}
	apiGroupVersionSet = map[schema.GroupVersion]bool{}
)

func init() {
	for _, gv := range apiGroupVersions {
		apiGroupVersionSet[gv] = true
	}
}

func LoadPolicyFromFile(filePath string) (*auditinternal.Policy, error) {
	if filePath == "" {
		return nil, fmt.Errorf("file path not specified")
	}
	policyDef, err := ioutil.ReadFile(filePath)
	if err != nil {
		return nil, fmt.Errorf("failed to read file path %q: %+v", filePath, err)
	}

	ret, err := LoadPolicyFromBytes(policyDef)
	if err != nil {
		return nil, fmt.Errorf("%v: from file %v", err.Error(), filePath)
	}

	return ret, nil
}

func LoadPolicyFromBytes(policyDef []byte) (*auditinternal.Policy, error) {
	policy := &auditinternal.Policy{}
	decoder := audit.Codecs.UniversalDecoder(apiGroupVersions...)

	_, gvk, err := decoder.Decode(policyDef, nil, policy)
	if err != nil {
		return nil, fmt.Errorf("failed decoding: %v", err)
	}

	// Ensure the policy file contained an apiVersion and kind.
	if !apiGroupVersionSet[schema.GroupVersion{Group: gvk.Group, Version: gvk.Version}] {
		return nil, fmt.Errorf("unknown group version field %v in policy", gvk)
	}

	if err := validation.ValidatePolicy(policy); err != nil {
		return nil, err.ToAggregate()
	}

	policyCnt := len(policy.Rules)
	if policyCnt == 0 {
		return nil, fmt.Errorf("loaded illegal policy with 0 rules")
	}
	glog.V(4).Infof("Loaded %d audit policy rules", policyCnt)
	return policy, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pprof_test

import (
	"bytes"
	"hash/crc32"
	"os/exec"
	"runtime"
	. "runtime/pprof"
	"strings"
	"testing"
	"unsafe"
)

func TestCPUProfile(t *testing.T) {
	switch runtime.GOOS {
	case "darwin":
		out, err := exec.Command("uname", "-a").CombinedOutput()
		if err != nil {
			t.Fatal(err)
		}
		vers := string(out)
		t.Logf("uname -a: %v", vers)
		if strings.Contains(vers, "Darwin Kernel Version 10.8.0") && strings.Contains(vers, "root:xnu-1504.15.3~1/RELEASE_X86_64") {
			t.Logf("skipping test on known-broken kernel (64-bit Snow Leopard)")
		}
	case "plan9":
		// unimplemented
		return
	}

	buf := make([]byte, 100000)
	var prof bytes.Buffer
	if err := StartCPUProfile(&prof); err != nil {
		t.Fatal(err)
	}
	// This loop takes about a quarter second on a 2 GHz laptop.
	// We only need to get one 100 Hz clock tick, so we've got
	// a 25x safety buffer.
	for i := 0; i < 1000; i++ {
		crc32.ChecksumIEEE(buf)
	}
	StopCPUProfile()

	// Convert []byte to []uintptr.
	bytes := prof.Bytes()
	val := *(*[]uintptr)(unsafe.Pointer(&bytes))
	val = val[:len(bytes)/int(unsafe.Sizeof(uintptr(0)))]

	if len(val) < 10 {
		t.Fatalf("profile too short: %#x", val)
	}
	if val[0] != 0 || val[1] != 3 || val[2] != 0 || val[3] != 1e6/100 || val[4] != 0 {
		t.Fatalf("unexpected header %#x", val[:5])
	}

	// Check that profile is well formed and contains ChecksumIEEE.
	found := false
	val = val[5:]
	for len(val) > 0 {
		if len(val) < 2 || val[0] < 1 || val[1] < 1 || uintptr(len(val)) < 2+val[1] {
			t.Fatalf("malformed profile.  leftover: %#x", val)
		}
		for _, pc := range val[2 : 2+val[1]] {
			f := runtime.FuncForPC(pc)
			if f == nil {
				continue
			}
			if strings.Contains(f.Name(), "ChecksumIEEE") {
				found = true
			}
		}
		val = val[2+val[1]:]
	}

	if !found {
		t.Fatal("did not find ChecksumIEEE in the profile")
	}
}
<commit_msg>runtime/pprof: skip test on 64-bit Mac OS X 10.6<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package pprof_test

import (
	"bytes"
	"hash/crc32"
	"os/exec"
	"runtime"
	. "runtime/pprof"
	"strings"
	"testing"
	"unsafe"
)

func TestCPUProfile(t *testing.T) {
	switch runtime.GOOS {
	case "darwin":
		out, err := exec.Command("uname", "-a").CombinedOutput()
		if err != nil {
			t.Fatal(err)
		}
		vers := string(out)
		t.Logf("uname -a: %v", vers)
		if strings.Contains(vers, "Darwin Kernel Version 10.8.0") && strings.Contains(vers, "root:xnu-1504.15.3~1/RELEASE_X86_64") {
			t.Logf("skipping test on known-broken kernel (64-bit Snow Leopard)")
			return
		}
	case "plan9":
		// unimplemented
		return
	}

	buf := make([]byte, 100000)
	var prof bytes.Buffer
	if err := StartCPUProfile(&prof); err != nil {
		t.Fatal(err)
	}
	// This loop takes about a quarter second on a 2 GHz laptop.
	// We only need to get one 100 Hz clock tick, so we've got
	// a 25x safety buffer.
	for i := 0; i < 1000; i++ {
		crc32.ChecksumIEEE(buf)
	}
	StopCPUProfile()

	// Convert []byte to []uintptr.
	bytes := prof.Bytes()
	val := *(*[]uintptr)(unsafe.Pointer(&bytes))
	val = val[:len(bytes)/int(unsafe.Sizeof(uintptr(0)))]

	if len(val) < 10 {
		t.Fatalf("profile too short: %#x", val)
	}
	if val[0] != 0 || val[1] != 3 || val[2] != 0 || val[3] != 1e6/100 || val[4] != 0 {
		t.Fatalf("unexpected header %#x", val[:5])
	}

	// Check that profile is well formed and contains ChecksumIEEE.
	found := false
	val = val[5:]
	for len(val) > 0 {
		if len(val) < 2 || val[0] < 1 || val[1] < 1 || uintptr(len(val)) < 2+val[1] {
			t.Fatalf("malformed profile.  leftover: %#x", val)
		}
		for _, pc := range val[2 : 2+val[1]] {
			f := runtime.FuncForPC(pc)
			if f == nil {
				continue
			}
			if strings.Contains(f.Name(), "ChecksumIEEE") {
				found = true
			}
		}
		val = val[2+val[1]:]
	}

	if !found {
		t.Fatal("did not find ChecksumIEEE in the profile")
	}
}
<|endoftext|>


<commit_before>package pachyderm

import (
	"context"
	"errors"
	"time"

	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/auth"
)

func (b *backend) pathAuthRenew(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
	if req.Auth == nil {
		return nil, errors.New("request auth was nil")
	}

	config, err := b.Config(ctx, req.Storage)
	if err != nil {
		return nil, err
	}
	if len(config.AdminToken) == 0 {
		return nil, errors.New("plugin is missing admin token")
	}
	if len(config.PachdAddress) == 0 {
		return nil, errors.New("plugin is missing pachd_address")
	}

	// Grab the user token
	userTokenRaw, ok := req.Auth.InternalData["user_token"]
	if !ok {
		return nil, errors.New("no internal user token found in the store")
	}
	userToken, ok := userTokenRaw.(string)
	if !ok {
		return nil, errors.New("stored user token is not a string")
	}

	ttl, maxTTL, err := b.SanitizeTTLStr(config.TTL, DefaultTTL)
	if err != nil {
		return nil, err
	}
	// Use the admin token to perform an action
	// for testing, hardcoding username to something else so that I can validate
	// renew has an effect:
	err = b.renewUserCredentials(ctx, config.PachdAddress, config.AdminToken, userToken, ttl)
	if err != nil {
		return nil, err
	}

	return framework.LeaseExtend(ttl, maxTTL, b.System())(ctx, req, d)
}

// renewUserCredentials extends the TTL of the Pachyderm authentication token
// 'userToken', using the vault plugin's Admin credentials. 'userToken' belongs
// to the user who is calling vault, and would like to extend their Pachyderm
// session.
func (b *backend) renewUserCredentials(ctx context.Context, pachdAddress string, adminToken string, userToken string, ttl time.Duration) error {
	// This is where we'd make the actual pachyderm calls to create the user
	// token using the admin token. For now, for testing purposes, we just do an action that only an
	// admin could do

	// Setup a single use client w the given admin token / address
	client, err := pclient.NewFromAddress(pachdAddress)
	if err != nil {
		return err
	}
	client = client.WithCtx(ctx)
	client.SetAuthToken(adminToken)

	_, err = client.AuthAPIClient.ModifyAdmins(client.Ctx(), &auth.ModifyAdminsRequest{
		Add: []string{"tweetybird"},
	})

	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Hook in auth renewal to vault plugin<commit_after>package pachyderm

import (
	"context"
	"errors"
	"time"

	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/auth"
)

func (b *backend) pathAuthRenew(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) {
	if req.Auth == nil {
		return nil, errors.New("request auth was nil")
	}

	config, err := b.Config(ctx, req.Storage)
	if err != nil {
		return nil, err
	}
	if len(config.AdminToken) == 0 {
		return nil, errors.New("plugin is missing admin token")
	}
	if len(config.PachdAddress) == 0 {
		return nil, errors.New("plugin is missing pachd_address")
	}

	// Grab the user token
	userTokenRaw, ok := req.Auth.InternalData["user_token"]
	if !ok {
		return nil, errors.New("no internal user token found in the store")
	}
	userToken, ok := userTokenRaw.(string)
	if !ok {
		return nil, errors.New("stored user token is not a string")
	}

	ttl, maxTTL, err := b.SanitizeTTLStr(config.TTL, DefaultTTL)
	if err != nil {
		return nil, err
	}
	// Use the admin token to perform an action
	// for testing, hardcoding username to something else so that I can validate
	// renew has an effect:
	err = b.renewUserCredentials(ctx, config.PachdAddress, config.AdminToken, userToken, ttl)
	if err != nil {
		return nil, err
	}

	return framework.LeaseExtend(ttl, maxTTL, b.System())(ctx, req, d)
}

// renewUserCredentials extends the TTL of the Pachyderm authentication token
// 'userToken', using the vault plugin's Admin credentials. 'userToken' belongs
// to the user who is calling vault, and would like to extend their Pachyderm
// session.
func (b *backend) renewUserCredentials(ctx context.Context, pachdAddress string, adminToken string, userToken string, ttl time.Duration) error {
	// Setup a single use client w the given admin token / address
	client, err := pclient.NewFromAddress(pachdAddress)
	if err != nil {
		return err
	}
	client = client.WithCtx(ctx)
	client.SetAuthToken(adminToken)

	_, err = client.AuthAPIClient.ExtendAuthToken(client.Ctx(), &auth.ExtendAuthTokenRequest{
		Token: userToken,
		TTL:   int64(ttl.Seconds()),
	})

	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"errors"
	"fmt"

	gostErrors "github.com/geodan/gost/src/errors"
	"github.com/geodan/gost/src/sensorthings/entities"
	"github.com/geodan/gost/src/sensorthings/models"
	"github.com/geodan/gost/src/sensorthings/odata"
)

// GetObservation returns an observation by id
func (a *APIv1) GetObservation(id interface{}, qo *odata.QueryOptions, path string) (*entities.Observation, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	o, err := a.db.GetObservation(id, qo)
	if err != nil {
		return nil, err
	}

	a.ProcessGetRequest(o, qo)
	return o, nil
}

// GetObservations return all observations by given QueryOptions
func (a *APIv1) GetObservations(qo *odata.QueryOptions, path string) (*models.ArrayResponse, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	observations, err := a.db.GetObservations(qo)
	return processObservations(a, observations, qo, path, err)
}

// GetObservationsByFeatureOfInterest returns all observation by given FeatureOfInterest end QueryOptions
func (a *APIv1) GetObservationsByFeatureOfInterest(foiID interface{}, qo *odata.QueryOptions, path string) (*models.ArrayResponse, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	observations, err := a.db.GetObservationsByFeatureOfInterest(foiID, qo)
	return processObservations(a, observations, qo, path, err)
}

// GetObservationsByDatastream returns all observations by given Datastream and QueryOptions
func (a *APIv1) GetObservationsByDatastream(datastreamID interface{}, qo *odata.QueryOptions, path string) (*models.ArrayResponse, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	observations, err := a.db.GetObservationsByDatastream(datastreamID, qo)
	return processObservations(a, observations, qo, path, err)
}

func processObservations(a *APIv1, observations []*entities.Observation, qo *odata.QueryOptions, path string, err error) (*models.ArrayResponse, error) {
	if err != nil {
		return nil, err
	}

	for idx, item := range observations {
		i := *item
		a.ProcessGetRequest(&i, qo)
		observations[idx] = &i
	}

	var numberOfObservations = len(observations)

	var data interface{} = observations
	return &models.ArrayResponse{
		Count:    numberOfObservations,
		NextLink: a.CreateNextLink(a.db.GetTotalObservations(), path, qo),
		Data:     &data,
	}, nil
}

// GetFoiIDByDatastreamID gets the location of the related lthing locatiob
func GetFoiIDByDatastreamID(gdb *models.Database, datastreamID interface{}) (string, error) {
	db := *gdb
	dId := toStringID(datastreamID)
	_, err := db.GetDatastream(dId, nil)
	if err != nil {
		return "", errors.New("Datastream not found")
	}

	thing, err := db.GetThingByDatastream(dId, nil)
	if err != nil {
		return "", errors.New("Thing by datastream not found")
	}

	l, err := db.GetLocationsByThing(thing.ID, nil)
	if err != nil || len(l) == 0 {
		return "", err
	}

	foi := &entities.FeatureOfInterest{}
	foi.Description = l[0].Description
	foi.EncodingType = l[0].EncodingType
	foi.Feature = l[0].Location

	nFoi, err := db.PostFeatureOfInterest(foi)
	if err != nil {
		return "", err
	}

	return toStringID(nFoi.ID), nil
}

// PostObservation checks for correctness of the observation and calls PostObservation on the database
func (a *APIv1) PostObservation(observation *entities.Observation) (*entities.Observation, []error) {
	_, err := observation.ContainsMandatoryParams()
	if err != nil {
		return nil, err
	}

	datastreamID := observation.Datastream.ID

	if observation.FeatureOfInterest == nil {
		foiID, err := GetFoiIDByDatastreamID(&a.db, toStringID(datastreamID))

		if err != nil {
			return nil, []error{gostErrors.NewBadRequestError(errors.New("Unable to link or create FeatureOfInterest for Observation. The linked Thing should have a location or supply (deep insert) a new FeatureOfInterest or link to a known FeatureOfInterest by id \"FeatureOfInterest\":{\"@iot.id\":30198}"))}
		}

		observation.FeatureOfInterest = &entities.FeatureOfInterest{}
		observation.FeatureOfInterest.ID = foiID
	} else if observation.FeatureOfInterest != nil && observation.FeatureOfInterest.ID == nil {
		if foi, err := a.PostFeatureOfInterest(observation.FeatureOfInterest); err != nil {
			return nil, []error{gostErrors.NewConflictRequestError(errors.New("Unable to create deep inserted FeatureOfInterest"))}
		} else {
			observation.FeatureOfInterest = foi
		}
	}

	no, err2 := a.db.PostObservation(observation)
	if err2 != nil {
		return nil, []error{err2}
	}

	no.SetAllLinks(a.config.GetExternalServerURI())

	json, _ := json.Marshal(no)
	s := string(json)

	//ToDo: MQTT TEST
	a.mqtt.Publish(fmt.Sprintf("Datastreams(%v)/Observations", datastreamID), s, 0)
	a.mqtt.Publish("Observations", s, 0)

	return no, nil
}

func toStringID(id interface{}) string {
	return fmt.Sprintf("%v", id)
}

// PostObservationByDatastream creates an Observation with a linked datastream by given datastream id and calls PostObservation on the database
func (a *APIv1) PostObservationByDatastream(datastreamID interface{}, observation *entities.Observation) (*entities.Observation, []error) {
	d := &entities.Datastream{}
	d.ID = datastreamID
	observation.Datastream = d
	return a.PostObservation(observation)
}

// PatchObservation todo
func (a *APIv1) PatchObservation(id interface{}, observation *entities.Observation) (*entities.Observation, error) {
	return nil, gostErrors.NewRequestNotImplemented(errors.New("not implemented yet"))
}

// DeleteObservation deletes a given Observation from the database
func (a *APIv1) DeleteObservation(id interface{}) error {
	return a.db.DeleteObservation(id)
}
<commit_msg>refactor postObservation<commit_after>package api

import (
	"encoding/json"
	"errors"
	"fmt"

	gostErrors "github.com/geodan/gost/src/errors"
	"github.com/geodan/gost/src/sensorthings/entities"
	"github.com/geodan/gost/src/sensorthings/models"
	"github.com/geodan/gost/src/sensorthings/odata"
)

// GetObservation returns an observation by id
func (a *APIv1) GetObservation(id interface{}, qo *odata.QueryOptions, path string) (*entities.Observation, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	o, err := a.db.GetObservation(id, qo)
	if err != nil {
		return nil, err
	}

	a.ProcessGetRequest(o, qo)
	return o, nil
}

// GetObservations return all observations by given QueryOptions
func (a *APIv1) GetObservations(qo *odata.QueryOptions, path string) (*models.ArrayResponse, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	observations, err := a.db.GetObservations(qo)
	return processObservations(a, observations, qo, path, err)
}

// GetObservationsByFeatureOfInterest returns all observation by given FeatureOfInterest end QueryOptions
func (a *APIv1) GetObservationsByFeatureOfInterest(foiID interface{}, qo *odata.QueryOptions, path string) (*models.ArrayResponse, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	observations, err := a.db.GetObservationsByFeatureOfInterest(foiID, qo)
	return processObservations(a, observations, qo, path, err)
}

// GetObservationsByDatastream returns all observations by given Datastream and QueryOptions
func (a *APIv1) GetObservationsByDatastream(datastreamID interface{}, qo *odata.QueryOptions, path string) (*models.ArrayResponse, error) {
	_, err := a.QueryOptionsSupported(qo, &entities.Observation{})
	if err != nil {
		return nil, err
	}

	observations, err := a.db.GetObservationsByDatastream(datastreamID, qo)
	return processObservations(a, observations, qo, path, err)
}

func processObservations(a *APIv1, observations []*entities.Observation, qo *odata.QueryOptions, path string, err error) (*models.ArrayResponse, error) {
	if err != nil {
		return nil, err
	}

	for idx, item := range observations {
		i := *item
		a.ProcessGetRequest(&i, qo)
		observations[idx] = &i
	}

	var numberOfObservations = len(observations)

	var data interface{} = observations
	return &models.ArrayResponse{
		Count:    numberOfObservations,
		NextLink: a.CreateNextLink(a.db.GetTotalObservations(), path, qo),
		Data:     &data,
	}, nil
}

// GetLocationByDatastreamID return Location object for Datastream
// todo: make 1 query instead of 3...
func GetLocationByDatastreamID(gdb *models.Database, datastreamID interface{}) (*entities.Location, error) {
	db := *gdb
	dId := toStringID(datastreamID)
	_, err := db.GetDatastream(dId, nil)
	if err != nil {
		return nil, errors.New("Datastream not found")
	}

	thing, err := db.GetThingByDatastream(dId, nil)
	if err != nil {
		return nil, errors.New("Thing by datastream not found")
	}

	l, err := db.GetLocationsByThing(thing.ID, nil)
	if err != nil || len(l) == 0 {
		return nil, err
	}

	// return the first location in the list
	return l[0], nil
}

// ConvertLocationToFoi converts a location to FOI
func ConvertLocationToFoi(l *entities.Location) *entities.FeatureOfInterest {
	foi := &entities.FeatureOfInterest{}
	foi.Description = l.Description
	foi.EncodingType = l.EncodingType
	foi.Feature = l.Location
	return foi
}

// GetFoiIDByDatastreamID gets the location of the related thing location
func CopyLocationToFoi(gdb *models.Database, datastreamID interface{}) (string, error) {
	db := *gdb
	l, err := GetLocationByDatastreamID(gdb, datastreamID)
	if err != nil {
		return "", err
	}

	foi := ConvertLocationToFoi(l)

	nFoi, err := db.PostFeatureOfInterest(foi)
	if err != nil {
		return "", err
	}

	return toStringID(nFoi.ID), nil
}

// PostObservation checks for correctness of the observation and calls PostObservation on the database
func (a *APIv1) PostObservation(observation *entities.Observation) (*entities.Observation, []error) {
	_, err := observation.ContainsMandatoryParams()
	if err != nil {
		return nil, err
	}

	datastreamID := observation.Datastream.ID

	// there is no foi posted: try to copy it from thing.location...
	if observation.FeatureOfInterest == nil {
		foiID, err := CopyLocationToFoi(&a.db, toStringID(datastreamID))

		if err != nil {
			errorMessage := "Unable to copy location of thing to featureofinterest."
			return nil, []error{gostErrors.NewBadRequestError(errors.New(errorMessage))}
		}

		observation.FeatureOfInterest = &entities.FeatureOfInterest{}
		observation.FeatureOfInterest.ID = foiID
	} else if observation.FeatureOfInterest != nil && observation.FeatureOfInterest.ID == nil {
		if foi, err := a.PostFeatureOfInterest(observation.FeatureOfInterest); err != nil {
			return nil, []error{gostErrors.NewConflictRequestError(errors.New("Unable to create deep inserted FeatureOfInterest"))}
		} else {
			observation.FeatureOfInterest = foi
		}
	}

	no, err2 := a.db.PostObservation(observation)
	if err2 != nil {
		return nil, []error{err2}
	}

	no.SetAllLinks(a.config.GetExternalServerURI())

	json, _ := json.Marshal(no)
	s := string(json)

	//ToDo: MQTT TEST
	a.mqtt.Publish(fmt.Sprintf("Datastreams(%v)/Observations", datastreamID), s, 0)
	a.mqtt.Publish("Observations", s, 0)

	return no, nil
}

func toStringID(id interface{}) string {
	return fmt.Sprintf("%v", id)
}

// PostObservationByDatastream creates an Observation with a linked datastream by given datastream id and calls PostObservation on the database
func (a *APIv1) PostObservationByDatastream(datastreamID interface{}, observation *entities.Observation) (*entities.Observation, []error) {
	d := &entities.Datastream{}
	d.ID = datastreamID
	observation.Datastream = d
	return a.PostObservation(observation)
}

// PatchObservation todo
func (a *APIv1) PatchObservation(id interface{}, observation *entities.Observation) (*entities.Observation, error) {
	return nil, gostErrors.NewRequestNotImplemented(errors.New("not implemented yet"))
}

// DeleteObservation deletes a given Observation from the database
func (a *APIv1) DeleteObservation(id interface{}) error {
	return a.db.DeleteObservation(id)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"math"
)

// BufferedSeriesIterator wraps an iterator with a look-back buffer.
type BufferedSeriesIterator struct {
	it    SeriesIterator
	buf   *sampleRing
	delta int64

	lastTime int64
	ok       bool
}

// NewBuffer returns a new iterator that buffers the values within the time range
// of the current element and the duration of delta before, initialized with an
// empty iterator. Use Reset() to set an actual iterator to be buffered.
func NewBuffer(delta int64) *BufferedSeriesIterator {
	return NewBufferIterator(&NoopSeriesIt, delta)
}

// NewBufferIterator returns a new iterator that buffers the values within the
// time range of the current element and the duration of delta before.
func NewBufferIterator(it SeriesIterator, delta int64) *BufferedSeriesIterator {
	bit := &BufferedSeriesIterator{
		buf:   newSampleRing(delta, 16),
		delta: delta,
	}
	bit.Reset(it)

	return bit
}

// Reset re-uses the buffer with a new iterator, resetting the buffered time
// delta to its original value.
func (b *BufferedSeriesIterator) Reset(it SeriesIterator) {
	b.it = it
	b.lastTime = math.MinInt64
	b.ok = true
	b.buf.reset()
	b.buf.delta = b.delta
	it.Next()
}

// ReduceDelta lowers the buffered time delta, for the current SeriesIterator only.
func (b *BufferedSeriesIterator) ReduceDelta(delta int64) bool {
	if delta > b.buf.delta {
		return false
	}
	b.buf.delta = delta
	return true
}

// PeekBack returns the nth previous element of the iterator. If there is none buffered,
// ok is false.
func (b *BufferedSeriesIterator) PeekBack(n int) (t int64, v float64, ok bool) {
	return b.buf.nthLast(n)
}

// Buffer returns an iterator over the buffered data. Invalidates previously
// returned iterators.
func (b *BufferedSeriesIterator) Buffer() SeriesIterator {
	return b.buf.iterator()
}

// Seek advances the iterator to the element at time t or greater.
func (b *BufferedSeriesIterator) Seek(t int64) bool {
	t0 := t - b.buf.delta

	// If the delta would cause us to seek backwards, preserve the buffer
	// and just continue regular advancement while filling the buffer on the way.
	if t0 > b.lastTime {
		b.buf.reset()

		b.ok = b.it.Seek(t0)
		if !b.ok {
			return false
		}
		b.lastTime, _ = b.Values()
	}

	if b.lastTime >= t {
		return true
	}
	for b.Next() {
		if b.lastTime >= t {
			return true
		}
	}

	return false
}

// Next advances the iterator to the next element.
func (b *BufferedSeriesIterator) Next() bool {
	if !b.ok {
		return false
	}

	// Add current element to buffer before advancing.
	b.buf.add(b.it.At())

	b.ok = b.it.Next()
	if b.ok {
		b.lastTime, _ = b.Values()
	}

	return b.ok
}

// Values returns the current element of the iterator.
func (b *BufferedSeriesIterator) Values() (int64, float64) {
	return b.it.At()
}

// Err returns the last encountered error.
func (b *BufferedSeriesIterator) Err() error {
	return b.it.Err()
}

type sample struct {
	t int64
	v float64
}

type sampleRing struct {
	delta int64

	buf []sample // lookback buffer
	i   int      // position of most recent element in ring buffer
	f   int      // position of first element in ring buffer
	l   int      // number of elements in buffer

	it sampleRingIterator
}

func newSampleRing(delta int64, sz int) *sampleRing {
	r := &sampleRing{delta: delta, buf: make([]sample, sz)}
	r.reset()

	return r
}

func (r *sampleRing) reset() {
	r.l = 0
	r.i = -1
	r.f = 0
}

// Returns the current iterator. Invalidates previously returned iterators.
func (r *sampleRing) iterator() SeriesIterator {
	r.it.r = r
	r.it.i = -1
	return &r.it
}

type sampleRingIterator struct {
	r *sampleRing
	i int
}

func (it *sampleRingIterator) Next() bool {
	it.i++
	return it.i < it.r.l
}

func (it *sampleRingIterator) Seek(int64) bool {
	return false
}

func (it *sampleRingIterator) Err() error {
	return nil
}

func (it *sampleRingIterator) At() (int64, float64) {
	return it.r.at(it.i)
}

func (r *sampleRing) at(i int) (int64, float64) {
	j := (r.f + i) % len(r.buf)
	s := r.buf[j]
	return s.t, s.v
}

// add adds a sample to the ring buffer and frees all samples that fall
// out of the delta range.
func (r *sampleRing) add(t int64, v float64) {
	l := len(r.buf)
	// Grow the ring buffer if it fits no more elements.
	if l == r.l {
		buf := make([]sample, 2*l)
		copy(buf[l+r.f:], r.buf[r.f:])
		copy(buf, r.buf[:r.f])

		r.buf = buf
		r.i = r.f
		r.f += l
		l = 2 * l
	} else {
		r.i++
		if r.i >= l {
			r.i -= l
		}
	}

	r.buf[r.i] = sample{t: t, v: v}
	r.l++

	// Free head of the buffer of samples that just fell out of the range.
	for r.buf[r.f].t < t-r.delta {
		r.f++
		if r.f >= l {
			r.f -= l
		}
		r.l--
	}
}

// nthLast returns the nth most recent element added to the ring.
func (r *sampleRing) nthLast(n int) (int64, float64, bool) {
	if n > r.l {
		return 0, 0, false
	}
	t, v := r.at(r.l - n)
	return t, v, true
}

func (r *sampleRing) samples() []sample {
	res := make([]sample, r.l)

	var k = r.f + r.l
	var j int
	if k > len(r.buf) {
		k = len(r.buf)
		j = r.l - k + r.f
	}

	n := copy(res, r.buf[r.f:k])
	copy(res[n:], r.buf[:j])

	return res
}
<commit_msg>Minor optimization for BufferedSeriesIterator: actually drop the samples falling outside of the new delta from the underlying sampleRing, when ReduceDelta is called. (#4849)<commit_after>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"math"
)

// BufferedSeriesIterator wraps an iterator with a look-back buffer.
type BufferedSeriesIterator struct {
	it    SeriesIterator
	buf   *sampleRing
	delta int64

	lastTime int64
	ok       bool
}

// NewBuffer returns a new iterator that buffers the values within the time range
// of the current element and the duration of delta before, initialized with an
// empty iterator. Use Reset() to set an actual iterator to be buffered.
func NewBuffer(delta int64) *BufferedSeriesIterator {
	return NewBufferIterator(&NoopSeriesIt, delta)
}

// NewBufferIterator returns a new iterator that buffers the values within the
// time range of the current element and the duration of delta before.
func NewBufferIterator(it SeriesIterator, delta int64) *BufferedSeriesIterator {
	bit := &BufferedSeriesIterator{
		buf:   newSampleRing(delta, 16),
		delta: delta,
	}
	bit.Reset(it)

	return bit
}

// Reset re-uses the buffer with a new iterator, resetting the buffered time
// delta to its original value.
func (b *BufferedSeriesIterator) Reset(it SeriesIterator) {
	b.it = it
	b.lastTime = math.MinInt64
	b.ok = true
	b.buf.reset()
	b.buf.delta = b.delta
	it.Next()
}

// ReduceDelta lowers the buffered time delta, for the current SeriesIterator only.
func (b *BufferedSeriesIterator) ReduceDelta(delta int64) bool {
	return b.buf.reduceDelta(delta)
}

// PeekBack returns the nth previous element of the iterator. If there is none buffered,
// ok is false.
func (b *BufferedSeriesIterator) PeekBack(n int) (t int64, v float64, ok bool) {
	return b.buf.nthLast(n)
}

// Buffer returns an iterator over the buffered data. Invalidates previously
// returned iterators.
func (b *BufferedSeriesIterator) Buffer() SeriesIterator {
	return b.buf.iterator()
}

// Seek advances the iterator to the element at time t or greater.
func (b *BufferedSeriesIterator) Seek(t int64) bool {
	t0 := t - b.buf.delta

	// If the delta would cause us to seek backwards, preserve the buffer
	// and just continue regular advancement while filling the buffer on the way.
	if t0 > b.lastTime {
		b.buf.reset()

		b.ok = b.it.Seek(t0)
		if !b.ok {
			return false
		}
		b.lastTime, _ = b.Values()
	}

	if b.lastTime >= t {
		return true
	}
	for b.Next() {
		if b.lastTime >= t {
			return true
		}
	}

	return false
}

// Next advances the iterator to the next element.
func (b *BufferedSeriesIterator) Next() bool {
	if !b.ok {
		return false
	}

	// Add current element to buffer before advancing.
	b.buf.add(b.it.At())

	b.ok = b.it.Next()
	if b.ok {
		b.lastTime, _ = b.Values()
	}

	return b.ok
}

// Values returns the current element of the iterator.
func (b *BufferedSeriesIterator) Values() (int64, float64) {
	return b.it.At()
}

// Err returns the last encountered error.
func (b *BufferedSeriesIterator) Err() error {
	return b.it.Err()
}

type sample struct {
	t int64
	v float64
}

type sampleRing struct {
	delta int64

	buf []sample // lookback buffer
	i   int      // position of most recent element in ring buffer
	f   int      // position of first element in ring buffer
	l   int      // number of elements in buffer

	it sampleRingIterator
}

func newSampleRing(delta int64, sz int) *sampleRing {
	r := &sampleRing{delta: delta, buf: make([]sample, sz)}
	r.reset()

	return r
}

func (r *sampleRing) reset() {
	r.l = 0
	r.i = -1
	r.f = 0
}

// Returns the current iterator. Invalidates previously returned iterators.
func (r *sampleRing) iterator() SeriesIterator {
	r.it.r = r
	r.it.i = -1
	return &r.it
}

type sampleRingIterator struct {
	r *sampleRing
	i int
}

func (it *sampleRingIterator) Next() bool {
	it.i++
	return it.i < it.r.l
}

func (it *sampleRingIterator) Seek(int64) bool {
	return false
}

func (it *sampleRingIterator) Err() error {
	return nil
}

func (it *sampleRingIterator) At() (int64, float64) {
	return it.r.at(it.i)
}

func (r *sampleRing) at(i int) (int64, float64) {
	j := (r.f + i) % len(r.buf)
	s := r.buf[j]
	return s.t, s.v
}

// add adds a sample to the ring buffer and frees all samples that fall
// out of the delta range.
func (r *sampleRing) add(t int64, v float64) {
	l := len(r.buf)
	// Grow the ring buffer if it fits no more elements.
	if l == r.l {
		buf := make([]sample, 2*l)
		copy(buf[l+r.f:], r.buf[r.f:])
		copy(buf, r.buf[:r.f])

		r.buf = buf
		r.i = r.f
		r.f += l
		l = 2 * l
	} else {
		r.i++
		if r.i >= l {
			r.i -= l
		}
	}

	r.buf[r.i] = sample{t: t, v: v}
	r.l++

	// Free head of the buffer of samples that just fell out of the range.
	tmin := t - r.delta
	for r.buf[r.f].t < tmin {
		r.f++
		if r.f >= l {
			r.f -= l
		}
		r.l--
	}
}

// reduceDelta lowers the buffered time delta, dropping any samples that are
// out of the new delta range.
func (r *sampleRing) reduceDelta(delta int64) bool {
	if delta > r.delta {
		return false
	}
	r.delta = delta

	if r.l == 0 {
		return true
	}

	// Free head of the buffer of samples that just fell out of the range.
	l := len(r.buf)
	tmin := r.buf[r.i].t - delta
	for r.buf[r.f].t < tmin {
		r.f++
		if r.f >= l {
			r.f -= l
		}
		r.l--
	}
	return true
}

// nthLast returns the nth most recent element added to the ring.
func (r *sampleRing) nthLast(n int) (int64, float64, bool) {
	if n > r.l {
		return 0, 0, false
	}
	t, v := r.at(r.l - n)
	return t, v, true
}

func (r *sampleRing) samples() []sample {
	res := make([]sample, r.l)

	var k = r.f + r.l
	var j int
	if k > len(r.buf) {
		k = len(r.buf)
		j = r.l - k + r.f
	}

	n := copy(res, r.buf[r.f:k])
	copy(res[n:], r.buf[:j])

	return res
}
<|endoftext|>


<commit_before>package policy

import (
	"bufio"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/scjalliance/resourceful/lease"
	"github.com/scjalliance/resourceful/strategy"
	"golang.org/x/crypto/sha3"
)

// TODO: Consider giving policies the ability to construct a "consumer" from
// any number of properties. Doing this would allow policies flexibility in
// describing the unit of consumption.
//
// We could use something like https://github.com/valyala/fasttemplate for
// this construction.

// Policy describes the matching conditions and rules for handling a particular
// resource.
//
// A policy is applied only if all of its conditions are matched.
type Policy struct {
	Resource   string            `json:"resource,omitempty"`
	Criteria   Criteria          `json:"criteria,omitempty"`
	Strategy   strategy.Strategy `json:"strategy,omitempty"`
	Limit      uint              `json:"limit,omitempty"`
	Duration   time.Duration     `json:"duration,omitempty"` // Time before a leased resource is automatically released
	Decay      time.Duration     `json:"decay,omitempty"`    // Time before a released resource is considered available again
	Refresh    lease.Refresh     `json:"refresh,omitempty"`  // Time between lease acquisitions while maintaining a lease
	Properties lease.Properties  `json:"override,omitempty"` // Properties for matching leases
}

// New returns a new policy for a particular resource with the given limit,
// duration and criteria.
func New(resource string, strat strategy.Strategy, limit uint, duration time.Duration, criteria Criteria) Policy {
	return Policy{
		Resource: resource,
		Criteria: criteria,
		Strategy: strat,
		Limit:    limit,
		Duration: duration,
	}
}

// MarshalJSON will encode the policy as JSON.
func (p *Policy) MarshalJSON() ([]byte, error) {
	type pol Policy
	type refresh struct {
		Active string `json:"active,omitempty"`
		Queued string `json:"queued,omitempty"`
	}
	return json.Marshal(&struct {
		*pol
		Duration string  `json:"duration"`
		Decay    string  `json:"decay"`
		Refresh  refresh `json:"refresh,omitempty"`
	}{
		pol:      (*pol)(p),
		Duration: p.Duration.String(),
		Decay:    p.Decay.String(),
		Refresh: refresh{
			Active: p.Refresh.Active.String(),
			Queued: p.Refresh.Queued.String(),
		},
	})
}

// UnmarshalJSON will decode JSON policy data.
func (p *Policy) UnmarshalJSON(data []byte) error {
	type pol Policy
	type refresh struct {
		Active string `json:"active,omitempty"`
		Queued string `json:"queued,omitempty"`
	}
	aux := &struct {
		*pol
		Duration string  `json:"duration"`
		Decay    string  `json:"decay"`
		Refresh  refresh `json:"refresh,omitempty"`
	}{
		pol: (*pol)(p),
	}
	var err error
	if err = json.Unmarshal(data, aux); err != nil {
		return err
	}
	if aux.Duration != "" {
		if p.Duration, err = time.ParseDuration(aux.Duration); err != nil {
			return err
		}
	}
	if aux.Decay != "" {
		if p.Decay, err = time.ParseDuration(aux.Decay); err != nil {
			return err
		}
	}
	if aux.Refresh.Active != "" {
		if p.Refresh.Active, err = time.ParseDuration(aux.Refresh.Active); err != nil {
			return err
		}
	}
	if aux.Refresh.Queued != "" {
		if p.Refresh.Queued, err = time.ParseDuration(aux.Refresh.Queued); err != nil {
			return err
		}
	}
	return nil
}

// Match returns true if the policy applies to a lease with the given
// properites.
//
// All of the policy's conditions must match for the policy to be applied.
func (p *Policy) Match(props lease.Properties) bool {
	return p.Criteria.Match(props)
}

// String returns a string representation of the policy.
func (p *Policy) String() string {
	var parts []string
	if p.Resource != "" {
		parts = append(parts, fmt.Sprintf("Resource: %q", p.Resource))
	}
	if len(p.Criteria) > 0 {
		parts = append(parts, fmt.Sprintf("Criteria: %q", p.Criteria.String()))
	}
	if p.Strategy != "" {
		parts = append(parts, fmt.Sprintf("Strategy: %s", p.Strategy))
	}
	if p.Limit != 0 {
		parts = append(parts, fmt.Sprintf("Limit: %d", p.Limit))
	}
	if p.Duration != 0 {
		parts = append(parts, fmt.Sprintf("Duration: %s", p.Duration))
	}
	if p.Decay != 0 {
		parts = append(parts, fmt.Sprintf("Decay: %s", p.Decay))
	}
	if p.Refresh.Active != 0 {
		parts = append(parts, fmt.Sprintf("Active Refresh: %s", p.Refresh.Active))
	}
	if p.Refresh.Queued != 0 {
		parts = append(parts, fmt.Sprintf("Queued Refresh: %s", p.Refresh.Queued))
	}
	if len(p.Properties) > 0 {
		parts = append(parts, fmt.Sprintf("Properties: %q", p.Properties))
	}
	return strings.Join(parts, " ")
}

// Hash returns a 224-bit hash of the policy.
func (p *Policy) Hash() Hash {
	var (
		hash = sha3.New224()
		w    = hashWriter{bufio.NewWriterSize(hash, hash.BlockSize())}
	)

	w.WriteString(p.Resource)
	w.WriteInt(len(p.Criteria))
	for _, c := range p.Criteria {
		w.WriteString(c.Key)
		w.WriteString(c.Comparison)
		w.WriteString(c.Value)
	}
	w.WriteString(string(p.Strategy))
	w.WriteInt(int(p.Limit))
	w.WriteDuration(p.Duration)
	w.WriteDuration(p.Decay)
	w.WriteDuration(p.Refresh.Active)
	w.WriteDuration(p.Refresh.Queued)
	w.WriteInt(len(p.Properties))
	for key, value := range p.Properties {
		w.WriteString(key)
		w.WriteString(value)
	}

	if err := w.Flush(); err != nil {
		panic(err)
	}

	var h Hash
	hash.Sum(h[:0])
	return h
}
<commit_msg>policy: Updated property serialization<commit_after>package policy

import (
	"bufio"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/scjalliance/resourceful/lease"
	"github.com/scjalliance/resourceful/strategy"
	"golang.org/x/crypto/sha3"
)

// TODO: Consider giving policies the ability to construct a "consumer" from
// any number of properties. Doing this would allow policies flexibility in
// describing the unit of consumption.
//
// We could use something like https://github.com/valyala/fasttemplate for
// this construction.

// Policy describes the matching conditions and rules for handling a particular
// resource.
//
// A policy is applied only if all of its conditions are matched.
type Policy struct {
	Resource   string            `json:"resource,omitempty"`   // Which resource pool this policy counts against
	Criteria   Criteria          `json:"criteria,omitempty"`   // Matching criteria for lease properties
	Strategy   strategy.Strategy `json:"strategy,omitempty"`   // Lease counting strategy
	Limit      uint              `json:"limit,omitempty"`      // Max concurrent leases
	Duration   time.Duration     `json:"duration,omitempty"`   // Time before a leased resource is automatically released
	Decay      time.Duration     `json:"decay,omitempty"`      // Time before a released resource is considered available again
	Refresh    lease.Refresh     `json:"refresh,omitempty"`    // Time between lease acquisitions while maintaining a lease
	Properties lease.Properties  `json:"properties,omitempty"` // Merged with each lease's properties
}

// New returns a new policy for a particular resource with the given limit,
// duration and criteria.
func New(resource string, strat strategy.Strategy, limit uint, duration time.Duration, criteria Criteria) Policy {
	return Policy{
		Resource: resource,
		Criteria: criteria,
		Strategy: strat,
		Limit:    limit,
		Duration: duration,
	}
}

// MarshalJSON will encode the policy as JSON.
func (p *Policy) MarshalJSON() ([]byte, error) {
	type pol Policy
	type refresh struct {
		Active string `json:"active,omitempty"`
		Queued string `json:"queued,omitempty"`
	}
	return json.Marshal(&struct {
		*pol
		Duration string  `json:"duration"`
		Decay    string  `json:"decay"`
		Refresh  refresh `json:"refresh,omitempty"`
	}{
		pol:      (*pol)(p),
		Duration: p.Duration.String(),
		Decay:    p.Decay.String(),
		Refresh: refresh{
			Active: p.Refresh.Active.String(),
			Queued: p.Refresh.Queued.String(),
		},
	})
}

// UnmarshalJSON will decode JSON policy data.
func (p *Policy) UnmarshalJSON(data []byte) error {
	type pol Policy
	type refresh struct {
		Active string `json:"active,omitempty"`
		Queued string `json:"queued,omitempty"`
	}
	aux := &struct {
		*pol
		Duration string  `json:"duration"`
		Decay    string  `json:"decay"`
		Refresh  refresh `json:"refresh,omitempty"`
	}{
		pol: (*pol)(p),
	}
	var err error
	if err = json.Unmarshal(data, aux); err != nil {
		return err
	}
	if aux.Duration != "" {
		if p.Duration, err = time.ParseDuration(aux.Duration); err != nil {
			return err
		}
	}
	if aux.Decay != "" {
		if p.Decay, err = time.ParseDuration(aux.Decay); err != nil {
			return err
		}
	}
	if aux.Refresh.Active != "" {
		if p.Refresh.Active, err = time.ParseDuration(aux.Refresh.Active); err != nil {
			return err
		}
	}
	if aux.Refresh.Queued != "" {
		if p.Refresh.Queued, err = time.ParseDuration(aux.Refresh.Queued); err != nil {
			return err
		}
	}
	return nil
}

// Match returns true if the policy applies to a lease with the given
// properites.
//
// All of the policy's conditions must match for the policy to be applied.
func (p *Policy) Match(props lease.Properties) bool {
	return p.Criteria.Match(props)
}

// String returns a string representation of the policy.
func (p *Policy) String() string {
	var parts []string
	if p.Resource != "" {
		parts = append(parts, fmt.Sprintf("Resource: %q", p.Resource))
	}
	if len(p.Criteria) > 0 {
		parts = append(parts, fmt.Sprintf("Criteria: %q", p.Criteria.String()))
	}
	if p.Strategy != "" {
		parts = append(parts, fmt.Sprintf("Strategy: %s", p.Strategy))
	}
	if p.Limit != 0 {
		parts = append(parts, fmt.Sprintf("Limit: %d", p.Limit))
	}
	if p.Duration != 0 {
		parts = append(parts, fmt.Sprintf("Duration: %s", p.Duration))
	}
	if p.Decay != 0 {
		parts = append(parts, fmt.Sprintf("Decay: %s", p.Decay))
	}
	if p.Refresh.Active != 0 {
		parts = append(parts, fmt.Sprintf("Active Refresh: %s", p.Refresh.Active))
	}
	if p.Refresh.Queued != 0 {
		parts = append(parts, fmt.Sprintf("Queued Refresh: %s", p.Refresh.Queued))
	}
	if len(p.Properties) > 0 {
		parts = append(parts, fmt.Sprintf("Properties: %q", p.Properties))
	}
	return strings.Join(parts, " ")
}

// Hash returns a 224-bit hash of the policy.
func (p *Policy) Hash() Hash {
	var (
		hash = sha3.New224()
		w    = hashWriter{bufio.NewWriterSize(hash, hash.BlockSize())}
	)

	w.WriteString(p.Resource)
	w.WriteInt(len(p.Criteria))
	for _, c := range p.Criteria {
		w.WriteString(c.Key)
		w.WriteString(c.Comparison)
		w.WriteString(c.Value)
	}
	w.WriteString(string(p.Strategy))
	w.WriteInt(int(p.Limit))
	w.WriteDuration(p.Duration)
	w.WriteDuration(p.Decay)
	w.WriteDuration(p.Refresh.Active)
	w.WriteDuration(p.Refresh.Queued)
	w.WriteInt(len(p.Properties))
	for key, value := range p.Properties {
		w.WriteString(key)
		w.WriteString(value)
	}

	if err := w.Flush(); err != nil {
		panic(err)
	}

	var h Hash
	hash.Sum(h[:0])
	return h
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syncer

import (
	"sync"
	"time"

	"logger"
	"metadata"
	client "third_party/code.google.com/p/google-api-go-client/drive/v2"
)

const (
	intervalSync   = 30 * time.Second // TODO: should be adaptive
	layoutDateTime = "2013-09-19T14:29:12.570Z"
)

type CachedSyncer struct {
	remoteService *client.Service
	metaService   *metadata.MetaService

	mu sync.RWMutex
}

func NewCachedSyncer(service *client.Service, metaService *metadata.MetaService) *CachedSyncer {
	return &CachedSyncer{
		remoteService: service,
		metaService:   metaService,
	}
}

func (d *CachedSyncer) Start() {
	go func() {
		for {
			d.Sync(false)
			<-time.After(intervalSync)
		}
	}()
}

func (d *CachedSyncer) Sync(isForce bool) (err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	logger.V("Started syncer...")
	err = d.syncInbound(isForce)
	if err != nil {
		logger.V("error during sync", err)
		return
	}
	logger.V("Done syncing...")
	return
}

func (d *CachedSyncer) syncOutbound(rootId string, isRecursive bool, isForce bool) error {
	panic("not implemented")
	return nil
}

func (d *CachedSyncer) syncInbound(isForce bool) (err error) {
	var largestChangeId int64
	largestChangeId, err = d.metaService.GetLargestChangeId()
	if isForce || err != nil {
		largestChangeId = 0
	} else {
		largestChangeId += 1
	}
	isInitialSync := largestChangeId == 0

	// retrieve metadata about root
	var rootFile *client.File
	if rootFile, err = d.remoteService.Files.Get(metadata.IdRootFolder).Do(); err != nil {
		return
	}

	data := buildMetadata(metadata.IdRootFolder, "", rootFile)
	if err = d.metaService.Save("", metadata.IdRootFolder, data, false, false); err != nil {
		return
	}
	pageToken := ""
	for {
		pageToken, err = d.mergeChanges(isInitialSync, rootFile.Id, largestChangeId, pageToken)
		if err != nil || pageToken == "" {
			return
		}
	}
	return
}

func (d *CachedSyncer) mergeChanges(isInitialSync bool, rootId string, startChangeId int64, pageToken string) (nextPageToken string, err error) {
	logger.V("merging changes starting with pageToken:", pageToken, "and startChangeId", startChangeId)

	req := d.remoteService.Changes.List()
	req.IncludeSubscribed(false)
	if pageToken != "" {
		req.PageToken(pageToken)
	} else if startChangeId > 0 { // can't set page token and start change mutually
		req.StartChangeId(startChangeId)
	}
	if isInitialSync {
		req.IncludeDeleted(false)
	}

	var changes *client.ChangeList
	if changes, err = req.Do(); err != nil {
		return
	}

	var largestId int64
	nextPageToken = changes.NextPageToken
	for _, item := range changes.Items {
		if err = d.mergeChange(rootId, item); err != nil {
			return
		}
		largestId = item.Id
	}
	if largestId > 0 {
		// persist largest change id
		d.metaService.SaveLargestChangeId(largestId)
	}
	return
}

func (d *CachedSyncer) mergeChange(rootId string, item *client.Change) (err error) {
	if item.Deleted || item.File.Labels.Trashed {
		// delete
		if d.metaService.Delete(item.FileId); err != nil {
			return
		}
	} else {
		if item.File.DownloadUrl == "" && item.File.MimeType != metadata.MimeTypeFolder {
			return
		}

		fileId := item.FileId
		parentId := ""
		if len(item.File.Parents) > 0 {
			parentId = item.File.Parents[0].Id
		}
		if parentId == rootId {
			parentId = metadata.IdRootFolder
		}
		metadata := buildMetadata(item.FileId, parentId, item.File)
		if err = d.metaService.Save(parentId, fileId, metadata, !metadata.IsFolder(), false); err != nil {
			return
		}
	}
	return
}

func buildMetadata(id string, parentId string, file *client.File) *metadata.CachedDriveFile {
	lastMod, _ := time.Parse(layoutDateTime, file.ModifiedDate)
	return &metadata.CachedDriveFile{
		Id:          id,
		ParentId:    parentId, // ignoring multiple parents
		Name:        file.Title,
		MimeType:    file.MimeType,
		FileSize:    file.FileSize,
		Md5Checksum: file.Md5Checksum,
		LastMod:     lastMod,
	}
}
<commit_msg>Determine if it's initial sync before incrementing largest change id.<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package syncer

import (
	"sync"
	"time"

	"logger"
	"metadata"
	client "third_party/code.google.com/p/google-api-go-client/drive/v2"
)

const (
	intervalSync   = 30 * time.Second // TODO: should be adaptive
	layoutDateTime = "2013-09-19T14:29:12.570Z"
)

type CachedSyncer struct {
	remoteService *client.Service
	metaService   *metadata.MetaService

	mu sync.RWMutex
}

func NewCachedSyncer(service *client.Service, metaService *metadata.MetaService) *CachedSyncer {
	return &CachedSyncer{
		remoteService: service,
		metaService:   metaService,
	}
}

func (d *CachedSyncer) Start() {
	go func() {
		for {
			d.Sync(false)
			<-time.After(intervalSync)
		}
	}()
}

func (d *CachedSyncer) Sync(isForce bool) (err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	logger.V("Started syncer...")
	err = d.syncInbound(isForce)
	if err != nil {
		logger.V("error during sync", err)
		return
	}
	logger.V("Done syncing...")
	return
}

func (d *CachedSyncer) syncOutbound(rootId string, isRecursive bool, isForce bool) error {
	panic("not implemented")
	return nil
}

func (d *CachedSyncer) syncInbound(isForce bool) (err error) {
	var largestChangeId int64
	largestChangeId, err = d.metaService.GetLargestChangeId()
	isInitialSync := largestChangeId == 0
	if isForce || err != nil {
		largestChangeId = 0
	} else {
		largestChangeId += 1
	}

	// retrieve metadata about root
	var rootFile *client.File
	if rootFile, err = d.remoteService.Files.Get(metadata.IdRootFolder).Do(); err != nil {
		return
	}

	data := buildMetadata(metadata.IdRootFolder, "", rootFile)
	if err = d.metaService.Save("", metadata.IdRootFolder, data, false, false); err != nil {
		return
	}
	pageToken := ""
	for {
		pageToken, err = d.mergeChanges(isInitialSync, rootFile.Id, largestChangeId, pageToken)
		if err != nil || pageToken == "" {
			return
		}
	}
	return
}

func (d *CachedSyncer) mergeChanges(isInitialSync bool, rootId string, startChangeId int64, pageToken string) (nextPageToken string, err error) {
	logger.V("merging changes starting with pageToken:", pageToken, "and startChangeId", startChangeId)

	req := d.remoteService.Changes.List()
	req.IncludeSubscribed(false)
	if pageToken != "" {
		req.PageToken(pageToken)
	} else if startChangeId > 0 { // can't set page token and start change mutually
		req.StartChangeId(startChangeId)
	}
	if isInitialSync {
		req.IncludeDeleted(false)
	}

	var changes *client.ChangeList
	if changes, err = req.Do(); err != nil {
		return
	}

	var largestId int64
	nextPageToken = changes.NextPageToken
	for _, item := range changes.Items {
		if err = d.mergeChange(rootId, item); err != nil {
			return
		}
		largestId = item.Id
	}
	if largestId > 0 {
		// persist largest change id
		d.metaService.SaveLargestChangeId(largestId)
	}
	return
}

func (d *CachedSyncer) mergeChange(rootId string, item *client.Change) (err error) {
	if item.Deleted || item.File.Labels.Trashed {
		// delete
		if d.metaService.Delete(item.FileId); err != nil {
			return
		}
	} else {
		if item.File.DownloadUrl == "" && item.File.MimeType != metadata.MimeTypeFolder {
			return
		}

		fileId := item.FileId
		parentId := ""
		if len(item.File.Parents) > 0 {
			parentId = item.File.Parents[0].Id
		}
		if parentId == rootId {
			parentId = metadata.IdRootFolder
		}
		metadata := buildMetadata(item.FileId, parentId, item.File)
		if err = d.metaService.Save(parentId, fileId, metadata, !metadata.IsFolder(), false); err != nil {
			return
		}
	}
	return
}

func buildMetadata(id string, parentId string, file *client.File) *metadata.CachedDriveFile {
	lastMod, _ := time.Parse(layoutDateTime, file.ModifiedDate)
	return &metadata.CachedDriveFile{
		Id:          id,
		ParentId:    parentId, // ignoring multiple parents
		Name:        file.Title,
		MimeType:    file.MimeType,
		FileSize:    file.FileSize,
		Md5Checksum: file.Md5Checksum,
		LastMod:     lastMod,
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/internal/trace"
	"google.golang.org/api/googleapi"
)

var crc32cTable = crc32.MakeTable(crc32.Castagnoli)

// ReaderObjectAttrs are attributes about the object being read. These are populated
// during the New call. This struct only holds a subset of object attributes: to
// get the full set of attributes, use ObjectHandle.Attrs.
//
// Each field is read-only.
type ReaderObjectAttrs struct {
	// Size is the length of the object's content.
	Size int64

	// StartOffset is the byte offset within the object
	// from which reading begins.
	// This value is only non-zero for range requests.
	StartOffset int64

	// ContentType is the MIME type of the object's content.
	ContentType string

	// ContentEncoding is the encoding of the object's content.
	ContentEncoding string

	// CacheControl specifies whether and for how long browser and Internet
	// caches are allowed to cache your objects.
	CacheControl string

	// LastModified is the time that the object was last modified.
	LastModified time.Time

	// Generation is the generation number of the object's content.
	Generation int64

	// Metageneration is the version of the metadata for this object at
	// this generation. This field is used for preconditions and for
	// detecting changes in metadata. A metageneration number is only
	// meaningful in the context of a particular generation of a
	// particular object.
	Metageneration int64
}

// NewReader creates a new Reader to read the contents of the
// object.
// ErrObjectNotExist will be returned if the object is not found.
//
// The caller must call Close on the returned Reader when done reading.
func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) {
	return o.NewRangeReader(ctx, 0, -1)
}

// NewRangeReader reads part of an object, reading at most length bytes
// starting at the given offset. If length is negative, the object is read
// until the end. If offset is negative, the object is read abs(offset) bytes
// from the end, and length must also be negative to indicate all remaining
// bytes will be read.
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (r *Reader, err error) {
	ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader")
	defer func() { trace.EndSpan(ctx, err) }()

	if err := o.validate(); err != nil {
		return nil, err
	}
	if offset < 0 && length >= 0 {
		return nil, fmt.Errorf("storage: invalid offset %d < 0 requires negative length", offset)
	}
	if o.conds != nil {
		if err := o.conds.validate("NewRangeReader"); err != nil {
			return nil, err
		}
	}
	u := &url.URL{
		Scheme: o.c.scheme,
		Host:   o.c.readHost,
		Path:   fmt.Sprintf("/%s/%s", o.bucket, o.object),
	}
	verb := "GET"
	if length == 0 {
		verb = "HEAD"
	}
	req, err := http.NewRequest(verb, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)
	if o.userProject != "" {
		req.Header.Set("X-Goog-User-Project", o.userProject)
	}
	if o.readCompressed {
		req.Header.Set("Accept-Encoding", "gzip")
	}
	if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil {
		return nil, err
	}

	gen := o.gen

	// Define a function that initiates a Read with offset and length, assuming we
	// have already read seen bytes.
	reopen := func(seen int64) (*http.Response, error) {
		start := offset + seen
		if length < 0 && start < 0 {
			req.Header.Set("Range", fmt.Sprintf("bytes=%d", start))
		} else if length < 0 && start > 0 {
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-", start))
		} else if length > 0 {
			// The end character isn't affected by how many bytes we've seen.
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
		}
		// We wait to assign conditions here because the generation number can change in between reopen() runs.
		req.URL.RawQuery = conditionsQuery(gen, o.conds)
		var res *http.Response
		err = runWithRetry(ctx, func() error {
			res, err = o.c.hc.Do(req)
			if err != nil {
				return err
			}
			if res.StatusCode == http.StatusNotFound {
				res.Body.Close()
				return ErrObjectNotExist
			}
			if res.StatusCode < 200 || res.StatusCode > 299 {
				body, _ := ioutil.ReadAll(res.Body)
				res.Body.Close()
				return &googleapi.Error{
					Code:   res.StatusCode,
					Header: res.Header,
					Body:   string(body),
				}
			}
			if start > 0 && length != 0 && res.StatusCode != http.StatusPartialContent {
				res.Body.Close()
				return errors.New("storage: partial request not satisfied")
			}
			// If a generation hasn't been specified, and this is the first response we get, let's record the
			// generation. In future requests we'll use this generation as a precondition to avoid data races.
			if gen < 0 && res.Header.Get("X-Goog-Generation") != "" {
				gen64, err := strconv.ParseInt(res.Header.Get("X-Goog-Generation"), 10, 64)
				if err != nil {
					return err
				}
				gen = gen64
			}
			return nil
		})
		if err != nil {
			return nil, err
		}
		return res, nil
	}

	res, err := reopen(0)
	if err != nil {
		return nil, err
	}
	var (
		size        int64 // total size of object, even if a range was requested.
		checkCRC    bool
		crc         uint32
		startOffset int64 // non-zero if range request.
	)
	if res.StatusCode == http.StatusPartialContent {
		cr := strings.TrimSpace(res.Header.Get("Content-Range"))
		if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") {
			return nil, fmt.Errorf("storage: invalid Content-Range %q", cr)
		}
		size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64)
		if err != nil {
			return nil, fmt.Errorf("storage: invalid Content-Range %q", cr)
		}

		dashIndex := strings.Index(cr, "-")
		if dashIndex >= 0 {
			startOffset, err = strconv.ParseInt(cr[len("bytes="):dashIndex], 10, 64)
			if err != nil {
				return nil, fmt.Errorf("storage: invalid Content-Range %q: %v", cr, err)
			}
		}
	} else {
		size = res.ContentLength
		// Check the CRC iff all of the following hold:
		// - We asked for content (length != 0).
		// - We got all the content (status != PartialContent).
		// - The server sent a CRC header.
		// - The Go http stack did not uncompress the file.
		// - We were not served compressed data that was uncompressed on download.
		// The problem with the last two cases is that the CRC will not match -- GCS
		// computes it on the compressed contents, but we compute it on the
		// uncompressed contents.
		if length != 0 && !res.Uncompressed && !uncompressedByServer(res) {
			crc, checkCRC = parseCRC32c(res)
		}
	}

	remain := res.ContentLength
	body := res.Body
	if length == 0 {
		remain = 0
		body.Close()
		body = emptyBody
	}
	var metaGen int64
	if res.Header.Get("X-Goog-Generation") != "" {
		metaGen, err = strconv.ParseInt(res.Header.Get("X-Goog-Metageneration"), 10, 64)
		if err != nil {
			return nil, err
		}
	}

	var lm time.Time
	if res.Header.Get("Last-Modified") != "" {
		lm, err = http.ParseTime(res.Header.Get("Last-Modified"))
		if err != nil {
			return nil, err
		}
	}

	attrs := ReaderObjectAttrs{
		Size:            size,
		ContentType:     res.Header.Get("Content-Type"),
		ContentEncoding: res.Header.Get("Content-Encoding"),
		CacheControl:    res.Header.Get("Cache-Control"),
		LastModified:    lm,
		StartOffset:     startOffset,
		Generation:      gen,
		Metageneration:  metaGen,
	}
	return &Reader{
		Attrs:    attrs,
		body:     body,
		size:     size,
		remain:   remain,
		wantCRC:  crc,
		checkCRC: checkCRC,
		reopen:   reopen,
	}, nil
}

func uncompressedByServer(res *http.Response) bool {
	// If the data is stored as gzip but is not encoded as gzip, then it
	// was uncompressed by the server.
	return res.Header.Get("X-Goog-Stored-Content-Encoding") == "gzip" &&
		res.Header.Get("Content-Encoding") != "gzip"
}

func parseCRC32c(res *http.Response) (uint32, bool) {
	const prefix = "crc32c="
	for _, spec := range res.Header["X-Goog-Hash"] {
		if strings.HasPrefix(spec, prefix) {
			c, err := decodeUint32(spec[len(prefix):])
			if err == nil {
				return c, true
			}
		}
	}
	return 0, false
}

var emptyBody = ioutil.NopCloser(strings.NewReader(""))

// Reader reads a Cloud Storage object.
// It implements io.Reader.
//
// Typically, a Reader computes the CRC of the downloaded content and compares it to
// the stored CRC, returning an error from Read if there is a mismatch. This integrity check
// is skipped if transcoding occurs. See https://cloud.google.com/storage/docs/transcoding.
type Reader struct {
	Attrs              ReaderObjectAttrs
	body               io.ReadCloser
	seen, remain, size int64
	checkCRC           bool   // should we check the CRC?
	wantCRC            uint32 // the CRC32c value the server sent in the header
	gotCRC             uint32 // running crc
	reopen             func(seen int64) (*http.Response, error)
}

// Close closes the Reader. It must be called when done reading.
func (r *Reader) Close() error {
	return r.body.Close()
}

func (r *Reader) Read(p []byte) (int, error) {
	n, err := r.readWithRetry(p)
	if r.remain != -1 {
		r.remain -= int64(n)
	}
	if r.checkCRC {
		r.gotCRC = crc32.Update(r.gotCRC, crc32cTable, p[:n])
		// Check CRC here. It would be natural to check it in Close, but
		// everybody defers Close on the assumption that it doesn't return
		// anything worth looking at.
		if err == io.EOF {
			if r.gotCRC != r.wantCRC {
				return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d",
					r.gotCRC, r.wantCRC)
			}
		}
	}
	return n, err
}

func (r *Reader) readWithRetry(p []byte) (int, error) {
	n := 0
	for len(p[n:]) > 0 {
		m, err := r.body.Read(p[n:])
		n += m
		r.seen += int64(m)
		if !shouldRetryRead(err) {
			return n, err
		}
		// Read failed, but we will try again. Send a ranged read request that takes
		// into account the number of bytes we've already seen.
		res, err := r.reopen(r.seen)
		if err != nil {
			// reopen already retries
			return n, err
		}
		r.body.Close()
		r.body = res.Body
	}
	return n, nil
}

func shouldRetryRead(err error) bool {
	if err == nil {
		return false
	}
	return strings.HasSuffix(err.Error(), "INTERNAL_ERROR") && strings.Contains(reflect.TypeOf(err).String(), "http2")
}

// Size returns the size of the object in bytes.
// The returned value is always the same and is not affected by
// calls to Read or Close.
//
// Deprecated: use Reader.Attrs.Size.
func (r *Reader) Size() int64 {
	return r.Attrs.Size
}

// Remain returns the number of bytes left to read, or -1 if unknown.
func (r *Reader) Remain() int64 {
	return r.remain
}

// ContentType returns the content type of the object.
//
// Deprecated: use Reader.Attrs.ContentType.
func (r *Reader) ContentType() string {
	return r.Attrs.ContentType
}

// ContentEncoding returns the content encoding of the object.
//
// Deprecated: use Reader.Attrs.ContentEncoding.
func (r *Reader) ContentEncoding() string {
	return r.Attrs.ContentEncoding
}

// CacheControl returns the cache control of the object.
//
// Deprecated: use Reader.Attrs.CacheControl.
func (r *Reader) CacheControl() string {
	return r.Attrs.CacheControl
}

// LastModified returns the value of the Last-Modified header.
//
// Deprecated: use Reader.Attrs.LastModified.
func (r *Reader) LastModified() (time.Time, error) {
	return r.Attrs.LastModified, nil
}
<commit_msg>storage: fix response header check in reader.<commit_after>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"reflect"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/internal/trace"
	"google.golang.org/api/googleapi"
)

var crc32cTable = crc32.MakeTable(crc32.Castagnoli)

// ReaderObjectAttrs are attributes about the object being read. These are populated
// during the New call. This struct only holds a subset of object attributes: to
// get the full set of attributes, use ObjectHandle.Attrs.
//
// Each field is read-only.
type ReaderObjectAttrs struct {
	// Size is the length of the object's content.
	Size int64

	// StartOffset is the byte offset within the object
	// from which reading begins.
	// This value is only non-zero for range requests.
	StartOffset int64

	// ContentType is the MIME type of the object's content.
	ContentType string

	// ContentEncoding is the encoding of the object's content.
	ContentEncoding string

	// CacheControl specifies whether and for how long browser and Internet
	// caches are allowed to cache your objects.
	CacheControl string

	// LastModified is the time that the object was last modified.
	LastModified time.Time

	// Generation is the generation number of the object's content.
	Generation int64

	// Metageneration is the version of the metadata for this object at
	// this generation. This field is used for preconditions and for
	// detecting changes in metadata. A metageneration number is only
	// meaningful in the context of a particular generation of a
	// particular object.
	Metageneration int64
}

// NewReader creates a new Reader to read the contents of the
// object.
// ErrObjectNotExist will be returned if the object is not found.
//
// The caller must call Close on the returned Reader when done reading.
func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) {
	return o.NewRangeReader(ctx, 0, -1)
}

// NewRangeReader reads part of an object, reading at most length bytes
// starting at the given offset. If length is negative, the object is read
// until the end. If offset is negative, the object is read abs(offset) bytes
// from the end, and length must also be negative to indicate all remaining
// bytes will be read.
func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (r *Reader, err error) {
	ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader")
	defer func() { trace.EndSpan(ctx, err) }()

	if err := o.validate(); err != nil {
		return nil, err
	}
	if offset < 0 && length >= 0 {
		return nil, fmt.Errorf("storage: invalid offset %d < 0 requires negative length", offset)
	}
	if o.conds != nil {
		if err := o.conds.validate("NewRangeReader"); err != nil {
			return nil, err
		}
	}
	u := &url.URL{
		Scheme: o.c.scheme,
		Host:   o.c.readHost,
		Path:   fmt.Sprintf("/%s/%s", o.bucket, o.object),
	}
	verb := "GET"
	if length == 0 {
		verb = "HEAD"
	}
	req, err := http.NewRequest(verb, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)
	if o.userProject != "" {
		req.Header.Set("X-Goog-User-Project", o.userProject)
	}
	if o.readCompressed {
		req.Header.Set("Accept-Encoding", "gzip")
	}
	if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil {
		return nil, err
	}

	gen := o.gen

	// Define a function that initiates a Read with offset and length, assuming we
	// have already read seen bytes.
	reopen := func(seen int64) (*http.Response, error) {
		start := offset + seen
		if length < 0 && start < 0 {
			req.Header.Set("Range", fmt.Sprintf("bytes=%d", start))
		} else if length < 0 && start > 0 {
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-", start))
		} else if length > 0 {
			// The end character isn't affected by how many bytes we've seen.
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1))
		}
		// We wait to assign conditions here because the generation number can change in between reopen() runs.
		req.URL.RawQuery = conditionsQuery(gen, o.conds)
		var res *http.Response
		err = runWithRetry(ctx, func() error {
			res, err = o.c.hc.Do(req)
			if err != nil {
				return err
			}
			if res.StatusCode == http.StatusNotFound {
				res.Body.Close()
				return ErrObjectNotExist
			}
			if res.StatusCode < 200 || res.StatusCode > 299 {
				body, _ := ioutil.ReadAll(res.Body)
				res.Body.Close()
				return &googleapi.Error{
					Code:   res.StatusCode,
					Header: res.Header,
					Body:   string(body),
				}
			}
			if start > 0 && length != 0 && res.StatusCode != http.StatusPartialContent {
				res.Body.Close()
				return errors.New("storage: partial request not satisfied")
			}
			// If a generation hasn't been specified, and this is the first response we get, let's record the
			// generation. In future requests we'll use this generation as a precondition to avoid data races.
			if gen < 0 && res.Header.Get("X-Goog-Generation") != "" {
				gen64, err := strconv.ParseInt(res.Header.Get("X-Goog-Generation"), 10, 64)
				if err != nil {
					return err
				}
				gen = gen64
			}
			return nil
		})
		if err != nil {
			return nil, err
		}
		return res, nil
	}

	res, err := reopen(0)
	if err != nil {
		return nil, err
	}
	var (
		size        int64 // total size of object, even if a range was requested.
		checkCRC    bool
		crc         uint32
		startOffset int64 // non-zero if range request.
	)
	if res.StatusCode == http.StatusPartialContent {
		cr := strings.TrimSpace(res.Header.Get("Content-Range"))
		if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") {
			return nil, fmt.Errorf("storage: invalid Content-Range %q", cr)
		}
		size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64)
		if err != nil {
			return nil, fmt.Errorf("storage: invalid Content-Range %q", cr)
		}

		dashIndex := strings.Index(cr, "-")
		if dashIndex >= 0 {
			startOffset, err = strconv.ParseInt(cr[len("bytes="):dashIndex], 10, 64)
			if err != nil {
				return nil, fmt.Errorf("storage: invalid Content-Range %q: %v", cr, err)
			}
		}
	} else {
		size = res.ContentLength
		// Check the CRC iff all of the following hold:
		// - We asked for content (length != 0).
		// - We got all the content (status != PartialContent).
		// - The server sent a CRC header.
		// - The Go http stack did not uncompress the file.
		// - We were not served compressed data that was uncompressed on download.
		// The problem with the last two cases is that the CRC will not match -- GCS
		// computes it on the compressed contents, but we compute it on the
		// uncompressed contents.
		if length != 0 && !res.Uncompressed && !uncompressedByServer(res) {
			crc, checkCRC = parseCRC32c(res)
		}
	}

	remain := res.ContentLength
	body := res.Body
	if length == 0 {
		remain = 0
		body.Close()
		body = emptyBody
	}
	var metaGen int64
	if res.Header.Get("X-Goog-Metageneration") != "" {
		metaGen, err = strconv.ParseInt(res.Header.Get("X-Goog-Metageneration"), 10, 64)
		if err != nil {
			return nil, err
		}
	}

	var lm time.Time
	if res.Header.Get("Last-Modified") != "" {
		lm, err = http.ParseTime(res.Header.Get("Last-Modified"))
		if err != nil {
			return nil, err
		}
	}

	attrs := ReaderObjectAttrs{
		Size:            size,
		ContentType:     res.Header.Get("Content-Type"),
		ContentEncoding: res.Header.Get("Content-Encoding"),
		CacheControl:    res.Header.Get("Cache-Control"),
		LastModified:    lm,
		StartOffset:     startOffset,
		Generation:      gen,
		Metageneration:  metaGen,
	}
	return &Reader{
		Attrs:    attrs,
		body:     body,
		size:     size,
		remain:   remain,
		wantCRC:  crc,
		checkCRC: checkCRC,
		reopen:   reopen,
	}, nil
}

func uncompressedByServer(res *http.Response) bool {
	// If the data is stored as gzip but is not encoded as gzip, then it
	// was uncompressed by the server.
	return res.Header.Get("X-Goog-Stored-Content-Encoding") == "gzip" &&
		res.Header.Get("Content-Encoding") != "gzip"
}

func parseCRC32c(res *http.Response) (uint32, bool) {
	const prefix = "crc32c="
	for _, spec := range res.Header["X-Goog-Hash"] {
		if strings.HasPrefix(spec, prefix) {
			c, err := decodeUint32(spec[len(prefix):])
			if err == nil {
				return c, true
			}
		}
	}
	return 0, false
}

var emptyBody = ioutil.NopCloser(strings.NewReader(""))

// Reader reads a Cloud Storage object.
// It implements io.Reader.
//
// Typically, a Reader computes the CRC of the downloaded content and compares it to
// the stored CRC, returning an error from Read if there is a mismatch. This integrity check
// is skipped if transcoding occurs. See https://cloud.google.com/storage/docs/transcoding.
type Reader struct {
	Attrs              ReaderObjectAttrs
	body               io.ReadCloser
	seen, remain, size int64
	checkCRC           bool   // should we check the CRC?
	wantCRC            uint32 // the CRC32c value the server sent in the header
	gotCRC             uint32 // running crc
	reopen             func(seen int64) (*http.Response, error)
}

// Close closes the Reader. It must be called when done reading.
func (r *Reader) Close() error {
	return r.body.Close()
}

func (r *Reader) Read(p []byte) (int, error) {
	n, err := r.readWithRetry(p)
	if r.remain != -1 {
		r.remain -= int64(n)
	}
	if r.checkCRC {
		r.gotCRC = crc32.Update(r.gotCRC, crc32cTable, p[:n])
		// Check CRC here. It would be natural to check it in Close, but
		// everybody defers Close on the assumption that it doesn't return
		// anything worth looking at.
		if err == io.EOF {
			if r.gotCRC != r.wantCRC {
				return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d",
					r.gotCRC, r.wantCRC)
			}
		}
	}
	return n, err
}

func (r *Reader) readWithRetry(p []byte) (int, error) {
	n := 0
	for len(p[n:]) > 0 {
		m, err := r.body.Read(p[n:])
		n += m
		r.seen += int64(m)
		if !shouldRetryRead(err) {
			return n, err
		}
		// Read failed, but we will try again. Send a ranged read request that takes
		// into account the number of bytes we've already seen.
		res, err := r.reopen(r.seen)
		if err != nil {
			// reopen already retries
			return n, err
		}
		r.body.Close()
		r.body = res.Body
	}
	return n, nil
}

func shouldRetryRead(err error) bool {
	if err == nil {
		return false
	}
	return strings.HasSuffix(err.Error(), "INTERNAL_ERROR") && strings.Contains(reflect.TypeOf(err).String(), "http2")
}

// Size returns the size of the object in bytes.
// The returned value is always the same and is not affected by
// calls to Read or Close.
//
// Deprecated: use Reader.Attrs.Size.
func (r *Reader) Size() int64 {
	return r.Attrs.Size
}

// Remain returns the number of bytes left to read, or -1 if unknown.
func (r *Reader) Remain() int64 {
	return r.remain
}

// ContentType returns the content type of the object.
//
// Deprecated: use Reader.Attrs.ContentType.
func (r *Reader) ContentType() string {
	return r.Attrs.ContentType
}

// ContentEncoding returns the content encoding of the object.
//
// Deprecated: use Reader.Attrs.ContentEncoding.
func (r *Reader) ContentEncoding() string {
	return r.Attrs.ContentEncoding
}

// CacheControl returns the cache control of the object.
//
// Deprecated: use Reader.Attrs.CacheControl.
func (r *Reader) CacheControl() string {
	return r.Attrs.CacheControl
}

// LastModified returns the value of the Last-Modified header.
//
// Deprecated: use Reader.Attrs.LastModified.
func (r *Reader) LastModified() (time.Time, error) {
	return r.Attrs.LastModified, nil
}
<|endoftext|>