<commit_before>/*
Copyright 2018 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"k8s.io/klog"

	"github.com/ceph/ceph-csi/pkg/util"
	"github.com/container-storage-interface/spec/lib/go/csi"
	"k8s.io/kubernetes/pkg/util/mount"
)

type volumeID string

func execCommand(program string, args ...string) (stdout, stderr []byte, err error) {
	var (
		cmd           = exec.Command(program, args...) // nolint: gosec
		sanitizedArgs = util.StripSecretInArgs(args)
		stdoutBuf     bytes.Buffer
		stderrBuf     bytes.Buffer
	)

	cmd.Stdout = &stdoutBuf
	cmd.Stderr = &stderrBuf

	klog.V(4).Infof("cephfs: EXEC %s %s", program, sanitizedArgs)

	if err := cmd.Run(); err != nil {
		return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s",
			cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes())
	}

	return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
}

func execCommandErr(program string, args ...string) error {
	_, _, err := execCommand(program, args...)
	return err
}

//nolint: unparam
func execCommandJSON(v interface{}, program string, args ...string) error {
	stdout, _, err := execCommand(program, args...)
	if err != nil {
		return err
	}

	if err = json.Unmarshal(stdout, v); err != nil {
		return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err)
	}

	return nil
}

// Used in isMountPoint()
var dummyMount = mount.New("")

func isMountPoint(p string) (bool, error) {
	notMnt, err := dummyMount.IsLikelyNotMountPoint(p)
	if err != nil {
		return false, status.Error(codes.Internal, err.Error())
	}

	return !notMnt, nil
}

func pathExists(p string) bool {
	_, err := os.Stat(p)
	return err == nil
}

// Controller service request validation
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid CreateVolumeRequest: %v", err)
	}

	if req.GetName() == "" {
		return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
	}

	reqCaps := req.GetVolumeCapabilities()
	if reqCaps == nil {
		return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
	}

	for _, cap := range reqCaps {
		if cap.GetBlock() != nil {
			return status.Error(codes.Unimplemented, "block volume not supported")
		}
	}

	return nil
}

func (cs *ControllerServer) validateDeleteVolumeRequest() error {
	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid DeleteVolumeRequest: %v", err)
	}

	return nil
}

// Node service request validation
func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
	if req.GetVolumeCapability() == nil {
		return errors.New("volume capability missing in request")
	}

	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetStagingTargetPath() == "" {
		return errors.New("staging target path missing in request")
	}

	if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 {
		return errors.New("stage secrets cannot be nil or empty")
	}

	return nil
}

func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetStagingTargetPath() == "" {
		return errors.New("staging target path missing in request")
	}

	return nil
}

func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
	if req.GetVolumeCapability() == nil {
		return errors.New("volume capability missing in request")
	}

	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetTargetPath() == "" {
		return errors.New("target path missing in request")
	}

	if req.GetStagingTargetPath() == "" {
		return errors.New("staging target path missing in request")
	}

	return nil
}

func validateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error {
	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetTargetPath() == "" {
		return errors.New("target path missing in request")
	}

	return nil
}
<commit_msg>Add nil check for process<commit_after>/*
Copyright 2018 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/exec"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"k8s.io/klog"

	"github.com/ceph/ceph-csi/pkg/util"
	"github.com/container-storage-interface/spec/lib/go/csi"
	"k8s.io/kubernetes/pkg/util/mount"
)

type volumeID string

func execCommand(program string, args ...string) (stdout, stderr []byte, err error) {
	var (
		cmd           = exec.Command(program, args...) // nolint: gosec
		sanitizedArgs = util.StripSecretInArgs(args)
		stdoutBuf     bytes.Buffer
		stderrBuf     bytes.Buffer
	)

	cmd.Stdout = &stdoutBuf
	cmd.Stderr = &stderrBuf

	klog.V(4).Infof("cephfs: EXEC %s %s", program, sanitizedArgs)

	if err := cmd.Run(); err != nil {
		if cmd.Process == nil {
			return nil, nil, fmt.Errorf("cannot get process pid while running %s %v: %v: %s",
				program, sanitizedArgs, err, stderrBuf.Bytes())
		}
		return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s",
			cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes())
	}

	return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
}

func execCommandErr(program string, args ...string) error {
	_, _, err := execCommand(program, args...)
	return err
}

//nolint: unparam
func execCommandJSON(v interface{}, program string, args ...string) error {
	stdout, _, err := execCommand(program, args...)
	if err != nil {
		return err
	}

	if err = json.Unmarshal(stdout, v); err != nil {
		return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err)
	}

	return nil
}

// Used in isMountPoint()
var dummyMount = mount.New("")

func isMountPoint(p string) (bool, error) {
	notMnt, err := dummyMount.IsLikelyNotMountPoint(p)
	if err != nil {
		return false, status.Error(codes.Internal, err.Error())
	}

	return !notMnt, nil
}

func pathExists(p string) bool {
	_, err := os.Stat(p)
	return err == nil
}

// Controller service request validation
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid CreateVolumeRequest: %v", err)
	}

	if req.GetName() == "" {
		return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
	}

	reqCaps := req.GetVolumeCapabilities()
	if reqCaps == nil {
		return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
	}

	for _, cap := range reqCaps {
		if cap.GetBlock() != nil {
			return status.Error(codes.Unimplemented, "block volume not supported")
		}
	}

	return nil
}

func (cs *ControllerServer) validateDeleteVolumeRequest() error {
	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid DeleteVolumeRequest: %v", err)
	}

	return nil
}

// Node service request validation
func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
	if req.GetVolumeCapability() == nil {
		return errors.New("volume capability missing in request")
	}

	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetStagingTargetPath() == "" {
		return errors.New("staging target path missing in request")
	}

	if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 {
		return errors.New("stage secrets cannot be nil or empty")
	}

	return nil
}

func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetStagingTargetPath() == "" {
		return errors.New("staging target path missing in request")
	}

	return nil
}

func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
	if req.GetVolumeCapability() == nil {
		return errors.New("volume capability missing in request")
	}

	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetTargetPath() == "" {
		return errors.New("target path missing in request")
	}

	if req.GetStagingTargetPath() == "" {
		return errors.New("staging target path missing in request")
	}

	return nil
}

func validateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error {
	if req.GetVolumeId() == "" {
		return errors.New("volume ID missing in request")
	}

	if req.GetTargetPath() == "" {
		return errors.New("target path missing in request")
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubectl

import (
	"fmt"
	"strconv"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
)

type BasicReplicationController struct{}

func (BasicReplicationController) ParamNames() []GeneratorParam {
	return []GeneratorParam{
		{"labels", false},
		{"name", true},
		{"replicas", true},
		{"image", true},
		{"port", false},
		{"hostport", false},
	}
}

func (BasicReplicationController) Generate(params map[string]string) (runtime.Object, error) {
	// TODO: extract this flag to a central location.
	labelString, found := params["labels"]
	var labels map[string]string
	var err error
	if found && len(labelString) > 0 {
		labels, err = ParseLabels(labelString)
		if err != nil {
			return nil, err
		}
	} else {
		labels = map[string]string{
			"run": params["name"],
		}
	}
	count, err := strconv.Atoi(params["replicas"])
	if err != nil {
		return nil, err
	}
	controller := api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Name:   params["name"],
			Labels: labels,
		},
		Spec: api.ReplicationControllerSpec{
			Replicas: count,
			Selector: labels,
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: labels,
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  params["name"],
							Image: params["image"],
						},
					},
				},
			},
		},
	}

	port := -1
	hostPort := -1
	if len(params["port"]) > 0 {
		port, err = strconv.Atoi(params["port"])
		if err != nil {
			return nil, err
		}
	}

	if len(params["hostport"]) > 0 {
		hostPort, err = strconv.Atoi(params["hostport"])
		if err != nil {
			return nil, err
		}
		if hostPort > 0 && port < 0 {
			return nil, fmt.Errorf("--hostport requires --port to be specified")
		}
	}

	// Don't include the port if it was not specified.
	if port > 0 {
		controller.Spec.Template.Spec.Containers[0].Ports = []api.ContainerPort{
			{
				ContainerPort: port,
			},
		}
		if hostPort > 0 {
			controller.Spec.Template.Spec.Containers[0].Ports[0].HostPort = hostPort
		}
	}
	return &controller, nil
}
<commit_msg>Name is a required parameter for kubectl run, default-name is not<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubectl

import (
	"fmt"
	"strconv"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
)

type BasicReplicationController struct{}

func (BasicReplicationController) ParamNames() []GeneratorParam {
	return []GeneratorParam{
		{"labels", false},
		{"default-name", false},
		{"name", true},
		{"replicas", true},
		{"image", true},
		{"port", false},
		{"hostport", false},
	}
}

func (BasicReplicationController) Generate(params map[string]string) (runtime.Object, error) {
	// TODO: extract this flag to a central location.
	labelString, found := params["labels"]
	var labels map[string]string
	var err error
	if found && len(labelString) > 0 {
		labels, err = ParseLabels(labelString)
		if err != nil {
			return nil, err
		}
	} else {
		labels = map[string]string{
			"run": params["name"],
		}
	}
	count, err := strconv.Atoi(params["replicas"])
	if err != nil {
		return nil, err
	}
	controller := api.ReplicationController{
		ObjectMeta: api.ObjectMeta{
			Name:   params["name"],
			Labels: labels,
		},
		Spec: api.ReplicationControllerSpec{
			Replicas: count,
			Selector: labels,
			Template: &api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Labels: labels,
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:  params["name"],
							Image: params["image"],
						},
					},
				},
			},
		},
	}

	port := -1
	hostPort := -1
	if len(params["port"]) > 0 {
		port, err = strconv.Atoi(params["port"])
		if err != nil {
			return nil, err
		}
	}

	if len(params["hostport"]) > 0 {
		hostPort, err = strconv.Atoi(params["hostport"])
		if err != nil {
			return nil, err
		}
		if hostPort > 0 && port < 0 {
			return nil, fmt.Errorf("--hostport requires --port to be specified")
		}
	}

	// Don't include the port if it was not specified.
	if port > 0 {
		controller.Spec.Template.Spec.Containers[0].Ports = []api.ContainerPort{
			{
				ContainerPort: port,
			},
		}
		if hostPort > 0 {
			controller.Spec.Template.Spec.Containers[0].Ports[0].HostPort = hostPort
		}
	}
	return &controller, nil
}
<|endoftext|>


<commit_before>package queue

import (
	"runtime"
	"sync/atomic"
	"time"
	"unsafe"
)

type Options struct {
	// Size of the queue, must be a power-of-two number (2,4,8,16,32..)
	Size int
	// Optionally pre-allocate slots Val's in the ring; this is useful if you want
	// to use buffer without allocating heap memory on the hot path - allocate records for yourself
	// once, and then populate those records with appropriate values
	Allocate func() interface{}
}

// A specialized alternative to Go Channels. Lets you send values from one
// Go Routine to another in a safe way, along with many other cool use cases.
type Queue interface {
	// For publishers - get the next free slot to write data to
	NextFree() (*Slot, error)
	// For publishers - publish a slot after filling it with data
	Publish(slot *Slot) error

	// Receive up to queue length items in bulk, blocking if there are no
	// items available
	Drain(handler func(*Slot)) error
}

type Slot struct {
	s   int64
	ptr unsafe.Pointer
}

func (s *Slot) Get() interface{} {
	return *(*interface{})(atomic.LoadPointer(&s.ptr))
}

func (s *Slot) Set(v interface{}) {
	atomic.StorePointer(&s.ptr, unsafe.Pointer(&v))
}

// How to approach the queue being empty
type WaitStrategy interface {
	// Called when producers stick new stuff in the queue. A wait strategy can use this to
	// snooze go routines in WaitFor, and wake them up when this gets called.
	SignalAllWhenBlocking()
	// Called when the queue is empty. The implementation should wait for sequence to be <= dependentSequence,
	// or until it gets tired of waiting according to whatever criteria.
	// When this returns, it should return the current value of dependentSequence - which is allowed to be < sequence.
	WaitFor(sequence int64, dependentSequence *sequence) int64
}

// Default wait strategy - spinlock for 100 cycles, then fall back to letting the go scheduler
// schedule other goroutines a hundred times, and if we're still not done waiting it will start sleeping in
// nanosecond intervals.
type SleepWaitStrategy struct {
}

func (w *SleepWaitStrategy) SignalAllWhenBlocking() {

}
func (w *SleepWaitStrategy) WaitFor(sequence int64, dependentSequence *sequence) int64 {
	var availableSequence int64
	counter := 200

	//fmt.Printf("WaitFor(%d >= %d)\n", dependentSequence.value, sequence)
	for availableSequence = dependentSequence.value; availableSequence < sequence; availableSequence = dependentSequence.value {
		if counter > 100 {
			counter--
		} else if counter > 0 {
			counter--
			runtime.Gosched()
		} else {
			time.Sleep(time.Nanosecond)
		}
	}
	return availableSequence
}

// A thin wrapper around a int64, giving some convenience methods for ordered writes and CAS
//
// On Overflow: If the queue is processing 10 million messages per second,
// it will transfer 13140000000000 messages a year. int64 fits 9223372036854775807 before overflow,
// meaning at 10M/s, the queue can run for ~100K years without overflowing. Hence, the code does
// knowingly not account for this value wrapping around. Off-the-cuff, throughput may be able
// to reach the low billions before hitting actual physical limits (something something speed of light,
// something something nano metres), but even then the queue can run for thousands of years before wrapping.
type sequence struct {
	_lpad [56]byte
	value int64
	_rpad [56]byte
}

func (s *sequence) get() int64 {
	return atomic.LoadInt64(&s.value)
}

func (s *sequence) set(v int64) {
	atomic.StoreInt64(&s.value, v)
}
func (s *sequence) compareAndSet(old, new int64) bool {
	return atomic.CompareAndSwapInt64(&s.value, old, new)
}
func (s *sequence) add(delta int64) {
	atomic.AddInt64(&s.value, delta)
}

// The sequencer is the center piece of these queues - it is based entirely on the brilliant work at LMAX.
// Each queue has one sequencer, and it controls the entry of new items into the queue.
//
// The core abstraction is an infinite sequence of numbers, with various parts of the queue tracking
// which part of the number sequence they've reached. The sequencer sits in front of them all, controlling the
// entry into "new territory". Once the sequencer increments further into the number sequence, other pointers
// in the queue may increment up to the point the sequencer is at.
//
// The sequencer does this, while maintaining a key invariant: The infinite sequence of numbers is mapped onto
// a circular buffer, which is distinctly not infinite. Hence, the sequencer keeps track of all the secondary
// sequence pointers, and ensures the delta from the lowest pointer to the sequencer pointer never is greater
// than the size of the buffer.
//
// At the end of the day, the sequencer is really just:
//
//   func next(n) {
//       if cursor + n < min(otherPointersInTheQueue) {
//           cursor += n
//       }
//       return cursor
//   }
//
// The neat thing is that it does the above using some really clever techniques that make the sequencer
// safe for concurrent use, and extremely low overhead to boot.
type sequencer struct {
	bufferSize   int64
	waitStrategy WaitStrategy

	// This is the golden cursor - it points to the highest number the sequencer (and hence any other pointer)
	// has reached in our supposedly infinite sequence of numbers
	cursor *sequence

	// This is the other pointer (soon to be multiple, but one for now) in the queue - the sequencer makes sure
	// it doesn't get further ahead of this than bufferSize, otherwise we'd wrap around the buffer and overwrite
	// slots before they had been processed.
	gatingSequence      *sequence
	gatingSequenceCache *sequence

	// Tracks published slots. This could be implemented with a simple counter;
	// however, when we have multiple producers, they would need to block and wait on one
	// another to mark their items published (since we publish in the sequence order)
	// This data structure, instead, has a slot for each item in the ring, each slot gets
	// the highest "lap number" published for that slot. The wrapPoint code in next() ensures
	// we don't overrun.
	//
	// This means that if we have a slow publisher, other publishers can mark their items
	// available ahead of time by writing their sequences lap number into the appropriate slot,
	// meaning they don't have to wait for the slow publisher to publish.
	availableBuffer []int32

	indexMask  int64
	indexShift uint
}

// Get control of n items, returning the end item sequence
func (s *sequencer) next(n int64) int64 {
	for {
		current := s.cursor.get()
		next := current + n

		wrapPoint := next - s.bufferSize
		cachedGatingSequence := s.gatingSequenceCache.get()

		if wrapPoint > cachedGatingSequence || cachedGatingSequence > current {
			gatingSequence := min(s.gatingSequence.value, current)
			if wrapPoint > gatingSequence {
				s.waitStrategy.SignalAllWhenBlocking()
				time.Sleep(time.Nanosecond)
				continue
			}
			s.gatingSequenceCache.set(gatingSequence)
		} else if s.cursor.compareAndSet(current, next) {
			return next
		}
	}
}

func (s *sequencer) publish(lo, hi int64) {
	for l := lo; l <= hi; l++ {
		s.setAvailable(l)
	}
	s.waitStrategy.SignalAllWhenBlocking()
}

func (s *sequencer) setAvailable(sequence int64) {
	s.setAvailableBufferValue(s.calculateIndex(sequence), s.calculateAvailabilityFlag(sequence))
}

// Try and wait for the given sequence to be available. How long this will wait depends on
// the wait strategy used - in any case, the actual sequence reached is returned and may be less
// than the requested sequence.
func (s *sequencer) waitFor(sequence int64) int64 {
	published := s.waitStrategy.WaitFor(sequence, s.cursor)

	if published < sequence {
		return published
	}

	high := s.getHighestPublishedSequence(sequence, published)
	return high
}

func (s *sequencer) getHighestPublishedSequence(lowerBound, availableSequence int64) int64 {
	for sequence := lowerBound; sequence <= availableSequence; sequence++ {
		if !s.isAvailable(sequence) {
			return sequence - 1
		}
	}
	return availableSequence
}

func (s *sequencer) isAvailable(sequence int64) bool {
	return atomic.LoadInt32(&s.availableBuffer[s.calculateIndex(sequence)]) == s.calculateAvailabilityFlag(sequence)
}

// The availability "flag" is a "lap counter", sequence / ring size
func (s *sequencer) calculateAvailabilityFlag(sequence int64) int32 {
	return int32(sequence >> s.indexShift)
}

func (s *sequencer) calculateIndex(sequence int64) int {
	return int(sequence & s.indexMask)
}

func (s *sequencer) setAvailableBufferValue(index int, flag int32) {
	atomic.StoreInt32(&s.availableBuffer[index], flag)
}

func newSequencer(bufferSize int, ws WaitStrategy, initial int64, gatingSequence *sequence) *sequencer {
	s := &sequencer{
		bufferSize:   int64(bufferSize),
		waitStrategy: ws,
		cursor: &sequence{
			value: initial,
		},
		gatingSequence:      gatingSequence,
		gatingSequenceCache: &sequence{value: -1},
		availableBuffer:     make([]int32, bufferSize),
		indexMask:           int64(bufferSize - 1),
		indexShift:          log2(bufferSize),
	}

	for i := bufferSize - 1; i != 0; i-- {
		s.setAvailableBufferValue(i, -1)
	}
	s.setAvailableBufferValue(0, -1)

	return s
}

func isPowerOfTwo(x int) bool {
	return (x != 0) && (x&(x-1)) == 0
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

func log2(i int) uint {
	r := uint(0)
	for i >>= 1; i != 0; i >>= 1 {
		r++
	}
	return r
}
<commit_msg>Add a load barrier before reading the gating sequence.<commit_after>package queue

import (
	"runtime"
	"sync/atomic"
	"time"
	"unsafe"
)

type Options struct {
	// Size of the queue, must be a power-of-two number (2,4,8,16,32..)
	Size int
	// Optionally pre-allocate slots Val's in the ring; this is useful if you want
	// to use buffer without allocating heap memory on the hot path - allocate records for yourself
	// once, and then populate those records with appropriate values
	Allocate func() interface{}
}

// A specialized alternative to Go Channels. Lets you send values from one
// Go Routine to another in a safe way, along with many other cool use cases.
type Queue interface {
	// For publishers - get the next free slot to write data to
	NextFree() (*Slot, error)
	// For publishers - publish a slot after filling it with data
	Publish(slot *Slot) error

	// Receive up to queue length items in bulk, blocking if there are no
	// items available
	Drain(handler func(*Slot)) error
}

type Slot struct {
	s   int64
	ptr unsafe.Pointer
}

func (s *Slot) Get() interface{} {
	return *(*interface{})(atomic.LoadPointer(&s.ptr))
}

func (s *Slot) Set(v interface{}) {
	atomic.StorePointer(&s.ptr, unsafe.Pointer(&v))
}

// How to approach the queue being empty
type WaitStrategy interface {
	// Called when producers stick new stuff in the queue. A wait strategy can use this to
	// snooze go routines in WaitFor, and wake them up when this gets called.
	SignalAllWhenBlocking()
	// Called when the queue is empty. The implementation should wait for sequence to be <= dependentSequence,
	// or until it gets tired of waiting according to whatever criteria.
	// When this returns, it should return the current value of dependentSequence - which is allowed to be < sequence.
	WaitFor(sequence int64, dependentSequence *sequence) int64
}

// Default wait strategy - spinlock for 100 cycles, then fall back to letting the go scheduler
// schedule other goroutines a hundred times, and if we're still not done waiting it will start sleeping in
// nanosecond intervals.
type SleepWaitStrategy struct {
}

func (w *SleepWaitStrategy) SignalAllWhenBlocking() {

}
func (w *SleepWaitStrategy) WaitFor(sequence int64, dependentSequence *sequence) int64 {
	var availableSequence int64
	counter := 200

	//fmt.Printf("WaitFor(%d >= %d)\n", dependentSequence.value, sequence)
	for availableSequence = dependentSequence.value; availableSequence < sequence; availableSequence = dependentSequence.value {
		if counter > 100 {
			counter--
		} else if counter > 0 {
			counter--
			runtime.Gosched()
		} else {
			time.Sleep(time.Nanosecond)
		}
	}
	return availableSequence
}

// A thin wrapper around a int64, giving some convenience methods for ordered writes and CAS
//
// On Overflow: If the queue is processing 10 million messages per second,
// it will transfer 13140000000000 messages a year. int64 fits 9223372036854775807 before overflow,
// meaning at 10M/s, the queue can run for ~100K years without overflowing. Hence, the code does
// knowingly not account for this value wrapping around. Off-the-cuff, throughput may be able
// to reach the low billions before hitting actual physical limits (something something speed of light,
// something something nano metres), but even then the queue can run for thousands of years before wrapping.
type sequence struct {
	_lpad [56]byte
	value int64
	_rpad [56]byte
}

func (s *sequence) get() int64 {
	return atomic.LoadInt64(&s.value)
}

func (s *sequence) set(v int64) {
	atomic.StoreInt64(&s.value, v)
}
func (s *sequence) compareAndSet(old, new int64) bool {
	return atomic.CompareAndSwapInt64(&s.value, old, new)
}
func (s *sequence) add(delta int64) {
	atomic.AddInt64(&s.value, delta)
}

// The sequencer is the center piece of these queues - it is based entirely on the brilliant work at LMAX.
// Each queue has one sequencer, and it controls the entry of new items into the queue.
//
// The core abstraction is an infinite sequence of numbers, with various parts of the queue tracking
// which part of the number sequence they've reached. The sequencer sits in front of them all, controlling the
// entry into "new territory". Once the sequencer increments further into the number sequence, other pointers
// in the queue may increment up to the point the sequencer is at.
//
// The sequencer does this, while maintaining a key invariant: The infinite sequence of numbers is mapped onto
// a circular buffer, which is distinctly not infinite. Hence, the sequencer keeps track of all the secondary
// sequence pointers, and ensures the delta from the lowest pointer to the sequencer pointer never is greater
// than the size of the buffer.
//
// At the end of the day, the sequencer is really just:
//
//   func next(n) {
//       if cursor + n < min(otherPointersInTheQueue) {
//           cursor += n
//       }
//       return cursor
//   }
//
// The neat thing is that it does the above using some really clever techniques that make the sequencer
// safe for concurrent use, and extremely low overhead to boot.
type sequencer struct {
	bufferSize   int64
	waitStrategy WaitStrategy

	// This is the golden cursor - it points to the highest number the sequencer (and hence any other pointer)
	// has reached in our supposedly infinite sequence of numbers
	cursor *sequence

	// This is the other pointer (soon to be multiple, but one for now) in the queue - the sequencer makes sure
	// it doesn't get further ahead of this than bufferSize, otherwise we'd wrap around the buffer and overwrite
	// slots before they had been processed.
	gatingSequence      *sequence
	gatingSequenceCache *sequence

	// Tracks published slots. This could be implemented with a simple counter;
	// however, when we have multiple producers, they would need to block and wait on one
	// another to mark their items published (since we publish in the sequence order)
	// This data structure, instead, has a slot for each item in the ring, each slot gets
	// the highest "lap number" published for that slot. The wrapPoint code in next() ensures
	// we don't overrun.
	//
	// This means that if we have a slow publisher, other publishers can mark their items
	// available ahead of time by writing their sequences lap number into the appropriate slot,
	// meaning they don't have to wait for the slow publisher to publish.
	availableBuffer []int32

	indexMask  int64
	indexShift uint
}

// Get control of n items, returning the end item sequence
func (s *sequencer) next(n int64) int64 {
	for {
		current := s.cursor.get()
		next := current + n

		wrapPoint := next - s.bufferSize
		cachedGatingSequence := s.gatingSequenceCache.get()

		if wrapPoint > cachedGatingSequence || cachedGatingSequence > current {
			gatingSequence := min(atomic.LoadInt64(&s.gatingSequence.value), current)
			if wrapPoint > gatingSequence {
				s.waitStrategy.SignalAllWhenBlocking()
				time.Sleep(time.Nanosecond)
				continue
			}
			s.gatingSequenceCache.set(gatingSequence)
		} else if s.cursor.compareAndSet(current, next) {
			return next
		}
	}
}

func (s *sequencer) publish(lo, hi int64) {
	for l := lo; l <= hi; l++ {
		s.setAvailable(l)
	}
	s.waitStrategy.SignalAllWhenBlocking()
}

func (s *sequencer) setAvailable(sequence int64) {
	s.setAvailableBufferValue(s.calculateIndex(sequence), s.calculateAvailabilityFlag(sequence))
}

// Try and wait for the given sequence to be available. How long this will wait depends on
// the wait strategy used - in any case, the actual sequence reached is returned and may be less
// than the requested sequence.
func (s *sequencer) waitFor(sequence int64) int64 {
	published := s.waitStrategy.WaitFor(sequence, s.cursor)

	if published < sequence {
		return published
	}

	high := s.getHighestPublishedSequence(sequence, published)
	return high
}

func (s *sequencer) getHighestPublishedSequence(lowerBound, availableSequence int64) int64 {
	for sequence := lowerBound; sequence <= availableSequence; sequence++ {
		if !s.isAvailable(sequence) {
			return sequence - 1
		}
	}
	return availableSequence
}

func (s *sequencer) isAvailable(sequence int64) bool {
	return atomic.LoadInt32(&s.availableBuffer[s.calculateIndex(sequence)]) == s.calculateAvailabilityFlag(sequence)
}

// The availability "flag" is a "lap counter", sequence / ring size
func (s *sequencer) calculateAvailabilityFlag(sequence int64) int32 {
	return int32(sequence >> s.indexShift)
}

func (s *sequencer) calculateIndex(sequence int64) int {
	return int(sequence & s.indexMask)
}

func (s *sequencer) setAvailableBufferValue(index int, flag int32) {
	atomic.StoreInt32(&s.availableBuffer[index], flag)
}

func newSequencer(bufferSize int, ws WaitStrategy, initial int64, gatingSequence *sequence) *sequencer {
	s := &sequencer{
		bufferSize:   int64(bufferSize),
		waitStrategy: ws,
		cursor: &sequence{
			value: initial,
		},
		gatingSequence:      gatingSequence,
		gatingSequenceCache: &sequence{value: -1},
		availableBuffer:     make([]int32, bufferSize),
		indexMask:           int64(bufferSize - 1),
		indexShift:          log2(bufferSize),
	}

	for i := bufferSize - 1; i != 0; i-- {
		s.setAvailableBufferValue(i, -1)
	}
	s.setAvailableBufferValue(0, -1)

	return s
}

func isPowerOfTwo(x int) bool {
	return (x != 0) && (x&(x-1)) == 0
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

func log2(i int) uint {
	r := uint(0)
	for i >>= 1; i != 0; i >>= 1 {
		r++
	}
	return r
}
<|endoftext|>


<commit_before>package gc // import "a4.io/blobstash/pkg/stash/gc"

import (
	"context"
	"fmt"

	"github.com/vmihailenco/msgpack"
	"github.com/yuin/gopher-lua"

	"a4.io/blobsfile"
	"a4.io/blobstash/pkg/apps/luautil"
	"a4.io/blobstash/pkg/blob"
	bsLua "a4.io/blobstash/pkg/blobstore/lua"
	"a4.io/blobstash/pkg/extra"
	"a4.io/blobstash/pkg/filetree/filetreeutil/node"
	"a4.io/blobstash/pkg/hub"
	kvsLua "a4.io/blobstash/pkg/kvstore/lua"
	"a4.io/blobstash/pkg/luascripts"
	"a4.io/blobstash/pkg/stash"
	"a4.io/blobstash/pkg/stash/store"
)

func GC(ctx context.Context, h *hub.Hub, s *stash.Stash, dc store.DataContext, script string, existingRefs map[string]struct{}) (int, uint64, error) {

	// TODO(tsileo): take a logger
	refs := map[string]struct{}{}
	orderedRefs := []string{}

	L := lua.NewState()
	var skipped int

	// premark(<blob hash>) notify the GC that this blob is already in the root blobstore explicitely (to speedup huge GC)
	premark := func(L *lua.LState) int {
		// TODO(tsileo): debug logging here to help troubleshot GC issues
		ref := L.ToString(1)
		if _, ok := existingRefs[ref]; !ok {
			existingRefs[ref] = struct{}{}
		}
		return 0
	}

	// mark(<blob hash>) is the lowest-level func, it "mark"s a blob to be copied to the root blobstore
	mark := func(L *lua.LState) int {
		// TODO(tsileo): debug logging here to help troubleshot GC issues
		ref := L.ToString(1)
		if _, ok := existingRefs[ref]; ok {
			skipped++
			return 0
		}
		if _, ok := refs[ref]; !ok {
			refs[ref] = struct{}{}
			orderedRefs = append(orderedRefs, ref)
		}
		return 0
	}

	L.SetGlobal("premark", L.NewFunction(premark))
	L.SetGlobal("mark", L.NewFunction(mark))
	L.PreloadModule("json", loadJSON)
	L.PreloadModule("msgpack", loadMsgpack)
	L.PreloadModule("node", loadNode)
	kvsLua.Setup(L, s.KvStore(), ctx)
	bsLua.Setup(ctx, L, s.BlobStore())
	extra.Setup(L)

	// Setup two global:
	// - mark_kv(key, version)  -- version must be a String because we use nano ts
	// - mark_filetree_node(ref)
	if err := L.DoString(luascripts.Get("stash_gc.lua")); err != nil {
		return 0, 0, err
	}

	if err := L.DoString(script); err != nil {
		return 0, 0, err
	}
	fmt.Printf("refs=%q\n", orderedRefs)
	blobsCnt := 0
	totalSize := uint64(0)
	for _, ref := range orderedRefs {
		// FIXME(tsileo): stat before get/put

		// Get the marked blob from the blobstore proxy
		data, err := dc.StashBlobStore().Get(ctx, ref)
		if err != nil {
			if err == blobsfile.ErrBlobNotFound {
				continue
			}
			return 0, 0, err
		}

		// Save it in the root blobstore
		saved, err := s.Root().BlobStore().Put(ctx, &blob.Blob{Hash: ref, Data: data})
		if err != nil {
			return 0, 0, err
		}

		if saved {
			blobsCnt++
			totalSize += uint64(len(data))
		}
	}
	fmt.Printf("premarking helped skipped %d blobs, refs=%d blobs, saved %d blobs\n", skipped, len(orderedRefs), blobsCnt)

	return blobsCnt, totalSize, nil
}

// FIXME(tsileo): have a single share "Lua lib" for all the Lua interactions (GC, document store...)
func loadNode(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": nodeDecode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func nodeDecode(L *lua.LState) int {
	data := L.ToString(1)
	blob := []byte(data)
	if encoded, ok := node.IsNodeBlob(blob); ok {
		blob = encoded
	}
	out := map[string]interface{}{}
	if err := msgpack.Unmarshal(blob, &out); err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

// FIXME(tsileo): have a single share "Lua lib" for all the Lua interactions (GC, document store...)
func loadMsgpack(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": msgpackDecode,
		"encode": msgpackEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func msgpackEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	txt, err := msgpack.Marshal(data)
	if err != nil {
		panic(err)
	}
	L.Push(lua.LString(string(txt)))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func msgpackDecode(L *lua.LState) int {
	data := L.ToString(1)
	out := map[string]interface{}{}
	if err := msgpack.Unmarshal([]byte(data), &out); err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

func loadJSON(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": jsonDecode,
		"encode": jsonEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func jsonEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	L.Push(lua.LString(string(luautil.ToJSON(data))))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func jsonDecode(L *lua.LState) int {
	data := L.ToString(1)
	L.Push(luautil.FromJSON(L, []byte(data)))
	return 1
}
<commit_msg>stash: disable premark<commit_after>package gc // import "a4.io/blobstash/pkg/stash/gc"

import (
	"context"
	"fmt"

	"github.com/vmihailenco/msgpack"
	"github.com/yuin/gopher-lua"

	"a4.io/blobsfile"
	"a4.io/blobstash/pkg/apps/luautil"
	"a4.io/blobstash/pkg/blob"
	bsLua "a4.io/blobstash/pkg/blobstore/lua"
	"a4.io/blobstash/pkg/extra"
	"a4.io/blobstash/pkg/filetree/filetreeutil/node"
	"a4.io/blobstash/pkg/hub"
	kvsLua "a4.io/blobstash/pkg/kvstore/lua"
	"a4.io/blobstash/pkg/luascripts"
	"a4.io/blobstash/pkg/stash"
	"a4.io/blobstash/pkg/stash/store"
)

func GC(ctx context.Context, h *hub.Hub, s *stash.Stash, dc store.DataContext, script string, existingRefs map[string]struct{}) (int, uint64, error) {

	// TODO(tsileo): take a logger
	refs := map[string]struct{}{}
	orderedRefs := []string{}

	L := lua.NewState()
	var skipped int

	// premark(<blob hash>) notify the GC that this blob is already in the root blobstore explicitely (to speedup huge GC)
	premark := func(L *lua.LState) int {
		// TODO(tsileo): debug logging here to help troubleshot GC issues
		ref := L.ToString(1)
		if _, ok := existingRefs[ref]; !ok {
			existingRefs[ref] = struct{}{}
		}
		return 0
	}

	// mark(<blob hash>) is the lowest-level func, it "mark"s a blob to be copied to the root blobstore
	mark := func(L *lua.LState) int {
		// TODO(tsileo): debug logging here to help troubleshot GC issues
		ref := L.ToString(1)
		// if _, ok := existingRefs[ref]; ok {
		//	skipped++
		//	return 0
		// }
		if _, ok := refs[ref]; !ok {
			refs[ref] = struct{}{}
			orderedRefs = append(orderedRefs, ref)
		}
		return 0
	}

	L.SetGlobal("premark", L.NewFunction(premark))
	L.SetGlobal("mark", L.NewFunction(mark))
	L.PreloadModule("json", loadJSON)
	L.PreloadModule("msgpack", loadMsgpack)
	L.PreloadModule("node", loadNode)
	kvsLua.Setup(L, s.KvStore(), ctx)
	bsLua.Setup(ctx, L, s.BlobStore())
	extra.Setup(L)

	// Setup two global:
	// - mark_kv(key, version)  -- version must be a String because we use nano ts
	// - mark_filetree_node(ref)
	if err := L.DoString(luascripts.Get("stash_gc.lua")); err != nil {
		return 0, 0, err
	}

	if err := L.DoString(script); err != nil {
		return 0, 0, err
	}
	fmt.Printf("refs=%q\n", orderedRefs)
	blobsCnt := 0
	totalSize := uint64(0)
	for _, ref := range orderedRefs {
		// FIXME(tsileo): stat before get/put

		// Get the marked blob from the blobstore proxy
		data, err := dc.StashBlobStore().Get(ctx, ref)
		if err != nil {
			if err == blobsfile.ErrBlobNotFound {
				continue
			}
			return 0, 0, err
		}

		// Save it in the root blobstore
		saved, err := s.Root().BlobStore().Put(ctx, &blob.Blob{Hash: ref, Data: data})
		if err != nil {
			return 0, 0, err
		}

		if saved {
			blobsCnt++
			totalSize += uint64(len(data))
		}
	}
	fmt.Printf("premarking helped skipped %d blobs, refs=%d blobs, saved %d blobs\n", skipped, len(orderedRefs), blobsCnt)

	return blobsCnt, totalSize, nil
}

// FIXME(tsileo): have a single share "Lua lib" for all the Lua interactions (GC, document store...)
func loadNode(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": nodeDecode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func nodeDecode(L *lua.LState) int {
	data := L.ToString(1)
	blob := []byte(data)
	if encoded, ok := node.IsNodeBlob(blob); ok {
		blob = encoded
	}
	out := map[string]interface{}{}
	if err := msgpack.Unmarshal(blob, &out); err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

// FIXME(tsileo): have a single share "Lua lib" for all the Lua interactions (GC, document store...)
func loadMsgpack(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": msgpackDecode,
		"encode": msgpackEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func msgpackEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	txt, err := msgpack.Marshal(data)
	if err != nil {
		panic(err)
	}
	L.Push(lua.LString(string(txt)))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func msgpackDecode(L *lua.LState) int {
	data := L.ToString(1)
	out := map[string]interface{}{}
	if err := msgpack.Unmarshal([]byte(data), &out); err != nil {
		panic(err)
	}
	L.Push(luautil.InterfaceToLValue(L, out))
	return 1
}

func loadJSON(L *lua.LState) int {
	// register functions to the table
	mod := L.SetFuncs(L.NewTable(), map[string]lua.LGFunction{
		"decode": jsonDecode,
		"encode": jsonEncode,
	})
	// returns the module
	L.Push(mod)
	return 1
}

func jsonEncode(L *lua.LState) int {
	data := L.CheckAny(1)
	if data == nil {
		L.Push(lua.LNil)
		return 1
	}
	L.Push(lua.LString(string(luautil.ToJSON(data))))
	return 1
}

// TODO(tsileo): a note about empty list vs empty object
func jsonDecode(L *lua.LState) int {
	data := L.ToString(1)
	L.Push(luautil.FromJSON(L, []byte(data)))
	return 1
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"compress/zlib"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"archive/zip"
)

var (
	gPath string
)

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return false
}

func main() {
	gopaths := os.Getenv("GOPATH")
	if gopaths == "" {
		panic("GOPATH为空！")
	}
	sp := strings.Split(gopaths, ";")
	libLCLBinResDir := strings.TrimSpace(sp[0])
	for _, s := range sp {
		s = strings.TrimSpace(s) + "/src/github.com/ying32/liblclbinres"
		if fileExists(s) {
			libLCLBinResDir = s
			break
		}
	}

	fmt.Println("找到路径")
	if !fileExists(libLCLBinResDir) {
		if err := os.MkdirAll(libLCLBinResDir, 0666); err != nil {
			panic(err)
		}
	}

	if len(os.Args) > 1 {
		//
		zipFileName := os.Args[1]
		fmt.Println(zipFileName)
		if strings.ToLower(path.Ext(zipFileName)) != ".zip" {
			panic("输入正确的zip包，如：“liblcl-2.0.2.zip”")
		}
		zz, err := zip.OpenReader(zipFileName)
		if err != nil {
			panic(err)
		}
		defer zz.Close()
		for _, ff := range zz.File {
			//fmt.Println(ff.Name)
			switch ff.Name {
			case "linux64-gtk2/liblcl.so":
				genresByte(readZipData(ff), libLCLBinResDir+"/liblcl_linux_amd64.go")
			case "win32/liblcl.dll":
				genresByte(readZipData(ff), libLCLBinResDir+"/liblcl_windows_386.go")
			case "win64/liblcl.dll":
				genresByte(readZipData(ff), libLCLBinResDir+"/liblcl_windows_amd64.go")
			}
		}
		reg := regexp.MustCompile(`liblcl-(\d+\.\d+\.\d+)`)
		ss := reg.FindStringSubmatch(path.Base(zipFileName))
		if len(ss) >= 2 {
			writeVersion(libLCLBinResDir+"/version.go", ss[1])
		} else {
			panic("检测文件错误，不能生成version.go文件。")
		}

	} else {
		genresFile("../../Librarys/liblcl/win32/liblcl.dll", libLCLBinResDir+"/liblcl_windows_386.go")
		genresFile("../../Librarys/liblcl/win64/liblcl.dll", libLCLBinResDir+"/liblcl_windows_amd64.go")
		genresFile("../../Librarys/liblcl/linux64-gtk2/liblcl.so", libLCLBinResDir+"/liblcl_linux_amd64.go")
		writeVersion(libLCLBinResDir+"/version.go", "2.0.3")
	}

	// macOS不支持这种，也不需要支持这种
	//genres("../liblcl/macOS64-cocoa/liblcl.dylib", "liblcl_darwin_amd64")
}

func readZipData(ff *zip.File) []byte {
	if rr, err := ff.Open(); err == nil {
		defer rr.Close()
		bs, err := ioutil.ReadAll(rr)
		if err != nil {
			return nil
		}
		return bs
	}
	return nil
}

func writeVersion(filename, version string) {
	fmt.Println("genFile: ", filename)
	ioutil.WriteFile(filename, []byte(fmt.Sprintf("package liblclbinres\r\n\r\nconst Version = \"%s\"", version)), 0666)
}

//  zlib压缩
func ZlibCompress(input []byte) ([]byte, error) {
	var in bytes.Buffer
	w, err := zlib.NewWriterLevel(&in, zlib.BestCompression)
	if err != nil {
		return nil, err
	}
	_, err = w.Write(input)
	if err != nil {
		return nil, err
	}
	err = w.Close()
	if err != nil {
		return nil, err
	}
	return in.Bytes(), nil
}

func genresByte(input []byte, newFileName string) {
	fmt.Println("genFile: ", newFileName)
	if len(input) == 0 {
		fmt.Println("000000")
		return
	}

	crc32Val := crc32.ChecksumIEEE(input)

	//压缩
	bs, err := ZlibCompress(input)
	if err != nil {
		panic(err)
	}
	code := bytes.NewBuffer(nil)
	code.WriteString("package liblclbinres")
	code.WriteString("\r\n\r\n")
	code.WriteString(fmt.Sprintf("const CRC32Value = 0x%x\r\n\r\n", crc32Val))

	code.WriteString("var LCLBinRes = []byte(\"")
	for _, b := range bs {
		code.WriteString("\\x" + fmt.Sprintf("%.2x", b))
	}
	code.WriteString("\")\r\n")
	ioutil.WriteFile(newFileName, code.Bytes(), 0666)
}

// 生成字节的单元
func genresFile(fileName, newFileName string) {
	bs, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}
	genresByte(bs, newFileName)
}
<commit_msg>genbinres: Remove version requirements.<commit_after>package main

import (
	"bytes"
	"compress/zlib"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"archive/zip"
)

var (
	gPath string
)

func fileExists(path string) bool {
	_, err := os.Stat(path)
	if err == nil {
		return true
	}
	if os.IsNotExist(err) {
		return false
	}
	return false
}

func main() {
	gopaths := os.Getenv("GOPATH")
	if gopaths == "" {
		panic("GOPATH为空！")
	}
	sp := strings.Split(gopaths, ";")
	libLCLBinResDir := strings.TrimSpace(sp[0])
	for _, s := range sp {
		s = strings.TrimSpace(s) + "/src/github.com/ying32/liblclbinres"
		if fileExists(s) {
			libLCLBinResDir = s
			break
		}
	}

	fmt.Println("找到路径")
	if !fileExists(libLCLBinResDir) {
		if err := os.MkdirAll(libLCLBinResDir, 0666); err != nil {
			panic(err)
		}
	}

	if len(os.Args) > 1 {
		//
		zipFileName := os.Args[1]
		fmt.Println(zipFileName)
		if strings.ToLower(path.Ext(zipFileName)) != ".zip" {
			panic("输入正确的zip包，如：“liblcl-2.0.2.zip”")
		}
		zz, err := zip.OpenReader(zipFileName)
		if err != nil {
			panic(err)
		}
		defer zz.Close()
		for _, ff := range zz.File {
			//fmt.Println(ff.Name)
			switch ff.Name {
			case "linux64-gtk2/liblcl.so":
				genresByte(readZipData(ff), libLCLBinResDir+"/liblcl_linux_amd64.go")
			case "win32/liblcl.dll":
				genresByte(readZipData(ff), libLCLBinResDir+"/liblcl_windows_386.go")
			case "win64/liblcl.dll":
				genresByte(readZipData(ff), libLCLBinResDir+"/liblcl_windows_amd64.go")
			}
		}

	} else {
		genresFile("../../Librarys/liblcl/win32/liblcl.dll", libLCLBinResDir+"/liblcl_windows_386.go")
		genresFile("../../Librarys/liblcl/win64/liblcl.dll", libLCLBinResDir+"/liblcl_windows_amd64.go")
		genresFile("../../Librarys/liblcl/linux64-gtk2/liblcl.so", libLCLBinResDir+"/liblcl_linux_amd64.go")
	}

	// macOS不支持这种，也不需要支持这种
	//genres("../liblcl/macOS64-cocoa/liblcl.dylib", "liblcl_darwin_amd64")
}

func readZipData(ff *zip.File) []byte {
	if rr, err := ff.Open(); err == nil {
		defer rr.Close()
		bs, err := ioutil.ReadAll(rr)
		if err != nil {
			return nil
		}
		return bs
	}
	return nil
}

//  zlib压缩
func ZlibCompress(input []byte) ([]byte, error) {
	var in bytes.Buffer
	w, err := zlib.NewWriterLevel(&in, zlib.BestCompression)
	if err != nil {
		return nil, err
	}
	_, err = w.Write(input)
	if err != nil {
		return nil, err
	}
	err = w.Close()
	if err != nil {
		return nil, err
	}
	return in.Bytes(), nil
}

func genresByte(input []byte, newFileName string) {
	fmt.Println("genFile: ", newFileName)
	if len(input) == 0 {
		fmt.Println("000000")
		return
	}

	crc32Val := crc32.ChecksumIEEE(input)

	//压缩
	bs, err := ZlibCompress(input)
	if err != nil {
		panic(err)
	}
	code := bytes.NewBuffer(nil)
	code.WriteString("package liblclbinres")
	code.WriteString("\r\n\r\n")
	code.WriteString(fmt.Sprintf("const CRC32Value uint32 = 0x%x\r\n\r\n", crc32Val))

	code.WriteString("var LCLBinRes = []byte(\"")
	for _, b := range bs {
		code.WriteString("\\x" + fmt.Sprintf("%.2x", b))
	}
	code.WriteString("\")\r\n")
	ioutil.WriteFile(newFileName, code.Bytes(), 0666)
}

// 生成字节的单元
func genresFile(fileName, newFileName string) {
	bs, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}
	genresByte(bs, newFileName)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"
)

// StartedEvents can be used to track resources that have been started so
// that you can correlate their finish times.
type StartedEvents map[string]UsageEvent

func main() {
	org := flag.String("org", "", "Org GUID to report on (required)")
	flag.Parse()

	if *org == "" {
		flag.Usage()
		os.Exit(2)
	}

	processEvents(os.Stdin, *org, os.Stdout)
}

func processEvents(input io.Reader, org string, output io.Writer) {
	var (
		firstEvent, lastEvent time.Time
		startedEvents         = make(StartedEvents)
	)

	encoder := csv.NewWriter(output)
	decoder := json.NewDecoder(input)

	// Array open bracket.
	if _, err := decoder.Token(); err != nil {
		log.Fatal(err)
	}

	for decoder.More() {
		var usageEvent UsageEvent
		err := decoder.Decode(&usageEvent)
		if err != nil {
			log.Fatal(err)
		}

		// Record (roughly) the beginning and end of the reporting period.
		lastEvent = usageEvent.MetaData.CreatedAt
		if firstEvent.IsZero() {
			firstEvent = usageEvent.MetaData.CreatedAt
		}

		if usageEvent.Entity.OrgGuid != org {
			continue
		}

		switch {
		case usageEvent.Entity.AppGuid != "":
			processApp(usageEvent, startedEvents, firstEvent, encoder)
		case usageEvent.Entity.ServiceInstanceGuid != "":
			processService(usageEvent, startedEvents, firstEvent, encoder)
		}
	}

	// Array close bracket.
	if _, err := decoder.Token(); err != nil {
		log.Fatal(err)
	}

	// Fake end events for resources that are still running at the end of the
	// reporting period.
	for _, usageEvent := range startedEvents {
		usageEvent.MetaData.CreatedAt = lastEvent
		switch {
		case usageEvent.Entity.AppGuid != "":
			usageEvent.Entity.State = "STOPPED"
			processApp(usageEvent, startedEvents, firstEvent, encoder)
		case usageEvent.Entity.ServiceInstanceGuid != "":
			usageEvent.Entity.State = "DELETED"
			processService(usageEvent, startedEvents, firstEvent, encoder)
		}
	}

	encoder.Flush()
	if err := encoder.Error(); err != nil {
		log.Fatal(err)
	}
}

func processApp(
	usageEvent UsageEvent,
	startedEvents StartedEvents,
	firstEvent time.Time,
	encoder *csv.Writer,
) {
	guid := usageEvent.Entity.AppGuid
	var startTime time.Time

	switch usageEvent.Entity.State {
	case "STARTED":
		if previous, ok := startedEvents[guid]; ok {
			// change to an existing app (e.g. scale) record new start/stop
			startTime = previous.MetaData.CreatedAt
		}
		startedEvents[guid] = usageEvent
	case "STOPPED":
		if previous, ok := startedEvents[guid]; ok {
			startTime = previous.MetaData.CreatedAt
			delete(startedEvents, guid)
		} else {
			startTime = firstEvent
		}
	}

	if !startTime.IsZero() {
		encoder.Write([]string{
			usageEvent.Entity.AppName,
			usageEvent.Entity.SpaceName,
			fmt.Sprintf("%d", usageEvent.Entity.InstanceCount),
			fmt.Sprintf("%d", usageEvent.Entity.MemoryPerInstance),
			fmt.Sprintf("%.0f", usageEvent.MetaData.CreatedAt.Sub(startTime).Seconds()),
			fmt.Sprintf("%s", startTime),
		})
	}
}

func processService(
	usageEvent UsageEvent,
	startedEvents StartedEvents,
	firstEvent time.Time,
	encoder *csv.Writer,
) {
	// Ignore user provided services
	if usageEvent.Entity.ServiceInstanceType != "managed_service_instance" {
		return
	}

	guid := usageEvent.Entity.ServiceInstanceGuid
	var startTime time.Time

	switch usageEvent.Entity.State {
	case "CREATED":
		startedEvents[guid] = usageEvent
	case "UPDATED": // change of service plan is like creating a new service
		if previous, ok := startedEvents[guid]; ok {
			startTime = previous.MetaData.CreatedAt
			startedEvents[guid] = usageEvent
		} else {
			startTime = firstEvent
		}
	case "DELETED":
		if previous, ok := startedEvents[guid]; ok {
			startTime = previous.MetaData.CreatedAt
			delete(startedEvents, guid)
		} else {
			startTime = firstEvent
		}
	}

	if !startTime.IsZero() {
		encoder.Write([]string{
			usageEvent.Entity.ServiceInstanceName,
			usageEvent.Entity.SpaceName,
			usageEvent.Entity.ServiceLabel,
			usageEvent.Entity.ServicePlanName,
			fmt.Sprintf("%.0f", usageEvent.MetaData.CreatedAt.Sub(startTime).Seconds()),
			fmt.Sprintf("%s", startTime),
		})
	}
}
<commit_msg>Add start and finish time for proces_events<commit_after>package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"time"
)

// StartedEvents can be used to track resources that have been started so
// that you can correlate their finish times.
type StartedEvents map[string]UsageEvent

func main() {
	org := flag.String("org", "", "Org GUID to report on (required)")
	start := flag.String("start", "", "RFC3339 date to start reporting on")
	finish := flag.String("finish", "", "RFC3339 date to finish reporting on")
	flag.Parse()

	startTime, err := time.Parse(time.RFC3339, *start)
	if err != nil {
		log.Fatal(err)
	}
	finishTime, err := time.Parse(time.RFC3339, *finish)
	if err != nil {
		log.Fatal(err)
	}

	if *org == "" {
		flag.Usage()
		os.Exit(2)
	}

	processEvents(os.Stdin, *org, os.Stdout, startTime, finishTime)
}

func processEvents(input io.Reader, org string, output io.Writer, startTime, finishTime time.Time) {
	var (
		firstEvent, lastEvent time.Time
		startedEvents         = make(StartedEvents)
	)

	encoder := csv.NewWriter(output)
	decoder := json.NewDecoder(input)

	// Array open bracket.
	if _, err := decoder.Token(); err != nil {
		log.Fatal(err)
	}

	for decoder.More() {
		var usageEvent UsageEvent
		err := decoder.Decode(&usageEvent)
		if err != nil {
			log.Fatal(err)
		}

		// Record (roughly) the beginning and end of the reporting period.
		lastEvent = usageEvent.MetaData.CreatedAt
		if firstEvent.IsZero() {
			firstEvent = usageEvent.MetaData.CreatedAt
		}

		if usageEvent.Entity.OrgGuid != org {
			continue
		}

		if usageEvent.MetaData.CreatedAt.Before(startTime) || usageEvent.MetaData.CreatedAt.After(finishTime) {
			continue
		}

		switch {
		case usageEvent.Entity.AppGuid != "":
			processApp(usageEvent, startedEvents, firstEvent, encoder)
		case usageEvent.Entity.ServiceInstanceGuid != "":
			processService(usageEvent, startedEvents, firstEvent, encoder)
		}
	}

	// Array close bracket.
	if _, err := decoder.Token(); err != nil {
		log.Fatal(err)
	}

	// Fake end events for resources that are still running at the end of the
	// reporting period.
	for _, usageEvent := range startedEvents {
		usageEvent.MetaData.CreatedAt = lastEvent
		switch {
		case usageEvent.Entity.AppGuid != "":
			usageEvent.Entity.State = "STOPPED"
			processApp(usageEvent, startedEvents, firstEvent, encoder)
		case usageEvent.Entity.ServiceInstanceGuid != "":
			usageEvent.Entity.State = "DELETED"
			processService(usageEvent, startedEvents, firstEvent, encoder)
		}
	}

	encoder.Flush()
	if err := encoder.Error(); err != nil {
		log.Fatal(err)
	}
}

func processApp(
	usageEvent UsageEvent,
	startedEvents StartedEvents,
	firstEvent time.Time,
	encoder *csv.Writer,
) {
	guid := usageEvent.Entity.AppGuid
	var startTime time.Time

	switch usageEvent.Entity.State {
	case "STARTED":
		if previous, ok := startedEvents[guid]; ok {
			// change to an existing app (e.g. scale) record new start/stop
			startTime = previous.MetaData.CreatedAt
		}
		startedEvents[guid] = usageEvent
	case "STOPPED":
		if previous, ok := startedEvents[guid]; ok {
			startTime = previous.MetaData.CreatedAt
			delete(startedEvents, guid)
		} else {
			startTime = firstEvent
		}
	}

	if !startTime.IsZero() {
		encoder.Write([]string{
			usageEvent.Entity.AppName,
			usageEvent.Entity.SpaceName,
			fmt.Sprintf("%d", usageEvent.Entity.InstanceCount),
			fmt.Sprintf("%d", usageEvent.Entity.MemoryPerInstance),
			fmt.Sprintf("%.0f", usageEvent.MetaData.CreatedAt.Sub(startTime).Seconds()),
			fmt.Sprintf("%s", startTime),
		})
	}
}

func processService(
	usageEvent UsageEvent,
	startedEvents StartedEvents,
	firstEvent time.Time,
	encoder *csv.Writer,
) {
	// Ignore user provided services
	if usageEvent.Entity.ServiceInstanceType != "managed_service_instance" {
		return
	}

	guid := usageEvent.Entity.ServiceInstanceGuid
	var startTime time.Time

	switch usageEvent.Entity.State {
	case "CREATED":
		startedEvents[guid] = usageEvent
	case "UPDATED": // change of service plan is like creating a new service
		if previous, ok := startedEvents[guid]; ok {
			startTime = previous.MetaData.CreatedAt
			startedEvents[guid] = usageEvent
		} else {
			startTime = firstEvent
		}
	case "DELETED":
		if previous, ok := startedEvents[guid]; ok {
			startTime = previous.MetaData.CreatedAt
			delete(startedEvents, guid)
		} else {
			startTime = firstEvent
		}
	}

	if !startTime.IsZero() {
		encoder.Write([]string{
			usageEvent.Entity.ServiceInstanceName,
			usageEvent.Entity.SpaceName,
			usageEvent.Entity.ServiceLabel,
			usageEvent.Entity.ServicePlanName,
			fmt.Sprintf("%.0f", usageEvent.MetaData.CreatedAt.Sub(startTime).Seconds()),
			fmt.Sprintf("%s", startTime),
		})
	}
}
<|endoftext|>


<commit_before>package quota

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared"
)

/*
#include <linux/fs.h>
#include <linux/dqblk_xfs.h>
#include <linux/quota.h>
#include <sys/ioctl.h>
#include <sys/quota.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdbool.h>
#include <unistd.h>

#ifndef FS_XFLAG_PROJINHERIT
struct fsxattr {
	__u32		fsx_xflags;
	__u32		fsx_extsize;
	__u32		fsx_nextents;
	__u32		fsx_projid;
	unsigned char	fsx_pad[12];
};
#define FS_XFLAG_PROJINHERIT 0x00000200
#endif

#ifndef QIF_DQBLKSIZE_BITS
struct if_dqinfo {
	__u64 dqi_bgrace;
	__u64 dqi_igrace;
	__u32 dqi_flags;
	__u32 dqi_valid;
};

struct if_dqblk {
	__u64 dqb_bhardlimit;
	__u64 dqb_bsoftlimit;
	__u64 dqb_curspace;
	__u64 dqb_ihardlimit;
	__u64 dqb_isoftlimit;
	__u64 dqb_curinodes;
	__u64 dqb_btime;
	__u64 dqb_itime;
	__u32 dqb_valid;
};
#define QIF_DQBLKSIZE_BITS 10
#endif

#ifndef FS_IOC_FSGETXATTR
#define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr)
#endif

#ifndef FS_IOC_FSSETXATTR
#define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr)
#endif

#ifndef PRJQUOTA
#define PRJQUOTA 2
#endif

int quota_supported(char *dev_path) {
	struct if_dqinfo dqinfo;

	return quotactl(QCMD(Q_GETINFO, PRJQUOTA), dev_path, 0, (caddr_t)&dqinfo);
}

int64_t quota_get_usage(char *dev_path, uint32_t id) {
	struct if_dqblk quota;

	if (quotactl(QCMD(Q_GETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)&quota) < 0) {
		return -1;
	}

	return quota.dqb_curspace;
}

int quota_set(char *dev_path, uint32_t id, uint64_t hard_bytes) {
	struct if_dqblk quota;
	fs_disk_quota_t xfsquota;

	if (quotactl(QCMD(Q_GETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)&quota) < 0) {
		return -1;
	}

	quota.dqb_bhardlimit = hard_bytes;
	if (quotactl(QCMD(Q_SETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)&quota) < 0) {
		xfsquota.d_version = FS_DQUOT_VERSION;
		xfsquota.d_id = id;
		xfsquota.d_flags = FS_PROJ_QUOTA;
		xfsquota.d_fieldmask = FS_DQ_BHARD;
		xfsquota.d_blk_hardlimit = hard_bytes * 1024 / 512;

		if (quotactl(QCMD(Q_XSETQLIM, PRJQUOTA), dev_path, id, (caddr_t)&xfsquota) < 0) {
			return -1;
		}
	}

	return 0;
}

int quota_set_path(char *path, uint32_t id, bool inherit) {
	struct fsxattr attr;
	int fd;
	int ret;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return -1;

	ret = ioctl(fd, FS_IOC_FSGETXATTR, &attr);
	if (ret < 0) {
		close(fd);
		return -1;
	}

	if (inherit) {
		attr.fsx_xflags |= FS_XFLAG_PROJINHERIT;
	}

	attr.fsx_projid = id;

	ret = ioctl(fd, FS_IOC_FSSETXATTR, &attr);
	if (ret < 0) {
		close(fd);
		return -1;
	}

	close(fd);
	return 0;
}

int32_t quota_get_path(char *path) {
	struct fsxattr attr;
	int fd;
	int ret;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return -1;

	ret = ioctl(fd, FS_IOC_FSGETXATTR, &attr);
	if (ret < 0) {
		close(fd);
		return -1;
	}

	close(fd);
	return attr.fsx_projid;
}

*/
import "C"

var errNoDevice = fmt.Errorf("Couldn't find backing device for mountpoint")

func devForPath(path string) (string, error) {
	// Get major/minor
	var stat unix.Stat_t
	err := unix.Lstat(path, &stat)
	if err != nil {
		return "", err
	}

	devMajor := unix.Major(uint64(stat.Dev))
	devMinor := unix.Minor(uint64(stat.Dev))

	// Parse mountinfo for it
	mountinfo, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return "", err
	}
	defer mountinfo.Close()

	scanner := bufio.NewScanner(mountinfo)
	for scanner.Scan() {
		line := scanner.Text()

		tokens := strings.Fields(line)
		if len(tokens) < 5 {
			continue
		}

		if tokens[2] == fmt.Sprintf("%d:%d", devMajor, devMinor) {
			if shared.PathExists(tokens[len(tokens)-2]) {
				return tokens[len(tokens)-2], nil
			}
		}
	}

	return "", errNoDevice
}

// Supported check if the given path supports project quotas
func Supported(path string) (bool, error) {
	// Get the backing device
	devPath, err := devForPath(path)
	if err != nil {
		return false, err
	}

	// Call quotactl through CGo
	cDevPath := C.CString(devPath)
	defer C.free(unsafe.Pointer(cDevPath))

	return C.quota_supported(cDevPath) == 0, nil
}

// GetProject returns the project quota ID for the given path
func GetProject(path string) (uint32, error) {
	// Call ioctl through CGo
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	id := C.quota_get_path(cPath)
	if id < 0 {
		return 0, fmt.Errorf("Failed to get project from '%s'", path)
	}

	return uint32(id), nil
}

// SetProject recursively sets the project quota ID (and project inherit flag on directories) for the given path.
func SetProject(path string, id uint32) error {
	err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		inherit := false

		if info.IsDir() {
			inherit = true // Only can set FS_XFLAG_PROJINHERIT on directories.
		}

		// Call ioctl through CGo.
		cPath := C.CString(filePath)
		defer C.free(unsafe.Pointer(cPath))

		if C.quota_set_path(cPath, C.uint32_t(id), C.bool(inherit)) != 0 {
			// Currently project ID cannot be set on non-regular files after file creation.
			// However if the parent directory has a project and the inherit flag set on it then
			// non-regular files do get accounted for under the parent's project, so we do still try
			// and set the post-create project on non-regular files in case at some point in the future
			// this inconsistency in behavior is fixed. However because it doesn't work today we will
			// ignore any errors setting project on non-regular files.
			if !info.Mode().IsRegular() {
				return nil
			}

			return fmt.Errorf(`Failed to set project ID "%d" on %q (inherit %t)`, id, filePath, inherit)
		}

		return nil
	})

	return err
}

// DeleteProject unsets the project id from the path and clears the quota for the project id
func DeleteProject(path string, id uint32) error {
	// Unset the project from the path
	err := SetProject(path, 0)
	if err != nil {
		return err
	}

	// Unset the quota on the project
	err = SetProjectQuota(path, id, 0)
	if err != nil {
		return err
	}

	return nil
}

// GetProjectUsage returns the current consumption
func GetProjectUsage(path string, id uint32) (int64, error) {
	// Get the backing device
	devPath, err := devForPath(path)
	if err != nil {
		return -1, err
	}

	// Call quotactl through CGo
	cDevPath := C.CString(devPath)
	defer C.free(unsafe.Pointer(cDevPath))

	size := C.quota_get_usage(cDevPath, C.uint32_t(id))
	if size < 0 {
		return -1, fmt.Errorf("Failed to get project consumption for id '%d' on '%s'", id, devPath)
	}

	return int64(size), nil
}

// SetProjectQuota sets the quota on the project id
func SetProjectQuota(path string, id uint32, bytes int64) error {
	// Get the backing device
	devPath, err := devForPath(path)
	if err != nil {
		return err
	}

	// Call quotactl through CGo
	cDevPath := C.CString(devPath)
	defer C.free(unsafe.Pointer(cDevPath))

	if C.quota_set(cDevPath, C.uint32_t(id), C.uint64_t(bytes/1024)) != 0 {
		return fmt.Errorf("Failed to set project quota for id '%d' on '%s'", id, devPath)
	}

	return nil
}
<commit_msg>lxd/storage/quota/projectquota: Only set quota on directories and regular files<commit_after>package quota

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared"
)

/*
#include <linux/fs.h>
#include <linux/dqblk_xfs.h>
#include <linux/quota.h>
#include <sys/ioctl.h>
#include <sys/quota.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdint.h>
#include <stdlib.h>
#include <stdbool.h>
#include <unistd.h>

#ifndef FS_XFLAG_PROJINHERIT
struct fsxattr {
	__u32		fsx_xflags;
	__u32		fsx_extsize;
	__u32		fsx_nextents;
	__u32		fsx_projid;
	unsigned char	fsx_pad[12];
};
#define FS_XFLAG_PROJINHERIT 0x00000200
#endif

#ifndef QIF_DQBLKSIZE_BITS
struct if_dqinfo {
	__u64 dqi_bgrace;
	__u64 dqi_igrace;
	__u32 dqi_flags;
	__u32 dqi_valid;
};

struct if_dqblk {
	__u64 dqb_bhardlimit;
	__u64 dqb_bsoftlimit;
	__u64 dqb_curspace;
	__u64 dqb_ihardlimit;
	__u64 dqb_isoftlimit;
	__u64 dqb_curinodes;
	__u64 dqb_btime;
	__u64 dqb_itime;
	__u32 dqb_valid;
};
#define QIF_DQBLKSIZE_BITS 10
#endif

#ifndef FS_IOC_FSGETXATTR
#define FS_IOC_FSGETXATTR _IOR ('X', 31, struct fsxattr)
#endif

#ifndef FS_IOC_FSSETXATTR
#define FS_IOC_FSSETXATTR _IOW ('X', 32, struct fsxattr)
#endif

#ifndef PRJQUOTA
#define PRJQUOTA 2
#endif

int quota_supported(char *dev_path) {
	struct if_dqinfo dqinfo;

	return quotactl(QCMD(Q_GETINFO, PRJQUOTA), dev_path, 0, (caddr_t)&dqinfo);
}

int64_t quota_get_usage(char *dev_path, uint32_t id) {
	struct if_dqblk quota;

	if (quotactl(QCMD(Q_GETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)&quota) < 0) {
		return -1;
	}

	return quota.dqb_curspace;
}

int quota_set(char *dev_path, uint32_t id, uint64_t hard_bytes) {
	struct if_dqblk quota;
	fs_disk_quota_t xfsquota;

	if (quotactl(QCMD(Q_GETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)&quota) < 0) {
		return -1;
	}

	quota.dqb_bhardlimit = hard_bytes;
	if (quotactl(QCMD(Q_SETQUOTA, PRJQUOTA), dev_path, id, (caddr_t)&quota) < 0) {
		xfsquota.d_version = FS_DQUOT_VERSION;
		xfsquota.d_id = id;
		xfsquota.d_flags = FS_PROJ_QUOTA;
		xfsquota.d_fieldmask = FS_DQ_BHARD;
		xfsquota.d_blk_hardlimit = hard_bytes * 1024 / 512;

		if (quotactl(QCMD(Q_XSETQLIM, PRJQUOTA), dev_path, id, (caddr_t)&xfsquota) < 0) {
			return -1;
		}
	}

	return 0;
}

int quota_set_path(char *path, uint32_t id, bool inherit) {
	struct fsxattr attr;
	int fd;
	int ret;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return -1;

	ret = ioctl(fd, FS_IOC_FSGETXATTR, &attr);
	if (ret < 0) {
		close(fd);
		return -1;
	}

	if (inherit) {
		attr.fsx_xflags |= FS_XFLAG_PROJINHERIT;
	}

	attr.fsx_projid = id;

	ret = ioctl(fd, FS_IOC_FSSETXATTR, &attr);
	if (ret < 0) {
		close(fd);
		return -1;
	}

	close(fd);
	return 0;
}

int32_t quota_get_path(char *path) {
	struct fsxattr attr;
	int fd;
	int ret;

	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0)
		return -1;

	ret = ioctl(fd, FS_IOC_FSGETXATTR, &attr);
	if (ret < 0) {
		close(fd);
		return -1;
	}

	close(fd);
	return attr.fsx_projid;
}

*/
import "C"

var errNoDevice = fmt.Errorf("Couldn't find backing device for mountpoint")

func devForPath(path string) (string, error) {
	// Get major/minor
	var stat unix.Stat_t
	err := unix.Lstat(path, &stat)
	if err != nil {
		return "", err
	}

	devMajor := unix.Major(uint64(stat.Dev))
	devMinor := unix.Minor(uint64(stat.Dev))

	// Parse mountinfo for it
	mountinfo, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return "", err
	}
	defer mountinfo.Close()

	scanner := bufio.NewScanner(mountinfo)
	for scanner.Scan() {
		line := scanner.Text()

		tokens := strings.Fields(line)
		if len(tokens) < 5 {
			continue
		}

		if tokens[2] == fmt.Sprintf("%d:%d", devMajor, devMinor) {
			if shared.PathExists(tokens[len(tokens)-2]) {
				return tokens[len(tokens)-2], nil
			}
		}
	}

	return "", errNoDevice
}

// Supported check if the given path supports project quotas
func Supported(path string) (bool, error) {
	// Get the backing device
	devPath, err := devForPath(path)
	if err != nil {
		return false, err
	}

	// Call quotactl through CGo
	cDevPath := C.CString(devPath)
	defer C.free(unsafe.Pointer(cDevPath))

	return C.quota_supported(cDevPath) == 0, nil
}

// GetProject returns the project quota ID for the given path
func GetProject(path string) (uint32, error) {
	// Call ioctl through CGo
	cPath := C.CString(path)
	defer C.free(unsafe.Pointer(cPath))

	id := C.quota_get_path(cPath)
	if id < 0 {
		return 0, fmt.Errorf("Failed to get project from '%s'", path)
	}

	return uint32(id), nil
}

// SetProject recursively sets the project quota ID (and project inherit flag on directories) for the given path.
func SetProject(path string, id uint32) error {
	err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		inherit := false
		if info.IsDir() {
			inherit = true // Only can set FS_XFLAG_PROJINHERIT on directories.
		} else if !info.Mode().IsRegular() {
			// Cannot set project ID on non-regular files after file creation. Infact trying to set
			// project ID on some file types just blocks forever (such as pipe files).
			// So skip them as they don't take up disk space anyway.
			return nil
		}

		// Call ioctl through CGo.
		cPath := C.CString(filePath)
		defer C.free(unsafe.Pointer(cPath))

		if C.quota_set_path(cPath, C.uint32_t(id), C.bool(inherit)) != 0 {
			return fmt.Errorf(`Failed to set project ID "%d" on %q (inherit %t)`, id, filePath, inherit)
		}

		return nil
	})

	return err
}

// DeleteProject unsets the project id from the path and clears the quota for the project id
func DeleteProject(path string, id uint32) error {
	// Unset the project from the path
	err := SetProject(path, 0)
	if err != nil {
		return err
	}

	// Unset the quota on the project
	err = SetProjectQuota(path, id, 0)
	if err != nil {
		return err
	}

	return nil
}

// GetProjectUsage returns the current consumption
func GetProjectUsage(path string, id uint32) (int64, error) {
	// Get the backing device
	devPath, err := devForPath(path)
	if err != nil {
		return -1, err
	}

	// Call quotactl through CGo
	cDevPath := C.CString(devPath)
	defer C.free(unsafe.Pointer(cDevPath))

	size := C.quota_get_usage(cDevPath, C.uint32_t(id))
	if size < 0 {
		return -1, fmt.Errorf("Failed to get project consumption for id '%d' on '%s'", id, devPath)
	}

	return int64(size), nil
}

// SetProjectQuota sets the quota on the project id
func SetProjectQuota(path string, id uint32, bytes int64) error {
	// Get the backing device
	devPath, err := devForPath(path)
	if err != nil {
		return err
	}

	// Call quotactl through CGo
	cDevPath := C.CString(devPath)
	defer C.free(unsafe.Pointer(cDevPath))

	if C.quota_set(cDevPath, C.uint32_t(id), C.uint64_t(bytes/1024)) != 0 {
		return fmt.Errorf("Failed to set project quota for id '%d' on '%s'", id, devPath)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/davecgh/go-spew/spew"
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"testing"
)

type testCase struct {
	in     []Point
	consol consolidation.Consolidator
	num    int
	out    []Point
}

func TestDataProcessor(t *testing.T) {
	cases := []testCase{
		{
			[]Point{
				{1, 1449178131},
				{2, 1449178141},
				{3, 1449178151},
				{4, 1449178161},
			},
			consolidation.Avg,
			2,
			[]Point{
				{1.5, 1449178141},
				{3.5, 1449178161},
			},
		},
	}
	for i, c := range cases {
		out := consolidate(c.in, c.num, c.consol)
		spew.Dump(out)
		continue
		if len(out) != len(c.out) {
			t.Fatalf("output for testcase %d mismatch: expected: %v, got: %v", i, c.out, out)

		} else {
			for j, p := range out {
				if p.Val != c.out[i].Val || p.Ts != c.out[i].Ts {
					t.Fatalf("output for testcase %d mismatch at point %d: expected: %v, got: %v", i, j, c.out[j], out[j])
				}
			}
		}
	}
}

type c struct {
	numPoints     uint32
	maxDataPoints uint32
	every         int
}

func TestAggEvery(t *testing.T) {
	cases := []c{
		{60, 80, 1},
		{70, 80, 1},
		{79, 80, 1},
		{80, 80, 1},
		{81, 80, 2},
		{120, 80, 2},
		{150, 80, 2},
		{158, 80, 2},
		{159, 80, 2},
		{160, 80, 2},
		{161, 80, 3},
		{165, 80, 3},
		{180, 80, 3},
	}
	for i, c := range cases {
		every := aggEvery(c.numPoints, c.maxDataPoints)
		if every != c.every {
			t.Fatalf("output for testcase %d mismatch: expected: %v, got: %v", i, c.every, every)
		}
	}
}
<commit_msg>fix tests<commit_after>package main

import (
	"github.com/raintank/raintank-metric/metric_tank/consolidation"
	"testing"
)

type testCase struct {
	in     []Point
	consol consolidation.Consolidator
	num    int
	out    []Point
}

func TestDataProcessor(t *testing.T) {
	cases := []testCase{
		{
			[]Point{
				{1, 1449178131},
				{2, 1449178141},
				{3, 1449178151},
				{4, 1449178161},
			},
			consolidation.Avg,
			2,
			[]Point{
				{1.5, 1449178141},
				{3.5, 1449178161},
			},
		},
	}
	for i, c := range cases {
		out := consolidate(c.in, c.num, c.consol)
		if len(out) != len(c.out) {
			t.Fatalf("output for testcase %d mismatch: expected: %v, got: %v", i, c.out, out)

		} else {
			for j, p := range out {
				if p.Val != c.out[j].Val || p.Ts != c.out[j].Ts {
					t.Fatalf("output for testcase %d mismatch at point %d: expected: %v, got: %v", i, j, c.out[j], out[j])
				}
			}
		}
	}
}

type c struct {
	numPoints     uint32
	maxDataPoints uint32
	every         int
}

func TestAggEvery(t *testing.T) {
	cases := []c{
		{60, 80, 1},
		{70, 80, 1},
		{79, 80, 1},
		{80, 80, 1},
		{81, 80, 2},
		{120, 80, 2},
		{150, 80, 2},
		{158, 80, 2},
		{159, 80, 2},
		{160, 80, 2},
		{161, 80, 3},
		{165, 80, 3},
		{180, 80, 3},
	}
	for i, c := range cases {
		every := aggEvery(c.numPoints, c.maxDataPoints)
		if every != c.every {
			t.Fatalf("output for testcase %d mismatch: expected: %v, got: %v", i, c.every, every)
		}
	}
}
<|endoftext|>


<commit_before>package ga

import (
	"sort"
	"strings"
)

func GroupAnagrams(input []string) [][]string {
	n := len(input)
	lc := make(map[string][]string, n)
	for _, str := range input {
		// old school
		// sort str alphabetically
		k := sortString(str)
		if v, exists := lc[k]; !exists {
			lc[k] = []string{str}
		} else {
			lc[k] = append(v, str)
		}
	}
	out := make([][]string, 0, len(lc))
	for _, v := range lc {
		out = append(out, v)
	}

	return out
}

func sortString(w string) string {
	s := strings.Split(w, "")
	sort.Strings(s)
	return strings.Join(s, "")
}
<commit_msg>using bucket to count<commit_after>package ga

import (
	"sort"
	"strconv"
	"strings"
)

func GroupAnagrams(input []string) [][]string {
	// return ga1(input)
	return ga2(input)
}

// time complex O(nKLogK) K is the max length of str in input
// space Complexity O(nK)
func ga1(input []string) [][]string {
	n := len(input)
	lc := make(map[string][]string, n)
	for _, str := range input {
		// old school
		// sort str alphabetically
		k := sortString(str)
		if v, exists := lc[k]; !exists {
			lc[k] = []string{str}
		} else {
			lc[k] = append(v, str)
		}
	}
	out := make([][]string, 0, len(lc))
	for _, v := range lc {
		out = append(out, v)
	}

	return out
}

func sortString(w string) string {
	s := strings.Split(w, "")
	sort.Strings(s)
	return strings.Join(s, "")
}

func ga2(str []string) [][]string {
	// use bucket to count existence as key without sorting
	n := len(str)
	lc := make(map[string][]string, n)
	for _, st := range str {
		bucket := make([]int, 26)
		for i := range st {
			bucket[st[i]-'a']++
		}
		// build bucket as key
		var sb strings.Builder
		for i := 0; i < 26; i++ {
			sb.WriteRune('#')
			sb.WriteString(strconv.FormatInt(int64(bucket[i]), 10))
		}
		key := sb.String()
		cv := lc[key]
		lc[key] = append(cv, st)
	}
	out := make([][]string, 0, len(lc))
	for _, v := range lc {
		out = append(out, v)
	}
	return out
}
<|endoftext|>


<commit_before>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    //"os"
    "fmt"
    "container/heap"
    "container/vector"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt *PTask) Type() string {
    return "PTask"
}
func (pt *PTask) SetFunc(f func(int64)) {
    pt.F = f
}
func (pt *PTask) Func() func(int64) {
    return pt.F
}
func (pt *PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(RegisteredTask).Task().(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}

//  A priority queue based on the "container/vector" package.
//  Ideally, an array-based priority queue implementation should have
//  fast dequeues and slow enqueues. I fear the vector.Vector class
//  gives slow equeues and slow dequeues.
type VectorPriorityQueue struct {
    v *vector.Vector
}

func NewVectorPriorityQueue() *VectorPriorityQueue {
    var vpq = new(VectorPriorityQueue)
    vpq.v = new(vector.Vector)
    return vpq
}

func (vpq *VectorPriorityQueue) Len() int {
    return vpq.v.Len()
}
type etypeStopIter struct {
}
func (e etypeStopIter) String() string {
    return "STOPITER"
}
func (vpq *VectorPriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        break
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                break
            default:
                panic(r)
            }
        }
        vpq.v.Insert(i, task)
    } ()
    vpq.v.Do(func (telm interface{}) {
        if task.Task().(PrioritizedTask).Key() > telm.(RegisteredTask).Task().(PrioritizedTask).Key() {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}
func (vpq *VectorPriorityQueue) Dequeue() RegisteredTask {
    var head = vpq.v.At(0).(RegisteredTask)
    vpq.v.Delete(0)
    return head
}
func (vpq *VectorPriorityQueue) SetKey(id int64, k float64) {
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                var rtask = vpq.v.At(i).(RegisteredTask)
                vpq.v.Delete(i)
                rtask.Task().(PrioritizedTask).SetKey(k)
                vpq.Enqueue(rtask)
            default:
                panic(r)
            }
        }
    } ()
    vpq.v.Do(func (telm interface{}) {
        if telm.(RegisteredTask).Id() != id {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}

type ArrayPriorityQueue struct {
    v          []RegisteredTask
    head, tail int
}

func NewArrayPriorityQueue() *ArrayPriorityQueue {
    var apq = new(ArrayPriorityQueue)
    apq.v = make([]RegisteredTask, 10)
    return apq
}

func (apq *ArrayPriorityQueue) Len() int {
    return apq.tail - apq.head
}

func (apq *ArrayPriorityQueue) Enqueue(task RegisteredTask) {
    var key = task.Task().(PrioritizedTask).Key()
    var insertoffset = registeredTaskSearch(
            apq.v[apq.head:apq.tail],
            func(t RegisteredTask) bool {
                return t.Task().(PrioritizedTask).Key() < key
            })
    if apq.tail != len(apq.v) {
        for j := apq.tail ; j > insertoffset ; j-- {
            apq.v[j] = apq.v[j-1]
        }
        apq.v[apq.head+insertoffset] = task
        apq.tail++
        return
    }
    var newv = apq.v
    if apq.head < len(apq.v)/2 {
        newv = make([]RegisteredTask, 2* len(apq.v))
    }
    var i, j int
    j = 0
    for i = apq.head ; i < apq.tail ; i++ {
        if apq.v[i].Task().(PrioritizedTask).Key() > key {
            break
        } else {
            newv[j] = apq.v[i]
            apq.v[i] = nil
        }
        j++
    }
    //fmt.Fprintf(os.Stderr, "Length %d index %d\n", len(newv), j)
    newv[j] = task
    j++
    for ; i < apq.tail ; i++ {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
    }
    apq.v = newv
    apq.head = 0
    apq.tail = j
}

func (apq *ArrayPriorityQueue) Dequeue() RegisteredTask {
    if apq.Len() == 0 {
        panic("empty")
    }
    var task = apq.v[apq.head]
    apq.v[apq.head] = nil
    apq.head++
    return task
}

func (apq *ArrayPriorityQueue) SetKey(id int64, k float64) {
}
<commit_msg>Clean up Equeue a little more<commit_after>package queues
/*
 *  Filename:    priority.go
 *  Package:     queues
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 22:18:57 PDT 2011
 *  Description:
 */
import (
    //"os"
    "fmt"
    "container/heap"
    "container/vector"
)

type PrioritizedTask interface {
    Task
    Key() float64
    SetKey(float64)
}
type PTask struct {
    F func(int64)
    P float64
}
func (pt *PTask) Type() string {
    return "PTask"
}
func (pt *PTask) SetFunc(f func(int64)) {
    pt.F = f
}
func (pt *PTask) Func() func(int64) {
    return pt.F
}
func (pt *PTask) Key() float64 {
    return pt.P
}
func (pt *PTask) SetKey(k float64) {
    pt.P = k
}

type pQueue struct {
    elements []RegisteredTask
}
func newPQueue() *pQueue {
    var h = new(pQueue)
    h.elements = make([]RegisteredTask, 0, 5)
    return h
}
func (h *pQueue) GetPTask(i int) PrioritizedTask {
    if n := len(h.elements) ; i < 0 || i >= n {
        panic("badindex")
    }
    return h.elements[i].Task().(PrioritizedTask)
}
func (h *pQueue) Len() int {
    return len(h.elements)
}
func (h *pQueue) Less(i, j int) bool {
    return h.GetPTask(i).Key() < h.GetPTask(j).Key()
}
func (h *pQueue) Swap(i, j int) {
    if n := len(h.elements) ; i < 0 || i >=n || j < 0 || j >= n {
        panic("badindex")
    }
    var tmp = h.elements[i]
    h.elements[i] = h.elements[j]
    h.elements[j] = tmp
}
func (h *pQueue) Push(x interface{}) {
    switch x.(RegisteredTask).Task().(type) {
    case PrioritizedTask:
        h.elements = append(h.elements, x.(RegisteredTask))
    default:
        panic("badtype")
    }
}
func (h *pQueue) Pop() interface{} {
    if len(h.elements) <= 0 {
        panic("empty")
    }
    var head = h.elements[0]
    h.elements = h.elements[1:]
    return head
}
func (h *pQueue) FindId(id int64) (int, RegisteredTask) {
    for i, elm := range h.elements {
        if elm.Id() == id {
            return i, elm
        }
    }
    return -1, nil
}

type PriorityQueue struct {
    h  *pQueue
}

func NewPriorityQueue() *PriorityQueue {
    var pq = new(PriorityQueue)
    pq.h = newPQueue()
    // No need to call heap.Init(pq.h) on an empty heap.
    return pq
}

func (pq *PriorityQueue) Len() int {
    return pq.h.Len()
}
func (pq *PriorityQueue) Dequeue() RegisteredTask {
    if pq.Len() <= 0 {
        panic("empty")
    }
    return heap.Pop(pq.h).(RegisteredTask)
}
func (pq *PriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        heap.Push(pq.h, task)
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
}
func (pq *PriorityQueue) SetKey(id int64, k float64) {
    var i, task = pq.h.FindId(id)
    if i < 0 {
        return
    }
    heap.Remove(pq.h, i)
    task.Task().(PrioritizedTask).SetKey(k)
    heap.Push(pq.h, task)
}

//  A priority queue based on the "container/vector" package.
//  Ideally, an array-based priority queue implementation should have
//  fast dequeues and slow enqueues. I fear the vector.Vector class
//  gives slow equeues and slow dequeues.
type VectorPriorityQueue struct {
    v *vector.Vector
}

func NewVectorPriorityQueue() *VectorPriorityQueue {
    var vpq = new(VectorPriorityQueue)
    vpq.v = new(vector.Vector)
    return vpq
}

func (vpq *VectorPriorityQueue) Len() int {
    return vpq.v.Len()
}
type etypeStopIter struct {
}
func (e etypeStopIter) String() string {
    return "STOPITER"
}
func (vpq *VectorPriorityQueue) Enqueue(task RegisteredTask) {
    switch task.Task().(type) {
    case PrioritizedTask:
        break
    default:
        panic(fmt.Sprintf("nokey %s", task.Task().Type()))
    }
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                break
            default:
                panic(r)
            }
        }
        vpq.v.Insert(i, task)
    } ()
    vpq.v.Do(func (telm interface{}) {
        if task.Task().(PrioritizedTask).Key() > telm.(RegisteredTask).Task().(PrioritizedTask).Key() {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}
func (vpq *VectorPriorityQueue) Dequeue() RegisteredTask {
    var head = vpq.v.At(0).(RegisteredTask)
    vpq.v.Delete(0)
    return head
}
func (vpq *VectorPriorityQueue) SetKey(id int64, k float64) {
    var i int
    defer func() {
        if r := recover(); r != nil {
            switch r.(type) {
            case etypeStopIter:
                var rtask = vpq.v.At(i).(RegisteredTask)
                vpq.v.Delete(i)
                rtask.Task().(PrioritizedTask).SetKey(k)
                vpq.Enqueue(rtask)
            default:
                panic(r)
            }
        }
    } ()
    vpq.v.Do(func (telm interface{}) {
        if telm.(RegisteredTask).Id() != id {
            i++
        } else {
            panic(etypeStopIter{})
        }
    })
}

type ArrayPriorityQueue struct {
    v          []RegisteredTask
    head, tail int
}

func NewArrayPriorityQueue() *ArrayPriorityQueue {
    var apq = new(ArrayPriorityQueue)
    apq.v = make([]RegisteredTask, 10)
    return apq
}

func (apq *ArrayPriorityQueue) Len() int {
    return apq.tail - apq.head
}

func (apq *ArrayPriorityQueue) Enqueue(task RegisteredTask) {
    var key = task.Task().(PrioritizedTask).Key()
    var insertoffset = registeredTaskSearch(
            apq.v[apq.head:apq.tail],
            func(t RegisteredTask) bool {
                return t.Task().(PrioritizedTask).Key() < key
            })
    if apq.tail != len(apq.v) {
        for j := apq.tail ; j > insertoffset ; j-- {
            apq.v[j] = apq.v[j-1]
        }
        apq.v[apq.head+insertoffset] = task
        apq.tail++
        return
    }
    var newv = apq.v
    if apq.head < len(apq.v)/2 {
        newv = make([]RegisteredTask, 2* len(apq.v))
    }
    var i, j int
    j = 0
    for i = apq.head ; i < apq.head+insertoffset ; i++ {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
    }
    //fmt.Fprintf(os.Stderr, "Length %d index %d\n", len(newv), j)
    newv[insertoffset] = task
    j++
    for ; i < apq.tail ; i++ {
        newv[j] = apq.v[i]
        apq.v[i] = nil
        j++
    }
    apq.v = newv
    apq.head = 0
    apq.tail = j
}

func (apq *ArrayPriorityQueue) Dequeue() RegisteredTask {
    if apq.Len() == 0 {
        panic("empty")
    }
    var task = apq.v[apq.head]
    apq.v[apq.head] = nil
    apq.head++
    return task
}

func (apq *ArrayPriorityQueue) SetKey(id int64, k float64) {
}
<|endoftext|>


<commit_before>package activity

import "github.com/awslabs/aws-sdk-go/gen/swf"

//LongRunningActivityFunc that creates all the coordination channels, starts heartbeating, and calls into
type LongRunningActivityCoordinator struct {
	ToStopActivity        chan struct{}
	ToAckStopActivity     chan struct{}
	ToStopHeartbeating    chan struct{}
	ToAckStopHeartbeating chan struct{}
}

type HandleCoordinatedActivity func(*LongRunningActivityCoordinator, *swf.ActivityTask, interface{})
<commit_msg>flesh out CoordinatedActivityHandler<commit_after>package activity

import (
	"log"
	"time"

	"github.com/awslabs/aws-sdk-go/gen/swf"
	. "github.com/sclasen/swfsm/sugar"
)

//LongRunningActivityFunc that creates all the coordination channels, starts heartbeating, and calls into
type LongRunningActivityCoordinator struct {
	HeartbeatInterval     time.Duration
	ToCancelActivity      chan struct{}
	ToAckCancelActivity   chan struct{}
	ToStopHeartbeating    chan struct{}
	ToAckStopHeartbeating chan struct{}
	HeartbeatErrors       chan error
}

type HandleCoordinatedActivity func(*LongRunningActivityCoordinator, *swf.ActivityTask, interface{})

//NewCoordinatedActivityHandler creates a LongRunningActivityFunc that will build a LongRunningActivityCoordinator and execute your HandleCoordinatedActivity
//
// * heartbeats the activity at the given interval
// * sends any errors heartbeating into the HeartbeatErrors channel, which is buffered by heartbeatErrorThreshold
// * if the send to HeartbeatErrors blocks because the buffer is full and not being consumed the task will eventually timeout.
// * if the heartbeat indicates the task was canceled, send on the ToCancelActivity channel and wait on ToAckCancelActivity channel
// * your HandleCoordinatedActivityHandler is responsible for responding to messages on ToCancelActivity, by stopping, acking the cancel to swf, and sending on ToAckCancel
// * if your HandleCoordinatedActivityHandler wishes to stop heartbeats, send on ToStopHeartbeating and recieve on ToAckStopHeartbeating.

func NewCoordinatedActivityHandler(swfOps SWFOps, heartbeatInterval time.Duration, heartbeatErrorThreshold int, handler HandleCoordinatedActivity) LongRunningActivityHandlerFunc {
	coordinator := &LongRunningActivityCoordinator{
		HeartbeatInterval:     heartbeatInterval,
		HeartbeatErrors:       make(chan error, heartbeatErrorThreshold),
		ToCancelActivity:      make(chan struct{}),
		ToAckCancelActivity:   make(chan struct{}),
		ToStopHeartbeating:    make(chan struct{}),
		ToAckStopHeartbeating: make(chan struct{}),
	}

	return func(activityTask *swf.ActivityTask, input interface{}) {
		go handler(coordinator, activityTask, input)
		for {
			select {
			case <-time.After(heartbeatInterval):
				status, err := swfOps.RecordActivityTaskHeartbeat(&swf.RecordActivityTaskHeartbeatInput{
					TaskToken: activityTask.TaskToken,
				})
				if err != nil {
					log.Printf("workflow-id=%s activity-id=%s activity-id=%s at=heartbeat-error error=%s ", LS(activityTask.WorkflowExecution.WorkflowID), LS(activityTask.ActivityType.Name), LS(activityTask.ActivityID), err.Error())
					coordinator.HeartbeatErrors <- err
				} else {
					if *status.CancelRequested {
						log.Printf("workflow-id=%s activity-id=%s activity-id=%s at=activity-cancel-requested", LS(activityTask.WorkflowExecution.WorkflowID), LS(activityTask.ActivityType.Name), LS(activityTask.ActivityID))
						coordinator.ToCancelActivity <- struct{}{}
						<-coordinator.ToAckCancelActivity
						log.Printf("workflow-id=%s activity-id=%s activity-id=%s at=activity-canceled", LS(activityTask.WorkflowExecution.WorkflowID), LS(activityTask.ActivityType.Name), LS(activityTask.ActivityID))
						return
					}
				}
			case <-coordinator.ToStopHeartbeating:
				log.Printf("workflow-id=%s activity-id=%s activity-id=%s at=stop-heartbeating", LS(activityTask.WorkflowExecution.WorkflowID), LS(activityTask.ActivityType.Name), LS(activityTask.ActivityID))
				coordinator.ToAckStopHeartbeating <- struct{}{}
				log.Printf("workflow-id=%s activity-id=%s activity-id=%s at=ack-stop-heartbeating", LS(activityTask.WorkflowExecution.WorkflowID), LS(activityTask.ActivityType.Name), LS(activityTask.ActivityID))
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package activitystreams

// ItemCollection represents an array of items
type ItemCollection []Item

// Item struct
type Item ObjectOrLink

// GetID returns the ObjectID corresponding to ItemCollection
func (i ItemCollection) GetID() *ObjectID {
	return nil
}

// GetLink returns the empty IRI
func (i ItemCollection) GetLink() IRI {
	return IRI("")
}

// GetType returns the ItemCollection's type
func (i ItemCollection) GetType() ActivityVocabularyType {
	return ActivityVocabularyType("")
}

// IsLink returns false for an ItemCollection object
func (i ItemCollection) IsLink() bool {
	return false
}

// IsObject returns true for a ItemCollection object
func (i ItemCollection) IsObject() bool {
	return false
}

// Append facilitates adding elements to Item arrays
// and ensures ItemCollection implements the Collection interface
func (i *ItemCollection) Append(o Item) error {
	oldLen := len(*i)
	d := make(ItemCollection, oldLen+1)
	for k, it := range *i {
		d[k] = it
	}
	d[oldLen] = o
	*i = d
	return nil
}

// Collection returns the current object as collection interface
func (i *ItemCollection) Collection() CollectionInterface {
	return i
}
<commit_msg>Add First() method for ItemCollection and using it to return the type<commit_after>package activitystreams

// ItemCollection represents an array of items
type ItemCollection []Item

// Item struct
type Item ObjectOrLink

// GetID returns the ObjectID corresponding to ItemCollection
func (i ItemCollection) GetID() *ObjectID {
	return nil
}

// GetLink returns the empty IRI
func (i ItemCollection) GetLink() IRI {
	return IRI("")
}

// GetType returns the ItemCollection's type
func (i ItemCollection) GetType() ActivityVocabularyType {
	return i.First().GetType()
}

// IsLink returns false for an ItemCollection object
func (i ItemCollection) IsLink() bool {
	return false
}

// IsObject returns true for a ItemCollection object
func (i ItemCollection) IsObject() bool {
	return false
}

// Append facilitates adding elements to Item arrays
// and ensures ItemCollection implements the Collection interface
func (i *ItemCollection) Append(o Item) error {
	oldLen := len(*i)
	d := make(ItemCollection, oldLen+1)
	for k, it := range *i {
		d[k] = it
	}
	d[oldLen] = o
	*i = d
	return nil
}

// First returns the ObjectID corresponding to ItemCollection
func (i ItemCollection) First() Item {
	if len(i) == 0 {
		return nil
	}
	return i[0]
}

// Collection returns the current object as collection interface
func (i *ItemCollection) Collection() CollectionInterface {
	return i
}
<|endoftext|>