<commit_before>// +build !windows

package filesystem

import (
	"os"
	"runtime"
)

const (
	// maximumReadContentMetadataWorkers provides an upper-bound on the number
	// of Directory.ReadContentMetadata worker Goroutines. It is chosen somewhat
	// arbitrarily, set high enough to allow parallelism but not so high that
	// performance scaling breaks down or that many-core systems are
	// overwhelmed.
	maximumReadContentMetadataWorkers = 4
)

// batchReadContentMetadataRequest is a request type encoding a batch of
// Directory.ReadContentMetadata operations to be performed serially.
type batchReadContentMetadataRequest struct {
	// directory is the directory object for which content metadata reads should
	// be performed.
	directory *Directory
	// names are the names of entries in the directory for which metadata should
	// be read.
	names []string
	// results is the pre-allocated storage space into which results should be
	// placed. It must have a length equal to name. In the case of entries that
	// are missing (i.e. those for which the fstatat operation returns an
	// os.IsNotExist error), a nil pointer should be stored.
	results []*Metadata
}

// batchReadContentMetadataResponse is a response type encoding the outcome of a
// batchReadContentMetadataRequest.
type batchReadContentMetadataResponse struct {
	// sawMissingEntry indicates whether or not any of the operations in the
	// batch saw an missing entry and set a corresponding nil pointer in the
	// results.
	sawMissingEntry bool
	// fatalError stores any fatal error (i.e. something other than an
	// os.IsNotExist error) that occurred for the batch. In this case, results
	// should be considered invalid.
	fatalError error
}

// handleBatchReadContentMetadataRequests is the entry point for
// ReadContentMetadata worker Goroutines. It processes requests until the
// requests channel is closed, at which point it closes its responses channel.
func handleBatchReadContentMetadataRequests(
	requests chan batchReadContentMetadataRequest,
	responses chan batchReadContentMetadataResponse,
) {
	// Loop until the requests channel is closed.
	for request := range requests {
		// Track missing entries and errors.
		var sawMissingEntry bool
		var fatalError error

		// Process each name in the request.
		for n, name := range request.names {
			if m, err := request.directory.ReadContentMetadata(name); err != nil {
				if os.IsNotExist(err) {
					request.results[n] = nil
					sawMissingEntry = true
					continue
				} else {
					fatalError = err
					break
				}
			} else {
				request.results[n] = m
			}
		}

		// Send a response.
		responses <- batchReadContentMetadataResponse{
			sawMissingEntry: sawMissingEntry,
			fatalError:      fatalError,
		}
	}

	// Indicate completion by closing our responses channel.
	close(responses)
}

// parallelReadContentMetadataRequest is a request type encoding a batch of
// Directory.ReadContentMetadata operations to be performed in parallel.
type parallelReadContentMetadataRequest struct {
	// directory is the directory object for which content metadata reads should
	// be performed.
	directory *Directory
	// names are the names of entries in the directory for which metadata should
	// be read.
	names []string
}

// parallelReadContentMetadataResponse is a response type encoding the outcome
// of a parallelReadContentMetadataRequest.
type parallelReadContentMetadataResponse struct {
	// results are the results for the request. They may be fewer than the names
	// in the corresponding request since entries which don't exist are ignored.
	results []*Metadata
	// fatalError stores any fatal error (i.e. something other than an
	// os.IsNotExist error) that occurred when processing the request. In this
	// case, results should be considered invalid.
	fatalError error
}

// parallelReadContentMetadataRequests is a queue for parallel
// Directory.ReadContentMetadata operations. Once a write to this channel
// succeeds, the writer knows that it should expect a response on
// parallelReadContentMetadataResponses.
var parallelReadContentMetadataRequests = make(chan parallelReadContentMetadataRequest)

// parallelReadContentMetadataResponses is the result channel for requests
// submitted to parallelReadContentMetadataRequests.
var parallelReadContentMetadataResponses = make(chan parallelReadContentMetadataResponse)

// handleParallelReadContentMetadataRequests is the entry point for parallel
// ReadContentMetadata request handling.
func handleParallelReadContentMetadataRequests() {
	// Compute the worker count.
	workerCount := runtime.NumCPU()
	if workerCount > maximumReadContentMetadataWorkers {
		workerCount = maximumReadContentMetadataWorkers
	}

	// Create request/response channels and start workers.
	batchRequestQueues := make([]chan batchReadContentMetadataRequest, workerCount)
	batchResponseQueues := make([]chan batchReadContentMetadataResponse, workerCount)
	for w := 0; w < workerCount; w++ {
		batchRequestQueues[w] = make(chan batchReadContentMetadataRequest, 1)
		batchResponseQueues[w] = make(chan batchReadContentMetadataResponse, 1)
		go handleBatchReadContentMetadataRequests(
			batchRequestQueues[w],
			batchResponseQueues[w],
		)
	}

	// Handle requests indefinitely.
	for request := range parallelReadContentMetadataRequests {
		// Allocate result storage.
		results := make([]*Metadata, len(request.names))

		// Compute the batch size. The additive term in the numerator of this
		// formula ensures that batchSize * workerCount >= len(names), while
		// still giving ~equal distribution across workers. Because of this
		// additive term, we have to check explicitly in the dispatch loop below
		// that we don't exceed the boundaries of the name slice.
		batchSize := (len(request.names) + (workerCount - 1)) / workerCount

		// Dispatch batch requests and track how many workers we use.
		workersUsed := 0
		for batchStart := 0; batchStart < len(request.names); batchStart += batchSize {
			batchStop := batchStart + batchSize
			if batchStop > len(request.names) {
				batchStop = len(request.names)
			}
			batchRequestQueues[workersUsed] <- batchReadContentMetadataRequest{
				directory: request.directory,
				names:     request.names[batchStart:batchStop],
				results:   results[batchStart:batchStop],
			}
			workersUsed++
		}

		// Wait for responses from all of the workers that we used.
		var fatalError error
		var sawMissingEntry bool
		for _, responseQueue := range batchResponseQueues[:workersUsed] {
			response := <-responseQueue
			sawMissingEntry = sawMissingEntry || response.sawMissingEntry
			if response.fatalError != nil && fatalError == nil {
				fatalError = response.fatalError
			}
		}

		// Handle the error case.
		if fatalError != nil {
			parallelReadContentMetadataResponses <- parallelReadContentMetadataResponse{
				fatalError: fatalError,
			}
			continue
		}

		// If missing entries were detected, shift them out of the slice.
		if sawMissingEntry {
			filteredResults := results[:0]
			for _, r := range results {
				if r != nil {
					filteredResults = append(filteredResults, r)
				}
			}
			results = filteredResults
		}

		// Success.
		parallelReadContentMetadataResponses <- parallelReadContentMetadataResponse{
			results: results,
		}
	}
}

// parallelMetadataDisabled tracks whether or not parallel metadata query
// operations have been disabled.
var parallelMetadataDisabled bool

func init() {
	// Disable parallel metadata query operations if this platform isn't
	// whitelisted for support. For now this whitelist is just the set of POSIX
	// platforms for which we run tests, because POSIX doesn't explicitly
	// guarantee that fstatat is safe for concurrent invocation on the same file
	// descriptor and we require confirmation of this behavior through
	// observation. We'll expand this list as we bring more builders online and
	// this code has more shakedown time.
	parallelMetadataDisabled = !(runtime.GOOS == "linux" || runtime.GOOS == "darwin")

	// Disable parallel metadata query operations if we don't have multiple
	// CPUs, because for a single-core system it only adds overhead.
	if runtime.NumCPU() < 2 {
		parallelMetadataDisabled = true
	}

	// If parallel metadata query operations aren't disabled, then start the
	// worker Goroutines.
	if !parallelMetadataDisabled {
		go handleParallelReadContentMetadataRequests()
	}
}
<commit_msg>Temporarily disabled parallel metadata query operations.<commit_after>// +build !windows

package filesystem

import (
	"os"
	"runtime"
)

const (
	// maximumReadContentMetadataWorkers provides an upper-bound on the number
	// of Directory.ReadContentMetadata worker Goroutines. It is chosen somewhat
	// arbitrarily, set high enough to allow parallelism but not so high that
	// performance scaling breaks down or that many-core systems are
	// overwhelmed.
	maximumReadContentMetadataWorkers = 4
)

// batchReadContentMetadataRequest is a request type encoding a batch of
// Directory.ReadContentMetadata operations to be performed serially.
type batchReadContentMetadataRequest struct {
	// directory is the directory object for which content metadata reads should
	// be performed.
	directory *Directory
	// names are the names of entries in the directory for which metadata should
	// be read.
	names []string
	// results is the pre-allocated storage space into which results should be
	// placed. It must have a length equal to name. In the case of entries that
	// are missing (i.e. those for which the fstatat operation returns an
	// os.IsNotExist error), a nil pointer should be stored.
	results []*Metadata
}

// batchReadContentMetadataResponse is a response type encoding the outcome of a
// batchReadContentMetadataRequest.
type batchReadContentMetadataResponse struct {
	// sawMissingEntry indicates whether or not any of the operations in the
	// batch saw an missing entry and set a corresponding nil pointer in the
	// results.
	sawMissingEntry bool
	// fatalError stores any fatal error (i.e. something other than an
	// os.IsNotExist error) that occurred for the batch. In this case, results
	// should be considered invalid.
	fatalError error
}

// handleBatchReadContentMetadataRequests is the entry point for
// ReadContentMetadata worker Goroutines. It processes requests until the
// requests channel is closed, at which point it closes its responses channel.
func handleBatchReadContentMetadataRequests(
	requests chan batchReadContentMetadataRequest,
	responses chan batchReadContentMetadataResponse,
) {
	// Loop until the requests channel is closed.
	for request := range requests {
		// Track missing entries and errors.
		var sawMissingEntry bool
		var fatalError error

		// Process each name in the request.
		for n, name := range request.names {
			if m, err := request.directory.ReadContentMetadata(name); err != nil {
				if os.IsNotExist(err) {
					request.results[n] = nil
					sawMissingEntry = true
					continue
				} else {
					fatalError = err
					break
				}
			} else {
				request.results[n] = m
			}
		}

		// Send a response.
		responses <- batchReadContentMetadataResponse{
			sawMissingEntry: sawMissingEntry,
			fatalError:      fatalError,
		}
	}

	// Indicate completion by closing our responses channel.
	close(responses)
}

// parallelReadContentMetadataRequest is a request type encoding a batch of
// Directory.ReadContentMetadata operations to be performed in parallel.
type parallelReadContentMetadataRequest struct {
	// directory is the directory object for which content metadata reads should
	// be performed.
	directory *Directory
	// names are the names of entries in the directory for which metadata should
	// be read.
	names []string
}

// parallelReadContentMetadataResponse is a response type encoding the outcome
// of a parallelReadContentMetadataRequest.
type parallelReadContentMetadataResponse struct {
	// results are the results for the request. They may be fewer than the names
	// in the corresponding request since entries which don't exist are ignored.
	results []*Metadata
	// fatalError stores any fatal error (i.e. something other than an
	// os.IsNotExist error) that occurred when processing the request. In this
	// case, results should be considered invalid.
	fatalError error
}

// parallelReadContentMetadataRequests is a queue for parallel
// Directory.ReadContentMetadata operations. Once a write to this channel
// succeeds, the writer knows that it should expect a response on
// parallelReadContentMetadataResponses.
var parallelReadContentMetadataRequests = make(chan parallelReadContentMetadataRequest)

// parallelReadContentMetadataResponses is the result channel for requests
// submitted to parallelReadContentMetadataRequests.
var parallelReadContentMetadataResponses = make(chan parallelReadContentMetadataResponse)

// handleParallelReadContentMetadataRequests is the entry point for parallel
// ReadContentMetadata request handling.
func handleParallelReadContentMetadataRequests() {
	// Compute the worker count.
	workerCount := runtime.NumCPU()
	if workerCount > maximumReadContentMetadataWorkers {
		workerCount = maximumReadContentMetadataWorkers
	}

	// Create request/response channels and start workers.
	batchRequestQueues := make([]chan batchReadContentMetadataRequest, workerCount)
	batchResponseQueues := make([]chan batchReadContentMetadataResponse, workerCount)
	for w := 0; w < workerCount; w++ {
		batchRequestQueues[w] = make(chan batchReadContentMetadataRequest, 1)
		batchResponseQueues[w] = make(chan batchReadContentMetadataResponse, 1)
		go handleBatchReadContentMetadataRequests(
			batchRequestQueues[w],
			batchResponseQueues[w],
		)
	}

	// Handle requests indefinitely.
	for request := range parallelReadContentMetadataRequests {
		// Allocate result storage.
		results := make([]*Metadata, len(request.names))

		// Compute the batch size. The additive term in the numerator of this
		// formula ensures that batchSize * workerCount >= len(names), while
		// still giving ~equal distribution across workers. Because of this
		// additive term, we have to check explicitly in the dispatch loop below
		// that we don't exceed the boundaries of the name slice.
		batchSize := (len(request.names) + (workerCount - 1)) / workerCount

		// Dispatch batch requests and track how many workers we use.
		workersUsed := 0
		for batchStart := 0; batchStart < len(request.names); batchStart += batchSize {
			batchStop := batchStart + batchSize
			if batchStop > len(request.names) {
				batchStop = len(request.names)
			}
			batchRequestQueues[workersUsed] <- batchReadContentMetadataRequest{
				directory: request.directory,
				names:     request.names[batchStart:batchStop],
				results:   results[batchStart:batchStop],
			}
			workersUsed++
		}

		// Wait for responses from all of the workers that we used.
		var fatalError error
		var sawMissingEntry bool
		for _, responseQueue := range batchResponseQueues[:workersUsed] {
			response := <-responseQueue
			sawMissingEntry = sawMissingEntry || response.sawMissingEntry
			if response.fatalError != nil && fatalError == nil {
				fatalError = response.fatalError
			}
		}

		// Handle the error case.
		if fatalError != nil {
			parallelReadContentMetadataResponses <- parallelReadContentMetadataResponse{
				fatalError: fatalError,
			}
			continue
		}

		// If missing entries were detected, shift them out of the slice.
		if sawMissingEntry {
			filteredResults := results[:0]
			for _, r := range results {
				if r != nil {
					filteredResults = append(filteredResults, r)
				}
			}
			results = filteredResults
		}

		// Success.
		parallelReadContentMetadataResponses <- parallelReadContentMetadataResponse{
			results: results,
		}
	}
}

// parallelMetadataDisabled tracks whether or not parallel metadata query
// operations have been disabled.
var parallelMetadataDisabled bool

func init() {
	// Disable parallel metadata query operations if this platform isn't
	// whitelisted for support. For now this whitelist is just the set of POSIX
	// platforms for which we run tests, because POSIX doesn't explicitly
	// guarantee that fstatat is safe for concurrent invocation on the same file
	// descriptor and we require confirmation of this behavior through
	// observation. We'll expand this list as we bring more builders online and
	// this code has more shakedown time.
	parallelMetadataDisabled = !(runtime.GOOS == "linux" || runtime.GOOS == "darwin")

	// Disable parallel metadata query operations if we don't have multiple
	// CPUs, because for a single-core system it only adds overhead.
	if runtime.NumCPU() < 2 {
		parallelMetadataDisabled = true
	}

	// Temporarily disable parallel metadata query operations on all platforms
	// until later in the v0.9.0 release cycle.
	parallelMetadataDisabled = true

	// If parallel metadata query operations aren't disabled, then start the
	// worker Goroutines.
	if !parallelMetadataDisabled {
		go handleParallelReadContentMetadataRequests()
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
	"fmt"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

// PolicyStatic is the name of the static policy
const PolicyStatic policyName = "static"

var _ Policy = &staticPolicy{}

// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
//
// This policy allocates CPUs exclusively for a container if all the following
// conditions are met:
//
// - The pod QoS class is Guaranteed.
// - The CPU request is a positive integer.
//
// The static policy maintains the following sets of logical CPUs:
//
// - SHARED: Burstable, BestEffort, and non-integral Guaranteed containers
//   run here. Initially this contains all CPU IDs on the system. As
//   exclusive allocations are created and destroyed, this CPU set shrinks
//   and grows, accordingly. This is stored in the state as the default
//   CPU set.
//
// - RESERVED: A subset of the shared pool which is not exclusively
//   allocatable. The membership of this pool is static for the lifetime of
//   the Kubelet. The size of the reserved pool is
//   ceil(systemreserved.cpu + kubereserved.cpu).
//   Reserved CPUs are taken topologically starting with lowest-indexed
//   physical core, as reported by cAdvisor.
//
// - ASSIGNABLE: Equal to SHARED - RESERVED. Exclusive CPUs are allocated
//   from this pool.
//
// - EXCLUSIVE ALLOCATIONS: CPU sets assigned exclusively to one container.
//   These are stored as explicit assignments in the state.
//
// When an exclusive allocation is made, the static policy also updates the
// default cpuset in the state abstraction. The CPU manager's periodic
// reconcile loop takes care of rewriting the cpuset in cgroupfs for any
// containers that may be running in the shared pool. For this reason,
// applications running within exclusively-allocated containers must tolerate
// potentially sharing their allocated CPUs for up to the CPU manager
// reconcile period.
type staticPolicy struct {
	// cpu socket topology
	topology *topology.CPUTopology
	// set of CPUs that is not available for exclusive assignment
	reserved cpuset.CPUSet
}

// Ensure staticPolicy implements Policy interface
var _ Policy = &staticPolicy{}

// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int) Policy {
	allCPUs := topology.CPUDetails.CPUs()
	// takeByTopology allocates CPUs associated with low-numbered cores from
	// allCPUs.
	//
	// For example: Given a system with 8 CPUs available and HT enabled,
	// if numReservedCPUs=2, then reserved={0,4}
	reserved, _ := takeByTopology(topology, allCPUs, numReservedCPUs)

	if reserved.Size() != numReservedCPUs {
		panic(fmt.Sprintf("[cpumanager] unable to reserve the required amount of CPUs (size of %s did not equal %d)", reserved, numReservedCPUs))
	}

	glog.Infof("[cpumanager] reserved %d CPUs (\"%s\") not available for exclusive assignment", reserved.Size(), reserved)

	return &staticPolicy{
		topology: topology,
		reserved: reserved,
	}
}

func (p *staticPolicy) Name() string {
	return string(PolicyStatic)
}

func (p *staticPolicy) Start(s state.State) {
	if err := p.validateState(s); err != nil {
		glog.Errorf("[cpumanager] static policy invalid state: %s\n", err.Error())
		panic("[cpumanager] - please drain node and remove policy state file")
	}
}

func (p *staticPolicy) validateState(s state.State) error {
	tmpAssignments := s.GetCPUAssignments()
	tmpDefaultCPUset := s.GetDefaultCPUSet()

	// Default cpuset cannot be empty when assignments exist
	if tmpDefaultCPUset.IsEmpty() {
		if len(tmpAssignments) != 0 {
			return fmt.Errorf("default cpuset cannot be empty")
		}
		// state is empty initialize
		allCPUs := p.topology.CPUDetails.CPUs()
		s.SetDefaultCPUSet(allCPUs)
		return nil
	}

	// State has already been initialized from file (is not empty)
	// 1 Check if the reserved cpuset is not part of default cpuset because:
	// - kube/system reserved have changed (increased) - may lead to some containers not being able to start
	// - user tampered with file
	if !p.reserved.Intersection(tmpDefaultCPUset).Equals(p.reserved) {
		return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
			p.reserved.String(), tmpDefaultCPUset.String())
	}

	// 2. Check if state for static policy is consistent
	for cID, cset := range tmpAssignments {
		// None of the cpu in DEFAULT cset should be in s.assignments
		if !tmpDefaultCPUset.Intersection(cset).IsEmpty() {
			return fmt.Errorf("container id: %s cpuset: \"%s\" overlaps with default cpuset \"%s\"",
				cID, cset.String(), tmpDefaultCPUset.String())
		}
	}
	return nil
}

// assignableCPUs returns the set of unassigned CPUs minus the reserved set.
func (p *staticPolicy) assignableCPUs(s state.State) cpuset.CPUSet {
	return s.GetDefaultCPUSet().Difference(p.reserved)
}

func (p *staticPolicy) AddContainer(s state.State, pod *v1.Pod, container *v1.Container, containerID string) error {
	if numCPUs := guaranteedCPUs(pod, container); numCPUs != 0 {
		glog.Infof("[cpumanager] static policy: AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID)
		// container belongs in an exclusively allocated pool

		if _, ok := s.GetCPUSet(containerID); ok {
			glog.Infof("[cpumanager] static policy: container already present in state, skipping (container: %s, container id: %s)", container.Name, containerID)
			return nil
		}

		cpuset, err := p.allocateCPUs(s, numCPUs)
		if err != nil {
			glog.Errorf("[cpumanager] unable to allocate %d CPUs (container id: %s, error: %v)", numCPUs, containerID, err)
			return err
		}
		s.SetCPUSet(containerID, cpuset)
	}
	// container belongs in the shared pool (nothing to do; use default cpuset)
	return nil
}

func (p *staticPolicy) RemoveContainer(s state.State, containerID string) error {
	glog.Infof("[cpumanager] static policy: RemoveContainer (container id: %s)", containerID)
	if toRelease, ok := s.GetCPUSet(containerID); ok {
		s.Delete(containerID)
		// Mutate the shared pool, adding released cpus.
		s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(toRelease))
	}
	return nil
}

func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int) (cpuset.CPUSet, error) {
	glog.Infof("[cpumanager] allocateCpus: (numCPUs: %d)", numCPUs)
	result, err := takeByTopology(p.topology, p.assignableCPUs(s), numCPUs)
	if err != nil {
		return cpuset.NewCPUSet(), err
	}
	// Remove allocated CPUs from the shared CPUSet.
	s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result))

	glog.Infof("[cpumanager] allocateCPUs: returning \"%v\"", result)
	return result, nil
}

func guaranteedCPUs(pod *v1.Pod, container *v1.Container) int {
	if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed {
		return 0
	}
	cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
	if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
		return 0
	}
	// Safe downcast to do for all systems with < 2.1 billion CPUs.
	// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
	// https://golang.org/ref/spec#Numeric_types
	return int(cpuQuantity.Value())
}
<commit_msg>Remove duplicate check line<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
	"fmt"

	"github.com/golang/glog"
	"k8s.io/api/core/v1"
	v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

// PolicyStatic is the name of the static policy
const PolicyStatic policyName = "static"

// staticPolicy is a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
//
// This policy allocates CPUs exclusively for a container if all the following
// conditions are met:
//
// - The pod QoS class is Guaranteed.
// - The CPU request is a positive integer.
//
// The static policy maintains the following sets of logical CPUs:
//
// - SHARED: Burstable, BestEffort, and non-integral Guaranteed containers
//   run here. Initially this contains all CPU IDs on the system. As
//   exclusive allocations are created and destroyed, this CPU set shrinks
//   and grows, accordingly. This is stored in the state as the default
//   CPU set.
//
// - RESERVED: A subset of the shared pool which is not exclusively
//   allocatable. The membership of this pool is static for the lifetime of
//   the Kubelet. The size of the reserved pool is
//   ceil(systemreserved.cpu + kubereserved.cpu).
//   Reserved CPUs are taken topologically starting with lowest-indexed
//   physical core, as reported by cAdvisor.
//
// - ASSIGNABLE: Equal to SHARED - RESERVED. Exclusive CPUs are allocated
//   from this pool.
//
// - EXCLUSIVE ALLOCATIONS: CPU sets assigned exclusively to one container.
//   These are stored as explicit assignments in the state.
//
// When an exclusive allocation is made, the static policy also updates the
// default cpuset in the state abstraction. The CPU manager's periodic
// reconcile loop takes care of rewriting the cpuset in cgroupfs for any
// containers that may be running in the shared pool. For this reason,
// applications running within exclusively-allocated containers must tolerate
// potentially sharing their allocated CPUs for up to the CPU manager
// reconcile period.
type staticPolicy struct {
	// cpu socket topology
	topology *topology.CPUTopology
	// set of CPUs that is not available for exclusive assignment
	reserved cpuset.CPUSet
}

// Ensure staticPolicy implements Policy interface
var _ Policy = &staticPolicy{}

// NewStaticPolicy returns a CPU manager policy that does not change CPU
// assignments for exclusively pinned guaranteed containers after the main
// container process starts.
func NewStaticPolicy(topology *topology.CPUTopology, numReservedCPUs int) Policy {
	allCPUs := topology.CPUDetails.CPUs()
	// takeByTopology allocates CPUs associated with low-numbered cores from
	// allCPUs.
	//
	// For example: Given a system with 8 CPUs available and HT enabled,
	// if numReservedCPUs=2, then reserved={0,4}
	reserved, _ := takeByTopology(topology, allCPUs, numReservedCPUs)

	if reserved.Size() != numReservedCPUs {
		panic(fmt.Sprintf("[cpumanager] unable to reserve the required amount of CPUs (size of %s did not equal %d)", reserved, numReservedCPUs))
	}

	glog.Infof("[cpumanager] reserved %d CPUs (\"%s\") not available for exclusive assignment", reserved.Size(), reserved)

	return &staticPolicy{
		topology: topology,
		reserved: reserved,
	}
}

func (p *staticPolicy) Name() string {
	return string(PolicyStatic)
}

func (p *staticPolicy) Start(s state.State) {
	if err := p.validateState(s); err != nil {
		glog.Errorf("[cpumanager] static policy invalid state: %s\n", err.Error())
		panic("[cpumanager] - please drain node and remove policy state file")
	}
}

func (p *staticPolicy) validateState(s state.State) error {
	tmpAssignments := s.GetCPUAssignments()
	tmpDefaultCPUset := s.GetDefaultCPUSet()

	// Default cpuset cannot be empty when assignments exist
	if tmpDefaultCPUset.IsEmpty() {
		if len(tmpAssignments) != 0 {
			return fmt.Errorf("default cpuset cannot be empty")
		}
		// state is empty initialize
		allCPUs := p.topology.CPUDetails.CPUs()
		s.SetDefaultCPUSet(allCPUs)
		return nil
	}

	// State has already been initialized from file (is not empty)
	// 1 Check if the reserved cpuset is not part of default cpuset because:
	// - kube/system reserved have changed (increased) - may lead to some containers not being able to start
	// - user tampered with file
	if !p.reserved.Intersection(tmpDefaultCPUset).Equals(p.reserved) {
		return fmt.Errorf("not all reserved cpus: \"%s\" are present in defaultCpuSet: \"%s\"",
			p.reserved.String(), tmpDefaultCPUset.String())
	}

	// 2. Check if state for static policy is consistent
	for cID, cset := range tmpAssignments {
		// None of the cpu in DEFAULT cset should be in s.assignments
		if !tmpDefaultCPUset.Intersection(cset).IsEmpty() {
			return fmt.Errorf("container id: %s cpuset: \"%s\" overlaps with default cpuset \"%s\"",
				cID, cset.String(), tmpDefaultCPUset.String())
		}
	}
	return nil
}

// assignableCPUs returns the set of unassigned CPUs minus the reserved set.
func (p *staticPolicy) assignableCPUs(s state.State) cpuset.CPUSet {
	return s.GetDefaultCPUSet().Difference(p.reserved)
}

func (p *staticPolicy) AddContainer(s state.State, pod *v1.Pod, container *v1.Container, containerID string) error {
	if numCPUs := guaranteedCPUs(pod, container); numCPUs != 0 {
		glog.Infof("[cpumanager] static policy: AddContainer (pod: %s, container: %s, container id: %s)", pod.Name, container.Name, containerID)
		// container belongs in an exclusively allocated pool

		if _, ok := s.GetCPUSet(containerID); ok {
			glog.Infof("[cpumanager] static policy: container already present in state, skipping (container: %s, container id: %s)", container.Name, containerID)
			return nil
		}

		cpuset, err := p.allocateCPUs(s, numCPUs)
		if err != nil {
			glog.Errorf("[cpumanager] unable to allocate %d CPUs (container id: %s, error: %v)", numCPUs, containerID, err)
			return err
		}
		s.SetCPUSet(containerID, cpuset)
	}
	// container belongs in the shared pool (nothing to do; use default cpuset)
	return nil
}

func (p *staticPolicy) RemoveContainer(s state.State, containerID string) error {
	glog.Infof("[cpumanager] static policy: RemoveContainer (container id: %s)", containerID)
	if toRelease, ok := s.GetCPUSet(containerID); ok {
		s.Delete(containerID)
		// Mutate the shared pool, adding released cpus.
		s.SetDefaultCPUSet(s.GetDefaultCPUSet().Union(toRelease))
	}
	return nil
}

func (p *staticPolicy) allocateCPUs(s state.State, numCPUs int) (cpuset.CPUSet, error) {
	glog.Infof("[cpumanager] allocateCpus: (numCPUs: %d)", numCPUs)
	result, err := takeByTopology(p.topology, p.assignableCPUs(s), numCPUs)
	if err != nil {
		return cpuset.NewCPUSet(), err
	}
	// Remove allocated CPUs from the shared CPUSet.
	s.SetDefaultCPUSet(s.GetDefaultCPUSet().Difference(result))

	glog.Infof("[cpumanager] allocateCPUs: returning \"%v\"", result)
	return result, nil
}

func guaranteedCPUs(pod *v1.Pod, container *v1.Container) int {
	if v1qos.GetPodQOS(pod) != v1.PodQOSGuaranteed {
		return 0
	}
	cpuQuantity := container.Resources.Requests[v1.ResourceCPU]
	if cpuQuantity.Value()*1000 != cpuQuantity.MilliValue() {
		return 0
	}
	// Safe downcast to do for all systems with < 2.1 billion CPUs.
	// Per the language spec, `int` is guaranteed to be at least 32 bits wide.
	// https://golang.org/ref/spec#Numeric_types
	return int(cpuQuantity.Value())
}
<|endoftext|>


<commit_before>package v5

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
)

const (
	elasticsearchConfigSubDir = "elasticsearch/config"
)

func (p *Pilot) WriteConfig(pilot *v1alpha1.Pilot) error {
	esConfigPath := fmt.Sprintf("%s/%s", p.Options.ConfigDir, elasticsearchConfigSubDir)
	err := filepath.Walk(esConfigPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		relPath, err := filepath.Rel(esConfigPath, path)
		if err != nil {
			return err
		}
		dstPath := fmt.Sprintf("%s/%s", p.Options.ElasticsearchOptions.ConfigDir, relPath)
		relDir := filepath.Dir(relPath)
		stat, err := os.Stat(relDir)
		if os.IsNotExist(err) {
			err = os.MkdirAll(relDir, 0644)
			if err != nil {
				return err
			}
			stat, err = os.Stat(relDir)
		}
		if err != nil {
			return err
		}
		if !stat.IsDir() {
			return fmt.Errorf("path '%s' exists and is not a directory", relDir)
		}
		if err = copyFileContents(path, dstPath); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error writing config file: %s", err.Error())
	}

	return nil
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
// From: https://stackoverflow.com/a/21067803
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}
<commit_msg>Fix writeconfig filepath walk<commit_after>package v5

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
)

const (
	elasticsearchConfigSubDir = "elasticsearch/config"
)

func (p *Pilot) WriteConfig(pilot *v1alpha1.Pilot) error {
	esConfigPath := fmt.Sprintf("%s/%s", p.Options.ConfigDir, elasticsearchConfigSubDir)
	err := filepath.Walk(esConfigPath, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			return nil
		}
		if err != nil {
			return err
		}
		relPath, err := filepath.Rel(esConfigPath, path)
		if err != nil {
			return err
		}
		dstPath := fmt.Sprintf("%s/%s", p.Options.ElasticsearchOptions.ConfigDir, relPath)
		relDir := filepath.Dir(relPath)
		stat, err := os.Stat(relDir)
		if os.IsNotExist(err) {
			err = os.MkdirAll(relDir, 0644)
			if err != nil {
				return err
			}
			stat, err = os.Stat(relDir)
		}
		if err != nil {
			return err
		}
		if !stat.IsDir() {
			return fmt.Errorf("path '%s' exists and is not a directory", relDir)
		}
		if err = copyFileContents(path, dstPath); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error writing config file: %s", err.Error())
	}

	return nil
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
// From: https://stackoverflow.com/a/21067803
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}
<|endoftext|>


<commit_before>package provider

import (
	"encoding/json"
	"net/http"

	"context"

	"github.com/go-kit/kit/log"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
)

// Request // Response types
type loginRequest struct {
	Username string `json:"username"`
	Password string `json:"password"`
}

type loginResponse struct {
	User *User `json:"user"`
}

// MakeHTTPHandler mounts all of the service endpoints into an http.Handler.
// Useful in a profilesvc server.
func MakeHTTPHandler(ctx context.Context, s Service, logger log.Logger) http.Handler {
	r := mux.NewRouter()
	options := []httptransport.ServerOption{
		httptransport.ServerErrorLogger(logger),
		httptransport.ServerErrorEncoder(encodeError),
	}
	e := MakeServerEndpoints(s)

	r.Methods("POST").Path("/users/login/{id}").Handler(httptransport.NewServer(
		e.LoginEndpoint,
		decodeUserRequest,
		encodeResponse,
		options...,
	))

	return r
}

type errorer interface {
	error() error
}

func encodeResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if e, ok := response.(errorer); ok && e.error() != nil {
		encodeError(ctx, e.error(), w)
		return nil
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("X-Api-Correlation-Id", "1234")
	return json.NewEncoder(w).Encode(response)
}

func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("X-Api-Correlation-Id", "1234")
	w.WriteHeader(codeFrom(err))
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error": err.Error(),
	})
}

func codeFrom(err error) int {
	switch err {
	case ErrNotFound:
		return http.StatusNotFound
	case ErrNotFound:
		return http.StatusNotFound
	case ErrUnauthorized:
		return http.StatusUnauthorized
	default:
		return http.StatusInternalServerError
	}
	// if e, ok := err.(httptransport.Error); ok {
	// 	switch e.Err {domaind
	// 	switch e.Domain {
	// 	case httptransport.DomainDecode:
	// 		return http.StatusBadRequest
	// 	case httptransport.DomainDo:
	// 		return http.StatusServiceUnavailable
	// 	default:
	// 		return http.StatusInternalServerError
	// 	}
	// }
	// return http.StatusInternalServerError
}
<commit_msg>remove duplicated switch case<commit_after>package provider

import (
	"encoding/json"
	"net/http"

	"context"

	"github.com/go-kit/kit/log"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
)

// Request // Response types
type loginRequest struct {
	Username string `json:"username"`
	Password string `json:"password"`
}

type loginResponse struct {
	User *User `json:"user"`
}

// MakeHTTPHandler mounts all of the service endpoints into an http.Handler.
// Useful in a profilesvc server.
func MakeHTTPHandler(ctx context.Context, s Service, logger log.Logger) http.Handler {
	r := mux.NewRouter()
	options := []httptransport.ServerOption{
		httptransport.ServerErrorLogger(logger),
		httptransport.ServerErrorEncoder(encodeError),
	}
	e := MakeServerEndpoints(s)

	r.Methods("POST").Path("/users/login/{id}").Handler(httptransport.NewServer(
		e.LoginEndpoint,
		decodeUserRequest,
		encodeResponse,
		options...,
	))

	return r
}

type errorer interface {
	error() error
}

func encodeResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if e, ok := response.(errorer); ok && e.error() != nil {
		encodeError(ctx, e.error(), w)
		return nil
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("X-Api-Correlation-Id", "1234")
	return json.NewEncoder(w).Encode(response)
}

func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("X-Api-Correlation-Id", "1234")
	w.WriteHeader(codeFrom(err))
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error": err.Error(),
	})
}

func codeFrom(err error) int {
	switch err {
	case ErrNotFound:
		return http.StatusNotFound
	case ErrUnauthorized:
		return http.StatusUnauthorized
	default:
		return http.StatusInternalServerError
	}
	// if e, ok := err.(httptransport.Error); ok {
	// 	switch e.Err {domaind
	// 	switch e.Domain {
	// 	case httptransport.DomainDecode:
	// 		return http.StatusBadRequest
	// 	case httptransport.DomainDo:
	// 		return http.StatusServiceUnavailable
	// 	default:
	// 		return http.StatusInternalServerError
	// 	}
	// }
	// return http.StatusInternalServerError
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Package appengine provides basic functionality for Google App Engine.
//
// For more information on how to write Go apps for Google App Engine, see:
// https://developers.google.com/appengine/docs/go/
package appengine

import (
	"code.google.com/p/goprotobuf/proto"
	"github.com/golang/appengine/internal"
)

// IsDevAppServer reports whether the App Engine app is running in the
// development App Server.
func IsDevAppServer() bool {
	// TODO(dsymonds): Detect this.
	return false
}

// Context represents the context of an in-flight HTTP request.
type Context interface {
	// Debugf formats its arguments according to the format, analogous to fmt.Printf,
	// and records the text as a log message at Debug level.
	Debugf(format string, args ...interface{})

	// Infof is like Debugf, but at Info level.
	Infof(format string, args ...interface{})

	// Warningf is like Debugf, but at Warning level.
	Warningf(format string, args ...interface{})

	// Errorf is like Debugf, but at Error level.
	Errorf(format string, args ...interface{})

	// Criticalf is like Debugf, but at Critical level.
	Criticalf(format string, args ...interface{})

	// The remaining methods are for internal use only.
	// Developer-facing APIs wrap these methods to provide a more friendly API.

	// Internal use only.
	Call(service, method string, in, out proto.Message, opts *internal.CallOptions) error
	// Internal use only. Use AppID instead.
	FullyQualifiedAppID() string
	// Internal use only.
	Request() interface{}
}

// BlobKey is a key for a blobstore blob.
//
// Conceptually, this type belongs in the blobstore package, but it lives in
// the appengine package to avoid a circular dependency: blobstore depends on
// datastore, and datastore needs to refer to the BlobKey type.
type BlobKey string
<commit_msg>Add appengine.NewContext.<commit_after>// Copyright 2011 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Package appengine provides basic functionality for Google App Engine.
//
// For more information on how to write Go apps for Google App Engine, see:
// https://developers.google.com/appengine/docs/go/
package appengine

import (
	"net/http"

	"code.google.com/p/goprotobuf/proto"
	"github.com/golang/appengine/internal"
)

// IsDevAppServer reports whether the App Engine app is running in the
// development App Server.
func IsDevAppServer() bool {
	// TODO(dsymonds): Detect this.
	return false
}

// Context represents the context of an in-flight HTTP request.
type Context interface {
	// Debugf formats its arguments according to the format, analogous to fmt.Printf,
	// and records the text as a log message at Debug level.
	Debugf(format string, args ...interface{})

	// Infof is like Debugf, but at Info level.
	Infof(format string, args ...interface{})

	// Warningf is like Debugf, but at Warning level.
	Warningf(format string, args ...interface{})

	// Errorf is like Debugf, but at Error level.
	Errorf(format string, args ...interface{})

	// Criticalf is like Debugf, but at Critical level.
	Criticalf(format string, args ...interface{})

	// The remaining methods are for internal use only.
	// Developer-facing APIs wrap these methods to provide a more friendly API.

	// Internal use only.
	Call(service, method string, in, out proto.Message, opts *internal.CallOptions) error
	// Internal use only. Use AppID instead.
	FullyQualifiedAppID() string
	// Internal use only.
	Request() interface{}
}

// NewContext returns a context for an in-flight HTTP request.
func NewContext(req *http.Request) Context {
	// TODO(dsymonds): Restore this doc comment when it is true:
	//	Repeated calls will return the same value.
	return internal.NewContext(req)
}

// BlobKey is a key for a blobstore blob.
//
// Conceptually, this type belongs in the blobstore package, but it lives in
// the appengine package to avoid a circular dependency: blobstore depends on
// datastore, and datastore needs to refer to the BlobKey type.
type BlobKey string
<|endoftext|>


<commit_before>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package compression

import (
	"math/rand"
	"testing"

	"github.com/ava-labs/avalanchego/utils/units"
	"github.com/stretchr/testify/assert"
)

func TestGzipCompressDecompress(t *testing.T) {
	data := make([]byte, 4096)
	for i := 0; i < len(data); i++ {
		data[i] = byte(rand.Intn(256)) // #nosec G404
	}

	data2 := make([]byte, 4096)
	for i := 0; i < len(data); i++ {
		data2[i] = byte(rand.Intn(256)) // #nosec G404
	}

	compressor := NewGzipCompressor(2 * units.MiB)

	dataCompressed, err := compressor.Compress(data)
	assert.NoError(t, err)

	data2Compressed, err := compressor.Compress(data2)
	assert.NoError(t, err)

	dataDecompressed, err := compressor.Decompress(dataCompressed)
	assert.NoError(t, err)
	assert.EqualValues(t, data, dataDecompressed)

	data2Decompressed, err := compressor.Decompress(data2Compressed)
	assert.NoError(t, err)
	assert.EqualValues(t, data2, data2Decompressed)

	dataDecompressed, err = compressor.Decompress(dataCompressed)
	assert.NoError(t, err)
	assert.EqualValues(t, data, dataDecompressed)

	nonGzipData := []byte{1, 2, 3}
	_, err = compressor.Decompress(nonGzipData)
	assert.Error(t, err)
}
<commit_msg>add test for max compression size<commit_after>// (c) 2021, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package compression

import (
	"math/rand"
	"testing"

	"github.com/ava-labs/avalanchego/utils/units"
	"github.com/stretchr/testify/assert"
)

func TestGzipCompressDecompress(t *testing.T) {
	data := make([]byte, 4096)
	for i := 0; i < len(data); i++ {
		data[i] = byte(rand.Intn(256)) // #nosec G404
	}

	data2 := make([]byte, 4096)
	for i := 0; i < len(data); i++ {
		data2[i] = byte(rand.Intn(256)) // #nosec G404
	}

	compressor := NewGzipCompressor(2 * units.MiB)

	dataCompressed, err := compressor.Compress(data)
	assert.NoError(t, err)

	data2Compressed, err := compressor.Compress(data2)
	assert.NoError(t, err)

	dataDecompressed, err := compressor.Decompress(dataCompressed)
	assert.NoError(t, err)
	assert.EqualValues(t, data, dataDecompressed)

	data2Decompressed, err := compressor.Decompress(data2Compressed)
	assert.NoError(t, err)
	assert.EqualValues(t, data2, data2Decompressed)

	dataDecompressed, err = compressor.Decompress(dataCompressed)
	assert.NoError(t, err)
	assert.EqualValues(t, data, dataDecompressed)

	nonGzipData := []byte{1, 2, 3}
	_, err = compressor.Decompress(nonGzipData)
	assert.Error(t, err)
}

func TestGzipCompressBomb(t *testing.T) {
	data := make([]byte, 3*units.MiB)
	compressor := NewGzipCompressor(2 * units.MiB)
	_, err := compressor.Compress(data) // should be too large
	assert.Error(t, err)

	compressor2 := NewGzipCompressor(4 * units.MiB)
	dataCompressed, err := compressor2.Compress(data)
	assert.NoError(t, err)

	_, err = compressor.Decompress(dataCompressed) // should be too large
	assert.Error(t, err)
}
<|endoftext|>


<commit_before>package beaglebone

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/hybridgroup/gobot"
	"github.com/hybridgroup/gobot/platforms/gpio"
	"github.com/hybridgroup/gobot/platforms/i2c"
	"github.com/hybridgroup/gobot/sysfs"
)

var _ gobot.Adaptor = (*BeagleboneAdaptor)(nil)

var _ gpio.DigitalReader = (*BeagleboneAdaptor)(nil)
var _ gpio.DigitalWriter = (*BeagleboneAdaptor)(nil)
var _ gpio.AnalogReader = (*BeagleboneAdaptor)(nil)
var _ gpio.PwmWriter = (*BeagleboneAdaptor)(nil)
var _ gpio.ServoWriter = (*BeagleboneAdaptor)(nil)

var _ i2c.I2c = (*BeagleboneAdaptor)(nil)

var slots = "/sys/devices/bone_capemgr.*"
var ocp = "/sys/devices/ocp.*"
var usrLed = "/sys/devices/ocp.3/gpio-leds.8/leds/beaglebone:green:"

var glob = func(pattern string) (matches []string, err error) {
	return filepath.Glob(pattern)
}

var pins = map[string]int{
	"P8_3":  38,
	"P8_4":  39,
	"P8_5":  34,
	"P8_6":  35,
	"P8_7":  66,
	"P8_8":  67,
	"P8_9":  69,
	"P8_10": 68,
	"P8_11": 45,
	"P8_12": 44,
	"P8_13": 23,
	"P8_14": 26,
	"P8_15": 47,
	"P8_16": 46,
	"P8_17": 27,
	"P8_18": 65,
	"P8_19": 22,
	"P8_20": 63,
	"P8_21": 62,
	"P8_22": 37,
	"P8_23": 36,
	"P8_24": 33,
	"P8_25": 32,
	"P8_26": 61,
	"P8_27": 86,
	"P8_28": 88,
	"P8_29": 87,
	"P8_30": 89,
	"P8_31": 10,
	"P8_32": 11,
	"P8_33": 9,
	"P8_34": 81,
	"P8_35": 8,
	"P8_36": 80,
	"P8_37": 78,
	"P8_38": 79,
	"P8_39": 76,
	"P8_40": 77,
	"P8_41": 74,
	"P8_42": 75,
	"P8_43": 72,
	"P8_44": 73,
	"P8_45": 70,
	"P8_46": 71,
	"P9_11": 30,
	"P9_12": 60,
	"P9_13": 31,
	"P9_14": 50,
	"P9_15": 48,
	"P9_16": 51,
	"P9_17": 5,
	"P9_18": 4,
	"P9_19": 13,
	"P9_20": 12,
	"P9_21": 3,
	"P9_22": 2,
	"P9_23": 49,
	"P9_24": 15,
	"P9_25": 117,
	"P9_26": 14,
	"P9_27": 115,
	"P9_28": 113,
	"P9_29": 111,
	"P9_30": 112,
	"P9_31": 110,
}

var pwmPins = map[string]string{
	"P9_14": "P9_14",
	"P9_21": "P9_21",
	"P9_22": "P9_22",
	"P9_29": "P9_29",
	"P9_42": "P9_42",
	"P8_13": "P8_13",
	"P8_34": "P8_34",
	"P8_45": "P8_45",
	"P8_46": "P8_46",
}

var analogPins = map[string]string{
	"P9_39": "AIN0",
	"P9_40": "AIN1",
	"P9_37": "AIN2",
	"P9_38": "AIN3",
	"P9_33": "AIN4",
	"P8_36": "AIN5",
	"P8_35": "AIN6",
}

// BeagleboneAdaptor is the gobot.Adaptor representation for the Beaglebone
type BeagleboneAdaptor struct {
	name        string
	digitalPins []sysfs.DigitalPin
	pwmPins     map[string]*pwmPin
	i2cDevice   sysfs.I2cDevice
	ocp         string
	helper      string
	slots       string
}

// NewBeagleboneAdaptor returns a new BeagleboneAdaptor with specified name
func NewBeagleboneAdaptor(name string) *BeagleboneAdaptor {
	b := &BeagleboneAdaptor{
		name:        name,
		digitalPins: make([]sysfs.DigitalPin, 120),
		pwmPins:     make(map[string]*pwmPin),
	}

	g, _ := glob(ocp)
	b.ocp = g[0]
	g, _ = glob(slots)
	b.slots = fmt.Sprintf("%v/slots", g[0])

	return b
}

// Name returns the BeagleboneAdaptors name
func (b *BeagleboneAdaptor) Name() string { return b.name }

// Connect initializes the pwm and analog dts.
func (b *BeagleboneAdaptor) Connect() (errs []error) {
	if err := ensureSlot(b.slots, "cape-bone-iio"); err != nil {
		return []error{err}
	}

	if err := ensureSlot(b.slots, "am33xx_pwm"); err != nil {
		return []error{err}
	}

	g, err := glob(fmt.Sprintf("%v/helper.*", b.ocp))
	if err != nil {
		return []error{err}
	}
	b.helper = g[0]

	return
}

// Finalize releases all i2c devices and exported analog, digital, pwm pins.
func (b *BeagleboneAdaptor) Finalize() (errs []error) {
	for _, pin := range b.pwmPins {
		if pin != nil {
			if err := pin.release(); err != nil {
				errs = append(errs, err)
			}
		}
	}
	for _, pin := range b.digitalPins {
		if pin != nil {
			if err := pin.Unexport(); err != nil {
				errs = append(errs, err)
			}
		}
	}
	if b.i2cDevice != nil {
		if err := b.i2cDevice.Close(); err != nil {
			errs = append(errs, err)
		}
	}
	return
}

// PwmWrite writes the 0-254 value to the specified pin
func (b *BeagleboneAdaptor) PwmWrite(pin string, val byte) (err error) {
	return b.pwmWrite(pin, val)
}

// ServoWrite writes the 0-180 degree val to the specified pin.
func (b *BeagleboneAdaptor) ServoWrite(pin string, val byte) (err error) {
	i, err := b.pwmPin(pin)
	if err != nil {
		return err
	}
	period := 16666666.0
	duty := (gobot.FromScale(float64(val), 0, 180.0) * 0.115) + 0.05
	return b.pwmPins[i].pwmWrite(strconv.Itoa(int(period)), strconv.Itoa(int(period*duty)))
}

// DigitalRead returns a digital value from specified pin
func (b *BeagleboneAdaptor) DigitalRead(pin string) (val int, err error) {
	sysfsPin, err := b.digitalPin(pin, sysfs.IN)
	if err != nil {
		return
	}
	return sysfsPin.Read()
}

// DigitalWrite writes a digital value to specified pin.
// valid usr pin values are usr0, usr1, usr2 and usr3
func (b *BeagleboneAdaptor) DigitalWrite(pin string, val byte) (err error) {
	if strings.Contains(pin, "usr") {
		fi, err := sysfs.OpenFile(usrLed+pin+"/brightness", os.O_WRONLY|os.O_APPEND, 0666)
		defer fi.Close()
		if err != nil {
			return err
		}
		_, err = fi.WriteString(strconv.Itoa(int(val)))
		return err
	}
	sysfsPin, err := b.digitalPin(pin, sysfs.OUT)
	if err != nil {
		return err
	}
	return sysfsPin.Write(int(val))
}

// AnalogRead returns an analog value from specified pin
func (b *BeagleboneAdaptor) AnalogRead(pin string) (val int, err error) {
	analogPin, err := b.translateAnalogPin(pin)
	if err != nil {
		return
	}
	fi, err := sysfs.OpenFile(fmt.Sprintf("%v/%v", b.helper, analogPin), os.O_RDONLY, 0644)
	defer fi.Close()

	if err != nil {
		return
	}

	var buf = make([]byte, 1024)
	_, err = fi.Read(buf)
	if err != nil {
		return
	}

	val, _ = strconv.Atoi(strings.Split(string(buf), "\n")[0])
	return
}

// I2cStart starts a i2c device in specified address on i2c bus /dev/i2c-1
func (b *BeagleboneAdaptor) I2cStart(address int) (err error) {
	if b.i2cDevice == nil {
		b.i2cDevice, err = sysfs.NewI2cDevice("/dev/i2c-1", address)
	}
	return
}

// I2cWrite writes data to i2c device
func (b *BeagleboneAdaptor) I2cWrite(address int, data []byte) (err error) {
	if err = b.i2cDevice.SetAddress(address); err != nil {
		return
	}
	_, err = b.i2cDevice.Write(data)
	return
}

// I2cRead returns size bytes from the i2c device
func (b *BeagleboneAdaptor) I2cRead(address int, size int) (data []byte, err error) {
	if err = b.i2cDevice.SetAddress(address); err != nil {
		return
	}
	data = make([]byte, size)
	_, err = b.i2cDevice.Read(data)
	return
}

// translatePin converts digital pin name to pin position
func (b *BeagleboneAdaptor) translatePin(pin string) (value int, err error) {
	for key, value := range pins {
		if key == pin {
			return value, nil
		}
	}
	err = errors.New("Not a valid pin")
	return
}

// translatePwmPin converts pwm pin name to pin position
func (b *BeagleboneAdaptor) translatePwmPin(pin string) (value string, err error) {
	for key, value := range pwmPins {
		if key == pin {
			return value, nil
		}
	}
	err = errors.New("Not a valid pin")
	return
}

// translateAnalogPin converts analog pin name to pin position
func (b *BeagleboneAdaptor) translateAnalogPin(pin string) (value string, err error) {
	for key, value := range analogPins {
		if key == pin {
			return value, nil
		}
	}
	err = errors.New("Not a valid pin")
	return
}

// digitalPin retrieves digital pin value by name
func (b *BeagleboneAdaptor) digitalPin(pin string, dir string) (sysfsPin sysfs.DigitalPin, err error) {
	i, err := b.translatePin(pin)
	if err != nil {
		return
	}
	if b.digitalPins[i] == nil {
		b.digitalPins[i] = sysfs.NewDigitalPin(i)
		err := b.digitalPins[i].Export()
		if err != nil {
			return nil, err
		}
	}
	if err = b.digitalPins[i].Direction(dir); err != nil {
		return
	}
	return b.digitalPins[i], nil
}

// pwPin retrieves pwm pin value by name
func (b *BeagleboneAdaptor) pwmPin(pin string) (i string, err error) {
	i, err = b.translatePwmPin(pin)
	if err != nil {
		return
	}
	if b.pwmPins[i] == nil {
		err = ensureSlot(b.slots, fmt.Sprintf("bone_pwm_%v", pin))
		if err != nil {
			return
		}
		b.pwmPins[i], err = newPwmPin(i, b.ocp)
		if err != nil {
			return
		}
	}
	return
}

// pwmWrite writes pwm value to specified pin
func (b *BeagleboneAdaptor) pwmWrite(pin string, val byte) (err error) {
	i, err := b.pwmPin(pin)
	if err != nil {
		return
	}
	period := 500000.0
	duty := gobot.FromScale(float64(val), 0, 255.0)
	return b.pwmPins[i].pwmWrite(strconv.Itoa(int(period)), strconv.Itoa(int(period*duty)))
}

func ensureSlot(slots, item string) (err error) {
	fi, err := sysfs.OpenFile(slots, os.O_RDWR|os.O_APPEND, 0666)
	defer fi.Close()
	if err != nil {
		return
	}

	// ensure the slot is not already written into the capemanager
	// (from: https://github.com/mrmorphic/hwio/blob/master/module_bb_pwm.go#L190)
	scanner := bufio.NewScanner(fi)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Index(line, item) > 0 {
			return
		}
	}

	_, err = fi.WriteString(item)
	if err != nil {
		return err
	}
	fi.Sync()

	scanner = bufio.NewScanner(fi)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Index(line, item) > 0 {
			return
		}
	}
	return
}
<commit_msg>Update pin assignments to be correct for my beaglebone.<commit_after>package beaglebone

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/hybridgroup/gobot"
	"github.com/hybridgroup/gobot/platforms/gpio"
	"github.com/hybridgroup/gobot/platforms/i2c"
	"github.com/hybridgroup/gobot/sysfs"
)

var _ gobot.Adaptor = (*BeagleboneAdaptor)(nil)

var _ gpio.DigitalReader = (*BeagleboneAdaptor)(nil)
var _ gpio.DigitalWriter = (*BeagleboneAdaptor)(nil)
var _ gpio.AnalogReader = (*BeagleboneAdaptor)(nil)
var _ gpio.PwmWriter = (*BeagleboneAdaptor)(nil)
var _ gpio.ServoWriter = (*BeagleboneAdaptor)(nil)

var _ i2c.I2c = (*BeagleboneAdaptor)(nil)

var slots = "/sys/devices/bone_capemgr.*"
var ocp = "/sys/devices/ocp.*"
var usrLed = "/sys/devices/ocp.3/gpio-leds.8/leds/beaglebone:green:"

var glob = func(pattern string) (matches []string, err error) {
	return filepath.Glob(pattern)
}

var pins = map[string]int{
	"P8_3":  38,
	"P8_4":  39,
	"P8_5":  34,
	"P8_6":  35,
	"P8_7":  66,
	"P8_8":  67,
	"P8_9":  69,
	"P8_10": 68,
	"P8_11": 45,
	"P8_12": 44,
	"P8_13": 23,
	"P8_14": 26,
	"P8_15": 47,
	"P8_16": 46,
	"P8_17": 27,
	"P8_18": 65,
	"P8_19": 22,
	"P8_20": 63,
	"P8_21": 62,
	"P8_22": 37,
	"P8_23": 36,
	"P8_24": 33,
	"P8_25": 32,
	"P8_26": 61,
	"P8_27": 86,
	"P8_28": 88,
	"P8_29": 87,
	"P8_30": 89,
	"P8_31": 10,
	"P8_32": 11,
	"P8_33": 9,
	"P8_34": 81,
	"P8_35": 8,
	"P8_36": 80,
	"P8_37": 78,
	"P8_38": 79,
	"P8_39": 76,
	"P8_40": 77,
	"P8_41": 74,
	"P8_42": 75,
	"P8_43": 72,
	"P8_44": 73,
	"P8_45": 70,
	"P8_46": 71,
	"P9_11": 30,
	"P9_12": 60,
	"P9_13": 31,
	"P9_14": 50,
	"P9_15": 48,
	"P9_16": 51,
	"P9_17": 5,
	"P9_18": 4,
	"P9_19": 13,
	"P9_20": 12,
	"P9_21": 3,
	"P9_22": 2,
	"P9_23": 49,
	"P9_24": 15,
	"P9_25": 117,
	"P9_26": 14,
	"P9_27": 115,
	"P9_28": 113,
	"P9_29": 111,
	"P9_30": 112,
	"P9_31": 110,
}

var pwmPins = map[string]string{
	"P9_14": "P9_14",
	"P9_21": "P9_21",
	"P9_22": "P9_22",
	"P9_29": "P9_29",
	"P9_42": "P9_42",
	"P8_13": "P8_13",
	"P8_34": "P8_34",
	"P8_45": "P8_45",
	"P8_46": "P8_46",
}

var analogPins = map[string]string{
	"P9_39": "AIN0",
	"P9_40": "AIN1",
	"P9_37": "AIN2",
	"P9_38": "AIN3",
	"P9_33": "AIN4",
	"P9_36": "AIN5",
	"P9_35": "AIN6",
}

// BeagleboneAdaptor is the gobot.Adaptor representation for the Beaglebone
type BeagleboneAdaptor struct {
	name        string
	digitalPins []sysfs.DigitalPin
	pwmPins     map[string]*pwmPin
	i2cDevice   sysfs.I2cDevice
	ocp         string
	helper      string
	slots       string
}

// NewBeagleboneAdaptor returns a new BeagleboneAdaptor with specified name
func NewBeagleboneAdaptor(name string) *BeagleboneAdaptor {
	b := &BeagleboneAdaptor{
		name:        name,
		digitalPins: make([]sysfs.DigitalPin, 120),
		pwmPins:     make(map[string]*pwmPin),
	}

	g, _ := glob(ocp)
	b.ocp = g[0]
	g, _ = glob(slots)
	b.slots = fmt.Sprintf("%v/slots", g[0])

	return b
}

// Name returns the BeagleboneAdaptors name
func (b *BeagleboneAdaptor) Name() string { return b.name }

// Connect initializes the pwm and analog dts.
func (b *BeagleboneAdaptor) Connect() (errs []error) {
	if err := ensureSlot(b.slots, "cape-bone-iio"); err != nil {
		return []error{err}
	}

	if err := ensureSlot(b.slots, "am33xx_pwm"); err != nil {
		return []error{err}
	}

	g, err := glob(fmt.Sprintf("%v/helper.*", b.ocp))
	if err != nil {
		return []error{err}
	}
	b.helper = g[0]

	return
}

// Finalize releases all i2c devices and exported analog, digital, pwm pins.
func (b *BeagleboneAdaptor) Finalize() (errs []error) {
	for _, pin := range b.pwmPins {
		if pin != nil {
			if err := pin.release(); err != nil {
				errs = append(errs, err)
			}
		}
	}
	for _, pin := range b.digitalPins {
		if pin != nil {
			if err := pin.Unexport(); err != nil {
				errs = append(errs, err)
			}
		}
	}
	if b.i2cDevice != nil {
		if err := b.i2cDevice.Close(); err != nil {
			errs = append(errs, err)
		}
	}
	return
}

// PwmWrite writes the 0-254 value to the specified pin
func (b *BeagleboneAdaptor) PwmWrite(pin string, val byte) (err error) {
	return b.pwmWrite(pin, val)
}

// ServoWrite writes the 0-180 degree val to the specified pin.
func (b *BeagleboneAdaptor) ServoWrite(pin string, val byte) (err error) {
	i, err := b.pwmPin(pin)
	if err != nil {
		return err
	}
	period := 16666666.0
	duty := (gobot.FromScale(float64(val), 0, 180.0) * 0.115) + 0.05
	return b.pwmPins[i].pwmWrite(strconv.Itoa(int(period)), strconv.Itoa(int(period*duty)))
}

// DigitalRead returns a digital value from specified pin
func (b *BeagleboneAdaptor) DigitalRead(pin string) (val int, err error) {
	sysfsPin, err := b.digitalPin(pin, sysfs.IN)
	if err != nil {
		return
	}
	return sysfsPin.Read()
}

// DigitalWrite writes a digital value to specified pin.
// valid usr pin values are usr0, usr1, usr2 and usr3
func (b *BeagleboneAdaptor) DigitalWrite(pin string, val byte) (err error) {
	if strings.Contains(pin, "usr") {
		fi, err := sysfs.OpenFile(usrLed+pin+"/brightness", os.O_WRONLY|os.O_APPEND, 0666)
		defer fi.Close()
		if err != nil {
			return err
		}
		_, err = fi.WriteString(strconv.Itoa(int(val)))
		return err
	}
	sysfsPin, err := b.digitalPin(pin, sysfs.OUT)
	if err != nil {
		return err
	}
	return sysfsPin.Write(int(val))
}

// AnalogRead returns an analog value from specified pin
func (b *BeagleboneAdaptor) AnalogRead(pin string) (val int, err error) {
	analogPin, err := b.translateAnalogPin(pin)
	if err != nil {
		return
	}
	fi, err := sysfs.OpenFile(fmt.Sprintf("%v/%v", b.helper, analogPin), os.O_RDONLY, 0644)
	defer fi.Close()

	if err != nil {
		return
	}

	var buf = make([]byte, 1024)
	_, err = fi.Read(buf)
	if err != nil {
		return
	}

	val, _ = strconv.Atoi(strings.Split(string(buf), "\n")[0])
	return
}

// I2cStart starts a i2c device in specified address on i2c bus /dev/i2c-1
func (b *BeagleboneAdaptor) I2cStart(address int) (err error) {
	if b.i2cDevice == nil {
		b.i2cDevice, err = sysfs.NewI2cDevice("/dev/i2c-1", address)
	}
	return
}

// I2cWrite writes data to i2c device
func (b *BeagleboneAdaptor) I2cWrite(address int, data []byte) (err error) {
	if err = b.i2cDevice.SetAddress(address); err != nil {
		return
	}
	_, err = b.i2cDevice.Write(data)
	return
}

// I2cRead returns size bytes from the i2c device
func (b *BeagleboneAdaptor) I2cRead(address int, size int) (data []byte, err error) {
	if err = b.i2cDevice.SetAddress(address); err != nil {
		return
	}
	data = make([]byte, size)
	_, err = b.i2cDevice.Read(data)
	return
}

// translatePin converts digital pin name to pin position
func (b *BeagleboneAdaptor) translatePin(pin string) (value int, err error) {
	for key, value := range pins {
		if key == pin {
			return value, nil
		}
	}
	err = errors.New("Not a valid pin")
	return
}

// translatePwmPin converts pwm pin name to pin position
func (b *BeagleboneAdaptor) translatePwmPin(pin string) (value string, err error) {
	for key, value := range pwmPins {
		if key == pin {
			return value, nil
		}
	}
	err = errors.New("Not a valid pin")
	return
}

// translateAnalogPin converts analog pin name to pin position
func (b *BeagleboneAdaptor) translateAnalogPin(pin string) (value string, err error) {
	for key, value := range analogPins {
		if key == pin {
			return value, nil
		}
	}
	err = errors.New("Not a valid pin")
	return
}

// digitalPin retrieves digital pin value by name
func (b *BeagleboneAdaptor) digitalPin(pin string, dir string) (sysfsPin sysfs.DigitalPin, err error) {
	i, err := b.translatePin(pin)
	if err != nil {
		return
	}
	if b.digitalPins[i] == nil {
		b.digitalPins[i] = sysfs.NewDigitalPin(i)
		err := b.digitalPins[i].Export()
		if err != nil {
			return nil, err
		}
	}
	if err = b.digitalPins[i].Direction(dir); err != nil {
		return
	}
	return b.digitalPins[i], nil
}

// pwPin retrieves pwm pin value by name
func (b *BeagleboneAdaptor) pwmPin(pin string) (i string, err error) {
	i, err = b.translatePwmPin(pin)
	if err != nil {
		return
	}
	if b.pwmPins[i] == nil {
		err = ensureSlot(b.slots, fmt.Sprintf("bone_pwm_%v", pin))
		if err != nil {
			return
		}
		b.pwmPins[i], err = newPwmPin(i, b.ocp)
		if err != nil {
			return
		}
	}
	return
}

// pwmWrite writes pwm value to specified pin
func (b *BeagleboneAdaptor) pwmWrite(pin string, val byte) (err error) {
	i, err := b.pwmPin(pin)
	if err != nil {
		return
	}
	period := 500000.0
	duty := gobot.FromScale(float64(val), 0, 255.0)
	return b.pwmPins[i].pwmWrite(strconv.Itoa(int(period)), strconv.Itoa(int(period*duty)))
}

func ensureSlot(slots, item string) (err error) {
	fi, err := sysfs.OpenFile(slots, os.O_RDWR|os.O_APPEND, 0666)
	defer fi.Close()
	if err != nil {
		return
	}

	// ensure the slot is not already written into the capemanager
	// (from: https://github.com/mrmorphic/hwio/blob/master/module_bb_pwm.go#L190)
	scanner := bufio.NewScanner(fi)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Index(line, item) > 0 {
			return
		}
	}

	_, err = fi.WriteString(item)
	if err != nil {
		return err
	}
	fi.Sync()

	scanner = bufio.NewScanner(fi)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.Index(line, item) > 0 {
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>// Package buuid provides a type 4 uuid that can be encoded into a 16 byte
// binary representation as two 64 bit integers in little endian byte order.
package buuid

import (
	"encoding/binary"
	"fmt"

	"code.google.com/p/go-uuid/uuid"
)

const (
	EncodedLength = 16
)

var (
	endianness = binary.LittleEndian
	zero       = ID{}
)

// ID is a type 4 UUID.
type ID struct {
	part1 uint64
	part2 uint64
}

// Random creates a random ID (uses uuid.NewRandom() which uses crypto.Random()
// under the covers)
func Random() ID {
	id, err := Read(uuid.NewRandom())
	if err != nil {
		panic(fmt.Sprintf("Unable to generate random peer id: %s", err))
	}
	return id
}

// Read reads the ID from a 16-byte or larger buffer
func Read(b []byte) (ID, error) {
	if len(b) < EncodedLength {
		return zero, fmt.Errorf("Insufficient data to read id, data may be truncated")
	}
	return ID{
		endianness.Uint64(b[0:8]),
		endianness.Uint64(b[8:]),
	}, nil
}

// Write writes the ID to a 16-byte or larger buffer
func (id ID) Write(b []byte) error {
	if len(b) < EncodedLength {
		return fmt.Errorf("Insufficient room to write id")
	}
	endianness.PutUint64(b, id.part1)
	endianness.PutUint64(b[8:], id.part2)
	return nil
}

// ToBytes returns a 16-byte representation of this ID
func (id ID) ToBytes() []byte {
	b := make([]byte, EncodedLength)
	id.Write(b)
	return b
}

// FromString constructs an ID from the string-encoded version of a uuid.UUID.
func FromString(s string) (ID, error) {
	return Read(uuid.Parse(s))
}

// String() returns the string-encoded version like in uuid.UUID.
func (id ID) String() string {
	b := uuid.UUID(make([]byte, EncodedLength))
	id.Write([]byte(b))
	return b.String()
}
<commit_msg>Errcheck'ed github.com/getlantern/buuid<commit_after>// Package buuid provides a type 4 uuid that can be encoded into a 16 byte
// binary representation as two 64 bit integers in little endian byte order.
package buuid

import (
	"encoding/binary"
	"fmt"

	"code.google.com/p/go-uuid/uuid"

	"github.com/getlantern/golog"
)

const (
	EncodedLength = 16
)

var (
	endianness = binary.LittleEndian
	zero       = ID{}

	log = golog.LoggerFor("buuid")
)

// ID is a type 4 UUID.
type ID struct {
	part1 uint64
	part2 uint64
}

// Random creates a random ID (uses uuid.NewRandom() which uses crypto.Random()
// under the covers)
func Random() ID {
	id, err := Read(uuid.NewRandom())
	if err != nil {
		panic(fmt.Sprintf("Unable to generate random peer id: %s", err))
	}
	return id
}

// Read reads the ID from a 16-byte or larger buffer
func Read(b []byte) (ID, error) {
	if len(b) < EncodedLength {
		return zero, fmt.Errorf("Insufficient data to read id, data may be truncated")
	}
	return ID{
		endianness.Uint64(b[0:8]),
		endianness.Uint64(b[8:]),
	}, nil
}

// Write writes the ID to a 16-byte or larger buffer
func (id ID) Write(b []byte) error {
	if len(b) < EncodedLength {
		return fmt.Errorf("Insufficient room to write id")
	}
	endianness.PutUint64(b, id.part1)
	endianness.PutUint64(b[8:], id.part2)
	return nil
}

// ToBytes returns a 16-byte representation of this ID
func (id ID) ToBytes() []byte {
	b := make([]byte, EncodedLength)
	if err := id.Write(b); err != nil {
		log.Debugf("Unable to write as 16-bytes representation: %v", err)
	}
	return b
}

// FromString constructs an ID from the string-encoded version of a uuid.UUID.
func FromString(s string) (ID, error) {
	return Read(uuid.Parse(s))
}

// String() returns the string-encoded version like in uuid.UUID.
func (id ID) String() string {
	b := uuid.UUID(make([]byte, EncodedLength))
	if err := id.Write([]byte(b)); err != nil {
		log.Debugf("Unable to write as string: %v", err)
	}
	return b.String()
}
<|endoftext|>


<commit_before>package jsonapie;

import (
    . ".."
    "crypto/sha256"
    "hash"
    "errors"
    "fmt"
)

type ResourceCache struct{
    Resource, Cache Resource
    CacheName string
    HashFunc func() hash.Hash
}

type CacheRecord struct {
    Id string `jsonapi:"id"`
    Iders []Ider
}

func init() {
    // safety check to make sure ResourceCache is a Resource
    var t Resource;
    t = &ResourceCache{};
    _ = t;
}

func NewResourceCache(name string, resource, cache Resource) *ResourceCache {
    return &ResourceCache{
        Resource: resource,
        Cache: cache,
        CacheName: name,
        HashFunc: sha256.New,
    }
}

func(rc *ResourceCache) GenCacheKey(args ...string) (res string) {
    h := rc.HashFunc();
    for _,arg := range args {
        h.Write([]byte(arg));
    }
    r := []byte{}
    r = h.Sum(r);
    if len(r) == 0 {
        panic("Cache hash returned by HashFunc is not populating res");
    }
    res = fmt.Sprintf("val: %X", r);
    return
}

func(rc *ResourceCache) CacheFind(a *API, s Session, hkey string) (res *CacheRecord, err error) {
    r := Catch(func() {
        ider, err := rc.Cache.FindOne(a, s, hkey);
        Check(err);
        fmt.Printf("CACHE OUTPUT: %v\n", ider);
        res = ider.(*CacheRecord);
    });
    if r != nil {
        err = errors.New(fmt.Sprintf("Caught error from Cache: %#v\n", r));
    }
    return;
}

func(rc *ResourceCache) CacheCreate(a *API, s Session, hkey string, r *CacheRecord) {
    fmt.Printf("PUTTING INTO CACHE: %s - %#v\n", hkey, r);
    rc.Cache.Create(a,s,r,&hkey)
}

func(rc *ResourceCache) FindOne(a *API, s Session, id string) (Ider, error) {
    hkey := string(rc.GenCacheKey("FindOne", id));
    fmt.Printf("CHECKING CACHE\n");
    cacherecord, err := rc.CacheFind(a,s,hkey);
    if err == nil && cacherecord != nil {
        fmt.Printf("CACHE SUCCESS %#v\n", cacherecord);
        return cacherecord.Iders[0], nil;
    }
    fmt.Printf("CACHE FAILURE -- %s\n", err);
    ider, err := rc.Resource.FindOne(a,s,id);
    if err == nil && ider != nil {
        rc.CacheCreate(a,s,hkey, &CacheRecord{
            Iders: []Ider{ider},
        });
    }
    return ider, err;
}

func(rc *ResourceCache) FindMany(a *API, s Session, p *Paginator, ids []string) ([]Ider, error) {
    return rc.Resource.FindMany(a, s,p, ids);
}

func(rc *ResourceCache) FindManyByField(a *API, s Session, field string, value string) ([]Ider, error) {
    return rc.Resource.FindManyByField(a,s, field, value);
}

func(rc *ResourceCache) Delete(a *API, s Session, id string) error {
    return rc.Resource.Delete(a,s,id);
}

func(rc *ResourceCache) ParseJSON(a *API, s Session, raw []byte) (Ider, *string, *string, *OutputLinkageSet, error) {
    return rc.Resource.ParseJSON(a, s, raw);
}

func(rc *ResourceCache) Create(a *API, s Session, ider Ider, id *string) (RecordCreatedStatus, error) {
    return rc.Resource.Create(a,s, ider, id);
}
<commit_msg>Updating ResourceCache to use Logger<commit_after>package jsonapie;

import (
    . ".."
    "crypto/sha256"
    "hash"
    "errors"
    "fmt"
)

type ResourceCache struct{
    Resource, Cache Resource
    CacheName string
    HashFunc func() hash.Hash
}

type CacheRecord struct {
    Id string `jsonapi:"id"`
    Iders []Ider
}

func init() {
    // safety check to make sure ResourceCache is a Resource
    var t Resource;
    t = &ResourceCache{};
    _ = t;
}

func NewResourceCache(name string, resource, cache Resource) *ResourceCache {
    return &ResourceCache{
        Resource: resource,
        Cache: cache,
        CacheName: name,
        HashFunc: sha256.New,
    }
}

func(rc *ResourceCache) GenCacheKey(args ...string) (res string) {
    h := rc.HashFunc();
    for _,arg := range args {
        h.Write([]byte(arg));
    }
    r := []byte{}
    r = h.Sum(r);
    if len(r) == 0 {
        panic("Cache hash returned by HashFunc is not populating res");
    }
    res = fmt.Sprintf("%X", r);
    return
}

func(rc *ResourceCache) CacheFind(a *API, s Session, hkey string) (res *CacheRecord, err error) {
    r := Catch(func() {
        ider, err := rc.Cache.FindOne(a, s, hkey);
        Check(err);
        a.Logger.Printf("CACHE OUTPUT: %v\n", ider);
        res = ider.(*CacheRecord);
    });
    if r != nil {
        err = errors.New(fmt.Sprintf("Caught error from Cache: %#v\n", r));
    }
    return;
}

func(rc *ResourceCache) CacheCreate(a *API, s Session, hkey string, r *CacheRecord) {
    a.Logger.Printf("PUTTING INTO CACHE: %s - %#v\n", hkey, r);
    rc.Cache.Create(a,s,r,&hkey)
}

func(rc *ResourceCache) FindOne(a *API, s Session, id string) (Ider, error) {
    hkey := string(rc.GenCacheKey("FindOne", id));
    a.Logger.Printf("CHECKING CACHE\n");
    cacherecord, err := rc.CacheFind(a,s,hkey);
    if err == nil && cacherecord != nil {
        a.Logger.Printf("CACHE SUCCESS %#v\n", cacherecord);
        return cacherecord.Iders[0], nil;
    }
    a.Logger.Printf("CACHE FAILURE -- %s\n", err);
    ider, err := rc.Resource.FindOne(a,s,id);
    if err == nil && ider != nil {
        rc.CacheCreate(a,s,hkey, &CacheRecord{
            Iders: []Ider{ider},
        });
    }
    return ider, err;
}

func(rc *ResourceCache) FindMany(a *API, s Session, p *Paginator, ids []string) ([]Ider, error) {
    return rc.Resource.FindMany(a, s,p, ids);
}

func(rc *ResourceCache) FindManyByField(a *API, s Session, field string, value string) ([]Ider, error) {
    return rc.Resource.FindManyByField(a,s, field, value);
}

func(rc *ResourceCache) Delete(a *API, s Session, id string) error {
    return rc.Resource.Delete(a,s,id);
}

func(rc *ResourceCache) ParseJSON(a *API, s Session, raw []byte) (Ider, *string, *string, *OutputLinkageSet, error) {
    return rc.Resource.ParseJSON(a, s, raw);
}

func(rc *ResourceCache) Create(a *API, s Session, ider Ider, id *string) (RecordCreatedStatus, error) {
    return rc.Resource.Create(a,s, ider, id);
}
<|endoftext|>


<commit_before>package images

import (
	"io"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"
)

// Export a tarball of the images needed by a deployment.
func Export(opts *assets.AssetOpts, out io.Writer) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return errors.Wrapf(err, "error parsing auth, try running `docker login`")
	}
	if len(authConfigs.Configs) == 0 {
		return errors.Errorf("didn't find any valid auth configurations")
	}
	images := assets.Images(opts)
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		pulled := false
		var loopErr []error
		for registry, authConfig := range authConfigs.Configs {
			if err := client.PullImage(
				docker.PullImageOptions{
					Repository:        repository,
					Tag:               tag,
					InactivityTimeout: 5 * time.Second,
				},
				authConfig,
			); err != nil {
				loopErr = append(loopErr, errors.Wrapf(err, "error pulling from %s", registry))
				continue
			}
			pulled = true
			break
		}
		if !pulled {
			errStr := ""
			for _, err := range loopErr {
				errStr += err.Error() + "\n"
			}
			return errors.Errorf("errors pulling image %s:%s:\n%s", repository, tag, errStr)
		}
	}
	return client.ExportImages(docker.ExportImagesOptions{
		Names:        images,
		OutputStream: out,
	})
}

// Import a tarball of the images needed by a deployment such as the one
// created by Export and push those images to the registry specific in opts.
func Import(opts *assets.AssetOpts, in io.Reader) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return errors.Wrapf(err, "error parsing auth, try running `docker login`")
	}
	if len(authConfigs.Configs) == 0 {
		return errors.Errorf("didn't find any valid auth configurations")
	}
	if err := client.LoadImage(docker.LoadImageOptions{
		InputStream: in,
	}); err != nil {
		return err
	}
	registry := opts.Registry
	opts.Registry = "" // pretend we're using default images so we can get targets to tag
	images := assets.Images(opts)
	opts.Registry = registry
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		registryRepo := assets.AddRegistry(opts.Registry, repository)
		if err := client.TagImage(image, docker.TagImageOptions{
			Repo: registryRepo,
			Tag:  tag,
		},
		); err != nil {
			return errors.Wrapf(err, "error tagging image")
		}
		pushed := false
		var loopErr []error
		for registry, authConfig := range authConfigs.Configs {
			if err := client.PushImage(
				docker.PushImageOptions{
					Name:              registryRepo,
					Tag:               tag,
					Registry:          opts.Registry,
					InactivityTimeout: 5 * time.Second,
				},
				authConfig,
			); err != nil {
				loopErr = append(loopErr, errors.Wrapf(err, "error pushing to %s", registry))
				continue
			}
			pushed = true
			break
		}
		if !pushed {
			errStr := ""
			for _, err := range loopErr {
				errStr += err.Error() + "\n"
			}
			return errors.Errorf("errors pushing image %s:%s:\n%s", registryRepo, tag, errStr)
		}
	}
	return nil
}
<commit_msg>Return first error when pushing images.<commit_after>package images

import (
	"io"
	"time"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"
)

// Export a tarball of the images needed by a deployment.
func Export(opts *assets.AssetOpts, out io.Writer) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return errors.Wrapf(err, "error parsing auth, try running `docker login`")
	}
	if len(authConfigs.Configs) == 0 {
		return errors.Errorf("didn't find any valid auth configurations")
	}
	images := assets.Images(opts)
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		pulled := false
		var loopErr []error
		for registry, authConfig := range authConfigs.Configs {
			if err := client.PullImage(
				docker.PullImageOptions{
					Repository:        repository,
					Tag:               tag,
					InactivityTimeout: 5 * time.Second,
				},
				authConfig,
			); err != nil {
				loopErr = append(loopErr, errors.Wrapf(err, "error pulling from %s", registry))
				continue
			}
			pulled = true
			break
		}
		if !pulled {
			errStr := ""
			for _, err := range loopErr {
				errStr += err.Error() + "\n"
			}
			return errors.Errorf("errors pulling image %s:%s:\n%s", repository, tag, errStr)
		}
	}
	return client.ExportImages(docker.ExportImagesOptions{
		Names:        images,
		OutputStream: out,
	})
}

// Import a tarball of the images needed by a deployment such as the one
// created by Export and push those images to the registry specific in opts.
func Import(opts *assets.AssetOpts, in io.Reader) error {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		return err
	}
	authConfigs, err := docker.NewAuthConfigurationsFromDockerCfg()
	if err != nil {
		return errors.Wrapf(err, "error parsing auth, try running `docker login`")
	}
	if len(authConfigs.Configs) == 0 {
		return errors.Errorf("didn't find any valid auth configurations")
	}
	if err := client.LoadImage(docker.LoadImageOptions{
		InputStream: in,
	}); err != nil {
		return err
	}
	registry := opts.Registry
	opts.Registry = "" // pretend we're using default images so we can get targets to tag
	images := assets.Images(opts)
	opts.Registry = registry
	for _, image := range images {
		repository, tag := docker.ParseRepositoryTag(image)
		registryRepo := assets.AddRegistry(opts.Registry, repository)
		if err := client.TagImage(image, docker.TagImageOptions{
			Repo: registryRepo,
			Tag:  tag,
		},
		); err != nil {
			return errors.Wrapf(err, "error tagging image")
		}
		pushed := false
		var loopErr []error
		for registry, authConfig := range authConfigs.Configs {
			if err := client.PushImage(
				docker.PushImageOptions{
					Name:              registryRepo,
					Tag:               tag,
					Registry:          opts.Registry,
					InactivityTimeout: 5 * time.Second,
				},
				authConfig,
			); err != nil {
				loopErr = append(loopErr, errors.Wrapf(err, "error pushing to %s", registry))
				continue
			}
			pushed = true
			break
		}
		if !pushed {
			if len(loopErr) > 0 {
				return loopErr[0]
			} else {
				return errors.Errorf("failed to push images because there are no auth configs")
			}
		}
	}
	return nil
}
<|endoftext|>