<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cruntime contains code specific to container runtimes
package cruntime

import (
	"fmt"
	"os/exec"

	"github.com/blang/semver/v4"
	"github.com/pkg/errors"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/style"
	"k8s.io/minikube/pkg/minikube/sysinit"
)

// ContainerState is the run state of a container
type ContainerState int

const (
	// All is all of the states
	All ContainerState = iota
	// Running is only running
	Running
	// Paused is only paused
	Paused
)

func (cs ContainerState) String() string {
	return [...]string{"all", "running", "paused"}[cs]
}

// ValidRuntimes lists the supported container runtimes
func ValidRuntimes() []string {
	return []string{"docker", "cri-o", "containerd"}
}

// CommandRunner is the subset of command.Runner this package consumes
type CommandRunner interface {
	// RunCmd is a blocking method that runs a command
	// Use this if you don't need to stream stdout and stderr in real-time
	RunCmd(cmd *exec.Cmd) (*command.RunResult, error)
	// StartCmd is a non-blocking method that starts a command
	// Use WaitCmd to block until the command is complete
	// Use this if you need to stream stdout and/or stderr in real-time
	StartCmd(cmd *exec.Cmd) (*command.StartedCmd, error)
	// WaitCmd blocks until the started command completes
	WaitCmd(sc *command.StartedCmd) (*command.RunResult, error)
	// Copy is a convenience method that runs a command to copy a file
	Copy(assets.CopyableFile) error
	// Remove is a convenience method that runs a command to remove a file
	Remove(assets.CopyableFile) error
}

// Manager is a common interface for container runtimes
type Manager interface {
	// Name is a human readable name for a runtime
	Name() string
	// Version retrieves the current version of this runtime
	Version() (string, error)
	// Enable idempotently enables this runtime on a host
	Enable(bool, bool) error
	// Disable idempotently disables this runtime on a host
	Disable() error
	// Active returns whether or not a runtime is active on a host
	Active() bool
	// Available returns an error if it is not possible to use this runtime on a host
	Available() error
	// Style is an associated StyleEnum for Name()
	Style() style.Enum

	// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd")
	CGroupDriver() (string, error)
	// KubeletOptions returns kubelet options for a runtime.
	KubeletOptions() map[string]string
	// SocketPath returns the path to the socket file for a given runtime
	SocketPath() string

	// Load an image idempotently into the runtime on a host
	LoadImage(string) error
	// Pull an image to the runtime from the container registry
	PullImage(string) error
	// Build an image idempotently into the runtime on a host
	BuildImage(string, string, string, bool, []string, []string) error
	// Save an image from the runtime on a host
	SaveImage(string, string) error

	// ImageExists takes image name and image sha checks if an it exists
	ImageExists(string, string) bool
	// ListImages returns a list of images managed by this container runtime
	ListImages(ListImagesOptions) ([]string, error)

	// RemoveImage remove image based on name
	RemoveImage(string) error

	// ListContainers returns a list of containers managed by this container runtime
	ListContainers(ListContainersOptions) ([]string, error)
	// KillContainers removes containers based on ID
	KillContainers([]string) error
	// StopContainers stops containers based on ID
	StopContainers([]string) error
	// PauseContainers pauses containers based on ID
	PauseContainers([]string) error
	// UnpauseContainers unpauses containers based on ID
	UnpauseContainers([]string) error
	// ContainerLogCmd returns the command to retrieve the log for a container based on ID
	ContainerLogCmd(string, int, bool) string
	// SystemLogCmd returns the command to return the system logs
	SystemLogCmd(int) string
	// Preload preloads the container runtime with k8s images
	Preload(config.ClusterConfig) error
	// ImagesPreloaded returns true if all images have been preloaded
	ImagesPreloaded([]string) bool
}

// Config is runtime configuration
type Config struct {
	// Type of runtime to create ("docker, "crio", etc)
	Type string
	// Custom path to a socket file
	Socket string
	// Runner is the CommandRunner object to execute commands with
	Runner CommandRunner
	// ImageRepository image repository to download image from
	ImageRepository string
	// KubernetesVersion Kubernetes version
	KubernetesVersion semver.Version
	// InsecureRegistry list of insecure registries
	InsecureRegistry []string
}

// ListContainersOptions are the options to use for listing containers
type ListContainersOptions struct {
	// State is the container state to filter by (All, Running, Paused)
	State ContainerState
	// Name is a name filter
	Name string
	// Namespaces is the namespaces to look into
	Namespaces []string
}

// ListImagesOptions are the options to use for listing images
type ListImagesOptions struct {
}

// ErrContainerRuntimeNotRunning is thrown when container runtime is not running
var ErrContainerRuntimeNotRunning = errors.New("container runtime is not running")

// ErrServiceVersion is the error returned when disk image has incompatible version of service
type ErrServiceVersion struct {
	// Service is the name of the incompatible service
	Service string
	// Installed is the installed version of Service
	Installed string
	// Required is the minimum required version of Service
	Required string
}

// NewErrServiceVersion creates a new ErrServiceVersion
func NewErrServiceVersion(svc, required, installed string) *ErrServiceVersion {
	return &ErrServiceVersion{
		Service:   svc,
		Installed: installed,
		Required:  required,
	}
}

func (e ErrServiceVersion) Error() string {
	return fmt.Sprintf("service %q version is %v. Required: %v",
		e.Service, e.Installed, e.Required)
}

// New returns an appropriately configured runtime
func New(c Config) (Manager, error) {
	sm := sysinit.New(c.Runner)

	switch c.Type {
	case "", "docker":
		sp := c.Socket
		cs := ""
		// There is no more dockershim socket, in Kubernetes version 1.23 and beyond
		if sp == "" && c.KubernetesVersion.GTE(semver.Version{Major: 1, Minor: 23}) {
			sp = ExternalDockerCRISocket
			cs = "cri-dockerd.service"
		}
		return &Docker{
			Socket:            sp,
			Runner:            c.Runner,
			ImageRepository:   c.ImageRepository,
			KubernetesVersion: c.KubernetesVersion,
			Init:              sm,
			UseCRI:            (sp != ""), // !dockershim
			CRIService:        cs,
		}, nil
	case "crio", "cri-o":
		return &CRIO{
			Socket:            c.Socket,
			Runner:            c.Runner,
			ImageRepository:   c.ImageRepository,
			KubernetesVersion: c.KubernetesVersion,
			Init:              sm,
		}, nil
	case "containerd":
		return &Containerd{
			Socket:            c.Socket,
			Runner:            c.Runner,
			ImageRepository:   c.ImageRepository,
			KubernetesVersion: c.KubernetesVersion,
			Init:              sm,
			InsecureRegistry:  c.InsecureRegistry,
		}, nil
	default:
		return nil, fmt.Errorf("unknown runtime type: %q", c.Type)
	}
}

// ContainerStatusCommand works across container runtimes with good formatting
func ContainerStatusCommand() string {
	// Fallback to 'docker ps' if it fails (none driver)
	return "sudo `which crictl || echo crictl` ps -a || sudo docker ps -a"
}

// disableOthers disables all other runtimes except for me.
func disableOthers(me Manager, cr CommandRunner) error {
	// valid values returned by manager.Name()
	runtimes := []string{"containerd", "crio", "docker"}
	for _, name := range runtimes {
		r, err := New(Config{Type: name, Runner: cr})
		if err != nil {
			return fmt.Errorf("runtime(%s): %v", name, err)
		}

		// Don't disable myself.
		if r.Name() == me.Name() {
			continue
		}

		// Don't disable containerd if we are bound to it
		if me.Name() == "Docker" && r.Name() == "containerd" && dockerBoundToContainerd(cr) {
			klog.Infof("skipping containerd shutdown because we are bound to it")
			continue
		}
		// in case of docker, if other runtime are already not active we are sure it is disabled, nothing to do.
		// because #11515 for non-docker runtimes, we gotta ensure Docker is disabled and can not just check if it is not active
		// since it is enabled by default in the current base image and it keeps coming back to life
		if me.Name() == "Docker" && !r.Active() {
			continue
		}

		if err = r.Disable(); err != nil {
			klog.Warningf("disable failed: %v", err)
		}

		// Validate that the runtime really is offline - and that Active & Disable are properly written.
		if r.Active() {
			return fmt.Errorf("%s is still active", r.Name())
		}
	}
	return nil
}

var requiredContainerdVersion = semver.MustParse("1.4.0")

// compatibleWithVersion checks if current version of "runtime" is compatible with version "v"
func compatibleWithVersion(runtime, v string) error {
	if runtime == "containerd" {
		vv, err := semver.Make(v)
		if err != nil {
			return err
		}
		if requiredContainerdVersion.GT(vv) {
			return NewErrServiceVersion(runtime, requiredContainerdVersion.String(), vv.String())
		}
	}
	return nil
}

// CheckCompatibility checks if the container runtime managed by "cr" is compatible with current minikube code
// returns: NewErrServiceVersion if not
func CheckCompatibility(cr Manager) error {
	v, err := cr.Version()
	if err != nil {
		return errors.Wrap(err, "Failed to check container runtime version")
	}
	return compatibleWithVersion(cr.Name(), v)
}
<commit_msg>Use the socket systemd unit instead of the service<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package cruntime contains code specific to container runtimes
package cruntime

import (
	"fmt"
	"os/exec"

	"github.com/blang/semver/v4"
	"github.com/pkg/errors"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/style"
	"k8s.io/minikube/pkg/minikube/sysinit"
)

// ContainerState is the run state of a container
type ContainerState int

const (
	// All is all of the states
	All ContainerState = iota
	// Running is only running
	Running
	// Paused is only paused
	Paused
)

func (cs ContainerState) String() string {
	return [...]string{"all", "running", "paused"}[cs]
}

// ValidRuntimes lists the supported container runtimes
func ValidRuntimes() []string {
	return []string{"docker", "cri-o", "containerd"}
}

// CommandRunner is the subset of command.Runner this package consumes
type CommandRunner interface {
	// RunCmd is a blocking method that runs a command
	// Use this if you don't need to stream stdout and stderr in real-time
	RunCmd(cmd *exec.Cmd) (*command.RunResult, error)
	// StartCmd is a non-blocking method that starts a command
	// Use WaitCmd to block until the command is complete
	// Use this if you need to stream stdout and/or stderr in real-time
	StartCmd(cmd *exec.Cmd) (*command.StartedCmd, error)
	// WaitCmd blocks until the started command completes
	WaitCmd(sc *command.StartedCmd) (*command.RunResult, error)
	// Copy is a convenience method that runs a command to copy a file
	Copy(assets.CopyableFile) error
	// Remove is a convenience method that runs a command to remove a file
	Remove(assets.CopyableFile) error
}

// Manager is a common interface for container runtimes
type Manager interface {
	// Name is a human readable name for a runtime
	Name() string
	// Version retrieves the current version of this runtime
	Version() (string, error)
	// Enable idempotently enables this runtime on a host
	Enable(bool, bool) error
	// Disable idempotently disables this runtime on a host
	Disable() error
	// Active returns whether or not a runtime is active on a host
	Active() bool
	// Available returns an error if it is not possible to use this runtime on a host
	Available() error
	// Style is an associated StyleEnum for Name()
	Style() style.Enum

	// CGroupDriver returns cgroup driver ("cgroupfs" or "systemd")
	CGroupDriver() (string, error)
	// KubeletOptions returns kubelet options for a runtime.
	KubeletOptions() map[string]string
	// SocketPath returns the path to the socket file for a given runtime
	SocketPath() string

	// Load an image idempotently into the runtime on a host
	LoadImage(string) error
	// Pull an image to the runtime from the container registry
	PullImage(string) error
	// Build an image idempotently into the runtime on a host
	BuildImage(string, string, string, bool, []string, []string) error
	// Save an image from the runtime on a host
	SaveImage(string, string) error

	// ImageExists takes image name and image sha checks if an it exists
	ImageExists(string, string) bool
	// ListImages returns a list of images managed by this container runtime
	ListImages(ListImagesOptions) ([]string, error)

	// RemoveImage remove image based on name
	RemoveImage(string) error

	// ListContainers returns a list of containers managed by this container runtime
	ListContainers(ListContainersOptions) ([]string, error)
	// KillContainers removes containers based on ID
	KillContainers([]string) error
	// StopContainers stops containers based on ID
	StopContainers([]string) error
	// PauseContainers pauses containers based on ID
	PauseContainers([]string) error
	// UnpauseContainers unpauses containers based on ID
	UnpauseContainers([]string) error
	// ContainerLogCmd returns the command to retrieve the log for a container based on ID
	ContainerLogCmd(string, int, bool) string
	// SystemLogCmd returns the command to return the system logs
	SystemLogCmd(int) string
	// Preload preloads the container runtime with k8s images
	Preload(config.ClusterConfig) error
	// ImagesPreloaded returns true if all images have been preloaded
	ImagesPreloaded([]string) bool
}

// Config is runtime configuration
type Config struct {
	// Type of runtime to create ("docker, "crio", etc)
	Type string
	// Custom path to a socket file
	Socket string
	// Runner is the CommandRunner object to execute commands with
	Runner CommandRunner
	// ImageRepository image repository to download image from
	ImageRepository string
	// KubernetesVersion Kubernetes version
	KubernetesVersion semver.Version
	// InsecureRegistry list of insecure registries
	InsecureRegistry []string
}

// ListContainersOptions are the options to use for listing containers
type ListContainersOptions struct {
	// State is the container state to filter by (All, Running, Paused)
	State ContainerState
	// Name is a name filter
	Name string
	// Namespaces is the namespaces to look into
	Namespaces []string
}

// ListImagesOptions are the options to use for listing images
type ListImagesOptions struct {
}

// ErrContainerRuntimeNotRunning is thrown when container runtime is not running
var ErrContainerRuntimeNotRunning = errors.New("container runtime is not running")

// ErrServiceVersion is the error returned when disk image has incompatible version of service
type ErrServiceVersion struct {
	// Service is the name of the incompatible service
	Service string
	// Installed is the installed version of Service
	Installed string
	// Required is the minimum required version of Service
	Required string
}

// NewErrServiceVersion creates a new ErrServiceVersion
func NewErrServiceVersion(svc, required, installed string) *ErrServiceVersion {
	return &ErrServiceVersion{
		Service:   svc,
		Installed: installed,
		Required:  required,
	}
}

func (e ErrServiceVersion) Error() string {
	return fmt.Sprintf("service %q version is %v. Required: %v",
		e.Service, e.Installed, e.Required)
}

// New returns an appropriately configured runtime
func New(c Config) (Manager, error) {
	sm := sysinit.New(c.Runner)

	switch c.Type {
	case "", "docker":
		sp := c.Socket
		cs := ""
		// There is no more dockershim socket, in Kubernetes version 1.23 and beyond
		if sp == "" && c.KubernetesVersion.GTE(semver.Version{Major: 1, Minor: 23}) {
			sp = ExternalDockerCRISocket
			cs = "cri-dockerd.socket"
		}
		return &Docker{
			Socket:            sp,
			Runner:            c.Runner,
			ImageRepository:   c.ImageRepository,
			KubernetesVersion: c.KubernetesVersion,
			Init:              sm,
			UseCRI:            (sp != ""), // !dockershim
			CRIService:        cs,
		}, nil
	case "crio", "cri-o":
		return &CRIO{
			Socket:            c.Socket,
			Runner:            c.Runner,
			ImageRepository:   c.ImageRepository,
			KubernetesVersion: c.KubernetesVersion,
			Init:              sm,
		}, nil
	case "containerd":
		return &Containerd{
			Socket:            c.Socket,
			Runner:            c.Runner,
			ImageRepository:   c.ImageRepository,
			KubernetesVersion: c.KubernetesVersion,
			Init:              sm,
			InsecureRegistry:  c.InsecureRegistry,
		}, nil
	default:
		return nil, fmt.Errorf("unknown runtime type: %q", c.Type)
	}
}

// ContainerStatusCommand works across container runtimes with good formatting
func ContainerStatusCommand() string {
	// Fallback to 'docker ps' if it fails (none driver)
	return "sudo `which crictl || echo crictl` ps -a || sudo docker ps -a"
}

// disableOthers disables all other runtimes except for me.
func disableOthers(me Manager, cr CommandRunner) error {
	// valid values returned by manager.Name()
	runtimes := []string{"containerd", "crio", "docker"}
	for _, name := range runtimes {
		r, err := New(Config{Type: name, Runner: cr})
		if err != nil {
			return fmt.Errorf("runtime(%s): %v", name, err)
		}

		// Don't disable myself.
		if r.Name() == me.Name() {
			continue
		}

		// Don't disable containerd if we are bound to it
		if me.Name() == "Docker" && r.Name() == "containerd" && dockerBoundToContainerd(cr) {
			klog.Infof("skipping containerd shutdown because we are bound to it")
			continue
		}
		// in case of docker, if other runtime are already not active we are sure it is disabled, nothing to do.
		// because #11515 for non-docker runtimes, we gotta ensure Docker is disabled and can not just check if it is not active
		// since it is enabled by default in the current base image and it keeps coming back to life
		if me.Name() == "Docker" && !r.Active() {
			continue
		}

		if err = r.Disable(); err != nil {
			klog.Warningf("disable failed: %v", err)
		}

		// Validate that the runtime really is offline - and that Active & Disable are properly written.
		if r.Active() {
			return fmt.Errorf("%s is still active", r.Name())
		}
	}
	return nil
}

var requiredContainerdVersion = semver.MustParse("1.4.0")

// compatibleWithVersion checks if current version of "runtime" is compatible with version "v"
func compatibleWithVersion(runtime, v string) error {
	if runtime == "containerd" {
		vv, err := semver.Make(v)
		if err != nil {
			return err
		}
		if requiredContainerdVersion.GT(vv) {
			return NewErrServiceVersion(runtime, requiredContainerdVersion.String(), vv.String())
		}
	}
	return nil
}

// CheckCompatibility checks if the container runtime managed by "cr" is compatible with current minikube code
// returns: NewErrServiceVersion if not
func CheckCompatibility(cr Manager) error {
	v, err := cr.Version()
	if err != nil {
		return errors.Wrap(err, "Failed to check container runtime version")
	}
	return compatibleWithVersion(cr.Name(), v)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A record for the specific generation of the object from which our local
	// state is branched. If we have no local state, the contents of this
	// generation are exactly our contents.
	src storage.Object

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// 'src' above.
	localFile *os.File

	// true if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation.
//
// REQUIRES: o != nil
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	o *storage.Object) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket: bucket,
		src:    *o,
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	return op.name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, srcSize == 0
	if op.srcGeneration == 0 && op.srcSize != 0 {
		panic("Expected zero source size.")
	}

	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size int64, clobbered bool, err error) {
	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: op.name}
	o, bucketErr := op.bucket.StatObject(ctx, req)

	// Propagate errors.
	if bucketErr != nil {
		// Propagate errors. Special case: suppress gcs.NotFoundError, treating it
		// as a zero generation below.
		if _, ok := bucketErr.(*gcs.NotFoundError); !ok {
			err = fmt.Errorf("StatObject: %v", bucketErr)
			return
		}
	}

	// Find the generation number, or zero if not found.
	var currentGen int64
	if bucketErr == nil {
		currentGen = o.Generation
	}

	// We are clobbered iff the generation doesn't match our source generation.
	clobbered = (currentGen != op.srcGeneration)

	// If we have a file, it is authoritative for our size. Otherwise our source
	// size is authoritative.
	if op.localFile != nil {
		var fi os.FileInfo
		if fi, err = op.localFile.Stat(); err != nil {
			err = fmt.Errorf("Stat: %v", err)
			return
		}

		size = fi.Size()
	} else {
		size = op.srcSize
	}

	return
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n int64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return an error of type *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen int64, err error) {
	// Do we need to do anything?
	if !op.dirty {
		gen = op.srcGeneration
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = op.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	signedSrcGeneration := int64(op.srcGeneration)
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.name,
		},
		Contents:               op.localFile,
		GenerationPrecondition: &signedSrcGeneration,
	}

	o, err := op.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Make sure the server didn't return a zero generation number, since we use
	// that as a sentinel.
	if o.Generation == 0 {
		err = fmt.Errorf(
			"CreateObject returned invalid generation number: %v",
			o.Generation)

		return
	}

	gen = o.Generation

	// Update our state.
	op.srcGeneration = gen
	op.dirty = false

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation int64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Ensure that we clean up the file if we return in error from this method.
	defer func() {
		if err != nil {
			f.Close()
			f = nil
		}
	}()

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		req := &gcs.ReadObjectRequest{
			Name:       name,
			Generation: generation,
		}

		// Open for reading.
		var rc io.ReadCloser
		if rc, err = bucket.NewReader(ctx, req); err != nil {
			err = fmt.Errorf("NewReader: %v", err)
			return
		}

		// Copy to the file.
		if _, err = io.Copy(f, rc); err != nil {
			err = fmt.Errorf("Copy: %v", err)
			return
		}

		// Close.
		if err = rc.Close(); err != nil {
			err = fmt.Errorf("Close: %v", err)
			return
		}
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<commit_msg>Updated ObjectProxy.Name.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	// A record for the specific generation of the object from which our local
	// state is branched. If we have no local state, the contents of this
	// generation are exactly our contents.
	src storage.Object

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// 'src' above.
	localFile *os.File

	// true if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Create a view on the given GCS object generation.
//
// REQUIRES: o != nil
func NewObjectProxy(
	ctx context.Context,
	bucket gcs.Bucket,
	o *storage.Object) (op *ObjectProxy, err error) {
	// Set up the basic struct.
	op = &ObjectProxy{
		bucket: bucket,
		src:    *o,
	}

	return
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket, depending on whether the backing object has
// been deleted.
func (op *ObjectProxy) Name() string {
	return op.src.Name
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	// INVARIANT: If srcGeneration == 0, srcSize == 0
	if op.srcGeneration == 0 && op.srcSize != 0 {
		panic("Expected zero source size.")
	}

	// INVARIANT: If srcGeneration == 0, then dirty
	if op.srcGeneration == 0 && !op.dirty {
		panic("Expected dirty.")
	}

	// INVARIANT: If dirty, then localFile != nil
	if op.dirty && op.localFile == nil {
		panic("Expected non-nil localFile.")
	}
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() (err error) {
	// Make sure that when we exit no invariants are violated.
	defer func() {
		op.srcGeneration = 1
		op.localFile = nil
		op.dirty = false
	}()

	// If we have no local file, there's nothing to do.
	if op.localFile == nil {
		return
	}

	// Close the local file.
	if err = op.localFile.Close(); err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size int64, clobbered bool, err error) {
	// Stat the object in GCS.
	req := &gcs.StatObjectRequest{Name: op.name}
	o, bucketErr := op.bucket.StatObject(ctx, req)

	// Propagate errors.
	if bucketErr != nil {
		// Propagate errors. Special case: suppress gcs.NotFoundError, treating it
		// as a zero generation below.
		if _, ok := bucketErr.(*gcs.NotFoundError); !ok {
			err = fmt.Errorf("StatObject: %v", bucketErr)
			return
		}
	}

	// Find the generation number, or zero if not found.
	var currentGen int64
	if bucketErr == nil {
		currentGen = o.Generation
	}

	// We are clobbered iff the generation doesn't match our source generation.
	clobbered = (currentGen != op.srcGeneration)

	// If we have a file, it is authoritative for our size. Otherwise our source
	// size is authoritative.
	if op.localFile != nil {
		var fi os.FileInfo
		if fi, err = op.localFile.Stat(); err != nil {
			err = fmt.Errorf("Stat: %v", err)
			return
		}

		size = fi.Size()
	} else {
		size = op.srcSize
	}

	return
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Serve the read from the file.
	n, err = op.localFile.ReadAt(buf, offset)

	return
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	op.dirty = true
	n, err = op.localFile.WriteAt(buf, offset)

	return
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n int64) (err error) {
	// Make sure we have a local file.
	if err = op.ensureLocalFile(ctx); err != nil {
		err = fmt.Errorf("ensureLocalFile: %v", err)
		return
	}

	// Convert to signed, which is what os.File wants.
	if n > math.MaxInt64 {
		err = fmt.Errorf("Illegal offset: %v", n)
		return
	}

	op.dirty = true
	err = op.localFile.Truncate(int64(n))

	return
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return an error of type *gcs.PreconditionError.
func (op *ObjectProxy) Sync(ctx context.Context) (gen int64, err error) {
	// Do we need to do anything?
	if !op.dirty {
		gen = op.srcGeneration
		return
	}

	// Seek the file to the start so that it can be used as a reader for its full
	// contents below.
	_, err = op.localFile.Seek(0, 0)
	if err != nil {
		err = fmt.Errorf("Seek: %v", err)
		return
	}

	// Write a new generation of the object with the appropriate contents, using
	// an appropriate precondition.
	signedSrcGeneration := int64(op.srcGeneration)
	req := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: op.name,
		},
		Contents:               op.localFile,
		GenerationPrecondition: &signedSrcGeneration,
	}

	o, err := op.bucket.CreateObject(ctx, req)

	// Special case: handle precondition errors.
	if _, ok := err.(*gcs.PreconditionError); ok {
		err = &gcs.PreconditionError{
			Err: fmt.Errorf("CreateObject: %v", err),
		}

		return
	}

	// Propagate other errors more directly.
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Make sure the server didn't return a zero generation number, since we use
	// that as a sentinel.
	if o.Generation == 0 {
		err = fmt.Errorf(
			"CreateObject returned invalid generation number: %v",
			o.Generation)

		return
	}

	gen = o.Generation

	// Update our state.
	op.srcGeneration = gen
	op.dirty = false

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Set up an unlinked local temporary file for the given generation of the
// given object. Special case: generation == 0 means an empty file.
func makeLocalFile(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	generation int64) (f *os.File, err error) {
	// Create the file.
	f, err = ioutil.TempFile("", "object_proxy")
	if err != nil {
		err = fmt.Errorf("TempFile: %v", err)
		return
	}

	// Ensure that we clean up the file if we return in error from this method.
	defer func() {
		if err != nil {
			f.Close()
			f = nil
		}
	}()

	// Unlink the file so that its inode will be garbage collected when the file
	// is closed.
	if err = os.Remove(f.Name()); err != nil {
		err = fmt.Errorf("Remove: %v", err)
		return
	}

	// Fetch the object's contents if necessary.
	if generation != 0 {
		req := &gcs.ReadObjectRequest{
			Name:       name,
			Generation: generation,
		}

		// Open for reading.
		var rc io.ReadCloser
		if rc, err = bucket.NewReader(ctx, req); err != nil {
			err = fmt.Errorf("NewReader: %v", err)
			return
		}

		// Copy to the file.
		if _, err = io.Copy(f, rc); err != nil {
			err = fmt.Errorf("Copy: %v", err)
			return
		}

		// Close.
		if err = rc.Close(); err != nil {
			err = fmt.Errorf("Close: %v", err)
			return
		}
	}

	return
}

// Ensure that op.localFile is non-nil with an authoritative view of op's
// contents.
func (op *ObjectProxy) ensureLocalFile(ctx context.Context) (err error) {
	// Is there anything to do?
	if op.localFile != nil {
		return
	}

	// Set up the file.
	f, err := makeLocalFile(ctx, op.bucket, op.name, op.srcGeneration)
	if err != nil {
		err = fmt.Errorf("makeLocalFile: %v", err)
		return
	}

	op.localFile = f
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/registry/generic"
	etcdgeneric "k8s.io/kubernetes/pkg/registry/generic/etcd"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/storage"
)

type REST struct {
	etcdgeneric.Etcd
}

func NewStorage(s storage.Interface) *REST {
	prefix := "/services/specs"
	store := etcdgeneric.Etcd{
		NewFunc:     func() runtime.Object { return &api.Service{} },
		NewListFunc: func() runtime.Object { return &api.ServiceList{} },
		KeyRootFunc: func(ctx api.Context) string {
			return etcdgeneric.NamespaceKeyRootFunc(ctx, prefix)
		},
		KeyFunc: func(ctx api.Context, name string) (string, error) {
			return etcdgeneric.NamespaceKeyFunc(ctx, prefix, name)
		},
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*api.Service).Name, nil
		},
		PredicateFunc: func(label labels.Selector, field fields.Selector) generic.Matcher {
			return MatchServices(label, field)
		},
		EndpointName: "services",

		CreateStrategy: rest.Services,
		UpdateStrategy: rest.Services,

		Storage: s,
	}
	return &REST{store}
}

func MatchServices(label labels.Selector, field fields.Selector) generic.Matcher {
	return &generic.SelectionPredicate{label, field, ServiceAttributes}
}

func ServiceAttributes(obj runtime.Object) (objLabels labels.Set, objFields fields.Set, err error) {
	service, ok := obj.(*api.Service)
	if !ok {
		return nil, nil, fmt.Errorf("invalid object type %#v", obj)
	}
	return service.Labels, fields.Set{
		"metadata.name": service.Name,
	}, nil
}
<commit_msg>to fix go vet: composite literal uses unkeyed fields<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/rest"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/registry/generic"
	etcdgeneric "k8s.io/kubernetes/pkg/registry/generic/etcd"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/storage"
)

type REST struct {
	etcdgeneric.Etcd
}

func NewStorage(s storage.Interface) *REST {
	prefix := "/services/specs"
	store := etcdgeneric.Etcd{
		NewFunc:     func() runtime.Object { return &api.Service{} },
		NewListFunc: func() runtime.Object { return &api.ServiceList{} },
		KeyRootFunc: func(ctx api.Context) string {
			return etcdgeneric.NamespaceKeyRootFunc(ctx, prefix)
		},
		KeyFunc: func(ctx api.Context, name string) (string, error) {
			return etcdgeneric.NamespaceKeyFunc(ctx, prefix, name)
		},
		ObjectNameFunc: func(obj runtime.Object) (string, error) {
			return obj.(*api.Service).Name, nil
		},
		PredicateFunc: func(label labels.Selector, field fields.Selector) generic.Matcher {
			return MatchServices(label, field)
		},
		EndpointName: "services",

		CreateStrategy: rest.Services,
		UpdateStrategy: rest.Services,

		Storage: s,
	}
	return &REST{store}
}

func MatchServices(label labels.Selector, field fields.Selector) generic.Matcher {
	return &generic.SelectionPredicate{Label: label, Field: field, GetAttrs: ServiceAttributes}
}

func ServiceAttributes(obj runtime.Object) (objLabels labels.Set, objFields fields.Set, err error) {
	service, ok := obj.(*api.Service)
	if !ok {
		return nil, nil, fmt.Errorf("invalid object type %#v", obj)
	}
	return service.Labels, fields.Set{
		"metadata.name": service.Name,
	}, nil
}
<|endoftext|>


<commit_before>package "main"

import (
	"github.com/AutoRoute/l2"
)

<commit_msg>Started neighbor finding in binary. Awaiting import path to be fixed before testing.<commit_after>package main

import (
	"fmt"
	"github.com/AutoRoute/l2"
	"github.com/AutoRoute/node"
	"log"
	"net"
	"sync"
)

func PrintMessage(cs <-chan string, test string, wg *sync.WaitGroup) {
	defer wg.Done()
	msg := <-cs
	fmt.Printf("%q: Received: %v\n", test, msg)
}

func main() {
	var channels []<-chan string
	interfaces, err := net.Interfaces()

	if err != nil {
		log.Fatal(err)
	}

	for i := 0; i < len(interfaces); i++ {
		conn, err := l2.ConnectExistingDevice(interfaces[i].Name)
		if err != nil {
			log.Fatal(err)
		}

		public_key := node.pktest(fmt.Sprintf("test%d", i))
		nf := node.NewNeighborData(public_key)
		out, err := nf.Find(interfaces[i].HardwareAddr, conn)
		if err != nil {
			log.Fatal(err)
		}
		append(channels, out)
	}

	var wg sync.WaitGroup
	for i := 0; i < 2; i++ {
		for j := 0; j < len(channels); j++ {
			wg.Add(1)
			go PrintMessage(channels[i], fmt.Sprintf("test%d", i), &wg)
		}
		wg.Wait()
	}
}
<|endoftext|>


<commit_before>package ruler

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/rules"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex"
	"github.com/weaveworks/cortex/chunk"
	"github.com/weaveworks/cortex/distributor"
	"github.com/weaveworks/cortex/querier"
	"github.com/weaveworks/cortex/user"
)

// Config is the configuration for the recording rules server.
type Config struct {
	DistributorConfig distributor.Config
	ConfigsAPIURL     string
	ExternalURL       string
	// How frequently to evaluate rules by default.
	EvaluationInterval time.Duration
	// XXX: Currently single tenant only (which is awful) as the most
	// expedient way of getting *something* working.
	UserID string
}

// Ruler is a recording rules server.
type Ruler struct {
	cfg         Config
	chunkStore  chunk.Store
	distributor *distributor.Distributor

	configsAPIURL *url.URL
	externalURL   *url.URL
}

// Worker does a thing until it's told to stop.
type Worker interface {
	Run()
	Stop()
}

type worker struct {
	delay         time.Duration
	userID        string
	configsAPIURL *url.URL
	opts          *rules.ManagerOptions

	done       chan struct{}
	terminated chan struct{}
}

func (w *worker) Run() {
	defer close(w.terminated)
	var rs []rules.Rule
	var group *rules.Group
	tick := time.NewTicker(w.delay)
	defer tick.Stop()
	for {
		var err error
		select {
		case <-w.done:
			return
		default:
			select {
			case <-w.done:
				return
			case <-tick.C:
				if group == nil {
					rs, err = w.loadRules()
					if err != nil {
						log.Warnf("Could not get configuration for %v: %v", w.userID, err)
						continue
					}
					group = rules.NewGroup("default", w.delay, rs, w.opts)
				} else {
					group.Eval()
				}
			}
		}
	}
}

func (w *worker) loadRules() ([]rules.Rule, error) {
	cfg, err := getOrgConfig(w.configsAPIURL, w.userID)
	if err != nil {
		return nil, fmt.Errorf("Error fetching config: %v", err)
	}
	rs, err := loadRules(cfg.RulesFiles)
	if err != nil {
		return nil, fmt.Errorf("Error parsing rules: %v", err)
	}
	return rs, nil
}

func (w *worker) Stop() {
	close(w.done)
	<-w.terminated
}

// New returns a new Ruler.
func New(chunkStore chunk.Store, cfg Config) (*Ruler, error) {
	configsAPIURL, err := url.Parse(cfg.ConfigsAPIURL)
	if err != nil {
		return nil, err
	}
	externalURL, err := url.Parse(cfg.ExternalURL)
	if err != nil {
		return nil, err
	}

	d, err := distributor.New(cfg.DistributorConfig)
	if err != nil {
		return nil, err
	}
	return &Ruler{
		cfg:           cfg,
		chunkStore:    chunkStore,
		distributor:   d,
		configsAPIURL: configsAPIURL,
		externalURL:   externalURL,
	}, nil
}

// GetWorkerFor gets a rules recording worker for the given user.
// It will keep polling until it can construct one.
func (r *Ruler) GetWorkerFor(userID string) Worker {
	delay := time.Duration(r.cfg.EvaluationInterval)
	return &worker{
		delay:         delay,
		userID:        userID,
		configsAPIURL: r.configsAPIURL,
		opts:          r.getManagerOptions(userID),
	}
}

func (r *Ruler) getManagerOptions(userID string) *rules.ManagerOptions {
	ctx := user.WithID(context.Background(), userID)
	appender := appenderAdapter{appender: r.distributor, ctx: ctx}
	queryable := querier.NewQueryable(r.distributor, r.chunkStore)
	engine := promql.NewEngine(queryable, nil)
	return &rules.ManagerOptions{
		SampleAppender: appender,
		Notifier:       nil,
		QueryEngine:    engine,
		Context:        ctx,
		ExternalURL:    r.externalURL,
	}
}

// loadRules loads rules.
//
// Strongly inspired by `loadGroups` in Prometheus.
func loadRules(files map[string]string) ([]rules.Rule, error) {
	result := []rules.Rule{}
	for fn, content := range files {
		stmts, err := promql.ParseStmts(string(content))
		if err != nil {
			return nil, fmt.Errorf("error parsing %s: %s", fn, err)
		}

		for _, stmt := range stmts {
			var rule rules.Rule

			switch r := stmt.(type) {
			case *promql.AlertStmt:
				rule = rules.NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)

			case *promql.RecordStmt:
				rule = rules.NewRecordingRule(r.Name, r.Expr, r.Labels)

			default:
				panic("ruler.loadRules: unknown statement type")
			}
			result = append(result, rule)
		}
	}
	return result, nil
}

// appenderAdapter adapts cortex.SampleAppender to prometheus.SampleAppender
type appenderAdapter struct {
	appender cortex.SampleAppender
	ctx      context.Context
}

func (a appenderAdapter) Append(sample *model.Sample) error {
	return a.appender.Append(a.ctx, []*model.Sample{sample})
}

func (a appenderAdapter) NeedsThrottling() bool {
	// XXX: Just a guess. Who knows?
	return false
}

type cortexConfig struct {
	RulesFiles map[string]string `json:"rules_files"`
}

// getOrgConfig gets the organization's cortex config from a configs api server.
func getOrgConfig(configsAPIURL *url.URL, userID string) (*cortexConfig, error) {
	// TODO: Extract configs client logic into go client library (ala users)
	// TODO: Fix configs server so that we not need org ID in the URL to get authenticated org
	url := fmt.Sprintf("%s/api/configs/org/%s/cortex", configsAPIURL.String(), userID)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Scope-OrgID", userID)
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Invalid response from configs server: %v", res.StatusCode)
	}
	var config cortexConfig
	if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
		return nil, err
	}
	return &config, nil
}
<commit_msg>Clearer structure on the select loop<commit_after>package ruler

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/prometheus/common/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/rules"
	"golang.org/x/net/context"

	"github.com/weaveworks/cortex"
	"github.com/weaveworks/cortex/chunk"
	"github.com/weaveworks/cortex/distributor"
	"github.com/weaveworks/cortex/querier"
	"github.com/weaveworks/cortex/user"
)

// Config is the configuration for the recording rules server.
type Config struct {
	DistributorConfig distributor.Config
	ConfigsAPIURL     string
	ExternalURL       string
	// How frequently to evaluate rules by default.
	EvaluationInterval time.Duration
	// XXX: Currently single tenant only (which is awful) as the most
	// expedient way of getting *something* working.
	UserID string
}

// Ruler is a recording rules server.
type Ruler struct {
	cfg         Config
	chunkStore  chunk.Store
	distributor *distributor.Distributor

	configsAPIURL *url.URL
	externalURL   *url.URL
}

// Worker does a thing until it's told to stop.
type Worker interface {
	Run()
	Stop()
}

type worker struct {
	delay         time.Duration
	userID        string
	configsAPIURL *url.URL
	opts          *rules.ManagerOptions

	done       chan struct{}
	terminated chan struct{}
}

func (w *worker) Run() {
	defer close(w.terminated)
	var rs []rules.Rule
	var group *rules.Group
	tick := time.NewTicker(w.delay)
	defer tick.Stop()
	for {
		var err error
		select {
		case <-w.done:
			return
		default:
		}
		// Select on 'done' again to avoid live-locking.
		select {
		case <-w.done:
			return
		case <-tick.C:
			if group == nil {
				rs, err = w.loadRules()
				if err != nil {
					log.Warnf("Could not get configuration for %v: %v", w.userID, err)
					continue
				}
				group = rules.NewGroup("default", w.delay, rs, w.opts)
			} else {
				group.Eval()
			}
		}
	}
}

func (w *worker) loadRules() ([]rules.Rule, error) {
	cfg, err := getOrgConfig(w.configsAPIURL, w.userID)
	if err != nil {
		return nil, fmt.Errorf("Error fetching config: %v", err)
	}
	rs, err := loadRules(cfg.RulesFiles)
	if err != nil {
		return nil, fmt.Errorf("Error parsing rules: %v", err)
	}
	return rs, nil
}

func (w *worker) Stop() {
	close(w.done)
	<-w.terminated
}

// New returns a new Ruler.
func New(chunkStore chunk.Store, cfg Config) (*Ruler, error) {
	configsAPIURL, err := url.Parse(cfg.ConfigsAPIURL)
	if err != nil {
		return nil, err
	}
	externalURL, err := url.Parse(cfg.ExternalURL)
	if err != nil {
		return nil, err
	}

	d, err := distributor.New(cfg.DistributorConfig)
	if err != nil {
		return nil, err
	}
	return &Ruler{
		cfg:           cfg,
		chunkStore:    chunkStore,
		distributor:   d,
		configsAPIURL: configsAPIURL,
		externalURL:   externalURL,
	}, nil
}

// GetWorkerFor gets a rules recording worker for the given user.
// It will keep polling until it can construct one.
func (r *Ruler) GetWorkerFor(userID string) Worker {
	delay := time.Duration(r.cfg.EvaluationInterval)
	return &worker{
		delay:         delay,
		userID:        userID,
		configsAPIURL: r.configsAPIURL,
		opts:          r.getManagerOptions(userID),
	}
}

func (r *Ruler) getManagerOptions(userID string) *rules.ManagerOptions {
	ctx := user.WithID(context.Background(), userID)
	appender := appenderAdapter{appender: r.distributor, ctx: ctx}
	queryable := querier.NewQueryable(r.distributor, r.chunkStore)
	engine := promql.NewEngine(queryable, nil)
	return &rules.ManagerOptions{
		SampleAppender: appender,
		Notifier:       nil,
		QueryEngine:    engine,
		Context:        ctx,
		ExternalURL:    r.externalURL,
	}
}

// loadRules loads rules.
//
// Strongly inspired by `loadGroups` in Prometheus.
func loadRules(files map[string]string) ([]rules.Rule, error) {
	result := []rules.Rule{}
	for fn, content := range files {
		stmts, err := promql.ParseStmts(string(content))
		if err != nil {
			return nil, fmt.Errorf("error parsing %s: %s", fn, err)
		}

		for _, stmt := range stmts {
			var rule rules.Rule

			switch r := stmt.(type) {
			case *promql.AlertStmt:
				rule = rules.NewAlertingRule(r.Name, r.Expr, r.Duration, r.Labels, r.Annotations)

			case *promql.RecordStmt:
				rule = rules.NewRecordingRule(r.Name, r.Expr, r.Labels)

			default:
				panic("ruler.loadRules: unknown statement type")
			}
			result = append(result, rule)
		}
	}
	return result, nil
}

// appenderAdapter adapts cortex.SampleAppender to prometheus.SampleAppender
type appenderAdapter struct {
	appender cortex.SampleAppender
	ctx      context.Context
}

func (a appenderAdapter) Append(sample *model.Sample) error {
	return a.appender.Append(a.ctx, []*model.Sample{sample})
}

func (a appenderAdapter) NeedsThrottling() bool {
	// XXX: Just a guess. Who knows?
	return false
}

type cortexConfig struct {
	RulesFiles map[string]string `json:"rules_files"`
}

// getOrgConfig gets the organization's cortex config from a configs api server.
func getOrgConfig(configsAPIURL *url.URL, userID string) (*cortexConfig, error) {
	// TODO: Extract configs client logic into go client library (ala users)
	// TODO: Fix configs server so that we not need org ID in the URL to get authenticated org
	url := fmt.Sprintf("%s/api/configs/org/%s/cortex", configsAPIURL.String(), userID)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Scope-OrgID", userID)
	client := &http.Client{}
	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Invalid response from configs server: %v", res.StatusCode)
	}
	var config cortexConfig
	if err := json.NewDecoder(res.Body).Decode(&config); err != nil {
		return nil, err
	}
	return &config, nil
}
<|endoftext|>


<commit_before>package buffer

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
)

type file struct {
	file *os.File
	cap  int64
	len  int64
	io.Reader
	io.Writer
}

func NewFile(cap int64) Buffer {
	buf := &file{
		cap: cap,
	}
	return buf
}

func (buf *file) init() error {
	if buf.file == nil {
		if file, err := ioutil.TempFile("D:\\Downloads\\temp", "buffer"); err == nil {
			buf.file = file
			buf.Reader = NewWrapReader(file, buf.Cap())
			buf.Writer = NewWrapWriter(file, buf.Cap())
		} else {
			return err
		}
	}
	return nil
}

func (buf *file) Len() int64 {
	return buf.len
}

func (buf *file) Cap() int64 {
	return buf.cap
}

func (buf *file) Read(p []byte) (n int, err error) {
	if Empty(buf) {
		return 0, io.EOF
	}

	n, err = buf.Reader.Read(ShrinkToRead(buf, p))
	buf.len -= int64(n)

	if Empty(buf) {
		buf.Reset()
	}

	return n, err
}

// BUG(Dustin): Add short write
func (buf *file) Write(p []byte) (n int, err error) {
	if Full(buf) {
		return 0, bytes.ErrTooLarge
	}

	if err := buf.init(); err != nil {
		return 0, err
	}

	n, err = buf.Writer.Write(ShrinkToFit(buf, p))
	buf.len += int64(n)

	return n, err
}

func (buf *file) Reset() {
	if buf.file != nil {
		buf.file.Close()
		os.Remove(buf.file.Name())
		buf.file = nil
		buf.Reader = nil
		buf.Writer = nil
	}
}
<commit_msg>File Buffer now supports ReaderAt and WriterAt<commit_after>package buffer

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
)

type file struct {
	file *os.File
	cap  int64
	len  int64
	io.Reader
	io.ReaderAt
	io.Writer
	io.WriterAt
}

func NewFile(cap int64) Buffer {
	buf := &file{
		cap: cap,
	}
	return buf
}

func (buf *file) init() error {
	if buf.file == nil {
		if file, err := ioutil.TempFile("D:\\Downloads\\temp", "buffer"); err == nil {
			buf.file = file
			r := NewWrapReader(file, buf.Cap())
			buf.Reader = r
			buf.ReaderAt = r
			w := NewWrapWriter(file, buf.Cap())
			buf.Writer = w
			buf.WriterAt = w
		} else {
			return err
		}
	}
	return nil
}

func (buf *file) Len() int64 {
	return buf.len
}

func (buf *file) Cap() int64 {
	return buf.cap
}

func (buf *file) Read(p []byte) (n int, err error) {
	if Empty(buf) {
		return 0, io.EOF
	}

	n, err = buf.Reader.Read(ShrinkToRead(buf, p))
	buf.len -= int64(n)

	if Empty(buf) {
		buf.Reset()
	}

	return n, err
}

// BUG(Dustin): Add short write
func (buf *file) Write(p []byte) (n int, err error) {
	if Full(buf) {
		return 0, bytes.ErrTooLarge
	}

	if err := buf.init(); err != nil {
		return 0, err
	}

	n, err = buf.Writer.Write(ShrinkToFit(buf, p))
	buf.len += int64(n)

	return n, err
}

func (buf *file) Reset() {
	if buf.file != nil {
		buf.file.Close()
		os.Remove(buf.file.Name())
		buf.file = nil
		buf.Reader = nil
		buf.Writer = nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"unicode/utf8"
)

func open(f string) (*Text, error) {
	ex, err := exists(f)
	if err != nil {
		return nil, err
	}
	if !ex {
		return &Text{lines: []Line{Line{data: ""}}}, nil
	}
	file, err := os.Open(f)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	lines := make([]Line, 0)
	tabToSpace := false
	tabWidth := 4 // the default tab width

	findIndentLine := false
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		t := scanner.Text()
		if !findIndentLine {
			r, _ := utf8.DecodeRuneInString(t)
			if r == ' ' || r == '\t' {
				findIndentLine = true
				if r == ' ' {
					tabToSpace = true
					// calculate tab width
					tabWidth = 0
					remain := t
					for len(remain) != 0 {
						r, rlen := utf8.DecodeRuneInString(remain)
						remain = remain[rlen:]
						if r != ' ' {
							break
						}
						tabWidth++
					}
				}
			}
		}
		lines = append(lines, Line{t})
	}

	// if file created with `touch` cmd, scanner could not scan anything,
	// which cause no line in text that makes program panic.
	if len(lines) == 0 {
		lines = append(lines, Line{""})
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return &Text{lines, tabToSpace, tabWidth, false}, nil
}

// parseFileArg parses file argument. Which should look like "filepath:line:offset"
// the correspond return values are (filepath, linenum, offset, error).
// final ":" is (if exists) properly ignored for convinience.
// if the linenum is given, but 0 or negative, it will be 1.
// if the offset is given, but negative, it will be 0.
// when only filepath is given, linenum and offset will be set to -1.
func parseFileArg(farg string) (string, int, int, error) {
	if strings.HasSuffix(farg, ":") {
		farg = farg[:len(farg)-1]
	}
	finfo := strings.Split(farg, ":")
	f := finfo[0]
	l, o := -1, -1
	err := error(nil)

	if len(finfo) >= 4 {
		return "", -1, -1, errors.New("too many colons in file argument")
	}

	if len(finfo) == 1 {
		return f, -1, -1, nil
	}
	if len(finfo) >= 2 {
		l, err = strconv.Atoi(finfo[1])
		if err != nil {
			return "", -1, -1, err
		}
		if len(finfo) == 3 {
			o, err = strconv.Atoi(finfo[2])
			if err != nil {
				return "", -1, -1, err
			}
		}
	}

	if l < 0 {
		l = 0
	}
	if o < 0 {
		o = 0
	}
	return f, l, o, nil
}

func save(f string, t *Text) error {
	file, err := os.Create(f)
	if err != nil {
		return err
	}
	defer file.Close()
	for _, line := range t.lines {
		file.WriteString(line.data + "\n")
	}
	return nil
}

// exists returns whether the given file or directory exists or not
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// extend file name at very before the file extension.
// extendFileName("/home/yongbin/test.txt", "_tor") -> "/home/yongbin/test_tor.txt"
func extendFileName(f, e string) string {
	fdir := filepath.Dir(f)
	fname := filepath.Base(f)
	fext := filepath.Ext(f)
	var froot string
	if fext == "" {
		froot = fname
	} else {
		froot = fname[:len(fname)-len(fext)]
	}
	return filepath.Join(fdir, froot+e+fext)
}
<commit_msg>add comment and slightly tuned open function<commit_after>package main

import (
	"bufio"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"unicode/utf8"
)

// open read a file and return it as *Text.
// If the file not exist, it will return *Text with one empty line.
func open(f string) (*Text, error) {
	ex, err := exists(f)
	if err != nil {
		return nil, err
	}
	if !ex {
		return &Text{lines: []Line{Line{data: ""}}}, nil
	}
	file, err := os.Open(f)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	lines := make([]Line, 0)

	// tor use tab(shown as 4 space) for indentation as default.
	// But when parse an exsit file, follow the file's rule.
	tabToSpace := false
	tabWidth := 4

	findIndentLine := false
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		t := scanner.Text()
		if !findIndentLine {
			r, _ := utf8.DecodeRuneInString(t)
			if r == ' ' || r == '\t' {
				findIndentLine = true
				if r == ' ' {
					tabToSpace = true
					// calculate tab width
					tabWidth = 0
					remain := t
					for len(remain) != 0 {
						r, rlen := utf8.DecodeRuneInString(remain)
						remain = remain[rlen:]
						if r != ' ' {
							break
						}
						tabWidth++
					}
				}
			}
		}
		lines = append(lines, Line{t})
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}

	// if file created with `touch` cmd, scanner could not scan anything,
	// which cause no line in text that makes program panic.
	if len(lines) == 0 {
		lines = append(lines, Line{""})
	}

	return &Text{lines, tabToSpace, tabWidth, false}, nil
}

// parseFileArg parses file argument. Which should look like "filepath:line:offset"
// the correspond return values are (filepath, linenum, offset, error).
// final ":" is (if exists) properly ignored for convinience.
// if the linenum is given, but 0 or negative, it will be 1.
// if the offset is given, but negative, it will be 0.
// when only filepath is given, linenum and offset will be set to -1.
func parseFileArg(farg string) (string, int, int, error) {
	if strings.HasSuffix(farg, ":") {
		farg = farg[:len(farg)-1]
	}
	finfo := strings.Split(farg, ":")
	f := finfo[0]
	l, o := -1, -1
	err := error(nil)

	if len(finfo) >= 4 {
		return "", -1, -1, errors.New("too many colons in file argument")
	}

	if len(finfo) == 1 {
		return f, -1, -1, nil
	}
	if len(finfo) >= 2 {
		l, err = strconv.Atoi(finfo[1])
		if err != nil {
			return "", -1, -1, err
		}
		if len(finfo) == 3 {
			o, err = strconv.Atoi(finfo[2])
			if err != nil {
				return "", -1, -1, err
			}
		}
	}

	if l < 0 {
		l = 0
	}
	if o < 0 {
		o = 0
	}
	return f, l, o, nil
}

func save(f string, t *Text) error {
	file, err := os.Create(f)
	if err != nil {
		return err
	}
	defer file.Close()
	for _, line := range t.lines {
		file.WriteString(line.data + "\n")
	}
	return nil
}

// exists returns whether the given file or directory exists or not
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

// extend file name at very before the file extension.
// extendFileName("/home/yongbin/test.txt", "_tor") -> "/home/yongbin/test_tor.txt"
func extendFileName(f, e string) string {
	fdir := filepath.Dir(f)
	fname := filepath.Base(f)
	fext := filepath.Ext(f)
	var froot string
	if fext == "" {
		froot = fname
	} else {
		froot = fname[:len(fname)-len(fext)]
	}
	return filepath.Join(fdir, froot+e+fext)
}
<|endoftext|>


<commit_before>package the_platinum_searcher

import (
	"io/ioutil"
	"os"

	"path/filepath"
	"strings"
)

type find struct {
	Out    chan *GrepParams
	Option *Option
}

func Find(root string, pattern *Pattern, out chan *GrepParams, option *Option) {
	find := find{
		Out:    out,
		Option: option,
	}
	find.Start(root, pattern)
}

func (f *find) Start(root string, pattern *Pattern) {
	if f.Option.SearchStream {
		f.findStream(pattern)
	} else {
		f.findFile(root, pattern)
	}
}

func (f *find) findStream(pattern *Pattern) {
	// TODO: File type is fixed in ASCII because it can not determine the character code.
	f.Out <- &GrepParams{"", ASCII, pattern}
	close(f.Out)
}

func (f *find) findFile(root string, pattern *Pattern) {

	var ignores ignoreMatchers
	if f.Option.NoPtIgnore == false {
		if homePtIgnore := homePtIgnore(); homePtIgnore != nil {
			ignores = append(ignores, homePtIgnore)
		}
	}

	if f.Option.NoGlobalGitIgnore == false {
		ignores = append(ignores, globalGitIgnore())
	}

	ignores = append(ignores, genericIgnore(f.Option.Ignore))
	Walk(root, ignores, f.Option.Follow, func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers) {
		if info.IsDir() {
			if depth > f.Option.Depth+1 {
				return filepath.SkipDir, ignores
			}
			//Current Directory skipping should be checked first before loading ignores
			//within this directory
			if !isRoot(depth) && isHidden(info.Name()) {
				return filepath.SkipDir, ignores
			} else {
				if ignores.Match(path, info.IsDir(), depth) {
					return filepath.SkipDir, ignores
				}
			}
			ignores = append(ignores, newIgnoreMatchers(path, f.Option.VcsIgnores(), depth+2)...)
			return nil, ignores
		}
		if !info.follow && info.IsSymlink() {
			return nil, ignores
		}
		if !isRoot(depth) && isHidden(info.Name()) {
			return nil, ignores
		}

		if ignores.Match(path, info.IsDir(), depth) {
			return nil, ignores
		}

		if pattern.FileRegexp != nil && !pattern.FileRegexp.MatchString(path) {
			return nil, ignores
		}
		fileType := UNKNOWN
		if f.Option.FilesWithRegexp == "" {
			fileType = IdentifyType(path)
			if fileType == ERROR || fileType == BINARY {
				return nil, ignores
			}
		}
		f.Out <- &GrepParams{path, fileType, pattern}
		return nil, ignores
	})
	close(f.Out)
}

type WalkFunc func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers)

func Walk(root string, ignores ignoreMatchers, follow bool, walkFn WalkFunc) error {
	info, err := os.Lstat(root)
	fileInfo := newFileInfo(root, info, follow)
	if err != nil {
		walkError, _ := walkFn(root, fileInfo, 1, nil, err)
		return walkError
	}
	return walk(root, fileInfo, 1, ignores, walkFn)
}

func walkOnGoRoutine(path string, info *FileInfo, notify chan int, depth int, parentIgnore ignoreMatchers, walkFn WalkFunc) {
	walk(path, info, depth, parentIgnore, walkFn)
	notify <- 0
}

func walk(path string, info *FileInfo, depth int, parentIgnores ignoreMatchers, walkFn WalkFunc) error {
	err, ig := walkFn(path, info, depth, parentIgnores, nil)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDir() {
		return nil
	}

	list, err := ioutil.ReadDir(path)
	if err != nil {
		walkError, _ := walkFn(path, info, depth, ig, err)
		return walkError
	}

	depth++
	notify := make(chan int, len(list))
	for _, l := range list {
		fileInfo := newFileInfo(path, l, info.follow)
		if isDirectRoot(depth) {
			go walkOnGoRoutine(filepath.Join(path, fileInfo.Name()), fileInfo, notify, depth, ig, walkFn)

		} else {
			walk(filepath.Join(path, fileInfo.Name()), fileInfo, depth, ig, walkFn)
		}
	}
	if isDirectRoot(depth) {
		for i := 0; i < cap(notify); i++ {
			<-notify
		}
	}
	return nil
}

func isRoot(depth int) bool {
	return depth == 1
}

func isDirectRoot(depth int) bool {
	return depth == 2
}

func isHidden(name string) bool {
	return strings.HasPrefix(name, ".") && len(name) > 1
}

func contains(path string, patterns *[]string) bool {
	for _, p := range *patterns {
		if p == path {
			return true
		}
	}
	return false
}
<commit_msg>Fixed a bug that happens panic when global gitignore file dosen't defined.<commit_after>package the_platinum_searcher

import (
	"io/ioutil"
	"os"

	"path/filepath"
	"strings"
)

type find struct {
	Out    chan *GrepParams
	Option *Option
}

func Find(root string, pattern *Pattern, out chan *GrepParams, option *Option) {
	find := find{
		Out:    out,
		Option: option,
	}
	find.Start(root, pattern)
}

func (f *find) Start(root string, pattern *Pattern) {
	if f.Option.SearchStream {
		f.findStream(pattern)
	} else {
		f.findFile(root, pattern)
	}
}

func (f *find) findStream(pattern *Pattern) {
	// TODO: File type is fixed in ASCII because it can not determine the character code.
	f.Out <- &GrepParams{"", ASCII, pattern}
	close(f.Out)
}

func (f *find) findFile(root string, pattern *Pattern) {

	var ignores ignoreMatchers
	if f.Option.NoPtIgnore == false {
		if ignore := homePtIgnore(); ignore != nil {
			ignores = append(ignores, ignore)
		}
	}

	if f.Option.NoGlobalGitIgnore == false {
		if ignore := globalGitIgnore(); ignore != nil {
			ignores = append(ignores, ignore)
		}
	}

	ignores = append(ignores, genericIgnore(f.Option.Ignore))
	Walk(root, ignores, f.Option.Follow, func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers) {
		if info.IsDir() {
			if depth > f.Option.Depth+1 {
				return filepath.SkipDir, ignores
			}
			//Current Directory skipping should be checked first before loading ignores
			//within this directory
			if !isRoot(depth) && isHidden(info.Name()) {
				return filepath.SkipDir, ignores
			} else {
				if ignores.Match(path, info.IsDir(), depth) {
					return filepath.SkipDir, ignores
				}
			}
			ignores = append(ignores, newIgnoreMatchers(path, f.Option.VcsIgnores(), depth+2)...)
			return nil, ignores
		}
		if !info.follow && info.IsSymlink() {
			return nil, ignores
		}
		if !isRoot(depth) && isHidden(info.Name()) {
			return nil, ignores
		}

		if ignores.Match(path, info.IsDir(), depth) {
			return nil, ignores
		}

		if pattern.FileRegexp != nil && !pattern.FileRegexp.MatchString(path) {
			return nil, ignores
		}
		fileType := UNKNOWN
		if f.Option.FilesWithRegexp == "" {
			fileType = IdentifyType(path)
			if fileType == ERROR || fileType == BINARY {
				return nil, ignores
			}
		}
		f.Out <- &GrepParams{path, fileType, pattern}
		return nil, ignores
	})
	close(f.Out)
}

type WalkFunc func(path string, info *FileInfo, depth int, ignores ignoreMatchers, err error) (error, ignoreMatchers)

func Walk(root string, ignores ignoreMatchers, follow bool, walkFn WalkFunc) error {
	info, err := os.Lstat(root)
	fileInfo := newFileInfo(root, info, follow)
	if err != nil {
		walkError, _ := walkFn(root, fileInfo, 1, nil, err)
		return walkError
	}
	return walk(root, fileInfo, 1, ignores, walkFn)
}

func walkOnGoRoutine(path string, info *FileInfo, notify chan int, depth int, parentIgnore ignoreMatchers, walkFn WalkFunc) {
	walk(path, info, depth, parentIgnore, walkFn)
	notify <- 0
}

func walk(path string, info *FileInfo, depth int, parentIgnores ignoreMatchers, walkFn WalkFunc) error {
	err, ig := walkFn(path, info, depth, parentIgnores, nil)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDir() {
		return nil
	}

	list, err := ioutil.ReadDir(path)
	if err != nil {
		walkError, _ := walkFn(path, info, depth, ig, err)
		return walkError
	}

	depth++
	notify := make(chan int, len(list))
	for _, l := range list {
		fileInfo := newFileInfo(path, l, info.follow)
		if isDirectRoot(depth) {
			go walkOnGoRoutine(filepath.Join(path, fileInfo.Name()), fileInfo, notify, depth, ig, walkFn)

		} else {
			walk(filepath.Join(path, fileInfo.Name()), fileInfo, depth, ig, walkFn)
		}
	}
	if isDirectRoot(depth) {
		for i := 0; i < cap(notify); i++ {
			<-notify
		}
	}
	return nil
}

func isRoot(depth int) bool {
	return depth == 1
}

func isDirectRoot(depth int) bool {
	return depth == 2
}

func isHidden(name string) bool {
	return strings.HasPrefix(name, ".") && len(name) > 1
}

func contains(path string, patterns *[]string) bool {
	for _, p := range *patterns {
		if p == path {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
)

var Flags = []cli.Flag{
	cli.StringFlag{
		Name:  "module, m",
		Value: "",
		Usage: "Pattern to extract module names (example: '^module\\s+(\\S+)')",
	},
	cli.StringFlag{
		Name:  "pattern, p",
		Value: "",
		Usage: "Pattern to extract imports (example: '^import\\s+(\\S+)')",
	},
	cli.StringFlag{
		Name:  "start, s",
		Value: "",
		Usage: "Pattern to start matching",
	},
	cli.StringFlag{
		Name:  "end, e",
		Value: "",
		Usage: "Pattern to end matching",
	},
	cli.BoolFlag{
		Name:  "recursive, r",
		Usage: "Recursively inspect files in subdirectories",
	},
	cli.StringFlag{
		Name:  "digraph, g",
		Value: "",
		Usage: "Specify the name of directed graph for graphviz. If omited, the command does not print digraph line.",
	},
	cli.BoolFlag{
		Name:  "trimext, t",
		Usage: "Trim the file extension.",
	},
	cli.BoolFlag{
		Name:  "help, h",
		Usage: "Shows the help of the command",
	},
}
<commit_msg>improve flag.go: change the order of the flags<commit_after>package main

import (
	"github.com/codegangsta/cli"
)

var Flags = []cli.Flag{
	cli.StringFlag{
		Name:  "pattern, p",
		Value: "",
		Usage: "Pattern to extract imports (example: '^import\\s+(\\S+)')",
	},
	cli.StringFlag{
		Name:  "module, m",
		Value: "",
		Usage: "Pattern to extract module names (example: '^module\\s+(\\S+)')",
	},
	cli.StringFlag{
		Name:  "start, s",
		Value: "",
		Usage: "Pattern to start matching",
	},
	cli.StringFlag{
		Name:  "end, e",
		Value: "",
		Usage: "Pattern to end matching",
	},
	cli.BoolFlag{
		Name:  "recursive, r",
		Usage: "Recursively inspect files in subdirectories",
	},
	cli.StringFlag{
		Name:  "digraph, g",
		Value: "",
		Usage: "Specify the name of directed graph for graphviz. If omited, the command does not print digraph line.",
	},
	cli.BoolFlag{
		Name:  "trimext, t",
		Usage: "Trim the file extension.",
	},
	cli.BoolFlag{
		Name:  "help, h",
		Usage: "Shows the help of the command",
	},
}
<|endoftext|>


<commit_before>package flit

import (
	"errors"
	"fmt"
	"os"
	"os/user"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/ianremmler/dgrl"
)

type Flit struct {
	issues *dgrl.Branch
}

func New() *Flit {
	return &Flit{issues: dgrl.NewRoot()}
}

func (f *Flit) InitFile() error {
	issueFile, err := os.OpenFile("issues", os.O_RDWR|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	issueFile.Close()
	return nil
}

func (f *Flit) Load() error {
	issueFile, err := os.Open("issues")
	if err != nil {
		return err
	}
	defer issueFile.Close()
	issues := dgrl.NewParser().Parse(issueFile)
	if issues == nil {
		return errors.New("error parsing issue file")
	}
	f.issues = issues
	return nil
}

func (f *Flit) Store() error {
	if f.issues == nil {
		return errors.New("issues not initialized")
	}
	issueFile, err := os.Create("issues")
	if err != nil {
		return err
	}
	fmt.Fprintln(issueFile, f.issues)
	issueFile.Close()
	return nil
}

func (f *Flit) AppendIssues() error {
	if f.issues == nil {
		return errors.New("issues not initialized")
	}
	issueFile, err := os.OpenFile("issues", os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		return err
	}
	fmt.Fprintln(issueFile, f.issues)
	issueFile.Close()
	return nil
}

func (f *Flit) IssueIds() []string {
	if f.issues == nil {
		return []string{}
	}
	issueIds := []string{}
	for _, node := range f.issues.Kids() {
		if node.Type() == dgrl.BranchType {
			issueIds = append(issueIds, node.Key())
		}
	}
	return issueIds
}

func (f *Flit) NewIssue() (string, error) {
	if f.issues == nil {
		return "", errors.New("issues not initialized")
	}
	id := uuid.New()
	issue := dgrl.NewBranch(id)
	issue.Append(dgrl.NewLeaf("created", Stamp()))
	issue.Append(dgrl.NewLeaf("updated", Stamp()))
	issue.Append(dgrl.NewLeaf("closed", ""))
	issue.Append(dgrl.NewLeaf("summary", ""))
	issue.Append(dgrl.NewLeaf("tags", ""))
	issue.Append(dgrl.NewLeaf("status", ""))
	issue.Append(dgrl.NewLeaf("priority", ""))
	issue.Append(dgrl.NewLeaf("assigned", ""))
	issue.Append(dgrl.NewLongLeaf("description", "\n"))
	f.issues.Append(issue)

	return id, nil
}

func (f *Flit) Issue(id string) *dgrl.Branch {
	for _, node := range f.issues.Kids() {
		if node.Type() == dgrl.BranchType && node.Key() == id {
			return node.(*dgrl.Branch)
		}
	}
	return nil
}

func (f *Flit) Match(key, val string, doesMatch bool) []string {
	matches := []string{}
	for _, node := range f.issues.Kids() {
		if issue, ok := node.(*dgrl.Branch); ok {
			if IssueContains(issue, key, val) {
				matches = append(matches, issue.Key())
			}
		}
	}
	return matches
}

func Get(issue *dgrl.Branch, key string) (string, bool) {
	for _, kid := range issue.Kids() {
		if leaf, ok := kid.(*dgrl.Leaf); ok {
			if strings.Contains(leaf.Key(), key) {
				return leaf.Value(), true
			}
		}
	}
	return "", false
}

func IssueContains(issue *dgrl.Branch, key, val string) bool {
	if issueVal, ok := Get(issue, key); ok {
		if strings.Contains(issueVal, val) {
			return true
		}
	}
	return false
}

func Set(issue *dgrl.Branch, key, val string) bool {
	for _, kid := range issue.Kids() {
		if leaf, ok := kid.(*dgrl.Leaf); ok {
			if strings.Contains(leaf.Key(), key) {
				leaf.SetValue(val)
				return true
			}
		}
	}
	return false
}

func curTime() string {
	return time.Now().Format(time.RFC3339)
}

func curUser() (string, error) {
	user, err := user.Current()
	if err != nil {
		return "", err
	}
	return user.Username, nil
}

func Stamp() string {
	user, err := curUser()
	if err != nil {
		user = "?"
	}
	return user + " " + curTime()
}
<commit_msg>Fixed negated match.<commit_after>package flit

import (
	"errors"
	"fmt"
	"os"
	"os/user"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/ianremmler/dgrl"
)

type Flit struct {
	issues *dgrl.Branch
}

func New() *Flit {
	return &Flit{issues: dgrl.NewRoot()}
}

func (f *Flit) InitFile() error {
	issueFile, err := os.OpenFile("issues", os.O_RDWR|os.O_CREATE, 0666)
	if err != nil {
		return err
	}
	issueFile.Close()
	return nil
}

func (f *Flit) Load() error {
	issueFile, err := os.Open("issues")
	if err != nil {
		return err
	}
	defer issueFile.Close()
	issues := dgrl.NewParser().Parse(issueFile)
	if issues == nil {
		return errors.New("error parsing issue file")
	}
	f.issues = issues
	return nil
}

func (f *Flit) Store() error {
	if f.issues == nil {
		return errors.New("issues not initialized")
	}
	issueFile, err := os.Create("issues")
	if err != nil {
		return err
	}
	fmt.Fprintln(issueFile, f.issues)
	issueFile.Close()
	return nil
}

func (f *Flit) AppendIssues() error {
	if f.issues == nil {
		return errors.New("issues not initialized")
	}
	issueFile, err := os.OpenFile("issues", os.O_RDWR|os.O_APPEND, 0666)
	if err != nil {
		return err
	}
	fmt.Fprintln(issueFile, f.issues)
	issueFile.Close()
	return nil
}

func (f *Flit) IssueIds() []string {
	if f.issues == nil {
		return []string{}
	}
	issueIds := []string{}
	for _, node := range f.issues.Kids() {
		if node.Type() == dgrl.BranchType {
			issueIds = append(issueIds, node.Key())
		}
	}
	return issueIds
}

func (f *Flit) NewIssue() (string, error) {
	if f.issues == nil {
		return "", errors.New("issues not initialized")
	}
	id := uuid.New()
	issue := dgrl.NewBranch(id)
	issue.Append(dgrl.NewLeaf("created", Stamp()))
	issue.Append(dgrl.NewLeaf("updated", Stamp()))
	issue.Append(dgrl.NewLeaf("closed", ""))
	issue.Append(dgrl.NewLeaf("summary", ""))
	issue.Append(dgrl.NewLeaf("tags", ""))
	issue.Append(dgrl.NewLeaf("status", ""))
	issue.Append(dgrl.NewLeaf("priority", ""))
	issue.Append(dgrl.NewLeaf("assigned", ""))
	issue.Append(dgrl.NewLongLeaf("description", "\n"))
	f.issues.Append(issue)

	return id, nil
}

func (f *Flit) Issue(id string) *dgrl.Branch {
	for _, node := range f.issues.Kids() {
		if node.Type() == dgrl.BranchType && node.Key() == id {
			return node.(*dgrl.Branch)
		}
	}
	return nil
}

func (f *Flit) Match(key, val string, doesMatch bool) []string {
	matches := []string{}
	for _, node := range f.issues.Kids() {
		if issue, ok := node.(*dgrl.Branch); ok {
			if IssueContains(issue, key, val) == doesMatch {
				matches = append(matches, issue.Key())
			}
		}
	}
	return matches
}

func Get(issue *dgrl.Branch, key string) (string, bool) {
	for _, kid := range issue.Kids() {
		if leaf, ok := kid.(*dgrl.Leaf); ok {
			if strings.Contains(leaf.Key(), key) {
				return leaf.Value(), true
			}
		}
	}
	return "", false
}

func IssueContains(issue *dgrl.Branch, key, val string) bool {
	if issueVal, ok := Get(issue, key); ok {
		if strings.Contains(issueVal, val) {
			return true
		}
	}
	return false
}

func Set(issue *dgrl.Branch, key, val string) bool {
	for _, kid := range issue.Kids() {
		if leaf, ok := kid.(*dgrl.Leaf); ok {
			if strings.Contains(leaf.Key(), key) {
				leaf.SetValue(val)
				return true
			}
		}
	}
	return false
}

func curTime() string {
	return time.Now().Format(time.RFC3339)
}

func curUser() (string, error) {
	user, err := user.Current()
	if err != nil {
		return "", err
	}
	return user.Username, nil
}

func Stamp() string {
	user, err := curUser()
	if err != nil {
		user = "?"
	}
	return user + " " + curTime()
}
<|endoftext|>