<commit_before>//Package grim is the "GitHub Responder In MediaMath". We liked the acronym and awkwardly filled in the details to fit it. In short, it is a task runner that is triggered by GitHub push/pull request hooks that is intended as a much simpler and easy-to-use build server than the more modular alternatives (eg. Jenkins).
//grim provides the library functions to support this use case.
//grimd is a daemon process that uses the grim library.
package grim

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"path/filepath"
	"fmt"
)

// Copyright 2015 MediaMath <http://www.mediamath.com>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Instance models the state of a configured Grim instance.
type Instance struct {
	configRoot *string
	queue      *sqsQueue
}

// SetConfigRoot sets the base path of the configuration directory and clears any previously read config values from memory.
func (i *Instance) SetConfigRoot(path string) {
	i.configRoot = &path
	i.queue = nil
}

// PrepareGrimQueue creates or reuses the Amazon SQS queue named in the config.
func (i *Instance) PrepareGrimQueue(logger *log.Logger) error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	if config.origServerID != config.grimServerID {
		logger.Printf(buildTruncatedMessage(config.truncateID))
	}

	queue, err := prepareSQSQueue(config.awsKey, config.awsSecret, config.awsRegion, config.grimQueueName)
	if err != nil {
		return fatalGrimErrorf("error preparing queue: %v", err)
	}

	i.queue = queue

	return nil
}

// PrepareRepos discovers all repos that are configured then sets up SNS and GitHub.
// It is an error to call this without calling PrepareGrimQueue first.
func (i *Instance) PrepareRepos() error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	repos := getAllConfiguredRepos(configRoot)

	var topicARNs []string
	for _, repo := range repos {
		localConfig, err := getEffectiveConfig(configRoot, repo.owner, repo.name)
		if err != nil {
			return fatalGrimErrorf("Error with config for %s/%s. %v", repo.owner, repo.name, err)
		}

		snsTopicARN, err := prepareSNSTopic(config.awsKey, config.awsSecret, config.awsRegion, localConfig.snsTopicName)
		if err != nil {
			return fatalGrimErrorf("error creating SNS Topic %s for %s/%s topic: %v", localConfig.snsTopicName, repo.owner, repo.name, err)
		}

		err = prepareSubscription(config.awsKey, config.awsSecret, config.awsRegion, snsTopicARN, i.queue.ARN)
		if err != nil {
			return fatalGrimErrorf("error subscribing Grim queue %q to SNS topic %q: %v", i.queue.ARN, snsTopicARN, err)
		}

		err = prepareAmazonSNSService(localConfig.gitHubToken, repo.owner, repo.name, snsTopicARN, config.awsKey, config.awsSecret, config.awsRegion)
		if err != nil {
			return fatalGrimErrorf("error creating configuring GitHub AmazonSNS service: %v", err)
		}

		topicARNs = append(topicARNs, snsTopicARN)
	}

	err = setPolicy(config.awsKey, config.awsSecret, config.awsRegion, i.queue.ARN, i.queue.URL, topicARNs)
	if err != nil {
		return fatalGrimErrorf("error setting policy for Grim queue %q with topics %v: %v", i.queue.ARN, topicARNs, err)
	}

	return nil
}

// BuildNextInGrimQueue creates or reuses an SQS queue as a source of work.
func (i *Instance) BuildNextInGrimQueue(logger *log.Logger) error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	globalConfig, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return grimErrorf("error while reading config: %v", err)
	}

	message, err := getNextMessage(globalConfig.awsKey, globalConfig.awsSecret, globalConfig.awsRegion, i.queue.URL)
	if err != nil {
		return grimErrorf("error retrieving message from Grim queue %q: %v", i.queue.URL, err)
	}

	if message != "" {
		hook, err := extractHookEvent(message)
		if err != nil {
			return grimErrorf("error extracting hook from message: %v", err)
		}

		if skipReason := shouldSkip(hook); skipReason != nil {
			logger.Printf("hook skipped %v: %s\n", skipReason, hook.Describe())
			return nil
		}
		logger.Printf("hook built: %s\n", hook.Describe())

		if hook.EventName == "pull_request" {
			sha, err := pollForMergeCommitSha(globalConfig.gitHubToken, hook.Owner, hook.Repo, hook.PrNumber)
			if err != nil {
				return grimErrorf("error getting merge commit sha: %v", err)
			} else if sha == "" {
				return grimErrorf("error getting merge commit sha: field empty")
			}
			hook.Ref = sha
		}

		localConfig, err := getEffectiveConfig(configRoot, hook.Owner, hook.Repo)
		if err != nil {
			return grimErrorf("error while reading config: %v", err)
		}

		if localConfig.usernameCanBuild(hook.UserName) {
			return buildForHook(configRoot, localConfig, *hook, logger)
		}
		return grimErrorf("username %q is not permitted to build", hook.UserName)
	}

	return nil
}

// BuildRef builds a git ref immediately.
func (i *Instance) BuildRef(owner, repo, ref string, logger *log.Logger) error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveConfig(configRoot, owner, repo)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	return buildForHook(configRoot, config, hookEvent{
		Owner: owner,
		Repo:  repo,
		Ref:   ref,
	}, logger)
}

func buildOnHook(configRoot string, resultPath string, config *effectiveConfig, hook hookEvent, basename string) (*executeResult, string, error) {
	return build(config.gitHubToken, configRoot, config.workspaceRoot, resultPath, config.pathToCloneIn, hook.Owner, hook.Repo, hook.Ref, hook.env(), basename)
}

func buildForHook(configRoot string, config *effectiveConfig, hook hookEvent, logger *log.Logger) error {
	return onHook(configRoot, config, hook, logger, buildOnHook)
}

type hookAction func(string, string, *effectiveConfig, hookEvent, string) (*executeResult, string, error)

func writeHookEvent(resultPath string, hook hookEvent) error {
	hookFile := filepath.Join(resultPath, "hook.json")
	hookBytes, marshalErr := json.Marshal(&hook)
	if marshalErr != nil {
		return marshalErr
	}

	ioutil.WriteFile(hookFile, hookBytes, 0644)
	return nil
}

func onHook(configRoot string, config *effectiveConfig, hook hookEvent, logger *log.Logger, action hookAction) error {
	basename := getTimeStamp()
	resultPath, err := makeTree(config.resultRoot, hook.Owner, hook.Repo, basename)
	if err != nil {
		return fatalGrimErrorf("error creating result path: %v", err)
	}

	// TODO: do something with this err
	writeHookEvent(resultPath, hook)

	notify(config, hook, "", resultPath, GrimPending, logger)

	result, ws, err := action(configRoot, resultPath, config, hook, basename)
	if err != nil {
		notify(config, hook, ws, resultPath, GrimError, logger)
		return fatalGrimErrorf("error during %v: %v", hook.Describe(), err)
	}

	var notifyError error
	if result.ExitCode == 0 {
		notifyError = notify(config, hook, ws, resultPath, GrimSuccess, logger)
	} else {
		notifyError = notify(config, hook, ws, resultPath, GrimFailure, logger)
	}

	return notifyError
}

func (i *Instance) checkGrimQueue() error {
	if i.queue == nil {
		return fatalGrimErrorf("the Grim queue must be prepared first")
	}

	return nil
}

const truncatedMessage = `%q shouldn't be over 15 characters and has been truncated
Please update your config.json file to have a shorter %q.
Or to use the server defaults, remove the entry %q`

func buildTruncatedMessage(truncateID string) string {
	return fmt.Sprintf(truncatedMessage, truncateID, truncateID, truncateID)
}

func shouldSkip(hook *hookEvent) *string {
	var message *string
	switch {
	case hook.Deleted:
		message = getStringPtr(fmt.Sprintf("because it was on a deleted branch"))
	case hook.EventName == "push":
	case hook.Action == "opened":
	case hook.Action == "reopened":
	case hook.Action == "synchronize":
		message = nil
	default:
		message = getStringPtr(fmt.Sprintf("because the action: %q was not 'opened', 'reopened', or 'synchronize'", hook.Action))
	}
	return message
}

func getStringPtr(s string) *string { return &s }
<commit_msg>refactored shouldSkip to use switch<commit_after>//Package grim is the "GitHub Responder In MediaMath". We liked the acronym and awkwardly filled in the details to fit it. In short, it is a task runner that is triggered by GitHub push/pull request hooks that is intended as a much simpler and easy-to-use build server than the more modular alternatives (eg. Jenkins).
//grim provides the library functions to support this use case.
//grimd is a daemon process that uses the grim library.
package grim

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"path/filepath"
	"fmt"
	"time"
)

// Copyright 2015 MediaMath <http://www.mediamath.com>.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Instance models the state of a configured Grim instance.
type Instance struct {
	configRoot *string
	queue      *sqsQueue
}

// SetConfigRoot sets the base path of the configuration directory and clears any previously read config values from memory.
func (i *Instance) SetConfigRoot(path string) {
	i.configRoot = &path
	i.queue = nil
}

// PrepareGrimQueue creates or reuses the Amazon SQS queue named in the config.
func (i *Instance) PrepareGrimQueue(logger *log.Logger) error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	if config.origServerID != config.grimServerID {
		logger.Printf(buildTruncatedMessage(config.truncateID))
	}

	queue, err := prepareSQSQueue(config.awsKey, config.awsSecret, config.awsRegion, config.grimQueueName)
	if err != nil {
		return fatalGrimErrorf("error preparing queue: %v", err)
	}

	i.queue = queue

	return nil
}

// PrepareRepos discovers all repos that are configured then sets up SNS and GitHub.
// It is an error to call this without calling PrepareGrimQueue first.
func (i *Instance) PrepareRepos() error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	repos := getAllConfiguredRepos(configRoot)

	var topicARNs []string
	for _, repo := range repos {
		localConfig, err := getEffectiveConfig(configRoot, repo.owner, repo.name)
		if err != nil {
			return fatalGrimErrorf("Error with config for %s/%s. %v", repo.owner, repo.name, err)
		}

		snsTopicARN, err := prepareSNSTopic(config.awsKey, config.awsSecret, config.awsRegion, localConfig.snsTopicName)
		if err != nil {
			return fatalGrimErrorf("error creating SNS Topic %s for %s/%s topic: %v", localConfig.snsTopicName, repo.owner, repo.name, err)
		}

		err = prepareSubscription(config.awsKey, config.awsSecret, config.awsRegion, snsTopicARN, i.queue.ARN)
		if err != nil {
			return fatalGrimErrorf("error subscribing Grim queue %q to SNS topic %q: %v", i.queue.ARN, snsTopicARN, err)
		}

		err = prepareAmazonSNSService(localConfig.gitHubToken, repo.owner, repo.name, snsTopicARN, config.awsKey, config.awsSecret, config.awsRegion)
		if err != nil {
			return fatalGrimErrorf("error creating configuring GitHub AmazonSNS service: %v", err)
		}

		topicARNs = append(topicARNs, snsTopicARN)
	}

	err = setPolicy(config.awsKey, config.awsSecret, config.awsRegion, i.queue.ARN, i.queue.URL, topicARNs)
	if err != nil {
		return fatalGrimErrorf("error setting policy for Grim queue %q with topics %v: %v", i.queue.ARN, topicARNs, err)
	}

	return nil
}

// BuildNextInGrimQueue creates or reuses an SQS queue as a source of work.
func (i *Instance) BuildNextInGrimQueue(logger *log.Logger) error {
	if err := i.checkGrimQueue(); err != nil {
		return err
	}

	configRoot := getEffectiveConfigRoot(i.configRoot)

	globalConfig, err := getEffectiveGlobalConfig(configRoot)
	if err != nil {
		return grimErrorf("error while reading config: %v", err)
	}

	message, err := getNextMessage(globalConfig.awsKey, globalConfig.awsSecret, globalConfig.awsRegion, i.queue.URL)
	if err != nil {
		return grimErrorf("error retrieving message from Grim queue %q: %v", i.queue.URL, err)
	}

	if message != "" {
		hook, err := extractHookEvent(message)
		if err != nil {
			return grimErrorf("error extracting hook from message: %v", err)
		}

		if skipReason := shouldSkip(hook); skipReason != nil {
			logger.Printf("hook skipped %v: %s\n", skipReason, hook.Describe())
			return nil
		}
		logger.Printf("hook built: %s\n", hook.Describe())

		if hook.EventName == "pull_request" {
			sha, err := pollForMergeCommitSha(globalConfig.gitHubToken, hook.Owner, hook.Repo, hook.PrNumber)
			if err != nil {
				return grimErrorf("error getting merge commit sha: %v", err)
			} else if sha == "" {
				return grimErrorf("error getting merge commit sha: field empty")
			}
			hook.Ref = sha
		}

		localConfig, err := getEffectiveConfig(configRoot, hook.Owner, hook.Repo)
		if err != nil {
			return grimErrorf("error while reading config: %v", err)
		}

		if localConfig.usernameCanBuild(hook.UserName) {
			return buildForHook(configRoot, localConfig, *hook, logger)
		}
		return grimErrorf("username %q is not permitted to build", hook.UserName)
	}

	return nil
}

// BuildRef builds a git ref immediately.
func (i *Instance) BuildRef(owner, repo, ref string, logger *log.Logger) error {
	configRoot := getEffectiveConfigRoot(i.configRoot)

	config, err := getEffectiveConfig(configRoot, owner, repo)
	if err != nil {
		return fatalGrimErrorf("error while reading config: %v", err)
	}

	return buildForHook(configRoot, config, hookEvent{
		Owner: owner,
		Repo:  repo,
		Ref:   ref,
	}, logger)
}

func buildOnHook(configRoot string, resultPath string, config *effectiveConfig, hook hookEvent, basename string) (*executeResult, string, error) {
	return build(config.gitHubToken, configRoot, config.workspaceRoot, resultPath, config.pathToCloneIn, hook.Owner, hook.Repo, hook.Ref, hook.env(), basename)
}

func buildForHook(configRoot string, config *effectiveConfig, hook hookEvent, logger *log.Logger) error {
	return onHook(configRoot, config, hook, logger, buildOnHook)
}

type hookAction func(string, string, *effectiveConfig, hookEvent, string) (*executeResult, string, error)

func writeHookEvent(resultPath string, hook hookEvent) error {
	hookFile := filepath.Join(resultPath, "hook.json")
	hookBytes, marshalErr := json.Marshal(&hook)
	if marshalErr != nil {
		return marshalErr
	}

	ioutil.WriteFile(hookFile, hookBytes, 0644)
	return nil
}

func onHook(configRoot string, config *effectiveConfig, hook hookEvent, logger *log.Logger, action hookAction) error {
	build := make(chan error, 1)
	defer close(build)

	go func() {
		build <- onHookBuild(configRoot, config, hook, logger, action)
	}()

	select {
	case err := <-build:
		return err
	case <- time.After(time.Second * time.Duration(config.timeout)):
		return fmt.Errorf("Build Timeout")
	}
}

func onHookBuild(configRoot string, config *effectiveConfig, hook hookEvent, logger *log.Logger, action hookAction) error {
	basename := getTimeStamp()
	resultPath, err := makeTree(config.resultRoot, hook.Owner, hook.Repo, basename)
	if err != nil {
		return fatalGrimErrorf("error creating result path: %v", err)
	}

	// TODO: do something with this err
	writeHookEvent(resultPath, hook)

	notify(config, hook, "", resultPath, GrimPending, logger)

	result, ws, err := action(configRoot, resultPath, config, hook, basename)
	if err != nil {
		notify(config, hook, ws, resultPath, GrimError, logger)
		return fatalGrimErrorf("error during %v: %v", hook.Describe(), err)
	}

	var notifyError error
	if result.ExitCode == 0 {
		notifyError = notify(config, hook, ws, resultPath, GrimSuccess, logger)
	} else {
		notifyError = notify(config, hook, ws, resultPath, GrimFailure, logger)
	}

	return notifyError
}

func (i *Instance) checkGrimQueue() error {
	if i.queue == nil {
		return fatalGrimErrorf("the Grim queue must be prepared first")
	}

	return nil
}

const truncatedMessage = `%q shouldn't be over 15 characters and has been truncated
Please update your config.json file to have a shorter %q.
Or to use the server defaults, remove the entry %q`

func buildTruncatedMessage(truncateID string) string {
	return fmt.Sprintf(truncatedMessage, truncateID, truncateID, truncateID)
}

func shouldSkip(hook *hookEvent) *string {
	var message *string
	switch {
	case hook.Deleted:
		message = getStringPtr(fmt.Sprintf("because it was on a deleted branch"))
	case hook.EventName == "push":
	case hook.Action == "opened":
	case hook.Action == "reopened":
	case hook.Action == "synchronize":
		message = nil
	default:
		message = getStringPtr(fmt.Sprintf("because the action: %q was not 'opened', 'reopened', or 'synchronize'", hook.Action))
	}
	return message
}

func getStringPtr(s string) *string { return &s }
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	docker "github.com/fsouza/go-dockerclient"
)

type stringslice []string

var (
	buildVersion            string
	version                 bool
	watch                   bool
	notifyCmd               string
	notifySigHUPContainerID string
	onlyExposed             bool
	onlyPublished           bool
	configFiles             stringslice
	configs                 ConfigFile
	interval                int
	endpoint                string
	tlsCert                 string
	tlsKey                  string
	tlsCaCert               string
	tlsVerify               bool
	tlsCertPath             string
	wg                      sync.WaitGroup
)

type Event struct {
	ContainerID string `json:"id"`
	Status      string `json:"status"`
	Image       string `json:"from"`
}

type Address struct {
	IP       string
	Port     string
	HostPort string
	Proto    string
	HostIP   string
}

type Volume struct {
	Path      string
	HostPath  string
	ReadWrite bool
}

type RuntimeContainer struct {
	ID        string
	Addresses []Address
	Gateway   string
	Name      string
	Image     DockerImage
	Env       map[string]string
	Volumes   map[string]Volume
	Node      SwarmNode
	Labels    map[string]string
	IP        string
}

type DockerImage struct {
	Registry   string
	Repository string
	Tag        string
}

type SwarmNode struct {
	ID      string
	Name    string
	Address Address
}

func (strings *stringslice) String() string {
	return "[]"
}

func (strings *stringslice) Set(value string) error {
	// TODO: Throw an error for duplicate `dest`
	*strings = append(*strings, value)
	return nil
}

func (i *DockerImage) String() string {
	ret := i.Repository
	if i.Registry != "" {
		ret = i.Registry + "/" + i.Repository
	}
	if i.Tag != "" {
		ret = ret + ":" + i.Tag
	}
	return ret
}

type Config struct {
	Template         string
	Dest             string
	Watch            bool
	NotifyCmd        string
	NotifyContainers map[string]docker.Signal
	OnlyExposed      bool
	OnlyPublished    bool
	Interval         int
}

type ConfigFile struct {
	Config []Config
}

type Context []*RuntimeContainer

func (c *Context) Env() map[string]string {
	return splitKeyValueSlice(os.Environ())
}

func (c *ConfigFile) filterWatches() ConfigFile {
	configWithWatches := []Config{}

	for _, config := range c.Config {
		if config.Watch {
			configWithWatches = append(configWithWatches, config)
		}
	}
	return ConfigFile{
		Config: configWithWatches,
	}
}

func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
	return r.ID == o.ID && r.Image == o.Image
}

func (r *RuntimeContainer) PublishedAddresses() []Address {
	mapped := []Address{}
	for _, address := range r.Addresses {
		if address.HostPort != "" {
			mapped = append(mapped, address)
		}
	}
	return mapped
}

func usage() {
	println(`Usage: docker-gen [options] template [dest]

Generate files from docker container meta-data

Options:`)
	flag.PrintDefaults()

	println(`
Arguments:
  template - path to a template to generate
  dest - path to a write the template.  If not specfied, STDOUT is used

Environment Variables:
  DOCKER_HOST - default value for -endpoint
`)
}

func tlsEnabled() bool {
	for _, v := range []string{tlsCert, tlsCaCert, tlsKey} {
		if e, err := pathExists(v); e && err == nil {
			return true
		}
	}
	return false
}

func NewDockerClient(endpoint string) (*docker.Client, error) {
	if strings.HasPrefix(endpoint, "unix:") {
		return docker.NewClient(endpoint)
	} else if tlsVerify || tlsEnabled() {
		if tlsVerify {
			if e, err := pathExists(tlsCaCert); !e || err != nil {
				return nil, errors.New("TLS verification was requested, but CA cert does not exist")
			}
		}

		return docker.NewTLSClient(endpoint, tlsCert, tlsKey, tlsCaCert)
	}
	return docker.NewClient(endpoint)
}

func generateFromContainers(client *docker.Client) {
	containers, err := getContainers(client)
	if err != nil {
		log.Printf("error listing containers: %s\n", err)
		return
	}
	for _, config := range configs.Config {
		changed := generateFile(config, containers)
		if !changed {
			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
			continue
		}
		runNotifyCmd(config)
		sendSignalToContainer(client, config)
	}
}

func runNotifyCmd(config Config) {
	if config.NotifyCmd == "" {
		return
	}

	log.Printf("Running '%s'", config.NotifyCmd)
	cmd := exec.Command("/bin/sh", "-c", config.NotifyCmd)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running notify command: %s, %s\n", config.NotifyCmd, err)
		log.Print(string(out))
	}
}

func sendSignalToContainer(client *docker.Client, config Config) {
	if len(config.NotifyContainers) < 1 {
		return
	}

	for container, signal := range config.NotifyContainers {
		log.Printf("Sending container '%s' signal '%v'", container, signal)
		killOpts := docker.KillContainerOptions{
			ID:     container,
			Signal: signal,
		}
		if err := client.KillContainer(killOpts); err != nil {
			log.Printf("Error sending signal to container: %s", err)
		}
	}
}

func loadConfig(file string) error {
	_, err := toml.DecodeFile(file, &configs)
	if err != nil {
		return err
	}
	return nil
}

func generateAtInterval(client *docker.Client, configs ConfigFile) {
	for _, config := range configs.Config {

		if config.Interval == 0 {
			continue
		}

		log.Printf("Generating every %d seconds", config.Interval)
		wg.Add(1)
		ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
		quit := make(chan struct{})
		configCopy := config
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ticker.C:
					containers, err := getContainers(client)
					if err != nil {
						log.Printf("Error listing containers: %s\n", err)
						continue
					}
					// ignore changed return value. always run notify command
					generateFile(configCopy, containers)
					runNotifyCmd(configCopy)
					sendSignalToContainer(client, configCopy)
				case <-quit:
					ticker.Stop()
					return
				}
			}
		}()
	}
}

func generateFromEvents(client *docker.Client, configs ConfigFile) {
	configs = configs.filterWatches()
	if len(configs.Config) == 0 {
		return
	}

	wg.Add(1)
	defer wg.Done()

	for {
		if client == nil {
			var err error
			endpoint, err := getEndpoint()
			if err != nil {
				log.Printf("Bad endpoint: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}

			client, err = NewDockerClient(endpoint)
			if err != nil {
				log.Printf("Unable to connect to docker daemon: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}
			generateFromContainers(client)
		}

		eventChan := make(chan *docker.APIEvents, 100)
		defer close(eventChan)

		watching := false
		for {

			if client == nil {
				break
			}
			err := client.Ping()
			if err != nil {
				log.Printf("Unable to ping docker daemon: %s", err)
				if watching {
					client.RemoveEventListener(eventChan)
					watching = false
					client = nil
				}
				time.Sleep(10 * time.Second)
				break

			}

			if !watching {
				err = client.AddEventListener(eventChan)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
				log.Println("Watching docker events")
			}

			select {

			case event := <-eventChan:
				if event == nil {
					if watching {
						client.RemoveEventListener(eventChan)
						watching = false
						client = nil
					}
					break
				}

				if event.Status == "start" || event.Status == "stop" || event.Status == "die" {
					log.Printf("Received event %s for container %s", event.Status, event.ID[:12])
					generateFromContainers(client)
				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}
}

func initFlags() {

	certPath := filepath.Join(os.Getenv("DOCKER_CERT_PATH"))
	if certPath == "" {
		certPath = filepath.Join(os.Getenv("HOME"), ".docker")
	}
	flag.BoolVar(&version, "version", false, "show version")
	flag.BoolVar(&watch, "watch", false, "watch for container changes")
	flag.BoolVar(&onlyExposed, "only-exposed", false, "only include containers with exposed ports")

	flag.BoolVar(&onlyPublished, "only-published", false,
		"only include containers with published ports (implies -only-exposed)")
	flag.StringVar(&notifyCmd, "notify", "", "run command after template is regenerated (e.g `restart xyz`)")
	flag.StringVar(&notifySigHUPContainerID, "notify-sighup", "",
		"send HUP signal to container.  Equivalent to `docker kill -s HUP container-ID`")
	flag.Var(&configFiles, "config", "config files with template directives. Config files will be merged if this option is specified multiple times.")
	flag.IntVar(&interval, "interval", 0, "notify command interval (secs)")
	flag.StringVar(&endpoint, "endpoint", "", "docker api endpoint (tcp|unix://..). Default unix:///var/run/docker.sock")
	flag.StringVar(&tlsCert, "tlscert", filepath.Join(certPath, "cert.pem"), "path to TLS client certificate file")
	flag.StringVar(&tlsKey, "tlskey", filepath.Join(certPath, "key.pem"), "path to TLS client key file")
	flag.StringVar(&tlsCaCert, "tlscacert", filepath.Join(certPath, "ca.pem"), "path to TLS CA certificate file")
	flag.BoolVar(&tlsVerify, "tlsverify", os.Getenv("DOCKER_TLS_VERIFY") != "", "verify docker daemon's TLS certicate")

	flag.Usage = usage
	flag.Parse()
}

func main() {
	initFlags()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if flag.NArg() < 1 && len(configFiles) == 0 {
		usage()
		os.Exit(1)
	}

	if len(configFiles) > 0 {
		for _, configFile := range configFiles {
			err := loadConfig(configFile)
			if err != nil {
				log.Fatalf("error loading config %s: %s\n", configFile, err)
			}
		}
	} else {
		config := Config{
			Template:         flag.Arg(0),
			Dest:             flag.Arg(1),
			Watch:            watch,
			NotifyCmd:        notifyCmd,
			NotifyContainers: make(map[string]docker.Signal),
			OnlyExposed:      onlyExposed,
			OnlyPublished:    onlyPublished,
			Interval:         interval,
		}
		if notifySigHUPContainerID != "" {
			config.NotifyContainers[notifySigHUPContainerID] = docker.SIGHUP
		}
		configs = ConfigFile{
			Config: []Config{config}}
	}

	endpoint, err := getEndpoint()
	if err != nil {
		log.Fatalf("Bad endpoint: %s", err)
	}

	client, err := NewDockerClient(endpoint)
	if err != nil {
		log.Fatalf("Unable to create docker client: %s", err)
	}

	generateFromContainers(client)
	generateAtInterval(client, configs)
	generateFromEvents(client, configs)
	wg.Wait()
}
<commit_msg>Add DOCKER_CERT_PATH/DOCKER_TLS_VERIFY to usage docs<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	docker "github.com/fsouza/go-dockerclient"
)

type stringslice []string

var (
	buildVersion            string
	version                 bool
	watch                   bool
	notifyCmd               string
	notifySigHUPContainerID string
	onlyExposed             bool
	onlyPublished           bool
	configFiles             stringslice
	configs                 ConfigFile
	interval                int
	endpoint                string
	tlsCert                 string
	tlsKey                  string
	tlsCaCert               string
	tlsVerify               bool
	tlsCertPath             string
	wg                      sync.WaitGroup
)

type Event struct {
	ContainerID string `json:"id"`
	Status      string `json:"status"`
	Image       string `json:"from"`
}

type Address struct {
	IP       string
	Port     string
	HostPort string
	Proto    string
	HostIP   string
}

type Volume struct {
	Path      string
	HostPath  string
	ReadWrite bool
}

type RuntimeContainer struct {
	ID        string
	Addresses []Address
	Gateway   string
	Name      string
	Image     DockerImage
	Env       map[string]string
	Volumes   map[string]Volume
	Node      SwarmNode
	Labels    map[string]string
	IP        string
}

type DockerImage struct {
	Registry   string
	Repository string
	Tag        string
}

type SwarmNode struct {
	ID      string
	Name    string
	Address Address
}

func (strings *stringslice) String() string {
	return "[]"
}

func (strings *stringslice) Set(value string) error {
	// TODO: Throw an error for duplicate `dest`
	*strings = append(*strings, value)
	return nil
}

func (i *DockerImage) String() string {
	ret := i.Repository
	if i.Registry != "" {
		ret = i.Registry + "/" + i.Repository
	}
	if i.Tag != "" {
		ret = ret + ":" + i.Tag
	}
	return ret
}

type Config struct {
	Template         string
	Dest             string
	Watch            bool
	NotifyCmd        string
	NotifyContainers map[string]docker.Signal
	OnlyExposed      bool
	OnlyPublished    bool
	Interval         int
}

type ConfigFile struct {
	Config []Config
}

type Context []*RuntimeContainer

func (c *Context) Env() map[string]string {
	return splitKeyValueSlice(os.Environ())
}

func (c *ConfigFile) filterWatches() ConfigFile {
	configWithWatches := []Config{}

	for _, config := range c.Config {
		if config.Watch {
			configWithWatches = append(configWithWatches, config)
		}
	}
	return ConfigFile{
		Config: configWithWatches,
	}
}

func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
	return r.ID == o.ID && r.Image == o.Image
}

func (r *RuntimeContainer) PublishedAddresses() []Address {
	mapped := []Address{}
	for _, address := range r.Addresses {
		if address.HostPort != "" {
			mapped = append(mapped, address)
		}
	}
	return mapped
}

func usage() {
	println(`Usage: docker-gen [options] template [dest]

Generate files from docker container meta-data

Options:`)
	flag.PrintDefaults()

	println(`
Arguments:
  template - path to a template to generate
  dest - path to a write the template.  If not specfied, STDOUT is used`)

	println(`
Environment Variables:
  DOCKER_HOST - default value for -endpoint
  DOCKER_CERT_PATH - directory path containing key.pem, cert.pm and ca.pem
  DOCKER_TLS_VERIFY - enable client TLS verification
`)
}

func tlsEnabled() bool {
	for _, v := range []string{tlsCert, tlsCaCert, tlsKey} {
		if e, err := pathExists(v); e && err == nil {
			return true
		}
	}
	return false
}

func NewDockerClient(endpoint string) (*docker.Client, error) {
	if strings.HasPrefix(endpoint, "unix:") {
		return docker.NewClient(endpoint)
	} else if tlsVerify || tlsEnabled() {
		if tlsVerify {
			if e, err := pathExists(tlsCaCert); !e || err != nil {
				return nil, errors.New("TLS verification was requested, but CA cert does not exist")
			}
		}

		return docker.NewTLSClient(endpoint, tlsCert, tlsKey, tlsCaCert)
	}
	return docker.NewClient(endpoint)
}

func generateFromContainers(client *docker.Client) {
	containers, err := getContainers(client)
	if err != nil {
		log.Printf("error listing containers: %s\n", err)
		return
	}
	for _, config := range configs.Config {
		changed := generateFile(config, containers)
		if !changed {
			log.Printf("Contents of %s did not change. Skipping notification '%s'", config.Dest, config.NotifyCmd)
			continue
		}
		runNotifyCmd(config)
		sendSignalToContainer(client, config)
	}
}

func runNotifyCmd(config Config) {
	if config.NotifyCmd == "" {
		return
	}

	log.Printf("Running '%s'", config.NotifyCmd)
	cmd := exec.Command("/bin/sh", "-c", config.NotifyCmd)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Printf("Error running notify command: %s, %s\n", config.NotifyCmd, err)
		log.Print(string(out))
	}
}

func sendSignalToContainer(client *docker.Client, config Config) {
	if len(config.NotifyContainers) < 1 {
		return
	}

	for container, signal := range config.NotifyContainers {
		log.Printf("Sending container '%s' signal '%v'", container, signal)
		killOpts := docker.KillContainerOptions{
			ID:     container,
			Signal: signal,
		}
		if err := client.KillContainer(killOpts); err != nil {
			log.Printf("Error sending signal to container: %s", err)
		}
	}
}

func loadConfig(file string) error {
	_, err := toml.DecodeFile(file, &configs)
	if err != nil {
		return err
	}
	return nil
}

func generateAtInterval(client *docker.Client, configs ConfigFile) {
	for _, config := range configs.Config {

		if config.Interval == 0 {
			continue
		}

		log.Printf("Generating every %d seconds", config.Interval)
		wg.Add(1)
		ticker := time.NewTicker(time.Duration(config.Interval) * time.Second)
		quit := make(chan struct{})
		configCopy := config
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ticker.C:
					containers, err := getContainers(client)
					if err != nil {
						log.Printf("Error listing containers: %s\n", err)
						continue
					}
					// ignore changed return value. always run notify command
					generateFile(configCopy, containers)
					runNotifyCmd(configCopy)
					sendSignalToContainer(client, configCopy)
				case <-quit:
					ticker.Stop()
					return
				}
			}
		}()
	}
}

func generateFromEvents(client *docker.Client, configs ConfigFile) {
	configs = configs.filterWatches()
	if len(configs.Config) == 0 {
		return
	}

	wg.Add(1)
	defer wg.Done()

	for {
		if client == nil {
			var err error
			endpoint, err := getEndpoint()
			if err != nil {
				log.Printf("Bad endpoint: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}

			client, err = NewDockerClient(endpoint)
			if err != nil {
				log.Printf("Unable to connect to docker daemon: %s", err)
				time.Sleep(10 * time.Second)
				continue
			}
			generateFromContainers(client)
		}

		eventChan := make(chan *docker.APIEvents, 100)
		defer close(eventChan)

		watching := false
		for {

			if client == nil {
				break
			}
			err := client.Ping()
			if err != nil {
				log.Printf("Unable to ping docker daemon: %s", err)
				if watching {
					client.RemoveEventListener(eventChan)
					watching = false
					client = nil
				}
				time.Sleep(10 * time.Second)
				break

			}

			if !watching {
				err = client.AddEventListener(eventChan)
				if err != nil && err != docker.ErrListenerAlreadyExists {
					log.Printf("Error registering docker event listener: %s", err)
					time.Sleep(10 * time.Second)
					continue
				}
				watching = true
				log.Println("Watching docker events")
			}

			select {

			case event := <-eventChan:
				if event == nil {
					if watching {
						client.RemoveEventListener(eventChan)
						watching = false
						client = nil
					}
					break
				}

				if event.Status == "start" || event.Status == "stop" || event.Status == "die" {
					log.Printf("Received event %s for container %s", event.Status, event.ID[:12])
					generateFromContainers(client)
				}
			case <-time.After(10 * time.Second):
				// check for docker liveness
			}

		}
	}
}

func initFlags() {

	certPath := filepath.Join(os.Getenv("DOCKER_CERT_PATH"))
	if certPath == "" {
		certPath = filepath.Join(os.Getenv("HOME"), ".docker")
	}
	flag.BoolVar(&version, "version", false, "show version")
	flag.BoolVar(&watch, "watch", false, "watch for container changes")
	flag.BoolVar(&onlyExposed, "only-exposed", false, "only include containers with exposed ports")

	flag.BoolVar(&onlyPublished, "only-published", false,
		"only include containers with published ports (implies -only-exposed)")
	flag.StringVar(&notifyCmd, "notify", "", "run command after template is regenerated (e.g `restart xyz`)")
	flag.StringVar(&notifySigHUPContainerID, "notify-sighup", "",
		"send HUP signal to container.  Equivalent to `docker kill -s HUP container-ID`")
	flag.Var(&configFiles, "config", "config files with template directives. Config files will be merged if this option is specified multiple times.")
	flag.IntVar(&interval, "interval", 0, "notify command interval (secs)")
	flag.StringVar(&endpoint, "endpoint", "", "docker api endpoint (tcp|unix://..). Default unix:///var/run/docker.sock")
	flag.StringVar(&tlsCert, "tlscert", filepath.Join(certPath, "cert.pem"), "path to TLS client certificate file")
	flag.StringVar(&tlsKey, "tlskey", filepath.Join(certPath, "key.pem"), "path to TLS client key file")
	flag.StringVar(&tlsCaCert, "tlscacert", filepath.Join(certPath, "ca.pem"), "path to TLS CA certificate file")
	flag.BoolVar(&tlsVerify, "tlsverify", os.Getenv("DOCKER_TLS_VERIFY") != "", "verify docker daemon's TLS certicate")

	flag.Usage = usage
	flag.Parse()
}

func main() {
	initFlags()

	if version {
		fmt.Println(buildVersion)
		return
	}

	if flag.NArg() < 1 && len(configFiles) == 0 {
		usage()
		os.Exit(1)
	}

	if len(configFiles) > 0 {
		for _, configFile := range configFiles {
			err := loadConfig(configFile)
			if err != nil {
				log.Fatalf("error loading config %s: %s\n", configFile, err)
			}
		}
	} else {
		config := Config{
			Template:         flag.Arg(0),
			Dest:             flag.Arg(1),
			Watch:            watch,
			NotifyCmd:        notifyCmd,
			NotifyContainers: make(map[string]docker.Signal),
			OnlyExposed:      onlyExposed,
			OnlyPublished:    onlyPublished,
			Interval:         interval,
		}
		if notifySigHUPContainerID != "" {
			config.NotifyContainers[notifySigHUPContainerID] = docker.SIGHUP
		}
		configs = ConfigFile{
			Config: []Config{config}}
	}

	endpoint, err := getEndpoint()
	if err != nil {
		log.Fatalf("Bad endpoint: %s", err)
	}

	client, err := NewDockerClient(endpoint)
	if err != nil {
		log.Fatalf("Unable to create docker client: %s", err)
	}

	generateFromContainers(client)
	generateAtInterval(client, configs)
	generateFromEvents(client, configs)
	wg.Wait()
}
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"testing"

	. "github.com/quintans/toolkit/collections"
	. "github.com/quintans/toolkit/ext"
)

const value1 = "World"

func TestPutAndGet(t *testing.T) {
	hmap := NewHashMap()
	k := Str("Hello")
	hmap.Put(k, value1)
	v, ok := hmap.Get(k)
	if !ok || value1 != v {
		t.Error("Expected "+value1+", got ", v)
	}
	v, ok = hmap.Get(Str("lixo"))
	if ok {
		t.Error("Expected to find nothing, got ", v)
	}
}

func TestResize(t *testing.T) {
	hmap := NewHashMap()
	loop := 20
	// Insert
	for i := 0; i < loop; i++ {
		hmap.Put(Str("Hello"+string(i)), i*10)
	}
	if hmap.Size() != loop {
		t.Error("Expected "+string(loop)+", got ", hmap.Size())
	}
	// Check
	for i := 0; i < loop; i++ {
		v, _ := hmap.Get(Str("Hello" + string(i)))
		k := i * 10
		if k != v {
			t.Error("Expected "+string(k)+", got ", v)
		}
	}
	// Delete
	for i := 0; i < loop; i++ {
		v := hmap.Delete(Str("Hello" + string(i)))
		k := i * 10
		if k != v {
			t.Error("Expected deletion of "+string(k)+", got ", v)
		}
	}
	if hmap.Size() != 0 {
		t.Error("Expected 0, got ", hmap.Size())
	}
}

var dics = []KeyValue{
	KeyValue{Str("Martim"), 9},
	KeyValue{Str("Paulo"), 41},
	KeyValue{Str("Monica"), 33},
	KeyValue{Str("Francisca"), 15},
}

func TestHashMapIterator(t *testing.T) {
	dic := NewHashMap()
	dic.Put(dics[0].Key, dics[0].Value)
	dic.Put(dics[1].Key, dics[1].Value)
	dic.Put(dics[2].Key, dics[2].Value)
	dic.Put(dics[3].Key, dics[3].Value)

	fmt.Println("========> Iterating through ", dics)
	for it := dic.Iterator(); it.HasNext(); {
		fmt.Println(it.Next())
	}

	dic.Delete(dics[1].Key)

	fmt.Println("========> Deleting ", dics[1])
	for it := dic.Iterator(); it.HasNext(); {
		fmt.Println(it.Next())
	}
}

func TestLinkedHashMapIterator(t *testing.T) {
	dic := NewLinkedHashMap()
	dic.Put(dics[0].Key, dics[0].Value)
	dic.Put(dics[1].Key, dics[1].Value)
	dic.Put(dics[2].Key, dics[2].Value)
	dic.Put(dics[3].Key, dics[3].Value)

	for k, v := range dic.Elements() {
		if !v.Key.Equals(dics[k].Key) {
			t.Errorf("Value %s at position %v does not match with %s\n", v.Key, k, dics[k].Key)
		}
	}
	dic.Delete(dics[1].Key)

	set := []KeyValue{dics[0], dics[2], dics[3]}
	for k, v := range dic.Elements() {
		if !v.Key.Equals(set[k].Key) {
			t.Errorf("Value %s at position %s does not match with %s after delete\n", v.Key, k, dics[k].Key)
		}
	}

	//	fmt.Println("========> Deleting ", dics[1])
	//	for it := dic.Iterator(); it.HasNext(); {
	//		fmt.Println(it.Next())
	//	}
}
<commit_msg>fix text format arg<commit_after>package test

import (
	"fmt"
	"testing"

	. "github.com/quintans/toolkit/collections"
	. "github.com/quintans/toolkit/ext"
)

const value1 = "World"

func TestPutAndGet(t *testing.T) {
	hmap := NewHashMap()
	k := Str("Hello")
	hmap.Put(k, value1)
	v, ok := hmap.Get(k)
	if !ok || value1 != v {
		t.Error("Expected "+value1+", got ", v)
	}
	v, ok = hmap.Get(Str("lixo"))
	if ok {
		t.Error("Expected to find nothing, got ", v)
	}
}

func TestResize(t *testing.T) {
	hmap := NewHashMap()
	loop := 20
	// Insert
	for i := 0; i < loop; i++ {
		hmap.Put(Str("Hello"+string(i)), i*10)
	}
	if hmap.Size() != loop {
		t.Error("Expected "+string(loop)+", got ", hmap.Size())
	}
	// Check
	for i := 0; i < loop; i++ {
		v, _ := hmap.Get(Str("Hello" + string(i)))
		k := i * 10
		if k != v {
			t.Error("Expected "+string(k)+", got ", v)
		}
	}
	// Delete
	for i := 0; i < loop; i++ {
		v := hmap.Delete(Str("Hello" + string(i)))
		k := i * 10
		if k != v {
			t.Error("Expected deletion of "+string(k)+", got ", v)
		}
	}
	if hmap.Size() != 0 {
		t.Error("Expected 0, got ", hmap.Size())
	}
}

var dics = []KeyValue{
	KeyValue{Str("Martim"), 9},
	KeyValue{Str("Paulo"), 41},
	KeyValue{Str("Monica"), 33},
	KeyValue{Str("Francisca"), 15},
}

func TestHashMapIterator(t *testing.T) {
	dic := NewHashMap()
	dic.Put(dics[0].Key, dics[0].Value)
	dic.Put(dics[1].Key, dics[1].Value)
	dic.Put(dics[2].Key, dics[2].Value)
	dic.Put(dics[3].Key, dics[3].Value)

	fmt.Println("========> Iterating through ", dics)
	for it := dic.Iterator(); it.HasNext(); {
		fmt.Println(it.Next())
	}

	dic.Delete(dics[1].Key)

	fmt.Println("========> Deleting ", dics[1])
	for it := dic.Iterator(); it.HasNext(); {
		fmt.Println(it.Next())
	}
}

func TestLinkedHashMapIterator(t *testing.T) {
	dic := NewLinkedHashMap()
	dic.Put(dics[0].Key, dics[0].Value)
	dic.Put(dics[1].Key, dics[1].Value)
	dic.Put(dics[2].Key, dics[2].Value)
	dic.Put(dics[3].Key, dics[3].Value)

	for k, v := range dic.Elements() {
		if !v.Key.Equals(dics[k].Key) {
			t.Errorf("Value %s at position %v does not match with %s\n", v.Key, k, dics[k].Key)
		}
	}
	dic.Delete(dics[1].Key)

	set := []KeyValue{dics[0], dics[2], dics[3]}
	for k, v := range dic.Elements() {
		if !v.Key.Equals(set[k].Key) {
			t.Errorf("Value %s at position %d does not match with %s after delete\n", v.Key, k, dics[k].Key)
		}
	}

	//	fmt.Println("========> Deleting ", dics[1])
	//	for it := dic.Iterator(); it.HasNext(); {
	//		fmt.Println(it.Next())
	//	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nodevstat

package collector

import (
	"errors"
	"fmt"
	"sync"
	"unsafe"

	"github.com/prometheus/client_golang/prometheus"
)

// #cgo LDFLAGS: -ldevstat -lkvm -lelf
// #include "devstat_freebsd.h"
import "C"

const (
	devstatSubsystem = "devstat"
)

type devstatCollector struct {
	mu      sync.Mutex
	devinfo *C.struct_devinfo

	bytes       typedDesc
	bytes_total typedDesc
	transfers   typedDesc
	duration    typedDesc
	busyTime    typedDesc
	blocks      typedDesc
}

func init() {
	Factories["devstat"] = NewDevstatCollector
}

// NewDevstatCollector returns a new Collector exposing Device stats.
func NewDevstatCollector() (Collector, error) {
	return &devstatCollector{
		devinfo: &C.struct_devinfo{},
		bytes: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "bytes_total"),
			"The total number of bytes in transactions.",
			[]string{"device", "type"}, nil,
		), prometheus.CounterValue},
		transfers: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "transfers_total"),
			"The total number of transactions.",
			[]string{"device", "type"}, nil,
		), prometheus.CounterValue},
		duration: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "duration_seconds_total"),
			"The total duration of transactions in seconds.",
			[]string{"device", "type"}, nil,
		), prometheus.CounterValue},
		busyTime: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "busy_time_seconds_total"),
			"Total time the device had one or more transactions outstanding in seconds.",
			[]string{"device"}, nil,
		), prometheus.CounterValue},
		blocks: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "blocks_transferred_total"),
			"The total number of blocks transferred.",
			[]string{"device"}, nil,
		), prometheus.CounterValue},
	}, nil
}

func (c *devstatCollector) Update(ch chan<- prometheus.Metric) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	var stats *C.Stats
	n := C._get_stats(c.devinfo, &stats)
	if n == -1 {
		return errors.New("devstat_getdevs failed")
	}

	base := unsafe.Pointer(stats)
	for i := C.int(0); i < n; i++ {
		offset := i * C.int(C.sizeof_Stats)
		stat := (*C.Stats)(unsafe.Pointer(uintptr(base) + uintptr(offset)))

		device := fmt.Sprintf("%s%d", C.GoString(&stat.device[0]), stat.unit)
		ch <- c.bytes.mustNewConstMetric(float64(stat.bytes.read), device, "read")
		ch <- c.bytes.mustNewConstMetric(float64(stat.bytes.write), device, "write")
		ch <- c.transfers.mustNewConstMetric(float64(stat.transfers.other), device, "other")
		ch <- c.transfers.mustNewConstMetric(float64(stat.transfers.read), device, "read")
		ch <- c.transfers.mustNewConstMetric(float64(stat.transfers.write), device, "write")
		ch <- c.duration.mustNewConstMetric(float64(stat.duration.other), device, "other")
		ch <- c.duration.mustNewConstMetric(float64(stat.duration.read), device, "read")
		ch <- c.duration.mustNewConstMetric(float64(stat.duration.write), device, "write")
		ch <- c.busyTime.mustNewConstMetric(float64(stat.busyTime), device)
		ch <- c.blocks.mustNewConstMetric(float64(stat.blocks), device)
	}
	C.free(unsafe.Pointer(stats))
	return nil
}
<commit_msg>Remove unused devstatCollector struct member 'bytes_total'.<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build !nodevstat

package collector

import (
	"errors"
	"fmt"
	"sync"
	"unsafe"

	"github.com/prometheus/client_golang/prometheus"
)

// #cgo LDFLAGS: -ldevstat -lkvm -lelf
// #include "devstat_freebsd.h"
import "C"

const (
	devstatSubsystem = "devstat"
)

type devstatCollector struct {
	mu      sync.Mutex
	devinfo *C.struct_devinfo

	bytes     typedDesc
	transfers typedDesc
	duration  typedDesc
	busyTime  typedDesc
	blocks    typedDesc
}

func init() {
	Factories["devstat"] = NewDevstatCollector
}

// NewDevstatCollector returns a new Collector exposing Device stats.
func NewDevstatCollector() (Collector, error) {
	return &devstatCollector{
		devinfo: &C.struct_devinfo{},
		bytes: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "bytes_total"),
			"The total number of bytes in transactions.",
			[]string{"device", "type"}, nil,
		), prometheus.CounterValue},
		transfers: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "transfers_total"),
			"The total number of transactions.",
			[]string{"device", "type"}, nil,
		), prometheus.CounterValue},
		duration: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "duration_seconds_total"),
			"The total duration of transactions in seconds.",
			[]string{"device", "type"}, nil,
		), prometheus.CounterValue},
		busyTime: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "busy_time_seconds_total"),
			"Total time the device had one or more transactions outstanding in seconds.",
			[]string{"device"}, nil,
		), prometheus.CounterValue},
		blocks: typedDesc{prometheus.NewDesc(
			prometheus.BuildFQName(Namespace, devstatSubsystem, "blocks_transferred_total"),
			"The total number of blocks transferred.",
			[]string{"device"}, nil,
		), prometheus.CounterValue},
	}, nil
}

func (c *devstatCollector) Update(ch chan<- prometheus.Metric) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	var stats *C.Stats
	n := C._get_stats(c.devinfo, &stats)
	if n == -1 {
		return errors.New("devstat_getdevs failed")
	}

	base := unsafe.Pointer(stats)
	for i := C.int(0); i < n; i++ {
		offset := i * C.int(C.sizeof_Stats)
		stat := (*C.Stats)(unsafe.Pointer(uintptr(base) + uintptr(offset)))

		device := fmt.Sprintf("%s%d", C.GoString(&stat.device[0]), stat.unit)
		ch <- c.bytes.mustNewConstMetric(float64(stat.bytes.read), device, "read")
		ch <- c.bytes.mustNewConstMetric(float64(stat.bytes.write), device, "write")
		ch <- c.transfers.mustNewConstMetric(float64(stat.transfers.other), device, "other")
		ch <- c.transfers.mustNewConstMetric(float64(stat.transfers.read), device, "read")
		ch <- c.transfers.mustNewConstMetric(float64(stat.transfers.write), device, "write")
		ch <- c.duration.mustNewConstMetric(float64(stat.duration.other), device, "other")
		ch <- c.duration.mustNewConstMetric(float64(stat.duration.read), device, "read")
		ch <- c.duration.mustNewConstMetric(float64(stat.duration.write), device, "write")
		ch <- c.busyTime.mustNewConstMetric(float64(stat.busyTime), device)
		ch <- c.blocks.mustNewConstMetric(float64(stat.blocks), device)
	}
	C.free(unsafe.Pointer(stats))
	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"os"
	"os/exec"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	checkoutCmd = &cobra.Command{
		Use:   "checkout",
		Short: "Checks out LFS files into the working copy",
		Run:   checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {

	// Parameters are filters
	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	var rootedpaths = make([]string, len(args))
	inchan := make(chan string, 1)
	outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
	if err != nil {
		Panic(err, "Could not checkout")
	}
	for _, arg := range args {
		inchan <- arg
		rootedpaths = append(rootedpaths, <-outchan)
	}
	close(inchan)
	checkoutWithIncludeExclude(rootedpaths, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
	tracerx.Printf("starting fetch/parallel checkout")
	checkoutFromFetchChan(nil, nil, c)
}

func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}
	// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
	pointers, err := lfs.ScanTree(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	// Map oid to multiple pointers
	mapping := make(map[string][]*lfs.WrappedPointer)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
		}
	}

	// Launch git update-index
	c := make(chan *lfs.WrappedPointer)
	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Feed it from in, which comes from fetch
	for p := range in {
		// Add all of the files for this oid
		for _, fp := range mapping[p.Oid] {
			c <- fp
		}
	}
	close(c)
	wait.Wait()
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer, 1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Count bytes for progress
	var totalBytes int64
	for _, pointer := range pointers {
		totalBytes += pointer.Size
	}
	progress := lfs.NewProgressMeter(len(pointers), totalBytes, false)
	progress.Start()
	totalBytes = 0
	for _, pointer := range pointers {
		totalBytes += pointer.Size
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			progress.Add(pointer.Name)
			c <- pointer
			// not strictly correct (parallel) but we don't have a callback & it's just local
			// plus only 1 slot in channel so it'll block & be close
			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
			progress.FinishTransfer(pointer.Name)
		} else {
			progress.Skip(pointer.Size)
		}
	}
	close(c)
	wait.Wait()
	progress.Finish()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Fire up the update-index command
	cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
	updateIdxStdin, err := cmd.StdinPipe()
	if err != nil {
		Panic(err, "Could not update the index")
	}

	if err := cmd.Start(); err != nil {
		Panic(err, "Could not update the index")
	}

	// Get a converter from repo-relative to cwd-relative
	// Since writing data & calling git update-index must be relative to cwd
	repopathchan := make(chan string, 1)
	cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
	if err != nil {
		Panic(err, "Could not convert file paths")
	}

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if err == lfs.NotAPointerError {
				// File has non-pointer content, leave it alone
				continue
			}
			Panic(err, "Problem accessing %v", pointer.Name)
		}
		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}
		// OK now we can (over)write the file content
		repopathchan <- pointer.Name
		cwdfilepath := <-cwdpathchan
		// smudge but set auto-download to false
		err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
		if err != nil {
			if err == lfs.DownloadDeclinedError {
				// acceptable error, data not local (fetch not run or include/exclude)
				LoggedError(err, "Skipped checkout for %v, content not local. Use fetch to download.", pointer.Name)
			} else {
				Panic(err, "Could not checkout file")
			}
		}

		updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
	}
	close(repopathchan)

	updateIdxStdin.Close()
	if err := cmd.Wait(); err != nil {
		Panic(err, "Error updating the git index")
	}

}
<commit_msg>ラララララ ラー ウウウ フフフ<commit_after>package commands

import (
	"io"
	"os"
	"os/exec"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	checkoutCmd = &cobra.Command{
		Use:   "checkout",
		Short: "Checks out LFS files into the working copy",
		Run:   checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {

	// Parameters are filters
	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	var rootedpaths = make([]string, len(args))
	inchan := make(chan string, 1)
	outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
	if err != nil {
		Panic(err, "Could not checkout")
	}
	for _, arg := range args {
		inchan <- arg
		rootedpaths = append(rootedpaths, <-outchan)
	}
	close(inchan)
	checkoutWithIncludeExclude(rootedpaths, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
	tracerx.Printf("starting fetch/parallel checkout")
	checkoutFromFetchChan(nil, nil, c)
}

func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}
	// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
	pointers, err := lfs.ScanTree(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	// Map oid to multiple pointers
	mapping := make(map[string][]*lfs.WrappedPointer)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
		}
	}

	// Launch git update-index
	c := make(chan *lfs.WrappedPointer)
	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Feed it from in, which comes from fetch
	for p := range in {
		// Add all of the files for this oid
		for _, fp := range mapping[p.Oid] {
			c <- fp
		}
	}
	close(c)
	wait.Wait()
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer, 1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Count bytes for progress
	var totalBytes int64
	for _, pointer := range pointers {
		totalBytes += pointer.Size
	}
	progress := lfs.NewProgressMeter(len(pointers), totalBytes, false)
	progress.Start()
	totalBytes = 0
	for _, pointer := range pointers {
		totalBytes += pointer.Size
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			progress.Add(pointer.Name)
			c <- pointer
			// not strictly correct (parallel) but we don't have a callback & it's just local
			// plus only 1 slot in channel so it'll block & be close
			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
			progress.FinishTransfer(pointer.Name)
		} else {
			progress.Skip(pointer.Size)
		}
	}
	close(c)
	wait.Wait()
	progress.Finish()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Don't fire up the update-index command until we have at least one file to
	// give it. Otherwise git interprets the lack of arguments to mean param-less update-index
	// which can trigger entire working copy to be re-examined, which triggers clean filters
	// and which has unexpected side effects (e.g. downloading filtered-out files)
	var cmd *exec.Cmd
	var updateIdxStdin io.WriteCloser
	var err error

	// Get a converter from repo-relative to cwd-relative
	// Since writing data & calling git update-index must be relative to cwd
	repopathchan := make(chan string, 1)
	cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
	if err != nil {
		Panic(err, "Could not convert file paths")
	}

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if err == lfs.NotAPointerError {
				// File has non-pointer content, leave it alone
				continue
			}
			Panic(err, "Problem accessing %v", pointer.Name)
		}
		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}
		// OK now we can (over)write the file content
		repopathchan <- pointer.Name
		cwdfilepath := <-cwdpathchan
		// smudge but set auto-download to false
		err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
		if err != nil {
			if err == lfs.DownloadDeclinedError {
				// acceptable error, data not local (fetch not run or include/exclude)
				LoggedError(err, "Skipped checkout for %v, content not local. Use fetch to download.", pointer.Name)
			} else {
				Panic(err, "Could not checkout file")
			}
		}

		if cmd == nil {
			// Fire up the update-index command
			cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
			updateIdxStdin, err = cmd.StdinPipe()
			if err != nil {
				Panic(err, "Could not update the index")
			}

			if err := cmd.Start(); err != nil {
				Panic(err, "Could not update the index")
			}

		}

		updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
	}
	close(repopathchan)

	if cmd != nil && updateIdxStdin != nil {
		updateIdxStdin.Close()
		if err := cmd.Wait(); err != nil {
			Panic(err, "Error updating the git index")
		}
	}

}
<|endoftext|>


<commit_before>package commands

import (
	"io"
	"os"
	"os/exec"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	checkoutCmd = &cobra.Command{
		Use: "checkout",
		Run: checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {
	requireInRepo()

	// Parameters are filters
	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	var rootedpaths []string

	inchan := make(chan string, 1)
	outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
	if err != nil {
		Panic(err, "Could not checkout")
	}
	for _, arg := range args {
		inchan <- arg
		rootedpaths = append(rootedpaths, <-outchan)
	}
	close(inchan)
	checkoutWithIncludeExclude(rootedpaths, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
	tracerx.Printf("starting fetch/parallel checkout")
	checkoutFromFetchChan(nil, nil, c)
}

func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}
	// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	// Map oid to multiple pointers
	mapping := make(map[string][]*lfs.WrappedPointer)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
		}
	}

	// Launch git update-index
	c := make(chan *lfs.WrappedPointer)

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Feed it from in, which comes from fetch
	for p := range in {
		// Add all of the files for this oid
		for _, fp := range mapping[p.Oid] {
			c <- fp
		}
	}
	close(c)
	wait.Wait()
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer, 1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Count bytes for progress
	var totalBytes int64
	for _, pointer := range pointers {
		totalBytes += pointer.Size
	}
	progress := lfs.NewProgressMeter(len(pointers), totalBytes, false)
	progress.Start()
	totalBytes = 0
	for _, pointer := range pointers {
		totalBytes += pointer.Size
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			progress.Add(pointer.Name)
			c <- pointer
			// not strictly correct (parallel) but we don't have a callback & it's just local
			// plus only 1 slot in channel so it'll block & be close
			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
			progress.FinishTransfer(pointer.Name)
		} else {
			progress.Skip(pointer.Size)
		}
	}
	close(c)
	wait.Wait()
	progress.Finish()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
// Callers of this function MUST NOT Panic or otherwise exit the process
// without waiting for this function to shut down.  If the process exits while
// update-index is in the middle of processing a file the git index can be left
// in a locked state.
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Get a converter from repo-relative to cwd-relative
	// Since writing data & calling git update-index must be relative to cwd
	repopathchan := make(chan string, 1)
	cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
	if err != nil {
		Panic(err, "Could not convert file paths")
	}

	// Don't fire up the update-index command until we have at least one file to
	// give it. Otherwise git interprets the lack of arguments to mean param-less update-index
	// which can trigger entire working copy to be re-examined, which triggers clean filters
	// and which has unexpected side effects (e.g. downloading filtered-out files)
	var cmd *exec.Cmd
	var updateIdxStdin io.WriteCloser

	// From this point on, git update-index is running. Code in this loop MUST
	// NOT Panic() or otherwise cause the process to exit. If the process exits
	// while update-index is in the middle of updating, the index can remain in a
	// locked state.

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if lfs.IsNotAPointerError(err) {
				// File has non-pointer content, leave it alone
				continue
			}
			LoggedError(err, "Problem accessing %v", pointer.Name)
			continue
		}

		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}

		repopathchan <- pointer.Name
		cwdfilepath := <-cwdpathchan

		err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
		if err != nil {
			if lfs.IsDownloadDeclinedError(err) {
				// acceptable error, data not local (fetch not run or include/exclude)
				LoggedError(err, "Skipped checkout for %v, content not local. Use fetch to download.", pointer.Name)
			} else {
				LoggedError(err, "Could not checkout file")
				continue
			}
		}

		if cmd == nil {
			// Fire up the update-index command
			cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
			updateIdxStdin, err = cmd.StdinPipe()
			if err != nil {
				Panic(err, "Could not update the index")
			}

			if err := cmd.Start(); err != nil {
				Panic(err, "Could not update the index")
			}

		}

		updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
	}
	close(repopathchan)

	if cmd != nil && updateIdxStdin != nil {
		updateIdxStdin.Close()
		if err := cmd.Wait(); err != nil {
			Panic(err, "Error updating the git index")
		}
	}
}
<commit_msg>アアー アアアア アーアア<commit_after>package commands

import (
	"io"
	"os"
	"os/exec"
	"sync"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	checkoutCmd = &cobra.Command{
		Use: "checkout",
		Run: checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {
	requireInRepo()

	// Parameters are filters
	// firstly convert any pathspecs to the root of the repo, in case this is being executed in a sub-folder
	var rootedpaths []string

	inchan := make(chan string, 1)
	outchan, err := lfs.ConvertCwdFilesRelativeToRepo(inchan)
	if err != nil {
		Panic(err, "Could not checkout")
	}
	for _, arg := range args {
		inchan <- arg
		rootedpaths = append(rootedpaths, <-outchan)
	}
	close(inchan)
	checkoutWithIncludeExclude(rootedpaths, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

// Checkout from items reported from the fetch process (in parallel)
func checkoutAllFromFetchChan(c chan *lfs.WrappedPointer) {
	tracerx.Printf("starting fetch/parallel checkout")
	checkoutFromFetchChan(nil, nil, c)
}

func checkoutFromFetchChan(include []string, exclude []string, in chan *lfs.WrappedPointer) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}
	// Need to ScanTree to identify multiple files with the same content (fetch will only report oids once)
	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	// Map oid to multiple pointers
	mapping := make(map[string][]*lfs.WrappedPointer)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			mapping[pointer.Oid] = append(mapping[pointer.Oid], pointer)
		}
	}

	// Launch git update-index
	c := make(chan *lfs.WrappedPointer)

	var wait sync.WaitGroup
	wait.Add(1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Feed it from in, which comes from fetch
	for p := range in {
		// Add all of the files for this oid
		for _, fp := range mapping[p.Oid] {
			c <- fp
		}
	}
	close(c)
	wait.Wait()
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref.Sha)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer, 1)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()

	// Count bytes for progress
	var totalBytes int64
	for _, pointer := range pointers {
		totalBytes += pointer.Size
	}
	progress := lfs.NewProgressMeter(len(pointers), totalBytes, false)
	progress.Start()
	totalBytes = 0
	for _, pointer := range pointers {
		totalBytes += pointer.Size
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			progress.Add(pointer.Name)
			c <- pointer
			// not strictly correct (parallel) but we don't have a callback & it's just local
			// plus only 1 slot in channel so it'll block & be close
			progress.TransferBytes("checkout", pointer.Name, pointer.Size, totalBytes, int(pointer.Size))
			progress.FinishTransfer(pointer.Name)
		} else {
			progress.Skip(pointer.Size)
		}
	}
	close(c)
	wait.Wait()
	progress.Finish()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
// Callers of this function MUST NOT Panic or otherwise exit the process
// without waiting for this function to shut down.  If the process exits while
// update-index is in the middle of processing a file the git index can be left
// in a locked state.
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Get a converter from repo-relative to cwd-relative
	// Since writing data & calling git update-index must be relative to cwd
	repopathchan := make(chan string, 1)
	cwdpathchan, err := lfs.ConvertRepoFilesRelativeToCwd(repopathchan)
	if err != nil {
		Panic(err, "Could not convert file paths")
	}

	// Don't fire up the update-index command until we have at least one file to
	// give it. Otherwise git interprets the lack of arguments to mean param-less update-index
	// which can trigger entire working copy to be re-examined, which triggers clean filters
	// and which has unexpected side effects (e.g. downloading filtered-out files)
	var cmd *exec.Cmd
	var updateIdxStdin io.WriteCloser

	// From this point on, git update-index is running. Code in this loop MUST
	// NOT Panic() or otherwise cause the process to exit. If the process exits
	// while update-index is in the middle of updating, the index can remain in a
	// locked state.

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if lfs.IsNotAPointerError(err) {
				// File has non-pointer content, leave it alone
				continue
			}
			LoggedError(err, "Problem accessing %v", pointer.Name)
			continue
		}

		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}

		repopathchan <- pointer.Name
		cwdfilepath := <-cwdpathchan

		err = lfs.PointerSmudgeToFile(cwdfilepath, pointer.Pointer, false, nil)
		if err != nil {
			if lfs.IsDownloadDeclinedError(err) {
				// acceptable error, data not local (fetch not run or include/exclude)
				LoggedError(err, "Skipped checkout for %v, content not local. Use fetch to download.", pointer.Name)
			} else {
				LoggedError(err, "Could not checkout file")
				continue
			}
		}

		if cmd == nil {
			// Fire up the update-index command
			cmd = exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
			updateIdxStdin, err = cmd.StdinPipe()
			if err != nil {
				Panic(err, "Could not update the index")
			}

			if err := cmd.Start(); err != nil {
				Panic(err, "Could not update the index")
			}

		}

		updateIdxStdin.Write([]byte(cwdfilepath + "\n"))
	}
	close(repopathchan)

	if cmd != nil && updateIdxStdin != nil {
		updateIdxStdin.Close()
		if err := cmd.Wait(); err != nil {
			outp, _ := cmd.CombinedOutput()
			LoggedError(err, "Error updating the git index\n%v", string(outp))
		}
	}
}
<|endoftext|>


<commit_before>package common

import "fmt"

type APIRequestError struct {
	Err error
}

func (e APIRequestError) Error() string {
	return "Request error: {{.Error}}\nTIP: If you are behind a firewall and require an HTTP proxy, verify the https_proxy environment variable is correctly set. Else, check your network connection."
}

func (e APIRequestError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"Error": e.Err,
	})
}

type InvalidSSLCertError struct {
	API string
}

func (e InvalidSSLCertError) Error() string {
	return "Invalid SSL Cert for {{.API}}\nTIP: Use 'cf api --skip-ssl-validation' to continue with an insecure API endpoint"
}

func (e InvalidSSLCertError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"API": e.API,
	})
}

type NoAPISetError struct {
	BinaryName string
}

func (e NoAPISetError) Error() string {
	return "No API endpoint set. Use '{{.LoginTip}}' or '{{.APITip}}' to target an endpoint."
}

func (e NoAPISetError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"LoginTip": fmt.Sprintf("%s login", e.BinaryName),
		"APITip":   fmt.Sprintf("%s api", e.BinaryName),
	})
}

type NotLoggedInError struct {
	BinaryName string
}

func (e NotLoggedInError) Error() string {
	return "Not logged in. Use '{{.CFLoginCommand}}' to log in."
}

func (e NotLoggedInError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"CFLoginCommand": fmt.Sprintf("%s login", e.BinaryName),
	})
}

type NoTargetedOrgError struct {
	BinaryName string
}

func (e NoTargetedOrgError) Error() string {
	return "No org targeted, use '{{.Command}}' to target an org."
}

func (e NoTargetedOrgError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"Command": fmt.Sprintf("%s login", e.BinaryName),
	})
}

type NoTargetedSpaceError struct {
	BinaryName string
}

func (e NoTargetedSpaceError) Error() string {
	return "No space targeted, use '{{.Command}}' to target a space"
}

func (e NoTargetedSpaceError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"Command": fmt.Sprintf("%s login", e.BinaryName),
	})
}

type ApplicationNotFoundError struct {
	Name string
}

func (e ApplicationNotFoundError) Error() string {
	return "App {{.AppName}} not found"
}

func (e ApplicationNotFoundError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"AppName": e.Name,
	})
}

type ServiceInstanceNotFoundError struct {
	Name string
}

func (e ServiceInstanceNotFoundError) Error() string {
	return "Service instance {{.ServiceInstance}} not found"
}

func (e ServiceInstanceNotFoundError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"ServiceInstance": e.Name,
	})
}
<commit_msg>better error for unset org or space<commit_after>package common

import "fmt"

type APIRequestError struct {
	Err error
}

func (e APIRequestError) Error() string {
	return "Request error: {{.Error}}\nTIP: If you are behind a firewall and require an HTTP proxy, verify the https_proxy environment variable is correctly set. Else, check your network connection."
}

func (e APIRequestError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"Error": e.Err,
	})
}

type InvalidSSLCertError struct {
	API string
}

func (e InvalidSSLCertError) Error() string {
	return "Invalid SSL Cert for {{.API}}\nTIP: Use 'cf api --skip-ssl-validation' to continue with an insecure API endpoint"
}

func (e InvalidSSLCertError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"API": e.API,
	})
}

type NoAPISetError struct {
	BinaryName string
}

func (e NoAPISetError) Error() string {
	return "No API endpoint set. Use '{{.LoginTip}}' or '{{.APITip}}' to target an endpoint."
}

func (e NoAPISetError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"LoginTip": fmt.Sprintf("%s login", e.BinaryName),
		"APITip":   fmt.Sprintf("%s api", e.BinaryName),
	})
}

type NotLoggedInError struct {
	BinaryName string
}

func (e NotLoggedInError) Error() string {
	return "Not logged in. Use '{{.CFLoginCommand}}' to log in."
}

func (e NotLoggedInError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"CFLoginCommand": fmt.Sprintf("%s login", e.BinaryName),
	})
}

type NoTargetedOrgError struct {
	BinaryName string
}

func (e NoTargetedOrgError) Error() string {
	return "No org targeted, use '{{.Command}}' to target an org."
}

func (e NoTargetedOrgError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"Command": fmt.Sprintf("%s target -o ORG", e.BinaryName),
	})
}

type NoTargetedSpaceError struct {
	BinaryName string
}

func (e NoTargetedSpaceError) Error() string {
	return "No space targeted, use '{{.Command}}' to target a space"
}

func (e NoTargetedSpaceError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"Command": fmt.Sprintf("%s target -s SPACE", e.BinaryName),
	})
}

type ApplicationNotFoundError struct {
	Name string
}

func (e ApplicationNotFoundError) Error() string {
	return "App {{.AppName}} not found"
}

func (e ApplicationNotFoundError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"AppName": e.Name,
	})
}

type ServiceInstanceNotFoundError struct {
	Name string
}

func (e ServiceInstanceNotFoundError) Error() string {
	return "Service instance {{.ServiceInstance}} not found"
}

func (e ServiceInstanceNotFoundError) Translate(translate func(string, ...interface{}) string) string {
	return translate(e.Error(), map[string]interface{}{
		"ServiceInstance": e.Name,
	})
}
<|endoftext|>


<commit_before>package obj

import (
	"fmt"
	"reflect"

	"github.com/polydawn/refmt/obj/atlas"
	. "github.com/polydawn/refmt/tok"
)

/*
	A lovely mechanism to stash marshalMachine objects pre-allocated and avoid mallocs.
	Works together with the Atlas: the Atlas says what kind of machinery is needed;
	the marshalSlab "allocates" it and returns it upon your request.
*/
type marshalSlab struct {
	atlas atlas.Atlas
	rows  []marshalSlabRow
}

type marshalSlabRow struct {
	ptrDerefDelegateMarshalMachine
	marshalMachinePrimitive
	marshalMachineWildcard
	marshalMachineMapWildcard
	marshalMachineSliceWildcard
	marshalMachineStructAtlas
	marshalMachineTransform

	errThunkMarshalMachine
}

// A thunk value that can be used to trigger `isNil` paths.
// (Substituting an 'invalid' kind reflect.Value with this is an easy way
// to emit a null without needing any additional special cases or error handling.)
var nil_rv reflect.Value = reflect.Zero(reflect.PtrTo(reflect.TypeOf(0)))

/*
	Return a reference to a machine from the slab.
	*You must release() when done.*

	Errors -- including "no info in Atlas for this type" -- are expressed by
	returning a machine that is a constantly-erroring thunk.
*/
func (slab *marshalSlab) requisitionMachine(rt reflect.Type) MarshalMachine {
	// Acquire a row.
	off := len(slab.rows)
	slab.grow()
	row := &slab.rows[off]

	// Indirect pointers as necessary.
	//  Keep count of how many times we do this; we'll use this again at the end.
	peelCount := 0
	for rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
		peelCount++
	}

	// Figure out what machinery to use at heart.
	mach := _yieldMarshalMachinePtr(row, slab.atlas, rt)
	// If nil answer, we had no match: yield an error thunk.
	if mach == nil {
		mach := &row.errThunkMarshalMachine
		mach.err = fmt.Errorf("no machine found")
		return mach
	}

	// If no indirection steps, return;
	//  otherwise wrap it in the ptrDeref machine and return that.
	if peelCount == 0 {
		return mach
	}
	row.ptrDerefDelegateMarshalMachine.MarshalMachine = mach
	row.ptrDerefDelegateMarshalMachine.peelCount = peelCount
	row.ptrDerefDelegateMarshalMachine.isNil = false
	return &row.ptrDerefDelegateMarshalMachine
}

var defaultCfg = &atlas.AtlasEntry{
	MapMorphism: &atlas.MapMorphism{
		atlas.KeySortMode_Default,
	},
}

func _yieldMarshalMachinePtr(row *marshalSlabRow, atl atlas.Atlas, rt reflect.Type) MarshalMachine {
	rtid := reflect.ValueOf(rt).Pointer()

	// Check primitives first; cheapest (and unoverridable).
	switch rtid {
	case rtid_bool,
		rtid_string,
		rtid_int, rtid_int8, rtid_int16, rtid_int32, rtid_int64,
		rtid_uint, rtid_uint8, rtid_uint16, rtid_uint32, rtid_uint64, rtid_uintptr,
		rtid_float32, rtid_float64,
		rtid_bytes:
		row.marshalMachinePrimitive.kind = rt.Kind()
		return &row.marshalMachinePrimitive
	}

	// Consult atlas second.
	if entry, ok := atl.Get(rtid); ok {
		// Switch across which of the union of configurations is applicable.
		switch {
		case entry.MarshalTransformFunc != nil:
			// Return a machine that calls the func(s), then later a real machine.
			// The entry.MarshalTransformTargetType is used to do a recursive lookup.
			// We can't just call the func here because we're still working off typeinfo
			// and don't have a real value to transform until later.
			row.marshalMachineTransform.trFunc = entry.MarshalTransformFunc
			// Pick delegate without growing stack.  (This currently means recursive transform won't fly.)
			row.marshalMachineTransform.delegate = _yieldMarshalMachinePtr(row, atl, entry.MarshalTransformTargetType)
			// If tags are in play: have the transformer machine glue that on.
			if entry.Tagged {
				row.marshalMachineTransform.tagged = true
				row.marshalMachineTransform.tag = entry.Tag
			}
			return &row.marshalMachineTransform
		case entry.StructMap != nil:
			row.marshalMachineStructAtlas.cfg = entry
			return &row.marshalMachineStructAtlas
		case entry.MapMorphism != nil:
			row.marshalMachineMapWildcard.morphism = entry.MapMorphism
			return &row.marshalMachineMapWildcard
		case atl.MapMorphism != nil:
			row.marshalMachineMapWildcard.morphism = atl.MapMorphism
			return &row.marshalMachineMapWildcard
		default:
			panic("invalid atlas entry")
		}
	}

	// If no specific behavior found, use default behavior based on kind.
	switch rt.Kind() {
	case reflect.Bool,
		reflect.String,
		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
		reflect.Float32, reflect.Float64:
		row.marshalMachinePrimitive.kind = rt.Kind()
		return &row.marshalMachinePrimitive
	case reflect.Slice:
		// un-typedef'd byte slices were handled already, but a typedef'd one still gets gets treated like a special kind:
		if rt.Elem().Kind() == reflect.Uint8 {
			row.marshalMachinePrimitive.kind = rt.Kind()
			return &row.marshalMachinePrimitive
		}
		return &row.marshalMachineSliceWildcard
	case reflect.Array:
		return &row.marshalMachineSliceWildcard.marshalMachineArrayWildcard
	case reflect.Map:
		row.marshalMachineMapWildcard.morphism = atl.MapMorphism
		return &row.marshalMachineMapWildcard
	case reflect.Struct:
		// TODO here we could also invoke automatic atlas autogen, if configured to be permitted
		mach := &row.errThunkMarshalMachine
		mach.err = fmt.Errorf("no atlas found for struct type %q", rt.String())
		return mach
	case reflect.Interface:
		return &row.marshalMachineWildcard
	case reflect.Func:
		panic(fmt.Errorf("functions cannot be marshalled!"))
	case reflect.Ptr:
		panic(fmt.Errorf("unreachable: ptrs must already be resolved"))
	default:
		panic(fmt.Errorf("excursion %s", rt.Kind()))
	}
}

func (s *marshalSlab) grow() {
	s.rows = append(s.rows, marshalSlabRow{})
}

func (s *marshalSlab) release() {
	s.rows = s.rows[0 : len(s.rows)-1]
}

type errThunkMarshalMachine struct {
	err error
}

func (m *errThunkMarshalMachine) Reset(_ *marshalSlab, _ reflect.Value, _ reflect.Type) error {
	return m.err
}
func (m *errThunkMarshalMachine) Step(d *Marshaller, s *marshalSlab, tok *Token) (done bool, err error) {
	return true, m.err
}
<commit_msg>the defaultCfg has been deposed.<commit_after>package obj

import (
	"fmt"
	"reflect"

	"github.com/polydawn/refmt/obj/atlas"
	. "github.com/polydawn/refmt/tok"
)

/*
	A lovely mechanism to stash marshalMachine objects pre-allocated and avoid mallocs.
	Works together with the Atlas: the Atlas says what kind of machinery is needed;
	the marshalSlab "allocates" it and returns it upon your request.
*/
type marshalSlab struct {
	atlas atlas.Atlas
	rows  []marshalSlabRow
}

type marshalSlabRow struct {
	ptrDerefDelegateMarshalMachine
	marshalMachinePrimitive
	marshalMachineWildcard
	marshalMachineMapWildcard
	marshalMachineSliceWildcard
	marshalMachineStructAtlas
	marshalMachineTransform

	errThunkMarshalMachine
}

// A thunk value that can be used to trigger `isNil` paths.
// (Substituting an 'invalid' kind reflect.Value with this is an easy way
// to emit a null without needing any additional special cases or error handling.)
var nil_rv reflect.Value = reflect.Zero(reflect.PtrTo(reflect.TypeOf(0)))

/*
	Return a reference to a machine from the slab.
	*You must release() when done.*

	Errors -- including "no info in Atlas for this type" -- are expressed by
	returning a machine that is a constantly-erroring thunk.
*/
func (slab *marshalSlab) requisitionMachine(rt reflect.Type) MarshalMachine {
	// Acquire a row.
	off := len(slab.rows)
	slab.grow()
	row := &slab.rows[off]

	// Indirect pointers as necessary.
	//  Keep count of how many times we do this; we'll use this again at the end.
	peelCount := 0
	for rt.Kind() == reflect.Ptr {
		rt = rt.Elem()
		peelCount++
	}

	// Figure out what machinery to use at heart.
	mach := _yieldMarshalMachinePtr(row, slab.atlas, rt)
	// If nil answer, we had no match: yield an error thunk.
	if mach == nil {
		mach := &row.errThunkMarshalMachine
		mach.err = fmt.Errorf("no machine found")
		return mach
	}

	// If no indirection steps, return;
	//  otherwise wrap it in the ptrDeref machine and return that.
	if peelCount == 0 {
		return mach
	}
	row.ptrDerefDelegateMarshalMachine.MarshalMachine = mach
	row.ptrDerefDelegateMarshalMachine.peelCount = peelCount
	row.ptrDerefDelegateMarshalMachine.isNil = false
	return &row.ptrDerefDelegateMarshalMachine
}

func _yieldMarshalMachinePtr(row *marshalSlabRow, atl atlas.Atlas, rt reflect.Type) MarshalMachine {
	rtid := reflect.ValueOf(rt).Pointer()

	// Check primitives first; cheapest (and unoverridable).
	switch rtid {
	case rtid_bool,
		rtid_string,
		rtid_int, rtid_int8, rtid_int16, rtid_int32, rtid_int64,
		rtid_uint, rtid_uint8, rtid_uint16, rtid_uint32, rtid_uint64, rtid_uintptr,
		rtid_float32, rtid_float64,
		rtid_bytes:
		row.marshalMachinePrimitive.kind = rt.Kind()
		return &row.marshalMachinePrimitive
	}

	// Consult atlas second.
	if entry, ok := atl.Get(rtid); ok {
		// Switch across which of the union of configurations is applicable.
		switch {
		case entry.MarshalTransformFunc != nil:
			// Return a machine that calls the func(s), then later a real machine.
			// The entry.MarshalTransformTargetType is used to do a recursive lookup.
			// We can't just call the func here because we're still working off typeinfo
			// and don't have a real value to transform until later.
			row.marshalMachineTransform.trFunc = entry.MarshalTransformFunc
			// Pick delegate without growing stack.  (This currently means recursive transform won't fly.)
			row.marshalMachineTransform.delegate = _yieldMarshalMachinePtr(row, atl, entry.MarshalTransformTargetType)
			// If tags are in play: have the transformer machine glue that on.
			if entry.Tagged {
				row.marshalMachineTransform.tagged = true
				row.marshalMachineTransform.tag = entry.Tag
			}
			return &row.marshalMachineTransform
		case entry.StructMap != nil:
			row.marshalMachineStructAtlas.cfg = entry
			return &row.marshalMachineStructAtlas
		case entry.MapMorphism != nil:
			row.marshalMachineMapWildcard.morphism = entry.MapMorphism
			return &row.marshalMachineMapWildcard
		case atl.MapMorphism != nil:
			row.marshalMachineMapWildcard.morphism = atl.MapMorphism
			return &row.marshalMachineMapWildcard
		default:
			panic("invalid atlas entry")
		}
	}

	// If no specific behavior found, use default behavior based on kind.
	switch rt.Kind() {
	case reflect.Bool,
		reflect.String,
		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
		reflect.Float32, reflect.Float64:
		row.marshalMachinePrimitive.kind = rt.Kind()
		return &row.marshalMachinePrimitive
	case reflect.Slice:
		// un-typedef'd byte slices were handled already, but a typedef'd one still gets gets treated like a special kind:
		if rt.Elem().Kind() == reflect.Uint8 {
			row.marshalMachinePrimitive.kind = rt.Kind()
			return &row.marshalMachinePrimitive
		}
		return &row.marshalMachineSliceWildcard
	case reflect.Array:
		return &row.marshalMachineSliceWildcard.marshalMachineArrayWildcard
	case reflect.Map:
		row.marshalMachineMapWildcard.morphism = atl.MapMorphism
		return &row.marshalMachineMapWildcard
	case reflect.Struct:
		// TODO here we could also invoke automatic atlas autogen, if configured to be permitted
		mach := &row.errThunkMarshalMachine
		mach.err = fmt.Errorf("no atlas found for struct type %q", rt.String())
		return mach
	case reflect.Interface:
		return &row.marshalMachineWildcard
	case reflect.Func:
		panic(fmt.Errorf("functions cannot be marshalled!"))
	case reflect.Ptr:
		panic(fmt.Errorf("unreachable: ptrs must already be resolved"))
	default:
		panic(fmt.Errorf("excursion %s", rt.Kind()))
	}
}

func (s *marshalSlab) grow() {
	s.rows = append(s.rows, marshalSlabRow{})
}

func (s *marshalSlab) release() {
	s.rows = s.rows[0 : len(s.rows)-1]
}

type errThunkMarshalMachine struct {
	err error
}

func (m *errThunkMarshalMachine) Reset(_ *marshalSlab, _ reflect.Value, _ reflect.Type) error {
	return m.err
}
func (m *errThunkMarshalMachine) Step(d *Marshaller, s *marshalSlab, tok *Token) (done bool, err error) {
	return true, m.err
}
<|endoftext|>