<commit_before>package containerd

import (
	"fmt"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/api/types/task"
	"github.com/docker/swarmkit/agent/exec"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type controller struct {
	task    *api.Task
	adapter *containerAdapter
	closed  chan struct{}
	err     error

	pulled     chan struct{} // closed after pull
	cancelPull func()        // cancels pull context if not nil
	pullErr    error         // pull error, protected by close of pulled
}

var _ exec.Controller = &controller{}

func newController(client *containerd.Client, task *api.Task, secrets exec.SecretGetter) (exec.Controller, error) {
	adapter, err := newContainerAdapter(client, task, secrets)
	if err != nil {
		return nil, err
	}

	return &controller{
		task:    task,
		adapter: adapter,
		closed:  make(chan struct{}),
	}, nil
}

// ContainerStatus returns the container-specific status for the task.
func (r *controller) ContainerStatus(ctx context.Context) (*api.ContainerStatus, error) {
	ctnr, err := r.adapter.inspect(ctx)
	if err != nil {
		if isUnknownContainer(err) {
			return nil, nil
		}

		return nil, err
	}

	status := &api.ContainerStatus{
		ContainerID: ctnr.ID,
		PID:         int32(ctnr.Pid),
	}

	if ec, ok := ctnr.ExitStatus.(exec.ExitCoder); ok {
		status.ExitCode = int32(ec.ExitCode())
	}

	return status, nil
}

// Update takes a recent task update and applies it to the container.
func (r *controller) Update(ctx context.Context, t *api.Task) error {
	log.G(ctx).Warnf("task updates not yet supported")
	// TODO(stevvooe): While assignment of tasks is idempotent, we do allow
	// updates of metadata, such as labelling, as well as any other properties
	// that make sense.
	return nil
}

// Prepare creates a container and ensures the image is pulled.
//
// If the container has already be created, exec.ErrTaskPrepared is returned.
func (r *controller) Prepare(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	//// Make sure all the networks that the task needs are created.
	// TODO(ijc)
	//if err := r.adapter.createNetworks(ctx); err != nil {
	//	return err
	//}

	//// Make sure all the volumes that the task needs are created.
	// TODO(ijc)
	//if err := r.adapter.createVolumes(ctx); err != nil {
	//	return err
	//}

	if r.pulled == nil {
		// Launches a re-entrant pull operation associated with controller,
		// dissociating the context from the caller's context. Allows pull
		// operation to be re-entrant on calls to prepare, resuming from the
		// same point after cancellation.
		var pctx context.Context

		r.pulled = make(chan struct{})
		pctx, r.cancelPull = context.WithCancel(context.Background()) // TODO(stevvooe): Bind a context to the entire controller.

		go func() {
			defer close(r.pulled)
			r.pullErr = r.adapter.pullImage(pctx)
		}()
	}

	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-r.pulled:
		if r.pullErr != nil {
			// NOTE(stevvooe): We always try to pull the image to make sure we have
			// the most up to date version. This will return an error, but we only
			// log it. If the image truly doesn't exist, the create below will
			// error out.
			//
			// This gives us some nice behavior where we use up to date versions of
			// mutable tags, but will still run if the old image is available but a
			// registry is down.
			//
			// If you don't want this behavior, lock down your image to an
			// immutable tag or digest.
			log.G(ctx).WithError(r.pullErr).Error("pulling image failed")
		}
	}

	if err := r.adapter.prepare(ctx); err != nil {
		if isContainerCreateNameConflict(err) {
			if _, err := r.adapter.inspect(ctx); err != nil {
				return err
			}

			// container is already created. success!
			return exec.ErrTaskPrepared
		}

		return errors.Wrap(err, "create container failed")
	}

	return nil
}

// Start the container. An error will be returned if the container is already started.
func (r *controller) Start(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	ctnr, err := r.adapter.inspect(ctx)
	if err != nil {
		return err
	}

	// Detect whether the container has *ever* been started. If so, we don't
	// issue the start.
	//
	// TODO(stevvooe): This is very racy. While reading inspect, another could
	// start the process and we could end up starting it twice.
	if ctnr.Status != containerd.Created {
		return exec.ErrTaskStarted
	}

	if err := r.adapter.start(ctx); err != nil {
		return errors.Wrap(err, "starting container failed")
	}

	// TODO(ijc): Wait for HealthCheck to report OK.

	return nil
}

// Wait on the container to exit.
func (r *controller) Wait(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	// check the initial state and report that.
	ctnr, err := r.adapter.inspect(ctx)
	if err != nil {
		return errors.Wrap(err, "inspecting container failed")
	}

	// TODO(ijc) this shouldn't be needed here, figure out why
	// .shutdown/.remove are not being called otherwise.
	shutdownWithExitStatus := func() error {
		err := r.adapter.shutdown(ctx)
		if err2 := r.adapter.remove(ctx); err != nil {
			// Just log it, report the exit status
			log.G(ctx).WithError(err2).Info("remove after wait failed")
		}
		return err
	}
	switch ctnr.Status {
	case containerd.Stopped:
		return shutdownWithExitStatus()
	}

	// We do not disable FailFast for this initial call (like we
	// do on the retry below) since we are still halfway through
	// setting up the container and if containerd goes away half
	// way through we consider that a failure.
	eventq, closed, err := r.adapter.events(ctx)
	if err != nil {
		return err
	}

	for {
		select {
		case event := <-eventq:
			switch event.Type {
			case task.Event_EXIT:
				return shutdownWithExitStatus()
			case task.Event_OOM, task.Event_CREATE, task.Event_START, task.Event_EXEC_ADDED, task.Event_PAUSED:
				continue
			default:
				return errors.Errorf("Unknown event type %s\n", event.Type.String())
			}
		case <-closed:
			// restart!
			log.G(ctx).Debugf("Restarting event stream")
			// We disable FailFast for this call so that gRPC will keep
			// retrying while we wait for containerd to come back. Otherwise
			// a temporary glitch in the connection (e.g. a containerd restart)
			// will result in the task being declared dead even though it is
			// likely to be recoverable.
			eventq, closed, err = r.adapter.events(ctx, grpc.FailFast(false))
			if err != nil {
				return err
			}

			// recheck the container state, if this fails then we may have missed a
			ctnr, err := r.adapter.inspect(ctx)
			if err != nil {
				return errors.Wrap(err, "inspecting container on event restart failed")
			}
			switch ctnr.Status {
			case containerd.Stopped:
				return shutdownWithExitStatus()
			}

		case <-ctx.Done():
			return ctx.Err()
		case <-r.closed:
			return r.err
		}
	}
}

// Shutdown the container cleanly.
func (r *controller) Shutdown(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	if r.cancelPull != nil {
		r.cancelPull()
	}

	if err := r.adapter.shutdown(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		return err
	}

	return nil
}

// Terminate the container, with force.
func (r *controller) Terminate(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	if r.cancelPull != nil {
		r.cancelPull()
	}

	if err := r.adapter.terminate(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		return err
	}

	return nil
}

// Remove the container and its resources.
func (r *controller) Remove(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	if r.cancelPull != nil {
		r.cancelPull()
	}

	// It may be necessary to shut down the task before removing it.
	if err := r.Shutdown(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		// This may fail if the task was already shut down.
		log.G(ctx).WithError(err).Debug("shutdown failed on removal")
	}

	// Try removing networks referenced in this task in case this
	// task is the last one referencing it
	// TODO(ijc)
	//if err := r.adapter.removeNetworks(ctx); err != nil {
	//	if isUnknownContainer(err) {
	//		return nil
	//	}

	//	return err
	//}

	if err := r.adapter.remove(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		return err
	}

	return nil
}

// Close the controller and clean up any ephemeral resources.
func (r *controller) Close() error {
	select {
	case <-r.closed:
		return r.err
	default:
		if r.cancelPull != nil {
			r.cancelPull()
		}

		r.err = exec.ErrControllerClosed
		close(r.closed)
	}
	return nil
}

func (r *controller) checkClosed() error {
	select {
	case <-r.closed:
		return r.err
	default:
		return nil
	}
}

type exitError struct {
	code  uint32
	cause error
}

func (e *exitError) Error() string {
	if e.cause != nil {
		return fmt.Sprintf("task: non-zero exit (%v): %v", e.code, e.cause)
	}
	return fmt.Sprintf("task: non-zero exit (%v)", e.code)
}

func (e *exitError) ExitCode() int {
	return int(e.code)
}

func (e *exitError) Cause() error {
	return e.cause
}

func makeExitError(exitStatus uint32, reason string) error {
	if exitStatus != 0 {
		var cause error
		if reason != "" {
			cause = errors.New(reason)
		}

		return &exitError{
			code:  exitStatus,
			cause: cause,
		}
	}

	return nil
}
<commit_msg>containerd: Do not shutdown container in Wait() method<commit_after>package containerd

import (
	"fmt"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/api/types/task"
	"github.com/docker/swarmkit/agent/exec"
	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/log"
	"github.com/pkg/errors"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type controller struct {
	task    *api.Task
	adapter *containerAdapter
	closed  chan struct{}
	err     error

	pulled     chan struct{} // closed after pull
	cancelPull func()        // cancels pull context if not nil
	pullErr    error         // pull error, protected by close of pulled
}

var _ exec.Controller = &controller{}

func newController(client *containerd.Client, task *api.Task, secrets exec.SecretGetter) (exec.Controller, error) {
	adapter, err := newContainerAdapter(client, task, secrets)
	if err != nil {
		return nil, err
	}

	return &controller{
		task:    task,
		adapter: adapter,
		closed:  make(chan struct{}),
	}, nil
}

// ContainerStatus returns the container-specific status for the task.
func (r *controller) ContainerStatus(ctx context.Context) (*api.ContainerStatus, error) {
	ctnr, err := r.adapter.inspect(ctx)
	if err != nil {
		if isUnknownContainer(err) {
			return nil, nil
		}

		return nil, err
	}

	status := &api.ContainerStatus{
		ContainerID: ctnr.ID,
		PID:         int32(ctnr.Pid),
	}

	if ec, ok := ctnr.ExitStatus.(exec.ExitCoder); ok {
		status.ExitCode = int32(ec.ExitCode())
	}

	return status, nil
}

// Update takes a recent task update and applies it to the container.
func (r *controller) Update(ctx context.Context, t *api.Task) error {
	log.G(ctx).Warnf("task updates not yet supported")
	// TODO(stevvooe): While assignment of tasks is idempotent, we do allow
	// updates of metadata, such as labelling, as well as any other properties
	// that make sense.
	return nil
}

// Prepare creates a container and ensures the image is pulled.
//
// If the container has already be created, exec.ErrTaskPrepared is returned.
func (r *controller) Prepare(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	//// Make sure all the networks that the task needs are created.
	// TODO(ijc)
	//if err := r.adapter.createNetworks(ctx); err != nil {
	//	return err
	//}

	//// Make sure all the volumes that the task needs are created.
	// TODO(ijc)
	//if err := r.adapter.createVolumes(ctx); err != nil {
	//	return err
	//}

	if r.pulled == nil {
		// Launches a re-entrant pull operation associated with controller,
		// dissociating the context from the caller's context. Allows pull
		// operation to be re-entrant on calls to prepare, resuming from the
		// same point after cancellation.
		var pctx context.Context

		r.pulled = make(chan struct{})
		pctx, r.cancelPull = context.WithCancel(context.Background()) // TODO(stevvooe): Bind a context to the entire controller.

		go func() {
			defer close(r.pulled)
			r.pullErr = r.adapter.pullImage(pctx)
		}()
	}

	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-r.pulled:
		if r.pullErr != nil {
			// NOTE(stevvooe): We always try to pull the image to make sure we have
			// the most up to date version. This will return an error, but we only
			// log it. If the image truly doesn't exist, the create below will
			// error out.
			//
			// This gives us some nice behavior where we use up to date versions of
			// mutable tags, but will still run if the old image is available but a
			// registry is down.
			//
			// If you don't want this behavior, lock down your image to an
			// immutable tag or digest.
			log.G(ctx).WithError(r.pullErr).Error("pulling image failed")
		}
	}

	if err := r.adapter.prepare(ctx); err != nil {
		if isContainerCreateNameConflict(err) {
			if _, err := r.adapter.inspect(ctx); err != nil {
				return err
			}

			// container is already created. success!
			return exec.ErrTaskPrepared
		}

		return errors.Wrap(err, "create container failed")
	}

	return nil
}

// Start the container. An error will be returned if the container is already started.
func (r *controller) Start(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	ctnr, err := r.adapter.inspect(ctx)
	if err != nil {
		return err
	}

	// Detect whether the container has *ever* been started. If so, we don't
	// issue the start.
	//
	// TODO(stevvooe): This is very racy. While reading inspect, another could
	// start the process and we could end up starting it twice.
	if ctnr.Status != containerd.Created {
		return exec.ErrTaskStarted
	}

	if err := r.adapter.start(ctx); err != nil {
		return errors.Wrap(err, "starting container failed")
	}

	// TODO(ijc): Wait for HealthCheck to report OK.

	return nil
}

// Wait on the container to exit.
func (r *controller) Wait(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	// check the initial state and report that.
	ctnr, err := r.adapter.inspect(ctx)
	if err != nil {
		return errors.Wrap(err, "inspecting container failed")
	}

	switch ctnr.Status {
	case containerd.Stopped:
		return ctnr.ExitStatus
	}

	// We do not disable FailFast for this initial call (like we
	// do on the retry below) since we are still halfway through
	// setting up the container and if containerd goes away half
	// way through we consider that a failure.
	eventq, closed, err := r.adapter.events(ctx)
	if err != nil {
		return err
	}

	for {
		select {
		case event := <-eventq:
			switch event.Type {
			case task.Event_EXIT:
				return makeExitError(event.ExitStatus, "")
			case task.Event_OOM, task.Event_CREATE, task.Event_START, task.Event_EXEC_ADDED, task.Event_PAUSED:
				continue
			default:
				return errors.Errorf("Unknown event type %s\n", event.Type.String())
			}
		case <-closed:
			// restart!
			log.G(ctx).Debugf("Restarting event stream")
			// We disable FailFast for this call so that gRPC will keep
			// retrying while we wait for containerd to come back. Otherwise
			// a temporary glitch in the connection (e.g. a containerd restart)
			// will result in the task being declared dead even though it is
			// likely to be recoverable.
			eventq, closed, err = r.adapter.events(ctx, grpc.FailFast(false))
			if err != nil {
				return err
			}

			// recheck the container state, if this fails then we may have missed a
			ctnr, err := r.adapter.inspect(ctx)
			if err != nil {
				return errors.Wrap(err, "inspecting container on event restart failed")
			}
			switch ctnr.Status {
			case containerd.Stopped:
				return ctnr.ExitStatus
			}

		case <-ctx.Done():
			return ctx.Err()
		case <-r.closed:
			return r.err
		}
	}
}

// Shutdown the container cleanly.
func (r *controller) Shutdown(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	if r.cancelPull != nil {
		r.cancelPull()
	}

	if err := r.adapter.shutdown(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		return err
	}

	return nil
}

// Terminate the container, with force.
func (r *controller) Terminate(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	if r.cancelPull != nil {
		r.cancelPull()
	}

	if err := r.adapter.terminate(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		return err
	}

	return nil
}

// Remove the container and its resources.
func (r *controller) Remove(ctx context.Context) error {
	ctx = log.WithModule(ctx, "containerd")

	if err := r.checkClosed(); err != nil {
		return err
	}

	if r.cancelPull != nil {
		r.cancelPull()
	}

	// It may be necessary to shut down the task before removing it.
	if err := r.Shutdown(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		// This may fail if the task was already shut down.
		log.G(ctx).WithError(err).Debug("shutdown failed on removal")
	}

	// Try removing networks referenced in this task in case this
	// task is the last one referencing it
	// TODO(ijc)
	//if err := r.adapter.removeNetworks(ctx); err != nil {
	//	if isUnknownContainer(err) {
	//		return nil
	//	}

	//	return err
	//}

	if err := r.adapter.remove(ctx); err != nil {
		if isUnknownContainer(err) {
			return nil
		}

		return err
	}

	return nil
}

// Close the controller and clean up any ephemeral resources.
func (r *controller) Close() error {
	select {
	case <-r.closed:
		return r.err
	default:
		if r.cancelPull != nil {
			r.cancelPull()
		}

		r.err = exec.ErrControllerClosed
		close(r.closed)
	}
	return nil
}

func (r *controller) checkClosed() error {
	select {
	case <-r.closed:
		return r.err
	default:
		return nil
	}
}

type exitError struct {
	code  uint32
	cause error
}

func (e *exitError) Error() string {
	if e.cause != nil {
		return fmt.Sprintf("task: non-zero exit (%v): %v", e.code, e.cause)
	}
	return fmt.Sprintf("task: non-zero exit (%v)", e.code)
}

func (e *exitError) ExitCode() int {
	return int(e.code)
}

func (e *exitError) Cause() error {
	return e.cause
}

func makeExitError(exitStatus uint32, reason string) error {
	if exitStatus != 0 {
		var cause error
		if reason != "" {
			cause = errors.New(reason)
		}

		return &exitError{
			code:  exitStatus,
			cause: cause,
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assets

import (
	"bytes"
	"fmt"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/blang/semver"
	"github.com/golang/glog"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/values"
	"k8s.io/kops/util/pkg/hashing"
	"k8s.io/kops/util/pkg/vfs"
	"regexp"
)

// RewriteManifests controls whether we rewrite manifests
// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))

// AssetBuilder discovers and remaps assets.
type AssetBuilder struct {
	ContainerAssets []*ContainerAsset
	FileAssets      []*FileAsset
	AssetsLocation  *kops.Assets
	// TODO we'd like to use cloudup.Phase here, but that introduces a go cyclic dependency
	Phase string

	// KubernetesVersion is the version of kubernetes we are installing
	KubernetesVersion semver.Version
}

// ContainerAsset models a container's location.
type ContainerAsset struct {
	// DockerImage will be the name of the container we should run.
	// This is used to copy a container to a ContainerRegistry.
	DockerImage string
	// CanonicalLocation will be the source location of the container.
	CanonicalLocation string
}

// FileAsset models a file's location.
type FileAsset struct {
	// FileURL is the URL of a file that is accessed by a Kubernetes cluster.
	FileURL *url.URL
	// CanonicalFileURL is the source URL of a file. This is used to copy a file to a FileRepository.
	CanonicalFileURL *url.URL
	// SHAValue is the SHA hash of the FileAsset.
	SHAValue string
}

// NewAssetBuilder creates a new AssetBuilder.
func NewAssetBuilder(cluster *kops.Cluster, phase string) *AssetBuilder {
	a := &AssetBuilder{
		AssetsLocation: cluster.Spec.Assets,
		Phase:          phase,
	}

	version, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		// This should have already been validated
		glog.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", cluster.Spec.KubernetesVersion, err)
	}
	a.KubernetesVersion = *version

	return a
}

// RemapManifest transforms a kubernetes manifest.
// Whenever we are building a Task that includes a manifest, we should pass it through RemapManifest first.
// This will:
// * rewrite the images if they are being redirected to a mirror, and ensure the image is uploaded
func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
	if !RewriteManifests.Enabled() {
		return data, nil
	}

	manifests, err := kubemanifest.LoadManifestsFrom(data)
	if err != nil {
		return nil, err
	}

	var yamlSeparator = []byte("\n---\n\n")
	var remappedManifests [][]byte
	for _, manifest := range manifests {
		if err := manifest.RemapImages(a.RemapImage); err != nil {
			return nil, fmt.Errorf("error remapping images: %v", err)
		}

		y, err := manifest.ToYAML()
		if err != nil {
			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
		}

		remappedManifests = append(remappedManifests, y)
	}

	return bytes.Join(remappedManifests, yamlSeparator), nil
}

// RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.
func (a *AssetBuilder) RemapImage(image string) (string, error) {
	asset := &ContainerAsset{}

	asset.DockerImage = image

	// The k8s.gcr.io prefix is an alias, but for CI builds we run from a docker load,
	// and we only double-tag from 1.10 onwards.
	// For versions prior to 1.10, remap k8s.gcr.io to the old name.
	// This also means that we won't start using the aliased names on existing clusters,
	// which could otherwise be surprising to users.
	if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(image, "k8s.gcr.io/") {
		image = "gcr.io/google_containers/" + strings.TrimPrefix(image, "k8s.gcr.io/")
	}

	if strings.HasPrefix(image, "kope/dns-controller:") {
		// To use user-defined DNS Controller:
		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
		// 3. make kops and create/apply cluster
		override := os.Getenv("DNSCONTROLLER_IMAGE")
		if override != "" {
			image = override
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil {
		containerProxy := *a.AssetsLocation.ContainerProxy
		normalized := image

		// If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/<image-name>
		// In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube
		if strings.Count(normalized, "/") == 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:"){
			normalized = containerProxy + "/" + normalized
		} else {
			var re = regexp.MustCompile(`^[^/]+`)
			normalized = re.ReplaceAllString(normalized, containerProxy)
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
		registryMirror := *a.AssetsLocation.ContainerRegistry
		normalized := image

		// Remove the 'standard' kubernetes image prefix, just for sanity
		if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(normalized, "gcr.io/google_containers/") {
			normalized = strings.TrimPrefix(normalized, "gcr.io/google_containers/")
		} else {
			normalized = strings.TrimPrefix(normalized, "k8s.gcr.io/")
		}

		// We can't nest arbitrarily
		// Some risk of collisions, but also -- and __ in the names appear to be blocked by docker hub
		normalized = strings.Replace(normalized, "/", "-", -1)
		asset.DockerImage = registryMirror + "/" + normalized

		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	a.ContainerAssets = append(a.ContainerAssets, asset)
	return image, nil
}

// RemapFileAndSHA returns a remapped url for the file, if AssetsLocation is defined.
// It also returns the SHA hash of the file.
func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
	if fileURL == nil {
		return nil, nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL: fileURL,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFileURL, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, nil, err
		}

		fileAsset.FileURL = normalizedFileURL

		glog.V(4).Infof("adding remapped file: %+v", fileAsset)
	}

	h, err := a.findHash(fileAsset)
	if err != nil {
		return nil, nil, err
	}
	fileAsset.SHAValue = h.Hex()

	a.FileAssets = append(a.FileAssets, fileAsset)
	glog.V(8).Infof("adding file: %+v", fileAsset)

	return fileAsset.FileURL, h, nil
}

// TODO - remove this method as CNI does now have a SHA file

// RemapFileAndSHAValue is used exclusively to remap the cni tarball, as the tarball does not have a sha file in object storage.
func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
	if fileURL == nil {
		return nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL:  fileURL,
		SHAValue: shaValue,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFile, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, err
		}

		fileAsset.FileURL = normalizedFile
		glog.V(4).Infof("adding remapped file: %q", fileAsset.FileURL.String())
	}

	a.FileAssets = append(a.FileAssets, fileAsset)

	return fileAsset.FileURL, nil
}

// FindHash returns the hash value of a FileAsset.
func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {

	// If the phase is "assets" we use the CanonicalFileURL,
	// but during other phases we use the hash from the FileRepository or the base kops path.
	// We do not want to just test for CanonicalFileURL as it is defined in
	// other phases, but is not used to test for the SHA.
	// This prevents a chicken and egg problem where the file is not yet in the FileRepository.
	//
	// assets phase -> get the sha file from the source / CanonicalFileURL
	// any other phase -> get the sha file from the kops base location or the FileRepository
	//
	// TLDR; we use the file.CanonicalFileURL during assets phase, and use file.FileUrl the
	// rest of the time. If not we get a chicken and the egg problem where we are reading the sha file
	// before it exists.
	u := file.FileURL
	if a.Phase == "assets" && file.CanonicalFileURL != nil {
		u = file.CanonicalFileURL
	}

	if u == nil {
		return nil, fmt.Errorf("file url is not defined")
	}

	for _, ext := range []string{".sha1"} {
		hashURL := u.String() + ext
		b, err := vfs.Context.ReadFile(hashURL)
		if err != nil {
			glog.Infof("error reading hash file %q: %v", hashURL, err)
			continue
		}
		hashString := strings.TrimSpace(string(b))
		glog.V(2).Infof("Found hash %q for %q", hashString, u)

		return hashing.FromString(hashString)
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		return nil, fmt.Errorf("you may have not staged your files correctly, please execute kops update cluster using the assets phase")
	}
	return nil, fmt.Errorf("cannot determine hash for %q (have you specified a valid file location?)", u)
}

func (a *AssetBuilder) normalizeURL(file *url.URL) (*url.URL, error) {

	if a.AssetsLocation == nil || a.AssetsLocation.FileRepository == nil {
		return nil, fmt.Errorf("assetLocation and fileRepository cannot be nil to normalize an file asset URL")
	}

	f := values.StringValue(a.AssetsLocation.FileRepository)

	if f == "" {
		return nil, fmt.Errorf("assetsLocation fileRepository cannot be an empty string")
	}

	fileRepo, err := url.Parse(f)
	if err != nil {
		return nil, fmt.Errorf("unable to parse file repository URL %q: %v", values.StringValue(a.AssetsLocation.FileRepository), err)
	}

	fileRepo.Path = path.Join(fileRepo.Path, file.Path)

	return fileRepo, nil
}
<commit_msg>Run gofmt<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package assets

import (
	"bytes"
	"fmt"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/blang/semver"
	"github.com/golang/glog"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/util"
	"k8s.io/kops/pkg/featureflag"
	"k8s.io/kops/pkg/kubemanifest"
	"k8s.io/kops/pkg/values"
	"k8s.io/kops/util/pkg/hashing"
	"k8s.io/kops/util/pkg/vfs"
)

// RewriteManifests controls whether we rewrite manifests
// Because manifest rewriting converts everything to and from YAML, we normalize everything by doing so
var RewriteManifests = featureflag.New("RewriteManifests", featureflag.Bool(true))

// AssetBuilder discovers and remaps assets.
type AssetBuilder struct {
	ContainerAssets []*ContainerAsset
	FileAssets      []*FileAsset
	AssetsLocation  *kops.Assets
	// TODO we'd like to use cloudup.Phase here, but that introduces a go cyclic dependency
	Phase string

	// KubernetesVersion is the version of kubernetes we are installing
	KubernetesVersion semver.Version
}

// ContainerAsset models a container's location.
type ContainerAsset struct {
	// DockerImage will be the name of the container we should run.
	// This is used to copy a container to a ContainerRegistry.
	DockerImage string
	// CanonicalLocation will be the source location of the container.
	CanonicalLocation string
}

// FileAsset models a file's location.
type FileAsset struct {
	// FileURL is the URL of a file that is accessed by a Kubernetes cluster.
	FileURL *url.URL
	// CanonicalFileURL is the source URL of a file. This is used to copy a file to a FileRepository.
	CanonicalFileURL *url.URL
	// SHAValue is the SHA hash of the FileAsset.
	SHAValue string
}

// NewAssetBuilder creates a new AssetBuilder.
func NewAssetBuilder(cluster *kops.Cluster, phase string) *AssetBuilder {
	a := &AssetBuilder{
		AssetsLocation: cluster.Spec.Assets,
		Phase:          phase,
	}

	version, err := util.ParseKubernetesVersion(cluster.Spec.KubernetesVersion)
	if err != nil {
		// This should have already been validated
		glog.Fatalf("unexpected error from ParseKubernetesVersion %s: %v", cluster.Spec.KubernetesVersion, err)
	}
	a.KubernetesVersion = *version

	return a
}

// RemapManifest transforms a kubernetes manifest.
// Whenever we are building a Task that includes a manifest, we should pass it through RemapManifest first.
// This will:
// * rewrite the images if they are being redirected to a mirror, and ensure the image is uploaded
func (a *AssetBuilder) RemapManifest(data []byte) ([]byte, error) {
	if !RewriteManifests.Enabled() {
		return data, nil
	}

	manifests, err := kubemanifest.LoadManifestsFrom(data)
	if err != nil {
		return nil, err
	}

	var yamlSeparator = []byte("\n---\n\n")
	var remappedManifests [][]byte
	for _, manifest := range manifests {
		if err := manifest.RemapImages(a.RemapImage); err != nil {
			return nil, fmt.Errorf("error remapping images: %v", err)
		}

		y, err := manifest.ToYAML()
		if err != nil {
			return nil, fmt.Errorf("error re-marshalling manifest: %v", err)
		}

		remappedManifests = append(remappedManifests, y)
	}

	return bytes.Join(remappedManifests, yamlSeparator), nil
}

// RemapImage normalizes a containers location if a user sets the AssetsLocation ContainerRegistry location.
func (a *AssetBuilder) RemapImage(image string) (string, error) {
	asset := &ContainerAsset{}

	asset.DockerImage = image

	// The k8s.gcr.io prefix is an alias, but for CI builds we run from a docker load,
	// and we only double-tag from 1.10 onwards.
	// For versions prior to 1.10, remap k8s.gcr.io to the old name.
	// This also means that we won't start using the aliased names on existing clusters,
	// which could otherwise be surprising to users.
	if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(image, "k8s.gcr.io/") {
		image = "gcr.io/google_containers/" + strings.TrimPrefix(image, "k8s.gcr.io/")
	}

	if strings.HasPrefix(image, "kope/dns-controller:") {
		// To use user-defined DNS Controller:
		// 1. DOCKER_REGISTRY=[your docker hub repo] make dns-controller-push
		// 2. export DNSCONTROLLER_IMAGE=[your docker hub repo]
		// 3. make kops and create/apply cluster
		override := os.Getenv("DNSCONTROLLER_IMAGE")
		if override != "" {
			image = override
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerProxy != nil {
		containerProxy := *a.AssetsLocation.ContainerProxy
		normalized := image

		// If the image name contains only a single / we need to determine if the image is located on docker-hub or if it's using a convenient URL like k8s.gcr.io/<image-name>
		// In case of a hub image it should be sufficient to just prepend the proxy url, producing eg docker-proxy.example.com/weaveworks/weave-kube
		if strings.Count(normalized, "/") == 1 && !strings.ContainsAny(strings.Split(normalized, "/")[0], ".:") {
			normalized = containerProxy + "/" + normalized
		} else {
			var re = regexp.MustCompile(`^[^/]+`)
			normalized = re.ReplaceAllString(normalized, containerProxy)
		}
	}

	if a.AssetsLocation != nil && a.AssetsLocation.ContainerRegistry != nil {
		registryMirror := *a.AssetsLocation.ContainerRegistry
		normalized := image

		// Remove the 'standard' kubernetes image prefix, just for sanity
		if !util.IsKubernetesGTE("1.10", a.KubernetesVersion) && strings.HasPrefix(normalized, "gcr.io/google_containers/") {
			normalized = strings.TrimPrefix(normalized, "gcr.io/google_containers/")
		} else {
			normalized = strings.TrimPrefix(normalized, "k8s.gcr.io/")
		}

		// We can't nest arbitrarily
		// Some risk of collisions, but also -- and __ in the names appear to be blocked by docker hub
		normalized = strings.Replace(normalized, "/", "-", -1)
		asset.DockerImage = registryMirror + "/" + normalized

		asset.CanonicalLocation = image

		// Run the new image
		image = asset.DockerImage
	}

	a.ContainerAssets = append(a.ContainerAssets, asset)
	return image, nil
}

// RemapFileAndSHA returns a remapped url for the file, if AssetsLocation is defined.
// It also returns the SHA hash of the file.
func (a *AssetBuilder) RemapFileAndSHA(fileURL *url.URL) (*url.URL, *hashing.Hash, error) {
	if fileURL == nil {
		return nil, nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL: fileURL,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFileURL, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, nil, err
		}

		fileAsset.FileURL = normalizedFileURL

		glog.V(4).Infof("adding remapped file: %+v", fileAsset)
	}

	h, err := a.findHash(fileAsset)
	if err != nil {
		return nil, nil, err
	}
	fileAsset.SHAValue = h.Hex()

	a.FileAssets = append(a.FileAssets, fileAsset)
	glog.V(8).Infof("adding file: %+v", fileAsset)

	return fileAsset.FileURL, h, nil
}

// TODO - remove this method as CNI does now have a SHA file

// RemapFileAndSHAValue is used exclusively to remap the cni tarball, as the tarball does not have a sha file in object storage.
func (a *AssetBuilder) RemapFileAndSHAValue(fileURL *url.URL, shaValue string) (*url.URL, error) {
	if fileURL == nil {
		return nil, fmt.Errorf("unable to remap a nil URL")
	}

	fileAsset := &FileAsset{
		FileURL:  fileURL,
		SHAValue: shaValue,
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		fileAsset.CanonicalFileURL = fileURL

		normalizedFile, err := a.normalizeURL(fileURL)
		if err != nil {
			return nil, err
		}

		fileAsset.FileURL = normalizedFile
		glog.V(4).Infof("adding remapped file: %q", fileAsset.FileURL.String())
	}

	a.FileAssets = append(a.FileAssets, fileAsset)

	return fileAsset.FileURL, nil
}

// FindHash returns the hash value of a FileAsset.
func (a *AssetBuilder) findHash(file *FileAsset) (*hashing.Hash, error) {

	// If the phase is "assets" we use the CanonicalFileURL,
	// but during other phases we use the hash from the FileRepository or the base kops path.
	// We do not want to just test for CanonicalFileURL as it is defined in
	// other phases, but is not used to test for the SHA.
	// This prevents a chicken and egg problem where the file is not yet in the FileRepository.
	//
	// assets phase -> get the sha file from the source / CanonicalFileURL
	// any other phase -> get the sha file from the kops base location or the FileRepository
	//
	// TLDR; we use the file.CanonicalFileURL during assets phase, and use file.FileUrl the
	// rest of the time. If not we get a chicken and the egg problem where we are reading the sha file
	// before it exists.
	u := file.FileURL
	if a.Phase == "assets" && file.CanonicalFileURL != nil {
		u = file.CanonicalFileURL
	}

	if u == nil {
		return nil, fmt.Errorf("file url is not defined")
	}

	for _, ext := range []string{".sha1"} {
		hashURL := u.String() + ext
		b, err := vfs.Context.ReadFile(hashURL)
		if err != nil {
			glog.Infof("error reading hash file %q: %v", hashURL, err)
			continue
		}
		hashString := strings.TrimSpace(string(b))
		glog.V(2).Infof("Found hash %q for %q", hashString, u)

		return hashing.FromString(hashString)
	}

	if a.AssetsLocation != nil && a.AssetsLocation.FileRepository != nil {
		return nil, fmt.Errorf("you may have not staged your files correctly, please execute kops update cluster using the assets phase")
	}
	return nil, fmt.Errorf("cannot determine hash for %q (have you specified a valid file location?)", u)
}

func (a *AssetBuilder) normalizeURL(file *url.URL) (*url.URL, error) {

	if a.AssetsLocation == nil || a.AssetsLocation.FileRepository == nil {
		return nil, fmt.Errorf("assetLocation and fileRepository cannot be nil to normalize an file asset URL")
	}

	f := values.StringValue(a.AssetsLocation.FileRepository)

	if f == "" {
		return nil, fmt.Errorf("assetsLocation fileRepository cannot be an empty string")
	}

	fileRepo, err := url.Parse(f)
	if err != nil {
		return nil, fmt.Errorf("unable to parse file repository URL %q: %v", values.StringValue(a.AssetsLocation.FileRepository), err)
	}

	fileRepo.Path = path.Join(fileRepo.Path, file.Path)

	return fileRepo, nil
}
<|endoftext|>


<commit_before>package operators

import (
	"math"
)

var (
	add = &Operator{
		Name:          "+",
		Precedence:    1,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] + args[1]
		},
	}
	sub = &Operator{
		Name:          "-",
		Precedence:    1,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] - args[1]
		},
	}
	neg = &Operator{
		Name:          "neg",
		Precedence:    2,
		Associativity: L,
		Args:          1,
		Operation: func(args []float64) float64 {
			return 0 - args[0]
		},
	}
	mul = &Operator{
		Name:          "*",
		Precedence:    2,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] * args[1]
		},
	}
	div = &Operator{
		Name:          "/",
		Precedence:    2,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] / args[1]
		},
	}
	pow = &Operator{
		Name:          "^",
		Precedence:    3,
		Associativity: R,
		Args:          2,
		Operation: func(args []float64) float64 {
			return math.Pow(args[0], args[1])
		},
	}
)

func init() {
	Register(add)
	Register(sub)
	Register(neg)
	Register(pow)
	Register(mul)
	Register(div)
}
<commit_msg>Added mod operator (%)<commit_after>package operators

import (
	"math"
)

var (
	add = &Operator{
		Name:          "+",
		Precedence:    1,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] + args[1]
		},
	}
	sub = &Operator{
		Name:          "-",
		Precedence:    1,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] - args[1]
		},
	}
	neg = &Operator{
		Name:          "neg",
		Precedence:    2,
		Associativity: L,
		Args:          1,
		Operation: func(args []float64) float64 {
			return 0 - args[0]
		},
	}
	mul = &Operator{
		Name:          "*",
		Precedence:    2,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] * args[1]
		},
	}
	div = &Operator{
		Name:          "/",
		Precedence:    2,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return args[0] / args[1]
		},
	}
	mod = &Operator{
		Name:          "%",
		Precedence:    2,
		Associativity: L,
		Args:          2,
		Operation: func(args []float64) float64 {
			return math.Mod(args[0], args[1])
		},
	}
	pow = &Operator{
		Name:          "^",
		Precedence:    3,
		Associativity: R,
		Args:          2,
		Operation: func(args []float64) float64 {
			return math.Pow(args[0], args[1])
		},
	}

)

func init() {
	Register(add)
	Register(sub)
	Register(neg)
	Register(pow)
	Register(mul)
    Register(mod)
	Register(div)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"flag"
	"github.com/br0r/hydra/config"
)

const BASE_DIR string = ".hydra"
const help string = `Usage:
hydra [OPTIONS] COMMAND

Commands:
  init - Create hydra project [--clean]
  start - Start hydra servers
  stop - Stops started servers
  ls - Show started servers
  logs [name] - Show logs for servers, or for specific server given by name
`

func initialize(c config.Config, clean bool) {
	if clean {
		_, e := exec.Command("rm", "-rf", ".hydra").Output()
		if e != nil {
			log.Fatal(e)
		}
	}

	exec.Command("mkdir", ".hydra").Output()

	for i := 0; i < len(c.Services); i += 1 {
		service := c.Services[i]
		p := service.Path
		name := service.Name
		base := path.Join(".hydra/", name)
		if _, err := os.Stat(base); os.IsNotExist(err) {
			fmt.Println("Creating", name)

			var e error
			if strings.Index(p, "git@") == 0 {
				e = exec.Command("git", "clone", p, base).Run()
			} else {
				e = exec.Command("cp", "-R", p, base).Run()
			}

			if e != nil {
				log.Fatal(e)
			}

			git := path.Join(base, ".git")
			exec.Command("rm", "-rf", git).Run()

			if service.Config.Dest != "" && service.Config.Src != "" {
				u := path.Join(base, service.Config.Dest)
				cf := service.Config.Src

				err := exec.Command("cp", cf, u).Run()
				if err != nil {
					log.Fatal(err)
				}
			}

			if service.Install != "" {
				args := strings.Split(service.Install, " ")
				cmd := exec.Command(args[0], args[1:]...)

				cmd.Dir = base
				e := cmd.Run()
				if e != nil {
					log.Fatal(e)
				}
			}
		}
	}
}

func start(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name

		cmdRunDir := path.Join(".hydra", name)
		pid_file := path.Join(cmdRunDir, "pid")
		log_file_path := path.Join(cmdRunDir, "log")

		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			fmt.Printf("%s is already started, run hydra stop if you want to restart\n", name)
			continue
		}

		env := os.Environ()
		for k, v := range service.Env {
			env = append(env, fmt.Sprintf("%s=%s", k, v))
		}

		fmt.Println("Starting", name)
		args := strings.Split(conf.Services[i].Start, " ")
		args = append(args, "2>&1 1> log")
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Env = env
		cmd.Dir = cmdRunDir

		log_file, e := os.Create(log_file_path)
		if e != nil {
			log.Fatal(e)
		}

		_, e = log_file.WriteString(fmt.Sprintf("%s:\n", name))
		if e != nil {
			log.Fatal(e)
		}

		defer log_file.Close()
		cmd.Stdout = log_file
		cmd.Stderr = log_file

		e = cmd.Start()
		pid := []byte(strconv.Itoa(cmd.Process.Pid))
		ioutil.WriteFile(pid_file, pid, 0440)

		if e != nil {
			os.Remove("pid")
			log.Fatal("Error with", name, e)
		}

	}
}

func kill(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name
		pid_file := path.Join(".hydra", name, "pid")
		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_file)
			if e != nil {
				log.Fatalf("Error when reading file %v", e)
			}
			pid := string(bytes)
			exec.Command("kill", pid).Run()
			if e == nil {
				fmt.Printf("Killed %s\n", name)
			}
			os.Remove(pid_file)
		}
	}
}

func ls(conf config.Config) {
	for _, service := range conf.Services {
		name := service.Name
		pid_path := path.Join(BASE_DIR, name, "pid")
		if _, err := os.Stat(pid_path); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_path)
			if e != nil {
				log.Fatal(e)
			}
			pid := string(bytes)
			fmt.Printf("%s running on pid: %s\n", name, pid)
		} else {
			fmt.Printf("%s is not running\n", name)
		}
	}
}

func logs(conf config.Config, servers []string) {
	for _, service := range conf.Services {
		if len(servers) > 0 {
			err := false
			for _, name := range servers {
				if name != service.Name {
					err = true
					break
				}
			}
			if err == true {
				continue
			}
		}

		logfile := path.Join(".hydra/", service.Name, "log")
		data, err := ioutil.ReadFile(logfile)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(data))
	}
}

func main() {
	var c config.Config = config.ReadConfig()
	var clean = flag.Bool("clean", false, "If we want a clean run")
	flag.Parse()

	if cmd := flag.Arg(0); cmd != "" {
		switch cmd {
		case "init":
			initialize(c, *clean)
		case "start":
			start(c)
		case "stop":
			kill(c)
		case "ls":
			ls(c)
		case "logs":
			logs(c, flag.Args()[1:])
		default:
			fmt.Println(help)
		}
	} else {
		fmt.Println(help)
	}
}
<commit_msg>added kill to clean<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"flag"
	"github.com/br0r/hydra/config"
)

const BASE_DIR string = ".hydra"
const help string = `Usage:
hydra [OPTIONS] COMMAND

Commands:
  init - Create hydra project [--clean]
  start - Start hydra servers
  stop - Stops started servers
  ls - Show started servers
  logs [name] - Show logs for servers, or for specific server given by name
`

func initialize(c config.Config, clean bool) {
	if clean {
		kill(c)
		_, e := exec.Command("rm", "-rf", ".hydra").Output()
		if e != nil {
			log.Fatal(e)
		}
	}

	exec.Command("mkdir", ".hydra").Output()

	for i := 0; i < len(c.Services); i += 1 {
		service := c.Services[i]
		p := service.Path
		name := service.Name
		base := path.Join(".hydra/", name)
		if _, err := os.Stat(base); os.IsNotExist(err) {
			fmt.Println("Creating", name)

			var e error
			if strings.Index(p, "git@") == 0 {
				e = exec.Command("git", "clone", p, base).Run()
			} else {
				e = exec.Command("cp", "-R", p, base).Run()
			}

			if e != nil {
				log.Fatal(e)
			}

			git := path.Join(base, ".git")
			exec.Command("rm", "-rf", git).Run()

			if service.Config.Dest != "" && service.Config.Src != "" {
				u := path.Join(base, service.Config.Dest)
				cf := service.Config.Src

				err := exec.Command("cp", cf, u).Run()
				if err != nil {
					log.Fatal(err)
				}
			}

			if service.Install != "" {
				args := strings.Split(service.Install, " ")
				cmd := exec.Command(args[0], args[1:]...)

				cmd.Dir = base
				e := cmd.Run()
				if e != nil {
					log.Fatal(e)
				}
			}
		}
	}
}

func start(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name

		cmdRunDir := path.Join(".hydra", name)
		pid_file := path.Join(cmdRunDir, "pid")
		log_file_path := path.Join(cmdRunDir, "log")

		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			fmt.Printf("%s is already started, run hydra stop if you want to restart\n", name)
			continue
		}

		env := os.Environ()
		for k, v := range service.Env {
			env = append(env, fmt.Sprintf("%s=%s", k, v))
		}

		fmt.Println("Starting", name)
		args := strings.Split(conf.Services[i].Start, " ")
		args = append(args, "2>&1 1> log")
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Env = env
		cmd.Dir = cmdRunDir

		log_file, e := os.Create(log_file_path)
		if e != nil {
			log.Fatal(e)
		}

		_, e = log_file.WriteString(fmt.Sprintf("%s:\n", name))
		if e != nil {
			log.Fatal(e)
		}

		defer log_file.Close()
		cmd.Stdout = log_file
		cmd.Stderr = log_file

		e = cmd.Start()
		pid := []byte(strconv.Itoa(cmd.Process.Pid))
		ioutil.WriteFile(pid_file, pid, 0440)

		if e != nil {
			os.Remove("pid")
			log.Fatal("Error with", name, e)
		}

	}
}

func kill(conf config.Config) {
	for i := 0; i < len(conf.Services); i += 1 {
		service := conf.Services[i]
		name := service.Name
		pid_file := path.Join(".hydra", name, "pid")
		if _, err := os.Stat(pid_file); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_file)
			if e != nil {
				log.Fatalf("Error when reading file %v", e)
			}
			pid := string(bytes)
			exec.Command("kill", pid).Run()
			if e == nil {
				fmt.Printf("Killed %s\n", name)
			}
			os.Remove(pid_file)
		}
	}
}

func ls(conf config.Config) {
	for _, service := range conf.Services {
		name := service.Name
		pid_path := path.Join(BASE_DIR, name, "pid")
		if _, err := os.Stat(pid_path); !os.IsNotExist(err) {
			bytes, e := ioutil.ReadFile(pid_path)
			if e != nil {
				log.Fatal(e)
			}
			pid := string(bytes)
			fmt.Printf("%s running on pid: %s\n", name, pid)
		} else {
			fmt.Printf("%s is not running\n", name)
		}
	}
}

func logs(conf config.Config, servers []string) {
	for _, service := range conf.Services {
		if len(servers) > 0 {
			err := false
			for _, name := range servers {
				if name != service.Name {
					err = true
					break
				}
			}
			if err == true {
				continue
			}
		}

		logfile := path.Join(".hydra/", service.Name, "log")
		data, err := ioutil.ReadFile(logfile)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(string(data))
	}
}

func main() {
	var c config.Config = config.ReadConfig()
	var clean = flag.Bool("clean", false, "If we want a clean run")
	flag.Parse()

	if cmd := flag.Arg(0); cmd != "" {
		switch cmd {
		case "init":
			initialize(c, *clean)
		case "start":
			start(c)
		case "stop":
			kill(c)
		case "ls":
			ls(c)
		case "logs":
			logs(c, flag.Args()[1:])
		default:
			fmt.Println(help)
		}
	} else {
		fmt.Println(help)
	}
}
<|endoftext|>


<commit_before><commit_msg>blockmanager: detect peers including OP_RETURN in filters<commit_after><|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/api/v1beta1"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	switch t := obj.(type) {
	case *api.ReplicationController:
		errors = validation.ValidateManifest(&t.DesiredState.PodTemplate.DesiredState.Manifest)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		errors = validation.ValidateManifest(&t.DesiredState.Manifest)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if ext != ".json" {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestApiExamples(t *testing.T) {
	expected := map[string]runtime.Object{
		"controller":       &api.ReplicationController{},
		"controller-list":  &api.ReplicationControllerList{},
		"pod":              &api.Pod{},
		"pod-list":         &api.PodList{},
		"service":          &api.Service{},
		"external-service": &api.Service{},
		"service-list":     &api.ServiceList{},
	}

	tested := 0
	err := walkJSONFiles("../api/examples", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

func TestExamples(t *testing.T) {
	expected := map[string]runtime.Object{
		"frontend-controller":    &api.ReplicationController{},
		"redis-slave-controller": &api.ReplicationController{},
		"redis-master":           &api.Pod{},
		"frontend-service":       &api.Service{},
		"redis-master-service":   &api.Service{},
		"redis-slave-service":    &api.Service{},
	}

	tested := 0
	err := walkJSONFiles("../examples/guestbook", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

var jsonRegexp = regexp.MustCompile("(?ms)^```\\w*\\n(\\{.+?\\})\\w*\\n^```")

func TestReadme(t *testing.T) {
	path := "../README.md"
	data, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatalf("Unable to read file: %v", err)
	}

	match := jsonRegexp.FindStringSubmatch(string(data))
	if match == nil {
		return
	}
	for _, json := range match[1:] {
		expectedType := &api.Pod{}
		if err := latest.Codec.DecodeInto([]byte(json), expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
		encoded, err := latest.Codec.Encode(expectedType)
		if err != nil {
			t.Errorf("Could not encode object: %v", err)
			continue
		}
		t.Logf("Found pod %s\n%s", json, encoded)
	}
}
<commit_msg>Make runtime less global for Codec<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package examples_test

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"testing"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/validation"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"
	"github.com/golang/glog"
)

func validateObject(obj runtime.Object) (errors []error) {
	switch t := obj.(type) {
	case *api.ReplicationController:
		errors = validation.ValidateManifest(&t.DesiredState.PodTemplate.DesiredState.Manifest)
	case *api.ReplicationControllerList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Service:
		errors = validation.ValidateService(t)
	case *api.ServiceList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	case *api.Pod:
		errors = validation.ValidateManifest(&t.DesiredState.Manifest)
	case *api.PodList:
		for i := range t.Items {
			errors = append(errors, validateObject(&t.Items[i])...)
		}
	default:
		return []error{fmt.Errorf("no validation defined for %#v", obj)}
	}
	return errors
}

func walkJSONFiles(inDir string, fn func(name, path string, data []byte)) error {
	err := filepath.Walk(inDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() && path != inDir {
			return filepath.SkipDir
		}
		name := filepath.Base(path)
		ext := filepath.Ext(name)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
		if ext != ".json" {
			return nil
		}
		glog.Infof("Testing %s", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return err
		}
		fn(name, path, data)
		return nil
	})
	return err
}

func TestApiExamples(t *testing.T) {
	expected := map[string]runtime.Object{
		"controller":       &api.ReplicationController{},
		"controller-list":  &api.ReplicationControllerList{},
		"pod":              &api.Pod{},
		"pod-list":         &api.PodList{},
		"service":          &api.Service{},
		"external-service": &api.Service{},
		"service-list":     &api.ServiceList{},
	}

	tested := 0
	err := walkJSONFiles("../api/examples", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

func TestExamples(t *testing.T) {
	expected := map[string]runtime.Object{
		"frontend-controller":    &api.ReplicationController{},
		"redis-slave-controller": &api.ReplicationController{},
		"redis-master":           &api.Pod{},
		"frontend-service":       &api.Service{},
		"redis-master-service":   &api.Service{},
		"redis-slave-service":    &api.Service{},
	}

	tested := 0
	err := walkJSONFiles("../examples/guestbook", func(name, path string, data []byte) {
		expectedType, found := expected[name]
		if !found {
			t.Errorf("%s does not have a test case defined", path)
			return
		}
		tested += 1
		if err := latest.Codec.DecodeInto(data, expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
	})
	if err != nil {
		t.Errorf("Expected no error, Got %v", err)
	}
	if tested != len(expected) {
		t.Errorf("Expected %d examples, Got %d", len(expected), tested)
	}
}

var jsonRegexp = regexp.MustCompile("(?ms)^```\\w*\\n(\\{.+?\\})\\w*\\n^```")

func TestReadme(t *testing.T) {
	path := "../README.md"
	data, err := ioutil.ReadFile(path)
	if err != nil {
		t.Fatalf("Unable to read file: %v", err)
	}

	match := jsonRegexp.FindStringSubmatch(string(data))
	if match == nil {
		return
	}
	for _, json := range match[1:] {
		expectedType := &api.Pod{}
		if err := latest.Codec.DecodeInto([]byte(json), expectedType); err != nil {
			t.Errorf("%s did not decode correctly: %v\n%s", path, err, string(data))
			return
		}
		if errors := validateObject(expectedType); len(errors) > 0 {
			t.Errorf("%s did not validate correctly: %v", path, errors)
		}
		encoded, err := latest.Codec.Encode(expectedType)
		if err != nil {
			t.Errorf("Could not encode object: %v", err)
			continue
		}
		t.Logf("Found pod %s\n%s", json, encoded)
	}
}
<|endoftext|>


<commit_before>// Wrapper for kasia.go templates designed for easy writing web applications.
package kview

import (
    "os"
    "io"
    "log"
    "path"
    "reflect"
    "fmt"
    "github.com/ziutek/kasia.go"
)

var (
    // You can modify this, if you store templates in a different directory.
    TemplatesDir = "templates"

    // You can modify this, if want a different error handling.
    ErrorHandler = func(name string, err os.Error) {
        log.Printf("%%View '%s' error. %s\n", name, err.String())
    }
)

type View interface {
    Copy() View
    Strict(bool)
    Div(string, View)
    Exec(io.Writer, ...interface{})
    Render(...interface{}) *kasia.NestedTemplate
}

// View definition
type KView struct {
    name    string
    tpl     *kasia.Template
    globals map[string]interface{}
}

// Returns a pointer to a page
func New(name string, globals ...map[string]interface{}) *KView {
    var (
        pg  KView
        err os.Error
    )
    pg.name = name
    pg.tpl, err = kasia.ParseFile(path.Join(TemplatesDir, name))
    if err != nil {
        ErrorHandler(name, err)
    }
    pg.globals = make(map[string]interface{})
    // First some default globals
    for k, v := range Globals {
        pg.globals[k] = v
    }
    // globals may default
    for _, g := range globals {
        for k, v := range g {
            pg.globals[k] = v
        }
    }
    return &pg
}

// Returns a pointer to a copy of the page
func (pg *KView) Copy() View {
    new_pg := *pg
    // Make a copy of globals
    new_pg.globals = make(map[string]interface{})
    for k, v := range pg.globals {
        new_pg.globals[k] = v
    }
    return &new_pg
}

// Set strig render flag
func (pg *KView) Strict(strict bool) {
    pg.tpl.Strict = strict
}

// Add subview
func (pg *KView) Div(name string, view View) {
    pg.globals[name] = view
}

func prepend(slice []interface{}, pre ...interface{}) (ret []interface{}) {
    ret = make([]interface{}, len(slice) + len(pre))
    copy(ret, pre)
    copy(ret[len(pre):], slice)
    return
}

// Render view to wr with data
func (pg *KView) Exec(wr io.Writer, ctx ...interface{}) {
    // Add globals to the bottom of the context stack
    ctx = prepend(ctx, pg.globals)
    err := pg.tpl.Run(wr, ctx...)
    if err != nil {
        ErrorHandler(pg.name, err)
    }
}

// Use this method in template text to render page inside other page.
func (pg *KView) Render(ctx ...interface{}) *kasia.NestedTemplate {
    if len(ctx) > 0 {
        // Check if render was called with full template context as first arg
        if ci, ok := ctx[0].(kasia.ContextItself); ok {
            // Rearange context, remove old globals
            ctx = append(ci[1:], ctx[1:])
        }
    }
    // Add globals to the bottom of the context stack
    ctx = prepend(ctx, pg.globals)
    return pg.tpl.Nested(ctx...)
}

// Some useful functions for globals.
// You can add there your functions/variables which will be visable in any
// view. See also globals parameter in New function.
var Globals = map[string]interface{} {
    "len": func(a interface{}) int {
        v := reflect.ValueOf(a)
        if v.Kind() == reflect.Array || v.Kind() == reflect.Slice {
            return v.Len()
        }
        return -1
    },
    "fmt": fmt.Sprintf,
}
<commit_msg>Updated to Go weekly.2011-11-02<commit_after>// Wrapper for kasia.go templates designed for easy writing web applications.
package kview

import (
	"io"
	"log"
	"path"
	"reflect"
	"fmt"
	"github.com/ziutek/kasia.go"
)

var (
	// You can modify this, if you store templates in a different directory.
	TemplatesDir = "templates"

	// You can modify this, if want a different error handling.
	ErrorHandler = func(name string, err error) {
		log.Printf("%%View '%s' error. %s\n", name, err.Error())
	}
)

type View interface {
	Copy() View
	Strict(bool)
	Div(string, View)
	Exec(io.Writer, ...interface{})
	Render(...interface{}) *kasia.NestedTemplate
}

// View definition
type KView struct {
	name    string
	tpl     *kasia.Template
	globals map[string]interface{}
}

// Returns a pointer to a page
func New(name string, globals ...map[string]interface{}) *KView {
	var (
		pg  KView
		err error
	)
	pg.name = name
	pg.tpl, err = kasia.ParseFile(path.Join(TemplatesDir, name))
	if err != nil {
		ErrorHandler(name, err)
	}
	pg.globals = make(map[string]interface{})
	// First some default globals
	for k, v := range Globals {
		pg.globals[k] = v
	}
	// globals may default
	for _, g := range globals {
		for k, v := range g {
			pg.globals[k] = v
		}
	}
	return &pg
}

// Returns a pointer to a copy of the page
func (pg *KView) Copy() View {
	new_pg := *pg
	// Make a copy of globals
	new_pg.globals = make(map[string]interface{})
	for k, v := range pg.globals {
		new_pg.globals[k] = v
	}
	return &new_pg
}

// Set strig render flag
func (pg *KView) Strict(strict bool) {
	pg.tpl.Strict = strict
}

// Add subview
func (pg *KView) Div(name string, view View) {
	pg.globals[name] = view
}

func prepend(slice []interface{}, pre ...interface{}) (ret []interface{}) {
	ret = make([]interface{}, len(slice)+len(pre))
	copy(ret, pre)
	copy(ret[len(pre):], slice)
	return
}

// Render view to wr with data
func (pg *KView) Exec(wr io.Writer, ctx ...interface{}) {
	// Add globals to the bottom of the context stack
	ctx = prepend(ctx, pg.globals)
	err := pg.tpl.Run(wr, ctx...)
	if err != nil {
		ErrorHandler(pg.name, err)
	}
}

// Use this method in template text to render page inside other page.
func (pg *KView) Render(ctx ...interface{}) *kasia.NestedTemplate {
	if len(ctx) > 0 {
		// Check if render was called with full template context as first arg
		if ci, ok := ctx[0].(kasia.ContextItself); ok {
			// Rearange context, remove old globals
			ctx = append(ci[1:], ctx[1:])
		}
	}
	// Add globals to the bottom of the context stack
	ctx = prepend(ctx, pg.globals)
	return pg.tpl.Nested(ctx...)
}

// Some useful functions for globals.
// You can add there your functions/variables which will be visable in any
// view. See also globals parameter in New function.
var Globals = map[string]interface{}{
	"len": func(a interface{}) int {
		v := reflect.ValueOf(a)
		if v.Kind() == reflect.Array || v.Kind() == reflect.Slice {
			return v.Len()
		}
		return -1
	},
	"fmt": fmt.Sprintf,
}
<|endoftext|>


<commit_before>package cli

import (
	"bufio"
	"context"
	"io"
	"log"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/taku-k/polymerase/pkg/storage/storagepb"
	"github.com/taku-k/polymerase/pkg/tempbackup/tempbackuppb"
)

var incBackupCmd = &cobra.Command{
	Use:   "inc-backup",
	Short: "Transfers incremental backups to a polymerase server",
	RunE:  cleanupTempDirRunE(runIncBackup),
}

func runIncBackup(cmd *cobra.Command, args []string) error {
	if len(args) > 0 {
		return usageAndError(cmd)
	}

	if db == "" {
		return errors.New("You should specify db with '--db' flag")
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	errCh := make(chan error, 1)
	finishCh := make(chan struct{})

	// Fetches latest to_lsn
	scli, _ := getStorageClient(ctx, db)
	res, err := scli.GetLatestToLSN(context.Background(), &storagepb.GetLatestToLSNRequest{Db: db})
	if err != nil {
		return errors.Wrapf(err, "Failed to get latest `to_lsn` with db=%s", db)
	}
	xtrabackupCfg.ToLsn = res.Lsn

	// Builds backup pipeline and start it
	r, err := buildBackupPipelineAndStart(ctx, errCh)
	if err != nil {
		return errors.Wrap(err, "Failed to build backup pipeline")
	}

	go transferIncBackup(ctx, r, db, errCh, finishCh)

	select {
	case err := <-errCh:
		cancel()
		<-ctx.Done()
		return err
	case <-finishCh:
		log.Println("Incremental backup succeeded")
		return nil
	}
}

func transferIncBackup(ctx context.Context, r io.Reader, db string, errCh chan error, finishCh chan struct{}) {
	bcli, _ := getTempBackupClient(ctx, db)
	stream, err := bcli.TransferIncBackup(ctx)
	if err != nil {
		errCh <- err
		return
	}

	buf := bufio.NewReader(r)
	chunk := make([]byte, 1<<20)
	var key string

	for {
		n, err := buf.Read(chunk)
		if err == io.EOF {
			reply, err := stream.CloseAndRecv()
			if err != nil {
				errCh <- err
				return
			}
			log.Println(reply)
			key = reply.Key
			break
		}
		if err != nil {
			errCh <- err
			return
		}
		stream.Send(&tempbackuppb.IncBackupContentStream{
			Content: chunk[:n],
			Db:      db,
			Lsn:     xtrabackupCfg.ToLsn,
		})
	}
	// Post xtrabackup_checkpoints
	res, err := postXtrabackupCP(ctx, bcli, key)
	if err != nil {
		errCh <- err
		return
	}
	log.Println(res)
	finishCh <- struct{}{}
	return
}
<commit_msg>Fix segmentation fault in case db is not found<commit_after>package cli

import (
	"bufio"
	"context"
	"io"
	"log"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/taku-k/polymerase/pkg/storage/storagepb"
	"github.com/taku-k/polymerase/pkg/tempbackup/tempbackuppb"
)

var incBackupCmd = &cobra.Command{
	Use:   "inc-backup",
	Short: "Transfers incremental backups to a polymerase server",
	RunE:  cleanupTempDirRunE(runIncBackup),
}

func runIncBackup(cmd *cobra.Command, args []string) error {
	if len(args) > 0 {
		return usageAndError(cmd)
	}

	if db == "" {
		return errors.New("You should specify db with '--db' flag")
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	errCh := make(chan error, 1)
	finishCh := make(chan struct{})

	// Fetches latest to_lsn
	scli, err := getStorageClient(ctx, db)
	if err != nil {
		return err
	}
	res, err := scli.GetLatestToLSN(context.Background(), &storagepb.GetLatestToLSNRequest{Db: db})
	if err != nil {
		return errors.Wrapf(err, "Failed to get latest `to_lsn` with db=%s", db)
	}
	xtrabackupCfg.ToLsn = res.Lsn

	// Builds backup pipeline and start it
	r, err := buildBackupPipelineAndStart(ctx, errCh)
	if err != nil {
		return errors.Wrap(err, "Failed to build backup pipeline")
	}

	go transferIncBackup(ctx, r, db, errCh, finishCh)

	select {
	case err := <-errCh:
		cancel()
		<-ctx.Done()
		return err
	case <-finishCh:
		log.Println("Incremental backup succeeded")
		return nil
	}
}

func transferIncBackup(ctx context.Context, r io.Reader, db string, errCh chan error, finishCh chan struct{}) {
	bcli, _ := getTempBackupClient(ctx, db)
	stream, err := bcli.TransferIncBackup(ctx)
	if err != nil {
		errCh <- err
		return
	}

	buf := bufio.NewReader(r)
	chunk := make([]byte, 1<<20)
	var key string

	for {
		n, err := buf.Read(chunk)
		if err == io.EOF {
			reply, err := stream.CloseAndRecv()
			if err != nil {
				errCh <- err
				return
			}
			log.Println(reply)
			key = reply.Key
			break
		}
		if err != nil {
			errCh <- err
			return
		}
		stream.Send(&tempbackuppb.IncBackupContentStream{
			Content: chunk[:n],
			Db:      db,
			Lsn:     xtrabackupCfg.ToLsn,
		})
	}
	// Post xtrabackup_checkpoints
	res, err := postXtrabackupCP(ctx, bcli, key)
	if err != nil {
		errCh <- err
		return
	}
	log.Println(res)
	finishCh <- struct{}{}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"runtime"

	"github.com/codegangsta/cli"
	"github.com/exercism/cli/api"
	"github.com/exercism/cli/cmd"
)

const (
	// Version is the current release of the command-line app.
	// We try to follow Semantic Versioning (http://semver.org),
	// but with the http://exercism.io app being a prototype, a
	// lot of things get out of hand.
	Version = "2.1.0"

	descDebug     = "Outputs useful debug information."
	descConfigure = "Writes config values to a JSON file."
	descDemo      = "Fetches a demo problem for each language track on exercism.io."
	descFetch     = "Fetches your current problems on exercism.io, as well as the next unstarted problem in each language."
	descRestore   = "Restores completed and current problems on from exercism.io, along with your most recent iteration for each."
	descSubmit    = "Submits a new iteration to a problem on exercism.io."
	descSkip      = "Skips a problem given a language and slug."
	descUnsubmit  = "Deletes the most recently submitted iteration."
	descTracks    = "List the available language tracks"
	descOpen      = "Opens the current submission of the specified exercise"

	descLongRestore = "Restore will pull the latest revisions of exercises that have already been submitted. It will *not* overwrite existing files. If you have made changes to a file and have not submitted it, and you're trying to restore the last submitted version, first move that file out of the way, then call restore."
	descDownload    = "Downloads and saves a specified submission into the local system"
	descList        = "Lists all available assignments for a given language"
)

func main() {
	api.UserAgent = fmt.Sprintf("github.com/exercism/cli v%s (%s/%s)", Version, runtime.GOOS, runtime.GOARCH)

	app := cli.NewApp()
	app.Name = "exercism"
	app.Usage = "A command line tool to interact with http://exercism.io"
	app.Version = Version
	app.HideVersion = true
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config, c",
			Usage:  "path to config file",
			EnvVar: "XDG_CONFIG_HOME,EXERCISM_CONFIG_FILE",
		},
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "turn on verbose logging",
		},
		cli.BoolFlag{
			Name:  "version",
			Usage: "print the version",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "debug",
			Usage:  descDebug,
			Action: cmd.Debug,
		},
		{
			Name:  "configure",
			Usage: descConfigure,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "path to exercises directory",
				},
				cli.StringFlag{
					Name:  "host, u",
					Usage: "exercism api host",
				},
				cli.StringFlag{
					Name:  "key, k",
					Usage: "exercism.io API key (see http://exercism.io/account)",
				},
				cli.StringFlag{
					Name:  "api, a",
					Usage: "exercism xapi host",
				},
			},
			Action: cmd.Configure,
		},
		{
			Name:      "demo",
			ShortName: "d",
			Usage:     descDemo,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "path to use for the demo exercises",
				},
			},
			Action: cmd.Demo,
		},
		{
			Name:      "fetch",
			ShortName: "f",
			Usage:     descFetch,
			Action:    cmd.Fetch,
		},
		{
			Name:        "restore",
			ShortName:   "r",
			Usage:       descRestore,
			Description: descLongRestore,
			Action:      cmd.Restore,
		},
		{
			Name:   "skip",
			Usage:  descSkip,
			Action: cmd.Skip,
		},
		{
			Name:      "submit",
			ShortName: "s",
			Usage:     descSubmit,
			Action:    cmd.Submit,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "test",
					Usage: "allow submision of test files",
				},
			},
		},
		{
			Name:      "unsubmit",
			ShortName: "u",
			Usage:     descUnsubmit,
			Action:    cmd.Unsubmit,
		},
		{
			Name:      "tracks",
			ShortName: "t",
			Usage:     descTracks,
			Action:    cmd.Tracks,
		},
		{
			Name:      "open",
			ShortName: "op",
			Usage:     descOpen,
			Action:    cmd.Open,
		},
		{
			Name:      "download",
			ShortName: "dl",
			Usage:     descDownload,
			Action:    cmd.Download,
		},
		{
			Name:      "list",
			ShortName: "li",
			Usage:     descList,
			Action:    cmd.List,
		},
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Fix typo on flag description<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"runtime"

	"github.com/codegangsta/cli"
	"github.com/exercism/cli/api"
	"github.com/exercism/cli/cmd"
)

const (
	// Version is the current release of the command-line app.
	// We try to follow Semantic Versioning (http://semver.org),
	// but with the http://exercism.io app being a prototype, a
	// lot of things get out of hand.
	Version = "2.1.0"

	descDebug     = "Outputs useful debug information."
	descConfigure = "Writes config values to a JSON file."
	descDemo      = "Fetches a demo problem for each language track on exercism.io."
	descFetch     = "Fetches your current problems on exercism.io, as well as the next unstarted problem in each language."
	descRestore   = "Restores completed and current problems on from exercism.io, along with your most recent iteration for each."
	descSubmit    = "Submits a new iteration to a problem on exercism.io."
	descSkip      = "Skips a problem given a language and slug."
	descUnsubmit  = "Deletes the most recently submitted iteration."
	descTracks    = "List the available language tracks"
	descOpen      = "Opens the current submission of the specified exercise"

	descLongRestore = "Restore will pull the latest revisions of exercises that have already been submitted. It will *not* overwrite existing files. If you have made changes to a file and have not submitted it, and you're trying to restore the last submitted version, first move that file out of the way, then call restore."
	descDownload    = "Downloads and saves a specified submission into the local system"
	descList        = "Lists all available assignments for a given language"
)

func main() {
	api.UserAgent = fmt.Sprintf("github.com/exercism/cli v%s (%s/%s)", Version, runtime.GOOS, runtime.GOARCH)

	app := cli.NewApp()
	app.Name = "exercism"
	app.Usage = "A command line tool to interact with http://exercism.io"
	app.Version = Version
	app.HideVersion = true
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "config, c",
			Usage:  "path to config file",
			EnvVar: "XDG_CONFIG_HOME,EXERCISM_CONFIG_FILE",
		},
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "turn on verbose logging",
		},
		cli.BoolFlag{
			Name:  "version",
			Usage: "print the version",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "debug",
			Usage:  descDebug,
			Action: cmd.Debug,
		},
		{
			Name:  "configure",
			Usage: descConfigure,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "path to exercises directory",
				},
				cli.StringFlag{
					Name:  "host, u",
					Usage: "exercism api host",
				},
				cli.StringFlag{
					Name:  "key, k",
					Usage: "exercism.io API key (see http://exercism.io/account)",
				},
				cli.StringFlag{
					Name:  "api, a",
					Usage: "exercism xapi host",
				},
			},
			Action: cmd.Configure,
		},
		{
			Name:      "demo",
			ShortName: "d",
			Usage:     descDemo,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "path to use for the demo exercises",
				},
			},
			Action: cmd.Demo,
		},
		{
			Name:      "fetch",
			ShortName: "f",
			Usage:     descFetch,
			Action:    cmd.Fetch,
		},
		{
			Name:        "restore",
			ShortName:   "r",
			Usage:       descRestore,
			Description: descLongRestore,
			Action:      cmd.Restore,
		},
		{
			Name:   "skip",
			Usage:  descSkip,
			Action: cmd.Skip,
		},
		{
			Name:      "submit",
			ShortName: "s",
			Usage:     descSubmit,
			Action:    cmd.Submit,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "test",
					Usage: "allow submission of test files",
				},
			},
		},
		{
			Name:      "unsubmit",
			ShortName: "u",
			Usage:     descUnsubmit,
			Action:    cmd.Unsubmit,
		},
		{
			Name:      "tracks",
			ShortName: "t",
			Usage:     descTracks,
			Action:    cmd.Tracks,
		},
		{
			Name:      "open",
			ShortName: "op",
			Usage:     descOpen,
			Action:    cmd.Open,
		},
		{
			Name:      "download",
			ShortName: "dl",
			Usage:     descDownload,
			Action:    cmd.Download,
		},
		{
			Name:      "list",
			ShortName: "li",
			Usage:     descList,
			Action:    cmd.List,
		},
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package gog

import (
	"bytes"
	"go/ast"
	"go/doc"
	"go/parser"
	"go/token"
	"log"
	"sort"

	"code.google.com/p/go.tools/go/loader"
	"code.google.com/p/go.tools/go/types"
)

type Doc struct {
	*SymbolKey

	Format string
	Data   string

	File string `json:",omitempty"`
	Span [2]int `json:",omitempty"`
}

func parseFiles(fset *token.FileSet, filenames []string) (map[string]*ast.File, error) {
	files := make(map[string]*ast.File)
	for _, f := range filenames {
		file, err := parser.ParseFile(fset, f, nil, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		files[f] = file
	}
	return files, nil
}

func (g *Grapher) emitDocs(pkgInfo *loader.PackageInfo) error {
	objOf := make(map[token.Position]types.Object, len(pkgInfo.Defs))
	for ident, obj := range pkgInfo.Defs {
		objOf[g.program.Fset.Position(ident.Pos())] = obj
	}

	var filenames []string
	for _, f := range pkgInfo.Files {
		filenames = append(filenames, g.program.Fset.Position(f.Name.Pos()).Filename)
	}
	sort.Strings(filenames)
	files, err := parseFiles(g.program.Fset, filenames)
	if err != nil {
		return err
	}

	// ignore errors because we assume that syntax checking has already occurred
	astPkg, _ := ast.NewPackage(g.program.Fset, files, nil, nil)

	docPkg := doc.New(astPkg, pkgInfo.Pkg.Path(), doc.AllDecls|doc.AllMethods)

	if docPkg.Doc != "" {
		// Find the file that defines package doc.
		for _, f := range sortedFiles(astPkg.Files) {
			if f.Doc != nil {
				err := g.emitDoc(types.NewPkgName(f.Package, pkgInfo.Pkg, pkgInfo.Pkg.Path()), f.Doc, docPkg.Doc)
				if err != nil {
					return err
				}
				break
			}
		}
	}

	emitDocForSpecs := func(pkgInfo *loader.PackageInfo, decl *ast.GenDecl, docstring string) error {
		for _, spec := range decl.Specs {
			switch spec := spec.(type) {
			case *ast.ValueSpec:
				for _, name := range spec.Names {
					g.emitDoc(objOf[g.program.Fset.Position(name.Pos())], firstNonNil(decl.Doc, spec.Doc, spec.Comment), docstring)
				}
			case *ast.TypeSpec:
				g.emitDoc(objOf[g.program.Fset.Position(spec.Name.Pos())], firstNonNil(decl.Doc, spec.Doc, spec.Comment), docstring)
			default:
				log.Panicf("unknown spec type %T", spec)
			}
		}

		return nil
	}

	for _, cnst := range docPkg.Consts {
		emitDocForSpecs(pkgInfo, cnst.Decl, cnst.Doc)
	}

	for _, vari := range docPkg.Vars {
		emitDocForSpecs(pkgInfo, vari.Decl, vari.Doc)
	}

	for _, fun := range docPkg.Funcs {
		g.emitDoc(objOf[g.program.Fset.Position(fun.Decl.Name.Pos())], fun.Decl.Doc, fun.Doc)
	}

	for _, typ := range docPkg.Types {
		emitDocForSpecs(pkgInfo, typ.Decl, typ.Doc)
		for _, cnst := range typ.Consts {
			emitDocForSpecs(pkgInfo, cnst.Decl, cnst.Doc)
		}
		for _, vari := range typ.Vars {
			emitDocForSpecs(pkgInfo, vari.Decl, vari.Doc)
		}
		for _, fun := range typ.Funcs {
			g.emitDoc(objOf[g.program.Fset.Position(fun.Decl.Name.Pos())], fun.Decl.Doc, fun.Doc)
		}
		for _, mth := range typ.Methods {
			g.emitDoc(objOf[g.program.Fset.Position(mth.Decl.Name.Pos())], mth.Decl.Doc, mth.Doc)
		}
	}

	return nil
}

func firstNonNil(comments ...*ast.CommentGroup) *ast.CommentGroup {
	for _, c := range comments {
		if c != nil {
			return c
		}
	}
	return nil
}

func (g *Grapher) emitDoc(obj types.Object, dc *ast.CommentGroup, docstring string) error {
	if obj == nil {
		return nil
	}
	if docstring == "" {
		return nil
	}

	key, _, err := g.symbolInfo(obj)
	if err != nil {
		return err
	}

	var htmlBuf bytes.Buffer
	doc.ToHTML(&htmlBuf, docstring, nil)

	var filename string
	var span [2]int
	if dc != nil {
		filename = g.program.Fset.Position(dc.Pos()).Filename
		span = makeSpan(g.program.Fset, dc)
	}

	g.addDoc(&Doc{
		SymbolKey: key,
		Format:    "text/html",
		Data:      htmlBuf.String(),
		File:      filename,
		Span:      span,
	})
	g.addDoc(&Doc{
		SymbolKey: key,
		Format:    "text/plain",
		Data:      docstring,
		File:      filename,
		Span:      span,
	})

	return nil
}

func (g *Grapher) addDoc(doc *Doc) {
	g.Docs = append(g.Docs, doc)
}
<commit_msg>only get methods tied to a type (fixes duplicate doc error)<commit_after>package gog

import (
	"bytes"
	"go/ast"
	"go/doc"
	"go/parser"
	"go/token"
	"log"
	"sort"

	"code.google.com/p/go.tools/go/loader"
	"code.google.com/p/go.tools/go/types"
)

type Doc struct {
	*SymbolKey

	Format string
	Data   string

	File string `json:",omitempty"`
	Span [2]int `json:",omitempty"`
}

func parseFiles(fset *token.FileSet, filenames []string) (map[string]*ast.File, error) {
	files := make(map[string]*ast.File)
	for _, f := range filenames {
		file, err := parser.ParseFile(fset, f, nil, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		files[f] = file
	}
	return files, nil
}

func (g *Grapher) emitDocs(pkgInfo *loader.PackageInfo) error {
	objOf := make(map[token.Position]types.Object, len(pkgInfo.Defs))
	for ident, obj := range pkgInfo.Defs {
		objOf[g.program.Fset.Position(ident.Pos())] = obj
	}

	var filenames []string
	for _, f := range pkgInfo.Files {
		filenames = append(filenames, g.program.Fset.Position(f.Name.Pos()).Filename)
	}
	sort.Strings(filenames)
	files, err := parseFiles(g.program.Fset, filenames)
	if err != nil {
		return err
	}

	// ignore errors because we assume that syntax checking has already occurred
	astPkg, _ := ast.NewPackage(g.program.Fset, files, nil, nil)

	docPkg := doc.New(astPkg, pkgInfo.Pkg.Path(), doc.AllDecls)

	if docPkg.Doc != "" {
		// Find the file that defines package doc.
		for _, f := range sortedFiles(astPkg.Files) {
			if f.Doc != nil {
				err := g.emitDoc(types.NewPkgName(f.Package, pkgInfo.Pkg, pkgInfo.Pkg.Path()), f.Doc, docPkg.Doc)
				if err != nil {
					return err
				}
				break
			}
		}
	}

	emitDocForSpecs := func(pkgInfo *loader.PackageInfo, decl *ast.GenDecl, docstring string) error {
		for _, spec := range decl.Specs {
			switch spec := spec.(type) {
			case *ast.ValueSpec:
				for _, name := range spec.Names {
					g.emitDoc(objOf[g.program.Fset.Position(name.Pos())], firstNonNil(decl.Doc, spec.Doc, spec.Comment), docstring)
				}
			case *ast.TypeSpec:
				g.emitDoc(objOf[g.program.Fset.Position(spec.Name.Pos())], firstNonNil(decl.Doc, spec.Doc, spec.Comment), docstring)
			default:
				log.Panicf("unknown spec type %T", spec)
			}
		}

		return nil
	}

	for _, cnst := range docPkg.Consts {
		emitDocForSpecs(pkgInfo, cnst.Decl, cnst.Doc)
	}

	for _, vari := range docPkg.Vars {
		emitDocForSpecs(pkgInfo, vari.Decl, vari.Doc)
	}

	for _, fun := range docPkg.Funcs {
		g.emitDoc(objOf[g.program.Fset.Position(fun.Decl.Name.Pos())], fun.Decl.Doc, fun.Doc)
	}

	for _, typ := range docPkg.Types {
		emitDocForSpecs(pkgInfo, typ.Decl, typ.Doc)
		for _, cnst := range typ.Consts {
			emitDocForSpecs(pkgInfo, cnst.Decl, cnst.Doc)
		}
		for _, vari := range typ.Vars {
			emitDocForSpecs(pkgInfo, vari.Decl, vari.Doc)
		}
		for _, fun := range typ.Funcs {
			g.emitDoc(objOf[g.program.Fset.Position(fun.Decl.Name.Pos())], fun.Decl.Doc, fun.Doc)
		}
		for _, mth := range typ.Methods {
			g.emitDoc(objOf[g.program.Fset.Position(mth.Decl.Name.Pos())], mth.Decl.Doc, mth.Doc)
		}
	}

	return nil
}

func firstNonNil(comments ...*ast.CommentGroup) *ast.CommentGroup {
	for _, c := range comments {
		if c != nil {
			return c
		}
	}
	return nil
}

func (g *Grapher) emitDoc(obj types.Object, dc *ast.CommentGroup, docstring string) error {
	if obj == nil {
		return nil
	}
	if docstring == "" {
		return nil
	}

	key, _, err := g.symbolInfo(obj)
	if err != nil {
		return err
	}

	var htmlBuf bytes.Buffer
	doc.ToHTML(&htmlBuf, docstring, nil)

	var filename string
	var span [2]int
	if dc != nil {
		filename = g.program.Fset.Position(dc.Pos()).Filename
		span = makeSpan(g.program.Fset, dc)
	}

	g.addDoc(&Doc{
		SymbolKey: key,
		Format:    "text/html",
		Data:      htmlBuf.String(),
		File:      filename,
		Span:      span,
	})
	g.addDoc(&Doc{
		SymbolKey: key,
		Format:    "text/plain",
		Data:      docstring,
		File:      filename,
		Span:      span,
	})

	return nil
}

func (g *Grapher) addDoc(doc *Doc) {
	g.Docs = append(g.Docs, doc)
}
<|endoftext|>