<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package images

import (
	gocontext "context"
	"errors"
	"fmt"
	"net/http/httptrace"
	"os"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/cmd/ctr/commands/content"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/pkg/progress"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/remotes"
	"github.com/containerd/containerd/remotes/docker"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
	"golang.org/x/sync/errgroup"
)

var pushCommand = cli.Command{
	Name:      "push",
	Usage:     "push an image to a remote",
	ArgsUsage: "[flags] <remote> [<local>]",
	Description: `Pushes an image reference from containerd.

	All resources associated with the manifest reference will be pushed.
	The ref is used to resolve to a locally existing image manifest.
	The image manifest must exist before push. Creating a new image
	manifest can be done through calculating the diff for layers,
	creating the associated configuration, and creating the manifest
	which references those resources.
`,
	Flags: append(commands.RegistryFlags, cli.StringFlag{
		Name:  "manifest",
		Usage: "digest of manifest",
	}, cli.StringFlag{
		Name:  "manifest-type",
		Usage: "media type of manifest digest",
		Value: ocispec.MediaTypeImageManifest,
	}, cli.StringSliceFlag{
		Name:  "platform",
		Usage: "push content from a specific platform",
		Value: &cli.StringSlice{},
	}, cli.IntFlag{
		Name:  "max-concurrent-uploaded-layers",
		Usage: "Set the max concurrent uploaded layers for each push",
	}),
	Action: func(context *cli.Context) error {
		var (
			ref   = context.Args().First()
			local = context.Args().Get(1)
			debug = context.GlobalBool("debug")
			desc  ocispec.Descriptor
		)
		if ref == "" {
			return errors.New("please provide a remote image reference to push")
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		if manifest := context.String("manifest"); manifest != "" {
			desc.Digest, err = digest.Parse(manifest)
			if err != nil {
				return fmt.Errorf("invalid manifest digest: %w", err)
			}
			desc.MediaType = context.String("manifest-type")
		} else {
			if local == "" {
				local = ref
			}
			img, err := client.ImageService().Get(ctx, local)
			if err != nil {
				return fmt.Errorf("unable to resolve image to manifest: %w", err)
			}
			desc = img.Target

			if pss := context.StringSlice("platform"); len(pss) == 1 {
				p, err := platforms.Parse(pss[0])
				if err != nil {
					return fmt.Errorf("invalid platform %q: %w", pss[0], err)
				}

				cs := client.ContentStore()
				if manifests, err := images.Children(ctx, cs, desc); err == nil && len(manifests) > 0 {
					matcher := platforms.NewMatcher(p)
					for _, manifest := range manifests {
						if manifest.Platform != nil && matcher.Match(*manifest.Platform) {
							if _, err := images.Children(ctx, cs, manifest); err != nil {
								return fmt.Errorf("no matching manifest: %w", err)
							}
							desc = manifest
							break
						}
					}
				}
			}
		}

		if context.Bool("http-trace") {
			ctx = httptrace.WithClientTrace(ctx, commands.NewDebugClientTrace(ctx))
		}
		resolver, err := commands.GetResolver(ctx, context)
		if err != nil {
			return err
		}
		ongoing := newPushJobs(commands.PushTracker)

		eg, ctx := errgroup.WithContext(ctx)

		// used to notify the progress writer
		doneCh := make(chan struct{})

		eg.Go(func() error {
			defer close(doneCh)

			log.G(ctx).WithField("image", ref).WithField("digest", desc.Digest).Debug("pushing")

			jobHandler := images.HandlerFunc(func(ctx gocontext.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
				ongoing.add(remotes.MakeRefKey(ctx, desc))
				return nil, nil
			})

			ropts := []containerd.RemoteOpt{
				containerd.WithResolver(resolver),
				containerd.WithImageHandler(jobHandler),
			}

			if context.IsSet("max-concurrent-uploaded-layers") {
				mcu := context.Int("max-concurrent-uploaded-layers")
				ropts = append(ropts, containerd.WithMaxConcurrentUploadedLayers(mcu))
			}

			return client.Push(ctx, ref, desc, ropts...)
		})

		// don't show progress if debug mode is set
		if !debug {
			eg.Go(func() error {
				var (
					ticker = time.NewTicker(100 * time.Millisecond)
					fw     = progress.NewWriter(os.Stdout)
					start  = time.Now()
					done   bool
				)

				defer ticker.Stop()

				for {
					select {
					case <-ticker.C:
						fw.Flush()

						tw := tabwriter.NewWriter(fw, 1, 8, 1, ' ', 0)

						content.Display(tw, ongoing.status(), start)
						tw.Flush()

						if done {
							fw.Flush()
							return nil
						}
					case <-doneCh:
						done = true
					case <-ctx.Done():
						done = true // allow ui to update once more
					}
				}
			})
		}
		return eg.Wait()
	},
}

type pushjobs struct {
	jobs    map[string]struct{}
	ordered []string
	tracker docker.StatusTracker
	mu      sync.Mutex
}

func newPushJobs(tracker docker.StatusTracker) *pushjobs {
	return &pushjobs{
		jobs:    make(map[string]struct{}),
		tracker: tracker,
	}
}

func (j *pushjobs) add(ref string) {
	j.mu.Lock()
	defer j.mu.Unlock()

	if _, ok := j.jobs[ref]; ok {
		return
	}
	j.ordered = append(j.ordered, ref)
	j.jobs[ref] = struct{}{}
}

func (j *pushjobs) status() []content.StatusInfo {
	j.mu.Lock()
	defer j.mu.Unlock()

	statuses := make([]content.StatusInfo, 0, len(j.jobs))
	for _, name := range j.ordered {
		si := content.StatusInfo{
			Ref: name,
		}

		status, err := j.tracker.GetStatus(name)
		if err != nil {
			si.Status = "waiting"
		} else {
			si.Offset = status.Offset
			si.Total = status.Total
			si.StartedAt = status.StartedAt
			si.UpdatedAt = status.UpdatedAt
			if status.Offset >= status.Total {
				if status.UploadUUID == "" {
					si.Status = "done"
				} else {
					si.Status = "committing"
				}
			} else {
				si.Status = "uploading"
			}
		}
		statuses = append(statuses, si)
	}

	return statuses
}
<commit_msg>ctr: flag to toggle non-distributable blob push<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package images

import (
	gocontext "context"
	"errors"
	"fmt"
	"net/http/httptrace"
	"os"
	"sync"
	"text/tabwriter"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cmd/ctr/commands"
	"github.com/containerd/containerd/cmd/ctr/commands/content"
	"github.com/containerd/containerd/images"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/pkg/progress"
	"github.com/containerd/containerd/platforms"
	"github.com/containerd/containerd/remotes"
	"github.com/containerd/containerd/remotes/docker"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
	"golang.org/x/sync/errgroup"
)

var pushCommand = cli.Command{
	Name:      "push",
	Usage:     "push an image to a remote",
	ArgsUsage: "[flags] <remote> [<local>]",
	Description: `Pushes an image reference from containerd.

	All resources associated with the manifest reference will be pushed.
	The ref is used to resolve to a locally existing image manifest.
	The image manifest must exist before push. Creating a new image
	manifest can be done through calculating the diff for layers,
	creating the associated configuration, and creating the manifest
	which references those resources.
`,
	Flags: append(commands.RegistryFlags, cli.StringFlag{
		Name:  "manifest",
		Usage: "digest of manifest",
	}, cli.StringFlag{
		Name:  "manifest-type",
		Usage: "media type of manifest digest",
		Value: ocispec.MediaTypeImageManifest,
	}, cli.StringSliceFlag{
		Name:  "platform",
		Usage: "push content from a specific platform",
		Value: &cli.StringSlice{},
	}, cli.IntFlag{
		Name:  "max-concurrent-uploaded-layers",
		Usage: "Set the max concurrent uploaded layers for each push",
	}, cli.BoolFlag{
		Name:  "allow-non-distributable-blobs",
		Usage: "Allow pushing blobs that are marked as non-distributable",
	}),
	Action: func(context *cli.Context) error {
		var (
			ref   = context.Args().First()
			local = context.Args().Get(1)
			debug = context.GlobalBool("debug")
			desc  ocispec.Descriptor
		)
		if ref == "" {
			return errors.New("please provide a remote image reference to push")
		}

		client, ctx, cancel, err := commands.NewClient(context)
		if err != nil {
			return err
		}
		defer cancel()

		if manifest := context.String("manifest"); manifest != "" {
			desc.Digest, err = digest.Parse(manifest)
			if err != nil {
				return fmt.Errorf("invalid manifest digest: %w", err)
			}
			desc.MediaType = context.String("manifest-type")
		} else {
			if local == "" {
				local = ref
			}
			img, err := client.ImageService().Get(ctx, local)
			if err != nil {
				return fmt.Errorf("unable to resolve image to manifest: %w", err)
			}
			desc = img.Target

			if pss := context.StringSlice("platform"); len(pss) == 1 {
				p, err := platforms.Parse(pss[0])
				if err != nil {
					return fmt.Errorf("invalid platform %q: %w", pss[0], err)
				}

				cs := client.ContentStore()
				if manifests, err := images.Children(ctx, cs, desc); err == nil && len(manifests) > 0 {
					matcher := platforms.NewMatcher(p)
					for _, manifest := range manifests {
						if manifest.Platform != nil && matcher.Match(*manifest.Platform) {
							if _, err := images.Children(ctx, cs, manifest); err != nil {
								return fmt.Errorf("no matching manifest: %w", err)
							}
							desc = manifest
							break
						}
					}
				}
			}
		}

		if context.Bool("http-trace") {
			ctx = httptrace.WithClientTrace(ctx, commands.NewDebugClientTrace(ctx))
		}
		resolver, err := commands.GetResolver(ctx, context)
		if err != nil {
			return err
		}
		ongoing := newPushJobs(commands.PushTracker)

		eg, ctx := errgroup.WithContext(ctx)

		// used to notify the progress writer
		doneCh := make(chan struct{})

		eg.Go(func() error {
			defer close(doneCh)

			log.G(ctx).WithField("image", ref).WithField("digest", desc.Digest).Debug("pushing")

			jobHandler := images.HandlerFunc(func(ctx gocontext.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
				if !context.Bool("allow-non-distributable-blobs") && images.IsNonDistributable(desc.MediaType) {
					return nil, nil
				}
				ongoing.add(remotes.MakeRefKey(ctx, desc))
				return nil, nil
			})

			handler := jobHandler
			if !context.Bool("allow-non-distributable-blobs") {
				handler = remotes.SkipNonDistributableBlobs(handler)
			}

			ropts := []containerd.RemoteOpt{
				containerd.WithResolver(resolver),
				containerd.WithImageHandler(handler),
			}

			if context.IsSet("max-concurrent-uploaded-layers") {
				mcu := context.Int("max-concurrent-uploaded-layers")
				ropts = append(ropts, containerd.WithMaxConcurrentUploadedLayers(mcu))
			}

			return client.Push(ctx, ref, desc, ropts...)
		})

		// don't show progress if debug mode is set
		if !debug {
			eg.Go(func() error {
				var (
					ticker = time.NewTicker(100 * time.Millisecond)
					fw     = progress.NewWriter(os.Stdout)
					start  = time.Now()
					done   bool
				)

				defer ticker.Stop()

				for {
					select {
					case <-ticker.C:
						fw.Flush()

						tw := tabwriter.NewWriter(fw, 1, 8, 1, ' ', 0)

						content.Display(tw, ongoing.status(), start)
						tw.Flush()

						if done {
							fw.Flush()
							return nil
						}
					case <-doneCh:
						done = true
					case <-ctx.Done():
						done = true // allow ui to update once more
					}
				}
			})
		}
		return eg.Wait()
	},
}

type pushjobs struct {
	jobs    map[string]struct{}
	ordered []string
	tracker docker.StatusTracker
	mu      sync.Mutex
}

func newPushJobs(tracker docker.StatusTracker) *pushjobs {
	return &pushjobs{
		jobs:    make(map[string]struct{}),
		tracker: tracker,
	}
}

func (j *pushjobs) add(ref string) {
	j.mu.Lock()
	defer j.mu.Unlock()

	if _, ok := j.jobs[ref]; ok {
		return
	}
	j.ordered = append(j.ordered, ref)
	j.jobs[ref] = struct{}{}
}

func (j *pushjobs) status() []content.StatusInfo {
	j.mu.Lock()
	defer j.mu.Unlock()

	statuses := make([]content.StatusInfo, 0, len(j.jobs))
	for _, name := range j.ordered {
		si := content.StatusInfo{
			Ref: name,
		}

		status, err := j.tracker.GetStatus(name)
		if err != nil {
			si.Status = "waiting"
		} else {
			si.Offset = status.Offset
			si.Total = status.Total
			si.StartedAt = status.StartedAt
			si.UpdatedAt = status.UpdatedAt
			if status.Offset >= status.Total {
				if status.UploadUUID == "" {
					si.Status = "done"
				} else {
					si.Status = "committing"
				}
			} else {
				si.Status = "uploading"
			}
		}
		statuses = append(statuses, si)
	}

	return statuses
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	"github.com/mesos/mesos-go/auth/sasl/mech"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/mesos/mesos-go/scheduler"
	"github.com/samuel/go-zookeeper/zk"
	"golang.org/x/net/context"

	"github.com/mesosphere/etcd-mesos/rpc"
	etcdscheduler "github.com/mesosphere/etcd-mesos/scheduler"
)

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Fatal(err)
	}
	if len(addr) < 1 {
		log.Fatalf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}

func main() {
	clusterName :=
		flag.String("cluster-name", "default", "Unique name of this etcd cluster")
	master :=
		flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
	zkFrameworkPersist :=
		flag.String("zk-framework-persist", "", "Zookeeper URI of the form zk://host1:port1,host2:port2/chroot/path")
	taskCount :=
		flag.Int("cluster-size", 5, "Total task count to run")
	adminPort :=
		flag.Int("admin-port", 23400, "Binding port for admin interface")
	reseedTimeout :=
		flag.Int("reseed-timeout", 240, "Seconds of etcd livelock to wait for before attempting a cluster re-seed")
	autoReseed :=
		flag.Bool("auto-reseed", true, "Perform automatic cluster reseed when the "+
			"cluster has been livelocked for -reseed-timeout seconds")
	artifactPort :=
		flag.Int("artifact-port", 12300, "Binding port for artifact server")
	sandboxDisk :=
		flag.Float64("sandbox-disk-limit", 4096, "Max disk usage for the etcd mesos sandbox in MB")
	sandboxCpu :=
		flag.Float64("sandbox-cpu-limit", 4, "Max cpu usage for the etcd mesos sandbox in MB")
	sandboxMem :=
		flag.Float64("sandbox-mem-limit", 2048, "Max memory usage for the etcd mesos sandbox in MB")
	executorPath :=
		flag.String("executor-bin", "./bin/etcd-mesos-executor", "Path to executor binary")
	etcdPath :=
		flag.String("etcd-bin", "./bin/etcd", "Path to etcd binary")
	etcdctlPath :=
		flag.String("etcdctl-bin", "./bin/etcdctl", "Path to etcdctl binary")
	address :=
		flag.String("address", "", "Binding address for scheduler and artifact server")
	driverPort :=
		flag.Int("driver-port", 0, "Binding port for scheduler driver")
	mesosAuthPrincipal :=
		flag.String("mesos-authentication-principal", "", "Mesos authentication principal")
	mesosAuthSecretFile :=
		flag.String("mesos-authentication-secret-file", "", "Mesos authentication secret file")
	authProvider :=
		flag.String("mesos-authentication-provider", sasl.ProviderName,
			fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
	singleInstancePerSlave :=
		flag.Bool("single-instance-per-slave", true, "Only allow one etcd instance to be started per slave")
	testMode :=
		flag.Bool("test-mode", false, "disable forced values for -single-instance-per-slave and "+
			"-zk-framework-persist")
	failoverTimeoutSeconds :=
		flag.Float64("failover-timeout-seconds", 60*60*24*7, "Mesos framework failover timeout in seconds")
	weburi := flag.String("framework-weburi", "", "A URI that points to a web-based interface for interacting with the framework.")

	flag.Parse()

	if *zkFrameworkPersist == "" && !*testMode {
		log.Fatal("No value provided for -zk-framework-persist ! This can be " +
			"overridden by experts using the -test-mode=true argument, but several " +
			"runtime guarantees no longer hold, and all tasks will be orphaned when " +
			"this process exits.")
	}

	if !*singleInstancePerSlave && !*testMode {
		log.Fatal("-single-instance-per-slave=false is dangerous because it may lead to " +
			"multiple etcd instances in the same cluster on a single node, amplifying " +
			"the cost of a single node being lost, livelock, and data loss.  This can " +
			"be overridden by passing the -test-mode=true argument, at your peril.")
	}

	if *address == "" {
		hostname, err := os.Hostname()
		if err == nil {
			*address = hostname
		} else {
			log.Errorf("Could not set default binding to hostname.  Defaulting to 127.0.0.1")
			*address = "127.0.0.1"
		}
	}

	if *weburi == "" {
		*weburi = fmt.Sprintf("http://%s:%d/stats", *address, *adminPort)
	}

	executorUris := []*mesos.CommandInfo_URI{}
	execUri, err := etcdscheduler.ServeExecutorArtifact(*executorPath, *address, *artifactPort)
	if err != nil {
		log.Errorf("Could not stat executor binary: %v", err)
		return
	}
	executorUris = append(executorUris, &mesos.CommandInfo_URI{
		Value:      execUri,
		Executable: proto.Bool(true),
	})
	etcdUri, err := etcdscheduler.ServeExecutorArtifact(*etcdPath, *address, *artifactPort)
	if err != nil {
		log.Errorf("Could not stat etcd binary: %v", err)
		return
	}
	executorUris = append(executorUris, &mesos.CommandInfo_URI{
		Value:      etcdUri,
		Executable: proto.Bool(true),
	})
	etcdctlUri, err := etcdscheduler.ServeExecutorArtifact(*etcdctlPath, *address, *artifactPort)
	if err != nil {
		log.Errorf("Could not stat etcd binary: %v", err)
		return
	}
	executorUris = append(executorUris, &mesos.CommandInfo_URI{
		Value:      etcdctlUri,
		Executable: proto.Bool(true),
	})

	go http.ListenAndServe(fmt.Sprintf("%s:%d", *address, *artifactPort), nil)
	log.V(2).Info("Serving executor artifacts...")

	bindingAddress := parseIP(*address)

	// chillFactor is the number of seconds that are slept for to allow for
	// convergence across the cluster during mutations.
	chillFactor := 10
	etcdScheduler := etcdscheduler.NewEtcdScheduler(
		*taskCount,
		chillFactor,
		*reseedTimeout,
		*autoReseed,
		executorUris,
		*singleInstancePerSlave,
		*sandboxDisk,
		*sandboxCpu,
		*sandboxMem,
	)
	etcdScheduler.ExecutorPath = *executorPath
	etcdScheduler.Master = *master
	etcdScheduler.ClusterName = *clusterName
	etcdScheduler.ZkConnect = *zkFrameworkPersist

	fwinfo := &mesos.FrameworkInfo{
		User:            proto.String(""), // Mesos-go will fill in user.
		Name:            proto.String("etcd-" + etcdScheduler.ClusterName),
		Checkpoint:      proto.Bool(true),
		FailoverTimeout: proto.Float64(*failoverTimeoutSeconds),
		WebuiUrl:        proto.String(*weburi),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    secret,
		}
	}

	zkServers, zkChroot, err := rpc.ParseZKURI(*zkFrameworkPersist)
	etcdScheduler.ZkServers = zkServers
	etcdScheduler.ZkChroot = zkChroot
	if err != nil && *zkFrameworkPersist != "" {
		log.Fatalf("Error parsing zookeeper URI of %s: %s", *zkFrameworkPersist, err)
	} else if *zkFrameworkPersist != "" {
		previous, err := rpc.GetPreviousFrameworkID(
			zkServers,
			zkChroot,
			etcdScheduler.ClusterName,
		)
		if err != nil && err != zk.ErrNoNode {
			log.Fatalf("Could not retrieve previous framework ID: %s", err)
		} else if err == zk.ErrNoNode {
			log.Info("No previous persisted framework ID exists in zookeeper.")
		} else {
			log.Infof("Found stored framework ID in Zookeeper, "+
				"attempting to re-use: %s", previous)
			fwinfo.Id = &mesos.FrameworkID{
				Value: proto.String(previous),
			}
		}
	}

	config := scheduler.DriverConfig{
		Scheduler:      etcdScheduler,
		Framework:      fwinfo,
		Master:         etcdScheduler.Master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		BindingPort:    uint16(*driverPort),
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}

	driver, err := scheduler.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	go etcdScheduler.SerialLauncher(driver)
	go etcdScheduler.PeriodicLaunchRequestor()
	go etcdScheduler.AdminHTTP(*adminPort, driver)

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s",
			stat.String(),
			err.Error())
	}
}
<commit_msg>Use configurable framework name<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"

	"github.com/gogo/protobuf/proto"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/auth"
	"github.com/mesos/mesos-go/auth/sasl"
	"github.com/mesos/mesos-go/auth/sasl/mech"
	mesos "github.com/mesos/mesos-go/mesosproto"
	"github.com/mesos/mesos-go/scheduler"
	"github.com/samuel/go-zookeeper/zk"
	"golang.org/x/net/context"

	"github.com/mesosphere/etcd-mesos/rpc"
	etcdscheduler "github.com/mesosphere/etcd-mesos/scheduler"
)

func parseIP(address string) net.IP {
	addr, err := net.LookupIP(address)
	if err != nil {
		log.Fatal(err)
	}
	if len(addr) < 1 {
		log.Fatalf("failed to parse IP from address '%v'", address)
	}
	return addr[0]
}

func main() {
	clusterName :=
		flag.String("cluster-name", "default", "Unique name of this etcd cluster")
	frameworkName :=
		flag.String("framework-name", "", "Mesos framework name for this etcd cluster")
	master :=
		flag.String("master", "127.0.0.1:5050", "Master address <ip:port>")
	zkFrameworkPersist :=
		flag.String("zk-framework-persist", "", "Zookeeper URI of the form zk://host1:port1,host2:port2/chroot/path")
	taskCount :=
		flag.Int("cluster-size", 5, "Total task count to run")
	adminPort :=
		flag.Int("admin-port", 23400, "Binding port for admin interface")
	reseedTimeout :=
		flag.Int("reseed-timeout", 240, "Seconds of etcd livelock to wait for before attempting a cluster re-seed")
	autoReseed :=
		flag.Bool("auto-reseed", true, "Perform automatic cluster reseed when the "+
			"cluster has been livelocked for -reseed-timeout seconds")
	artifactPort :=
		flag.Int("artifact-port", 12300, "Binding port for artifact server")
	sandboxDisk :=
		flag.Float64("sandbox-disk-limit", 4096, "Max disk usage for the etcd mesos sandbox in MB")
	sandboxCpu :=
		flag.Float64("sandbox-cpu-limit", 4, "Max cpu usage for the etcd mesos sandbox in MB")
	sandboxMem :=
		flag.Float64("sandbox-mem-limit", 2048, "Max memory usage for the etcd mesos sandbox in MB")
	executorPath :=
		flag.String("executor-bin", "./bin/etcd-mesos-executor", "Path to executor binary")
	etcdPath :=
		flag.String("etcd-bin", "./bin/etcd", "Path to etcd binary")
	etcdctlPath :=
		flag.String("etcdctl-bin", "./bin/etcdctl", "Path to etcdctl binary")
	address :=
		flag.String("address", "", "Binding address for scheduler and artifact server")
	driverPort :=
		flag.Int("driver-port", 0, "Binding port for scheduler driver")
	mesosAuthPrincipal :=
		flag.String("mesos-authentication-principal", "", "Mesos authentication principal")
	mesosAuthSecretFile :=
		flag.String("mesos-authentication-secret-file", "", "Mesos authentication secret file")
	authProvider :=
		flag.String("mesos-authentication-provider", sasl.ProviderName,
			fmt.Sprintf("Authentication provider to use, default is SASL that supports mechanisms: %+v", mech.ListSupported()))
	singleInstancePerSlave :=
		flag.Bool("single-instance-per-slave", true, "Only allow one etcd instance to be started per slave")
	testMode :=
		flag.Bool("test-mode", false, "disable forced values for -single-instance-per-slave and "+
			"-zk-framework-persist")
	failoverTimeoutSeconds :=
		flag.Float64("failover-timeout-seconds", 60*60*24*7, "Mesos framework failover timeout in seconds")
	weburi := flag.String("framework-weburi", "", "A URI that points to a web-based interface for interacting with the framework.")

	flag.Parse()

	if *frameworkName == "" {
		*frameworkName = "etcd-" + *clusterName
	}

	if *zkFrameworkPersist == "" && !*testMode {
		log.Fatal("No value provided for -zk-framework-persist ! This can be " +
			"overridden by experts using the -test-mode=true argument, but several " +
			"runtime guarantees no longer hold, and all tasks will be orphaned when " +
			"this process exits.")
	}

	if !*singleInstancePerSlave && !*testMode {
		log.Fatal("-single-instance-per-slave=false is dangerous because it may lead to " +
			"multiple etcd instances in the same cluster on a single node, amplifying " +
			"the cost of a single node being lost, livelock, and data loss.  This can " +
			"be overridden by passing the -test-mode=true argument, at your peril.")
	}

	if *address == "" {
		hostname, err := os.Hostname()
		if err == nil {
			*address = hostname
		} else {
			log.Errorf("Could not set default binding to hostname.  Defaulting to 127.0.0.1")
			*address = "127.0.0.1"
		}
	}

	if *weburi == "" {
		*weburi = fmt.Sprintf("http://%s:%d/stats", *address, *adminPort)
	}

	executorUris := []*mesos.CommandInfo_URI{}
	execUri, err := etcdscheduler.ServeExecutorArtifact(*executorPath, *address, *artifactPort)
	if err != nil {
		log.Errorf("Could not stat executor binary: %v", err)
		return
	}
	executorUris = append(executorUris, &mesos.CommandInfo_URI{
		Value:      execUri,
		Executable: proto.Bool(true),
	})
	etcdUri, err := etcdscheduler.ServeExecutorArtifact(*etcdPath, *address, *artifactPort)
	if err != nil {
		log.Errorf("Could not stat etcd binary: %v", err)
		return
	}
	executorUris = append(executorUris, &mesos.CommandInfo_URI{
		Value:      etcdUri,
		Executable: proto.Bool(true),
	})
	etcdctlUri, err := etcdscheduler.ServeExecutorArtifact(*etcdctlPath, *address, *artifactPort)
	if err != nil {
		log.Errorf("Could not stat etcd binary: %v", err)
		return
	}
	executorUris = append(executorUris, &mesos.CommandInfo_URI{
		Value:      etcdctlUri,
		Executable: proto.Bool(true),
	})

	go http.ListenAndServe(fmt.Sprintf("%s:%d", *address, *artifactPort), nil)
	log.V(2).Info("Serving executor artifacts...")

	bindingAddress := parseIP(*address)

	// chillFactor is the number of seconds that are slept for to allow for
	// convergence across the cluster during mutations.
	chillFactor := 10
	etcdScheduler := etcdscheduler.NewEtcdScheduler(
		*taskCount,
		chillFactor,
		*reseedTimeout,
		*autoReseed,
		executorUris,
		*singleInstancePerSlave,
		*sandboxDisk,
		*sandboxCpu,
		*sandboxMem,
	)
	etcdScheduler.ExecutorPath = *executorPath
	etcdScheduler.Master = *master
	etcdScheduler.ClusterName = *clusterName
	etcdScheduler.ZkConnect = *zkFrameworkPersist

	fwinfo := &mesos.FrameworkInfo{
		User:            proto.String(""), // Mesos-go will fill in user.
		Name:            proto.String(*frameworkName),
		Checkpoint:      proto.Bool(true),
		FailoverTimeout: proto.Float64(*failoverTimeoutSeconds),
		WebuiUrl:        proto.String(*weburi),
	}

	cred := (*mesos.Credential)(nil)
	if *mesosAuthPrincipal != "" {
		fwinfo.Principal = proto.String(*mesosAuthPrincipal)
		secret, err := ioutil.ReadFile(*mesosAuthSecretFile)
		if err != nil {
			log.Fatal(err)
		}
		cred = &mesos.Credential{
			Principal: proto.String(*mesosAuthPrincipal),
			Secret:    secret,
		}
	}

	zkServers, zkChroot, err := rpc.ParseZKURI(*zkFrameworkPersist)
	etcdScheduler.ZkServers = zkServers
	etcdScheduler.ZkChroot = zkChroot
	if err != nil && *zkFrameworkPersist != "" {
		log.Fatalf("Error parsing zookeeper URI of %s: %s", *zkFrameworkPersist, err)
	} else if *zkFrameworkPersist != "" {
		previous, err := rpc.GetPreviousFrameworkID(
			zkServers,
			zkChroot,
			etcdScheduler.ClusterName,
		)
		if err != nil && err != zk.ErrNoNode {
			log.Fatalf("Could not retrieve previous framework ID: %s", err)
		} else if err == zk.ErrNoNode {
			log.Info("No previous persisted framework ID exists in zookeeper.")
		} else {
			log.Infof("Found stored framework ID in Zookeeper, "+
				"attempting to re-use: %s", previous)
			fwinfo.Id = &mesos.FrameworkID{
				Value: proto.String(previous),
			}
		}
	}

	config := scheduler.DriverConfig{
		Scheduler:      etcdScheduler,
		Framework:      fwinfo,
		Master:         etcdScheduler.Master,
		Credential:     cred,
		BindingAddress: bindingAddress,
		BindingPort:    uint16(*driverPort),
		WithAuthContext: func(ctx context.Context) context.Context {
			ctx = auth.WithLoginProvider(ctx, *authProvider)
			ctx = sasl.WithBindingAddress(ctx, bindingAddress)
			return ctx
		},
	}

	driver, err := scheduler.NewMesosSchedulerDriver(config)

	if err != nil {
		log.Errorln("Unable to create a SchedulerDriver ", err.Error())
	}

	go etcdScheduler.SerialLauncher(driver)
	go etcdScheduler.PeriodicLaunchRequestor()
	go etcdScheduler.AdminHTTP(*adminPort, driver)

	if stat, err := driver.Run(); err != nil {
		log.Infof("Framework stopped with status %s and error: %s",
			stat.String(),
			err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/spf13/cobra"
	corev1 "k8s.io/api/core/v1"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/healthz"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/kubernetes-incubator/metrics-server/pkg/apiserver"
	"github.com/kubernetes-incubator/metrics-server/pkg/manager"
	"github.com/kubernetes-incubator/metrics-server/pkg/provider"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources/summary"
)

// NewCommandStartMetricsServer provides a CLI handler for the metrics server entrypoint
func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewMetricsServerOptions()

	cmd := &cobra.Command{
		Short: "Launch metrics-server",
		Long:  "Launch metrics-server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Run(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	flags := cmd.Flags()
	flags.DurationVar(&o.MetricResolution, "metric-resolution", o.MetricResolution, "The resolution at which metrics-server will retain metrics.")

	flags.BoolVar(&o.InsecureKubeletTLS, "kubelet-insecure-tls", o.InsecureKubeletTLS, "Do not verify CA of serving certificates presented by Kubelets")
	flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets (defaults to 10250)")
	flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)")
	flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node")

	o.SecureServing.AddFlags(flags)
	o.Authentication.AddFlags(flags)
	o.Authorization.AddFlags(flags)
	o.Features.AddFlags(flags)

	return cmd
}

type MetricsServerOptions struct {
	// genericoptions.ReccomendedOptions - EtcdOptions
	SecureServing  *genericoptions.SecureServingOptionsWithLoopback
	Authentication *genericoptions.DelegatingAuthenticationOptions
	Authorization  *genericoptions.DelegatingAuthorizationOptions
	Features       *genericoptions.FeatureOptions

	Kubeconfig string

	// Only to be used to for testing
	DisableAuthForTesting bool

	MetricResolution time.Duration

	KubeletPort                  int
	InsecureKubeletTLS           bool
	KubeletPreferredAddressTypes []string
}

// NewMetricsServerOptions constructs a new set of default options for metrics-server.
func NewMetricsServerOptions() *MetricsServerOptions {
	o := &MetricsServerOptions{
		SecureServing:  genericoptions.WithLoopback(genericoptions.NewSecureServingOptions()),
		Authentication: genericoptions.NewDelegatingAuthenticationOptions(),
		Authorization:  genericoptions.NewDelegatingAuthorizationOptions(),
		Features:       genericoptions.NewFeatureOptions(),

		MetricResolution:             60 * time.Second,
		KubeletPort:                  10250,
		KubeletPreferredAddressTypes: make([]string, len(summary.DefaultAddressTypePriority)),
	}

	for i, addrType := range summary.DefaultAddressTypePriority {
		o.KubeletPreferredAddressTypes[i] = string(addrType)
	}

	return o
}

func (o MetricsServerOptions) Config() (*apiserver.Config, error) {
	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig(apiserver.Codecs)
	if err := o.SecureServing.ApplyTo(serverConfig); err != nil {
		return nil, err
	}

	if !o.DisableAuthForTesting {
		if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
			return nil, err
		}
		if err := o.Authorization.ApplyTo(&serverConfig.Authorization); err != nil {
			return nil, err
		}
	}

	return &apiserver.Config{
		GenericConfig:  serverConfig,
		ProviderConfig: apiserver.ProviderConfig{},
	}, nil
}

func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error {
	// grab the config for the API server
	config, err := o.Config()
	if err != nil {
		return err
	}
	config.GenericConfig.EnableMetrics = true

	// set up the client config
	var clientConfig *rest.Config
	if len(o.Kubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.Kubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		clientConfig, err = loader.ClientConfig()
	} else {
		clientConfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return fmt.Errorf("unable to construct lister client config: %v", err)
	}

	// set up the informers
	kubeClient, err := kubernetes.NewForConfig(clientConfig)
	if err != nil {
		return fmt.Errorf("unable to construct lister client: %v", err)
	}
	// we should never need to resync, since we're not worried about missing events,
	// and resync is actually for regular interval-based reconciliation these days,
	// so set the default resync interval to 0
	informerFactory := informers.NewSharedInformerFactory(kubeClient, 0)

	// set up the source manager
	kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeletPort, o.InsecureKubeletTLS)
	kubeletClient, err := summary.KubeletClientFor(kubeletConfig)
	if err != nil {
		return fmt.Errorf("unable to construct a client to connect to the kubelets: %v", err)
	}

	// set up an address resolver according to the user's priorities
	addrPriority := make([]corev1.NodeAddressType, len(o.KubeletPreferredAddressTypes))
	for i, addrType := range o.KubeletPreferredAddressTypes {
		addrPriority[i] = corev1.NodeAddressType(addrType)
	}
	addrResolver := summary.NewPriorityNodeAddressResolver(addrPriority)

	sourceProvider := summary.NewSummaryProvider(informerFactory.Core().V1().Nodes().Lister(), kubeletClient, addrResolver)
	scrapeTimeout := time.Duration(float64(o.MetricResolution) * 0.90) // scrape timeout is 90% of the scrape interval
	sources.RegisterDurationMetrics(scrapeTimeout)
	sourceManager := sources.NewSourceManager(sourceProvider, scrapeTimeout)

	// set up the in-memory sink and provider
	metricSink, metricsProvider := provider.NewSinkProvider()

	// set up the general manager
	manager.RegisterDurationMetrics(o.MetricResolution)
	mgr := manager.NewManager(sourceManager, metricSink, o.MetricResolution)
	if err != nil {
		return fmt.Errorf("unable to create main manager: %v", err)
	}

	// inject the providers into the config
	config.ProviderConfig.Node = metricsProvider
	config.ProviderConfig.Pod = metricsProvider

	// complete the config to get an API server
	server, err := config.Complete(informerFactory).New()
	if err != nil {
		return err
	}

	// add health checks
	server.AddHealthzChecks(healthz.NamedCheck("healthz", mgr.CheckHealth))

	// run everything (the apiserver runs the shared informer factory for us)
	mgr.RunUntil(stopCh)
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<commit_msg>Fix typo in main command<commit_after>// Copyright 2018 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
	"fmt"
	"io"
	"net"
	"time"

	"github.com/spf13/cobra"
	corev1 "k8s.io/api/core/v1"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/healthz"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/kubernetes-incubator/metrics-server/pkg/apiserver"
	genericmetrics "github.com/kubernetes-incubator/metrics-server/pkg/apiserver/generic"
	"github.com/kubernetes-incubator/metrics-server/pkg/manager"
	"github.com/kubernetes-incubator/metrics-server/pkg/provider"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources"
	"github.com/kubernetes-incubator/metrics-server/pkg/sources/summary"
)

// NewCommandStartMetricsServer provides a CLI handler for the metrics server entrypoint
func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) *cobra.Command {
	o := NewMetricsServerOptions()

	cmd := &cobra.Command{
		Short: "Launch metrics-server",
		Long:  "Launch metrics-server",
		RunE: func(c *cobra.Command, args []string) error {
			if err := o.Run(stopCh); err != nil {
				return err
			}
			return nil
		},
	}

	flags := cmd.Flags()
	flags.DurationVar(&o.MetricResolution, "metric-resolution", o.MetricResolution, "The resolution at which metrics-server will retain metrics.")

	flags.BoolVar(&o.InsecureKubeletTLS, "kubelet-insecure-tls", o.InsecureKubeletTLS, "Do not verify CA of serving certificates presented by Kubelets")
	flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets (defaults to 10250)")
	flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)")
	flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node")

	o.SecureServing.AddFlags(flags)
	o.Authentication.AddFlags(flags)
	o.Authorization.AddFlags(flags)
	o.Features.AddFlags(flags)

	return cmd
}

type MetricsServerOptions struct {
	// genericoptions.ReccomendedOptions - EtcdOptions
	SecureServing  *genericoptions.SecureServingOptionsWithLoopback
	Authentication *genericoptions.DelegatingAuthenticationOptions
	Authorization  *genericoptions.DelegatingAuthorizationOptions
	Features       *genericoptions.FeatureOptions

	Kubeconfig string

	// Only to be used to for testing
	DisableAuthForTesting bool

	MetricResolution time.Duration

	KubeletPort                  int
	InsecureKubeletTLS           bool
	KubeletPreferredAddressTypes []string
}

// NewMetricsServerOptions constructs a new set of default options for metrics-server.
func NewMetricsServerOptions() *MetricsServerOptions {
	o := &MetricsServerOptions{
		SecureServing:  genericoptions.WithLoopback(genericoptions.NewSecureServingOptions()),
		Authentication: genericoptions.NewDelegatingAuthenticationOptions(),
		Authorization:  genericoptions.NewDelegatingAuthorizationOptions(),
		Features:       genericoptions.NewFeatureOptions(),

		MetricResolution:             60 * time.Second,
		KubeletPort:                  10250,
		KubeletPreferredAddressTypes: make([]string, len(summary.DefaultAddressTypePriority)),
	}

	for i, addrType := range summary.DefaultAddressTypePriority {
		o.KubeletPreferredAddressTypes[i] = string(addrType)
	}

	return o
}

func (o MetricsServerOptions) Config() (*apiserver.Config, error) {
	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1")}); err != nil {
		return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
	}

	serverConfig := genericapiserver.NewConfig(genericmetrics.Codecs)
	if err := o.SecureServing.ApplyTo(serverConfig); err != nil {
		return nil, err
	}

	if !o.DisableAuthForTesting {
		if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
			return nil, err
		}
		if err := o.Authorization.ApplyTo(&serverConfig.Authorization); err != nil {
			return nil, err
		}
	}

	return &apiserver.Config{
		GenericConfig:  serverConfig,
		ProviderConfig: genericmetrics.ProviderConfig{},
	}, nil
}

func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error {
	// grab the config for the API server
	config, err := o.Config()
	if err != nil {
		return err
	}
	config.GenericConfig.EnableMetrics = true

	// set up the client config
	var clientConfig *rest.Config
	if len(o.Kubeconfig) > 0 {
		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.Kubeconfig}
		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

		clientConfig, err = loader.ClientConfig()
	} else {
		clientConfig, err = rest.InClusterConfig()
	}
	if err != nil {
		return fmt.Errorf("unable to construct lister client config: %v", err)
	}

	// set up the informers
	kubeClient, err := kubernetes.NewForConfig(clientConfig)
	if err != nil {
		return fmt.Errorf("unable to construct lister client: %v", err)
	}
	// we should never need to resync, since we're not worried about missing events,
	// and resync is actually for regular interval-based reconciliation these days,
	// so set the default resync interval to 0
	informerFactory := informers.NewSharedInformerFactory(kubeClient, 0)

	// set up the source manager
	kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeletPort, o.InsecureKubeletTLS)
	kubeletClient, err := summary.KubeletClientFor(kubeletConfig)
	if err != nil {
		return fmt.Errorf("unable to construct a client to connect to the kubelets: %v", err)
	}

	// set up an address resolver according to the user's priorities
	addrPriority := make([]corev1.NodeAddressType, len(o.KubeletPreferredAddressTypes))
	for i, addrType := range o.KubeletPreferredAddressTypes {
		addrPriority[i] = corev1.NodeAddressType(addrType)
	}
	addrResolver := summary.NewPriorityNodeAddressResolver(addrPriority)

	sourceProvider := summary.NewSummaryProvider(informerFactory.Core().V1().Nodes().Lister(), kubeletClient, addrResolver)
	scrapeTimeout := time.Duration(float64(o.MetricResolution) * 0.90) // scrape timeout is 90% of the scrape interval
	sources.RegisterDurationMetrics(scrapeTimeout)
	sourceManager := sources.NewSourceManager(sourceProvider, scrapeTimeout)

	// set up the in-memory sink and provider
	metricSink, metricsProvider := provider.NewSinkProvider()

	// set up the general manager
	manager.RegisterDurationMetrics(o.MetricResolution)
	mgr := manager.NewManager(sourceManager, metricSink, o.MetricResolution)
	if err != nil {
		return fmt.Errorf("unable to create main manager: %v", err)
	}

	// inject the providers into the config
	config.ProviderConfig.Node = metricsProvider
	config.ProviderConfig.Pod = metricsProvider

	// complete the config to get an API server
	server, err := config.Complete(informerFactory).New()
	if err != nil {
		return err
	}

	// add health checks
	server.AddHealthzChecks(healthz.NamedCheck("healthz", mgr.CheckHealth))

	// run everything (the apiserver runs the shared informer factory for us)
	mgr.RunUntil(stopCh)
	return server.GenericAPIServer.PrepareRun().Run(stopCh)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

/*

Package query contains artifacts shared across other packages in query
processing.

*/
package query

// Error will eventually include code, message key, and internal error
// object (cause) and message
type Error interface {
	error
	Code() int32
	TranslationKey() string
	Cause() error
}

type ErrorChannel chan Error

func NewError(e error, internalMsg string) Error {
     return &err{cause: e, internalMsg: internalMsg}
}

type err struct {
	code        int32
	key         string
	cause       error
	internalMsg string
}

func (e *err) Error() string {
	switch {
	default:
		return "Unspecified error."
	case e.internalMsg != "":
		return e.internalMsg
	case e.cause != nil:
		return e.cause.Error()
	}
}

func (e *err) Code() int32 {
	return e.code
}

func (e *err) TranslationKey() string {
	return e.key
}

func (e *err) Cause() error {
	return e.cause
}
<commit_msg>go fmt<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

/*

Package query contains artifacts shared across other packages in query
processing.

*/
package query

// Error will eventually include code, message key, and internal error
// object (cause) and message
type Error interface {
	error
	Code() int32
	TranslationKey() string
	Cause() error
}

type ErrorChannel chan Error

func NewError(e error, internalMsg string) Error {
	return &err{cause: e, internalMsg: internalMsg}
}

type err struct {
	code        int32
	key         string
	cause       error
	internalMsg string
}

func (e *err) Error() string {
	switch {
	default:
		return "Unspecified error."
	case e.internalMsg != "":
		return e.internalMsg
	case e.cause != nil:
		return e.cause.Error()
	}
}

func (e *err) Code() int32 {
	return e.code
}

func (e *err) TranslationKey() string {
	return e.key
}

func (e *err) Cause() error {
	return e.cause
}
<|endoftext|>


<commit_before>package wdte_test

import (
	"strings"
	"testing"

	"github.com/DeedleFake/wdte"
	"github.com/DeedleFake/wdte/std"
)

func TestModule(t *testing.T) {
	const test = `
fib n => switch n {
	0 => 0;
	1 => 1;
	default => + (fib (- n 1)) (fib (- n 2));
};

main => (
	range 15
    -> map fib
	  -> print
	;
	[5; 2; (fib 7)]
	  -> map (+ 2)
		-> print
	;
);
`

	m, err := wdte.Parse(strings.NewReader(test), nil)
	if err != nil {
		t.Fatal(err)
	}

	std.Insert(m)

	m.Funcs["range"] = wdte.GoFunc(func(frame wdte.Frame, args ...wdte.Func) wdte.Func {
		frame = frame.WithID("range")

		a := args[0].Call(frame).(wdte.Number)

		r := make(wdte.Array, int(a))
		for i := range r {
			r[i] = wdte.Number(i)
		}

		return r
	})

	m.Funcs["map"] = wdte.GoFunc(func(frame wdte.Frame, args ...wdte.Func) wdte.Func {
		frame = frame.WithID("map")

		m := args[0].Call(frame)
		return wdte.GoFunc(func(frame wdte.Frame, args ...wdte.Func) wdte.Func {
			a := args[0].Call(frame).(wdte.Array)

			r := make(wdte.Array, len(a))
			for i := range r {
				r[i] = m.Call(frame, a[i].Call(frame))
				if _, ok := r[i].(error); ok {
					return r[i]
				}
			}
			return r
		})
	})

	m.Funcs["print"] = wdte.GoFunc(func(frame wdte.Frame, args ...wdte.Func) wdte.Func {
		frame = frame.WithID("print")

		if len(args) < 1 {
			return m.Funcs["print"]
		}

		a := args[0].Call(frame)
		t.Logf("%v", a)
		return a
	})

	//t.Log("Imports:")
	//for i := range m.Imports {
	//	t.Logf("\t%q", i)
	//}

	//t.Log("Funcs:")
	//for f := range m.Funcs {
	//	t.Logf("\t%q", f)
	//}

	if err, ok := m.Funcs["main"].Call(wdte.F()).(error); ok {
		t.Fatal(err)
	}
}
<commit_msg>wdte: Use standard library in tests.<commit_after>package wdte_test

import (
	"fmt"
	"strings"
	"testing"

	"github.com/DeedleFake/wdte"
	"github.com/DeedleFake/wdte/std"
	"github.com/DeedleFake/wdte/std/math"
	"github.com/DeedleFake/wdte/std/stream"
)

func TestModule(t *testing.T) {
	const test = `
'stream' => s;

fib n => switch n {
	0 => 0;
	1 => 1;
	default => + (fib (- n 1)) (fib (- n 2));
};

main => (
	s.range 15
    -> s.map fib
		-> s.collect
	  -> print
	;
	s.new [5; 2; (fib 7)]
	  -> s.map (+ 2)
		-> s.collect
		-> print
	;
);
`

	m, err := wdte.Parse(strings.NewReader(test), wdte.ImportFunc(func(from string) (*wdte.Module, error) {
		switch from {
		case "stream":
			return stream.Module(), nil
		case "math":
			return math.Module(), nil
		}

		return nil, fmt.Errorf("Unknown import: %q", from)
	}))
	if err != nil {
		t.Fatal(err)
	}
	std.Insert(m)

	m.Funcs["print"] = wdte.GoFunc(func(frame wdte.Frame, args ...wdte.Func) wdte.Func {
		frame = frame.WithID("print")

		if len(args) < 1 {
			return m.Funcs["print"]
		}

		a := args[0].Call(frame)
		t.Logf("%v", a)
		return a
	})

	//t.Log("Imports:")
	//for i := range m.Imports {
	//	t.Logf("\t%q", i)
	//}

	//t.Log("Funcs:")
	//for f := range m.Funcs {
	//	t.Logf("\t%q", f)
	//}

	if err, ok := m.Funcs["main"].Call(wdte.F()).(error); ok {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package discovery

import (
	"testing"

	"github.com/stretchr/testify/assert"

	vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin"
)

func TestNew(t *testing.T) {
	tests := []struct {
		name string
		impl string
		err  error
		typ  Discovery
	}{
		{
			name: "success",
			impl: "consul",
			err:  nil,
			typ:  &ConsulDiscovery{},
		},
		{
			name: "unregistered",
			impl: "unregistered",
			err:  ErrImplementationNotRegistered,
			typ:  nil,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			disco, err := New(tt.impl, &vtadminpb.Cluster{Id: "testid", Name: "testcluster"}, []string{})
			if tt.err != nil {
				assert.Error(t, err, tt.err.Error())
				return
			}

			assert.NoError(t, err)
			assert.IsType(t, tt.typ, disco)
		})
	}
}

func TestRegister(t *testing.T) {
	Register("testfactory", nil)

	defer func() {
		err := recover()
		assert.NotNil(t, err)
		assert.IsType(t, "", err)
		assert.Contains(t, err.(string), "factory already registered")
	}()

	// this one panics
	Register("testfactory", nil)
	assert.Equal(t, 1, 2, "double register should have panicked")
}
<commit_msg>Use timestamp in testfactory name so tests can be run repeatedly<commit_after>/*
Copyright 2020 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package discovery

import (
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	vtadminpb "vitess.io/vitess/go/vt/proto/vtadmin"
)

func TestNew(t *testing.T) {
	tests := []struct {
		name string
		impl string
		err  error
		typ  Discovery
	}{
		{
			name: "success",
			impl: "consul",
			err:  nil,
			typ:  &ConsulDiscovery{},
		},
		{
			name: "unregistered",
			impl: "unregistered",
			err:  ErrImplementationNotRegistered,
			typ:  nil,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			disco, err := New(tt.impl, &vtadminpb.Cluster{Id: "testid", Name: "testcluster"}, []string{})
			if tt.err != nil {
				assert.Error(t, err, tt.err.Error())
				return
			}

			assert.NoError(t, err)
			assert.IsType(t, tt.typ, disco)
		})
	}
}

func TestRegister(t *testing.T) {
	// Use a timestamp to allow running tests with `-count=N`.
	ts := time.Now().Unix()
	factoryName := fmt.Sprintf("testfactory-%d", ts)

	Register(factoryName, nil)

	defer func() {
		err := recover()
		assert.NotNil(t, err)
		assert.IsType(t, "", err)
		assert.Contains(t, err.(string), "factory already registered")
	}()

	// this one panics
	Register(factoryName, nil)
	assert.Equal(t, 1, 2, "double register should have panicked")
}
<|endoftext|>


<commit_before><commit_msg>cmd/mdlayher.com: bump GitHub and Medium cache timeouts<commit_after><|endoftext|>


<commit_before>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"

	"github.com/spf13/cobra"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/scheme"
	"sigs.k8s.io/yaml"

	"github.com/jetstack/cert-manager/cmd/ctl/pkg/factory"
	"github.com/jetstack/cert-manager/pkg/util"
	"github.com/jetstack/cert-manager/pkg/util/versionchecker"
)

// Version is a struct for version information
type Version struct {
	ClientVersion *util.Version           `json:"clientVersion,omitempty"`
	ServerVersion *versionchecker.Version `json:"serverVersion,omitempty"`
}

// Options is a struct to support version command
type Options struct {
	// If true, don't try to retrieve the installed version
	ClientOnly bool

	// If true, only prints the version number.
	Short bool

	// Output is the target output format for the version string. This may be of
	// value "", "json" or "yaml".
	Output string

	VersionChecker versionchecker.Interface

	genericclioptions.IOStreams
	*factory.Factory
}

// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
	return &Options{
		IOStreams: ioStreams,
	}
}

const versionLong = `Print the cert-manager kubectl plugin version and the deployed cert-manager version.

The kubectl plugin version is embedded in the binary and directly displayed. Determining
the the deployed cert-manager version is done by querying the cert-manger resources.
First, the tool looks at the labels of the cert-manager CRD resources. Then, it searches
for the labels of the resources related the the cert-manager webhook linked in the CRDs.
It also tries to derive the version from the docker image tag of that webhook service.
After gathering all this version information, the tool checks if all versions are the same
and returns that version. If no version information is found or the found versions differ,
an error will be displayed.

The '--client' flag can be used to disable the logic that tries to determine the installed
cert-manager version.

Some example uses:
	$ kubectl cert-manager version
or
	$ kubectl cert-manager version --client
or
	$ kubectl cert-manager version --short
or
	$ kubectl cert-manager version -o yaml
`

// NewCmdVersion returns a cobra command for fetching versions
func NewCmdVersion(ctx context.Context, ioStreams genericclioptions.IOStreams) *cobra.Command {
	o := NewOptions(ioStreams)

	cmd := &cobra.Command{
		Use:   "version",
		Short: "Print the cert-manager kubectl plugin version and the deployed cert-manager version",
		Long:  versionLong,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Validate())
			cmdutil.CheckErr(o.Run(ctx))
		},
	}

	cmd.Flags().BoolVar(&o.ClientOnly, "client", o.ClientOnly, "If true, shows client version only (no server required).")
	cmd.Flags().BoolVar(&o.Short, "short", o.Short, "If true, print just the version number.")
	cmd.Flags().StringVarP(&o.Output, "output", "o", o.Output, "One of 'yaml' or 'json'.")

	o.Factory = factory.New(cmd)

	return cmd
}

// Validate validates the provided options
func (o *Options) Validate() error {
	switch o.Output {
	case "", "yaml", "json":
		return nil
	default:
		return errors.New(`--output must be '', 'yaml' or 'json'`)
	}
}

// Complete takes the command arguments and factory and infers any remaining options.
func (o *Options) Complete(factory cmdutil.Factory) error {
	if o.ClientOnly {
		return nil
	}

	restConfig, err := factory.ToRESTConfig()
	if err != nil {
		return fmt.Errorf("cannot create the REST config: %v", err)
	}

	o.VersionChecker, err = versionchecker.New(restConfig, scheme.Scheme)
	if err != nil {
		return err
	}
	return nil
}

// Run executes version command
func (o *Options) Run(ctx context.Context) error {
	var (
		serverVersion *versionchecker.Version
		serverErr     error
		versionInfo   Version
	)

	clientVersion := util.VersionInfo()
	versionInfo.ClientVersion = &clientVersion

	if !o.ClientOnly {
		serverVersion, serverErr = o.VersionChecker.Version(ctx)
		versionInfo.ServerVersion = serverVersion
	}

	switch o.Output {
	case "":
		if o.Short {
			fmt.Fprintf(o.Out, "Client Version: %s\n", clientVersion.GitVersion)
			if serverVersion != nil {
				fmt.Fprintf(o.Out, "Server Version: %s\n", serverVersion.Detected)
			}
		} else {
			fmt.Fprintf(o.Out, "Client Version: %s\n", fmt.Sprintf("%#v", clientVersion))
			if serverVersion != nil {
				fmt.Fprintf(o.Out, "Server Version: %s\n", fmt.Sprintf("%#v", serverVersion))
			}
		}
	case "yaml":
		marshalled, err := yaml.Marshal(&versionInfo)
		if err != nil {
			return err
		}
		fmt.Fprint(o.Out, string(marshalled))
	case "json":
		marshalled, err := json.MarshalIndent(&versionInfo, "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(o.Out, string(marshalled))
	default:
		// There is a bug in the program if we hit this case.
		// However, we follow a policy of never panicking.
		return fmt.Errorf("VersionOptions were not validated: --output=%q should have been rejected", o.Output)
	}

	return serverErr
}
<commit_msg>Call Options.Complete to set up the version checker command<commit_after>/*
Copyright 2020 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"

	"github.com/spf13/cobra"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/scheme"
	"sigs.k8s.io/yaml"

	"github.com/jetstack/cert-manager/cmd/ctl/pkg/factory"
	"github.com/jetstack/cert-manager/pkg/util"
	"github.com/jetstack/cert-manager/pkg/util/versionchecker"
)

// Version is a struct for version information
type Version struct {
	ClientVersion *util.Version           `json:"clientVersion,omitempty"`
	ServerVersion *versionchecker.Version `json:"serverVersion,omitempty"`
}

// Options is a struct to support version command
type Options struct {
	// If true, don't try to retrieve the installed version
	ClientOnly bool

	// If true, only prints the version number.
	Short bool

	// Output is the target output format for the version string. This may be of
	// value "", "json" or "yaml".
	Output string

	VersionChecker versionchecker.Interface

	genericclioptions.IOStreams
	*factory.Factory
}

// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
	return &Options{
		IOStreams: ioStreams,
	}
}

const versionLong = `Print the cert-manager kubectl plugin version and the deployed cert-manager version.

The kubectl plugin version is embedded in the binary and directly displayed. Determining
the the deployed cert-manager version is done by querying the cert-manger resources.
First, the tool looks at the labels of the cert-manager CRD resources. Then, it searches
for the labels of the resources related the the cert-manager webhook linked in the CRDs.
It also tries to derive the version from the docker image tag of that webhook service.
After gathering all this version information, the tool checks if all versions are the same
and returns that version. If no version information is found or the found versions differ,
an error will be displayed.

The '--client' flag can be used to disable the logic that tries to determine the installed
cert-manager version.

Some example uses:
	$ kubectl cert-manager version
or
	$ kubectl cert-manager version --client
or
	$ kubectl cert-manager version --short
or
	$ kubectl cert-manager version -o yaml
`

// NewCmdVersion returns a cobra command for fetching versions
func NewCmdVersion(ctx context.Context, ioStreams genericclioptions.IOStreams) *cobra.Command {
	o := NewOptions(ioStreams)

	cmd := &cobra.Command{
		Use:   "version",
		Short: "Print the cert-manager kubectl plugin version and the deployed cert-manager version",
		Long:  versionLong,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Validate())
			cmdutil.CheckErr(o.Complete())
			cmdutil.CheckErr(o.Run(ctx))
		},
	}

	cmd.Flags().BoolVar(&o.ClientOnly, "client", o.ClientOnly, "If true, shows client version only (no server required).")
	cmd.Flags().BoolVar(&o.Short, "short", o.Short, "If true, print just the version number.")
	cmd.Flags().StringVarP(&o.Output, "output", "o", o.Output, "One of 'yaml' or 'json'.")

	o.Factory = factory.New(cmd)

	return cmd
}

// Validate validates the provided options
func (o *Options) Validate() error {
	switch o.Output {
	case "", "yaml", "json":
		return nil
	default:
		return errors.New(`--output must be '', 'yaml' or 'json'`)
	}
}

// Complete takes the command arguments and factory and infers any remaining options.
func (o *Options) Complete() error {
	if o.ClientOnly {
		return nil
	}

	versionChecker, err := versionchecker.New(o.RESTConfig, scheme.Scheme)
	if err != nil {
		return err
	}
	o.VersionChecker = versionChecker
	return nil
}

// Run executes version command
func (o *Options) Run(ctx context.Context) error {
	var (
		serverVersion *versionchecker.Version
		serverErr     error
		versionInfo   Version
	)

	clientVersion := util.VersionInfo()
	versionInfo.ClientVersion = &clientVersion

	if !o.ClientOnly {
		serverVersion, serverErr = o.VersionChecker.Version(ctx)
		versionInfo.ServerVersion = serverVersion
	}

	switch o.Output {
	case "":
		if o.Short {
			fmt.Fprintf(o.Out, "Client Version: %s\n", clientVersion.GitVersion)
			if serverVersion != nil {
				fmt.Fprintf(o.Out, "Server Version: %s\n", serverVersion.Detected)
			}
		} else {
			fmt.Fprintf(o.Out, "Client Version: %s\n", fmt.Sprintf("%#v", clientVersion))
			if serverVersion != nil {
				fmt.Fprintf(o.Out, "Server Version: %s\n", fmt.Sprintf("%#v", serverVersion))
			}
		}
	case "yaml":
		marshalled, err := yaml.Marshal(&versionInfo)
		if err != nil {
			return err
		}
		fmt.Fprint(o.Out, string(marshalled))
	case "json":
		marshalled, err := json.MarshalIndent(&versionInfo, "", "  ")
		if err != nil {
			return err
		}
		fmt.Fprintln(o.Out, string(marshalled))
	default:
		// There is a bug in the program if we hit this case.
		// However, we follow a policy of never panicking.
		return fmt.Errorf("VersionOptions were not validated: --output=%q should have been rejected", o.Output)
	}

	return serverErr
}
<|endoftext|>


<commit_before>/*
Freestore_client is a sample implementation of a freestore client.
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"os"
	"strings"
	"time"

	"github.com/mateusbraga/freestore/pkg/client"
	"github.com/mateusbraga/freestore/pkg/view"
)

var (
	nTotal = flag.Uint64("n", math.MaxUint64, "number of times to perform a read and write operation")
)

func main() {
	flag.Parse()

	initialView := getInitialView()
	freestoreClient := client.New(initialView)

	var finalValue interface{}
	var err error
	for i := uint64(0); i < *nTotal; i++ {
		startRead := time.Now()
		finalValue, err = freestoreClient.Read()
		endRead := time.Now()
		if err != nil {
			log.Fatalln(err)
		}

		startWrite := time.Now()
		err = freestoreClient.Write(finalValue)
		endWrite := time.Now()
		if err != nil {
			log.Fatalln(err)
		}

		fmt.Printf("%v: Read %v (%v)-> Write (%v)\n", i, finalValue, endRead.Sub(startRead), endWrite.Sub(startWrite))
	}
}

func getInitialView() *view.View {
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalln(err)
	}

	var process view.Process
	switch {
	case strings.Contains(hostname, "node-"): // emulab.net
		process = view.Process{"10.1.1.2:5000"}
	default:
		process = view.Process{"[::]:5000"}
	}

	initialView, err := client.GetCurrentView(process)
	if err != nil {
		log.Fatalln("Failed to get current view of process %v: %v\n", process, err)
	}
	return initialView
}
<commit_msg>Reduce output of freestore_client<commit_after>/*
Freestore_client is a sample implementation of a freestore client.
*/
package main

import (
	"flag"
	"fmt"
	"log"
	"math"
	"os"
	"strings"
	"time"

	"github.com/mateusbraga/freestore/pkg/client"
	"github.com/mateusbraga/freestore/pkg/view"
)

var (
	nTotal = flag.Uint64("n", math.MaxUint64, "number of times to perform a read and write operation")
)

func main() {
	flag.Parse()

	initialView := getInitialView()
	freestoreClient := client.New(initialView)

	var finalValue interface{}
	var err error
	for i := uint64(0); i < *nTotal; i++ {
		startRead := time.Now()
		finalValue, err = freestoreClient.Read()
		endRead := time.Now()
		if err != nil {
			log.Fatalln(err)
		}

		startWrite := time.Now()
		err = freestoreClient.Write(finalValue)
		endWrite := time.Now()
		if err != nil {
			log.Fatalln(err)
		}

		if i%100000 == 0 {
			fmt.Printf("%v: Read %v (%v)-> Write (%v)\n", i, finalValue, endRead.Sub(startRead), endWrite.Sub(startWrite))
		} else {
			fmt.Printf(".")
		}
	}
}

func getInitialView() *view.View {
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalln(err)
	}

	var process view.Process
	switch {
	case strings.Contains(hostname, "node-"): // emulab.net
		process = view.Process{"10.1.1.2:5000"}
	default:
		process = view.Process{"[::]:5000"}
	}

	initialView, err := client.GetCurrentView(process)
	if err != nil {
		log.Fatalln("Failed to get current view of process %v: %v\n", process, err)
	}
	return initialView
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/Azure/azure-sdk-for-go/storage"
)

// AnonGetBucketInfo - Get bucket metadata from azure anonymously.
func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
	if err != nil {
		return bucketInfo, azureToObjectError(traceError(err))
	}
	url.RawQuery = "restype=container"
	resp, err := http.Head(url.String())
	if err != nil {
		return bucketInfo, azureToObjectError(traceError(err), bucket)
	}
	resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
	}

	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
	if err != nil {
		return bucketInfo, traceError(err)
	}
	bucketInfo = BucketInfo{
		Name:    bucket,
		Created: t,
	}
	return bucketInfo, nil
}

// AnonPutObject - SendPUT request without authentication.
// This is needed when clients send PUT requests on objects that can be uploaded without auth.
func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
	// azure doesn't support anonymous put
	return ObjectInfo{}, traceError(NotImplemented{})
}

// AnonGetObject - SendGET request without authentication.
// This is needed when clients send GET requests on objects that can be downloaded without auth.
func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
	u := a.client.GetBlobURL(bucket, object)
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return azureToObjectError(traceError(err), bucket, object)
	}

	if length > 0 && startOffset > 0 {
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
	} else if startOffset > 0 {
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
	}

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return azureToObjectError(traceError(err), bucket, object)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
		return azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
	}

	_, err = io.Copy(writer, resp.Body)
	return traceError(err)
}

// AnonGetObjectInfo - Send HEAD request without authentication and convert the
// result to ObjectInfo.
func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
	resp, err := http.Head(a.client.GetBlobURL(bucket, object))
	if err != nil {
		return objInfo, azureToObjectError(traceError(err), bucket, object)
	}
	resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
	}

	var contentLength int64
	contentLengthStr := resp.Header.Get("Content-Length")
	if contentLengthStr != "" {
		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
		if err != nil {
			return objInfo, azureToObjectError(traceError(errUnexpected), bucket, object)
		}
	}

	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
	if err != nil {
		return objInfo, traceError(err)
	}

	objInfo.ModTime = t
	objInfo.Bucket = bucket
	objInfo.UserDefined = make(map[string]string)
	if resp.Header.Get("Content-Encoding") != "" {
		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
	}
	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
	objInfo.ETag = resp.Header.Get("Etag")
	objInfo.ModTime = t
	objInfo.Name = object
	objInfo.Size = contentLength
	return
}

// AnonListObjects - Use Azure equivalent ListBlobs.
func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
	params := storage.ListBlobsParameters{
		Prefix:     prefix,
		Marker:     marker,
		Delimiter:  delimiter,
		MaxResults: uint(maxKeys),
	}

	q := azureListBlobsGetParameters(params)
	q.Set("restype", "container")
	q.Set("comp", "list")

	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	url.RawQuery = q.Encode()

	resp, err := http.Get(url.String())
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	defer resp.Body.Close()

	var listResp storage.BlobListResponse

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	err = xml.Unmarshal(data, &listResp)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}

	result.IsTruncated = listResp.NextMarker != ""
	result.NextMarker = listResp.NextMarker
	for _, object := range listResp.Blobs {
		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
		if e != nil {
			continue
		}
		result.Objects = append(result.Objects, ObjectInfo{
			Bucket:          bucket,
			Name:            object.Name,
			ModTime:         t,
			Size:            object.Properties.ContentLength,
			ETag:            object.Properties.Etag,
			ContentType:     object.Properties.ContentType,
			ContentEncoding: object.Properties.ContentEncoding,
		})
	}
	result.Prefixes = listResp.BlobPrefixes
	return result, nil
}

// AnonListObjectsV2 - List objects in V2 mode, anonymously
func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) {
	params := storage.ListBlobsParameters{
		Prefix:     prefix,
		Marker:     continuationToken,
		Delimiter:  delimiter,
		MaxResults: uint(maxKeys),
	}

	q := azureListBlobsGetParameters(params)
	q.Set("restype", "container")
	q.Set("comp", "list")

	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	url.RawQuery = q.Encode()

	resp, err := http.Get(url.String())
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	defer resp.Body.Close()

	var listResp storage.BlobListResponse

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	err = xml.Unmarshal(data, &listResp)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}

	// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
	if listResp.NextMarker != "" {
		result.IsTruncated = true
		result.NextContinuationToken = listResp.NextMarker
	}
	for _, object := range listResp.Blobs {
		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
		if e != nil {
			continue
		}
		result.Objects = append(result.Objects, ObjectInfo{
			Bucket:          bucket,
			Name:            object.Name,
			ModTime:         t,
			Size:            object.Properties.ContentLength,
			ETag:            canonicalizeETag(object.Properties.Etag),
			ContentType:     object.Properties.ContentType,
			ContentEncoding: object.Properties.ContentEncoding,
		})
	}
	result.Prefixes = listResp.BlobPrefixes
	return result, nil
}
<commit_msg>gateway/azure: Parse error responses for anonymous requests (#4543)<commit_after>/*
 * Minio Cloud Storage, (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/Azure/azure-sdk-for-go/storage"
)

// Make anonymous HTTP request to azure endpoint.
func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response, error) {
	req, err := http.NewRequest(verb, urlStr, nil)
	if err != nil {
		return nil, err
	}
	if header != nil {
		req.Header = header
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}

	// 4XX and 5XX are error HTTP codes.
	if resp.StatusCode >= 400 && resp.StatusCode <= 511 {
		defer resp.Body.Close()
		respBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		if len(respBody) == 0 {
			// no error in response body, might happen in HEAD requests
			return nil, storage.AzureStorageServiceError{
				StatusCode: resp.StatusCode,
				Code:       resp.Status,
				Message:    "no response body was available for error status code",
			}
		}
		// Response contains Azure storage service error object.
		var storageErr storage.AzureStorageServiceError
		if err := xml.Unmarshal(respBody, &storageErr); err != nil {
			return nil, err
		}
		storageErr.StatusCode = resp.StatusCode
		return nil, storageErr
	}

	return resp, nil
}

// AnonGetBucketInfo - Get bucket metadata from azure anonymously.
func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
	if err != nil {
		return bucketInfo, azureToObjectError(traceError(err))
	}
	url.RawQuery = "restype=container"
	resp, err := azureAnonRequest(httpHEAD, url.String(), nil)
	if err != nil {
		return bucketInfo, azureToObjectError(traceError(err), bucket)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
	}

	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
	if err != nil {
		return bucketInfo, traceError(err)
	}
	bucketInfo = BucketInfo{
		Name:    bucket,
		Created: t,
	}
	return bucketInfo, nil
}

// AnonPutObject - SendPUT request without authentication.
// This is needed when clients send PUT requests on objects that can be uploaded without auth.
func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
	// azure doesn't support anonymous put
	return ObjectInfo{}, traceError(NotImplemented{})
}

// AnonGetObject - SendGET request without authentication.
// This is needed when clients send GET requests on objects that can be downloaded without auth.
func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
	h := make(http.Header)
	if length > 0 && startOffset > 0 {
		h.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
	} else if startOffset > 0 {
		h.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
	}

	resp, err := azureAnonRequest(httpGET, a.client.GetBlobURL(bucket, object), h)
	if err != nil {
		return azureToObjectError(traceError(err), bucket, object)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
		return azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
	}

	_, err = io.Copy(writer, resp.Body)
	return traceError(err)
}

// AnonGetObjectInfo - Send HEAD request without authentication and convert the
// result to ObjectInfo.
func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
	resp, err := azureAnonRequest(httpHEAD, a.client.GetBlobURL(bucket, object), nil)
	if err != nil {
		return objInfo, azureToObjectError(traceError(err), bucket, object)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
	}

	var contentLength int64
	contentLengthStr := resp.Header.Get("Content-Length")
	if contentLengthStr != "" {
		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
		if err != nil {
			return objInfo, azureToObjectError(traceError(errUnexpected), bucket, object)
		}
	}

	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
	if err != nil {
		return objInfo, traceError(err)
	}

	objInfo.ModTime = t
	objInfo.Bucket = bucket
	objInfo.UserDefined = make(map[string]string)
	if resp.Header.Get("Content-Encoding") != "" {
		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
	}
	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
	objInfo.ETag = resp.Header.Get("Etag")
	objInfo.ModTime = t
	objInfo.Name = object
	objInfo.Size = contentLength
	return
}

// AnonListObjects - Use Azure equivalent ListBlobs.
func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
	params := storage.ListBlobsParameters{
		Prefix:     prefix,
		Marker:     marker,
		Delimiter:  delimiter,
		MaxResults: uint(maxKeys),
	}

	q := azureListBlobsGetParameters(params)
	q.Set("restype", "container")
	q.Set("comp", "list")

	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	url.RawQuery = q.Encode()

	resp, err := azureAnonRequest(httpGET, url.String(), nil)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	defer resp.Body.Close()

	var listResp storage.BlobListResponse

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	err = xml.Unmarshal(data, &listResp)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}

	result.IsTruncated = listResp.NextMarker != ""
	result.NextMarker = listResp.NextMarker
	for _, object := range listResp.Blobs {
		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
		if e != nil {
			continue
		}
		result.Objects = append(result.Objects, ObjectInfo{
			Bucket:          bucket,
			Name:            object.Name,
			ModTime:         t,
			Size:            object.Properties.ContentLength,
			ETag:            object.Properties.Etag,
			ContentType:     object.Properties.ContentType,
			ContentEncoding: object.Properties.ContentEncoding,
		})
	}
	result.Prefixes = listResp.BlobPrefixes
	return result, nil
}

// AnonListObjectsV2 - List objects in V2 mode, anonymously
func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) {
	params := storage.ListBlobsParameters{
		Prefix:     prefix,
		Marker:     continuationToken,
		Delimiter:  delimiter,
		MaxResults: uint(maxKeys),
	}

	q := azureListBlobsGetParameters(params)
	q.Set("restype", "container")
	q.Set("comp", "list")

	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	url.RawQuery = q.Encode()

	resp, err := http.Get(url.String())
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	defer resp.Body.Close()

	var listResp storage.BlobListResponse

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}
	err = xml.Unmarshal(data, &listResp)
	if err != nil {
		return result, azureToObjectError(traceError(err))
	}

	// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
	if listResp.NextMarker != "" {
		result.IsTruncated = true
		result.NextContinuationToken = listResp.NextMarker
	}
	for _, object := range listResp.Blobs {
		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
		if e != nil {
			continue
		}
		result.Objects = append(result.Objects, ObjectInfo{
			Bucket:          bucket,
			Name:            object.Name,
			ModTime:         t,
			Size:            object.Properties.ContentLength,
			ETag:            canonicalizeETag(object.Properties.Etag),
			ContentType:     object.Properties.ContentType,
			ContentEncoding: object.Properties.ContentEncoding,
		})
	}
	result.Prefixes = listResp.BlobPrefixes
	return result, nil
}
<|endoftext|>