<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"net/url"
	"path"
	"strings"

	"github.com/spf13/pflag"

	"github.com/coreos/mantle/lang/maps"
	"github.com/coreos/mantle/sdk"
)

type storageSpec struct {
	BaseURL       string
	Title         string // Replace the bucket name in index page titles
	NamedPath     string // Copy to $BaseURL/$Board/$NamedPath
	VersionPath   bool   // Copy to $BaseURL/$Board/$Version
	DirectoryHTML bool
	IndexHTML     bool
}

type gceSpec struct {
	Project     string   // GCE project name
	Family      string   // A group name, also used as name prefix
	Description string   // Human readable-ish description
	Licenses    []string // Identifiers for tracking usage
	Image       string   // File name of image source
	Publish     string   // Write published image name to given file
	Limit       int      // Limit on # of old images to keep
}

type azureSpec struct {
	Image          string   // File name of image source
	StorageAccount string   // Storage account to use for image uploads
	Containers     []string // Containers to upload images to

	// Fields for azure.OSImage
	Label             string
	Description       string // Description of an image in this channel
	RecommendedVMSize string
	IconURI           string
	SmallIconURI      string
}

type channelSpec struct {
	BaseURL      string // Copy from $BaseURL/$Board/$Version
	Destinations []storageSpec
	GCE          gceSpec
	Azure        azureSpec
}

var (
	specBoard   string
	specChannel string
	specVersion string
	boards      = []string{"amd64-usr", "arm64-usr"}
	gceBoards   = []string{"amd64-usr"}
	azureBoards = []string{"amd64-usr"}
	specs       = map[string]channelSpec{
		"alpha": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/alpha/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://alpha.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-alpha",
				Title:         "alpha.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:     "gs://storage.core-os.net/coreos",
				NamedPath:   "alpha",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "alpha",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-alpha",
				Description: "CoreOS, CoreOS alpha",
				Licenses:    []string{"coreos-alpha"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
			Azure: azureSpec{
				Image:             "coreos_production_azure_image.vhd.bz2",
				StorageAccount:    "coreos",
				Containers:        []string{"publish", "pre-publish"},
				Label:             "CoreOS Alpha",
				Description:       "The Alpha channel closely tracks current development work and is released frequently. The newest versions of docker, etcd and fleet will be available for testing.",
				RecommendedVMSize: "Medium",
				IconURI:           "coreos-globe-color-lg-100px.png",
				SmallIconURI:      "coreos-globe-color-lg-45px.png",
			},
		},
		"beta": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/beta/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://beta.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-beta",
				Title:         "beta.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:   "gs://storage.core-os.net/coreos",
				NamedPath: "beta",
				IndexHTML: true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "beta",
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-beta",
				Description: "CoreOS, CoreOS beta",
				Licenses:    []string{"coreos-beta"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
			Azure: azureSpec{
				Image:             "coreos_production_azure_image.vhd.bz2",
				StorageAccount:    "coreos",
				Containers:        []string{"publish", "pre-publish"},
				Label:             "CoreOS Beta",
				Description:       "The Beta channel consists of promoted Alpha releases. Mix a few Beta machines into your production clusters to catch any bugs specific to your hardware or configuration.",
				RecommendedVMSize: "Medium",
				IconURI:           "coreos-globe-color-lg-100px.png",
				SmallIconURI:      "coreos-globe-color-lg-45px.png",
			},
		},
		"stable": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/stable/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://stable.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-stable",
				Title:         "stable.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-stable",
				Description: "CoreOS, CoreOS stable",
				Licenses:    []string{"coreos-stable"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
			Azure: azureSpec{
				Image:             "coreos_production_azure_image.vhd.bz2",
				StorageAccount:    "coreos",
				Containers:        []string{"publish", "pre-publish"},
				Label:             "CoreOS Stable",
				Description:       "The Stable channel should be used by production clusters. Versions of CoreOS are battle-tested within the Beta and Alpha channels before being promoted.",
				RecommendedVMSize: "Medium",
				IconURI:           "coreos-globe-color-lg-100px.png",
				SmallIconURI:      "coreos-globe-color-lg-45px.png",
			},
		},
	}
)

func AddSpecFlags(flags *pflag.FlagSet) {
	board := sdk.DefaultBoard()
	channels := strings.Join(maps.SortedKeys(specs), " ")
	versions, _ := sdk.VersionsFromManifest()
	flags.StringVarP(&specBoard, "board", "B",
		board, "target board")
	flags.StringVarP(&specChannel, "channel", "C",
		"alpha", "channels: "+channels)
	flags.StringVarP(&specVersion, "version", "V",
		versions.VersionID, "release version")
}

func ChannelSpec() channelSpec {
	if specBoard == "" {
		plog.Fatal("--board is required")
	}
	if specChannel == "" {
		plog.Fatal("--channel is required")
	}
	if specVersion == "" {
		plog.Fatal("--version is required")
	}

	spec, ok := specs[specChannel]
	if !ok {
		plog.Fatalf("Unknown channel: %s", specChannel)
	}

	boardOk := false
	for _, board := range boards {
		if specBoard == board {
			boardOk = true
			break
		}
	}
	if !boardOk {
		plog.Fatalf("Unknown board: %s", specBoard)
	}

	gceOk := false
	for _, board := range gceBoards {
		if specBoard == board {
			gceOk = true
			break
		}
	}
	if !gceOk {
		spec.GCE = gceSpec{}
	}

	azureOk := false
	for _, board := range azureBoards {
		if specBoard == board {
			azureOk = true
			break
		}
	}
	if !azureOk {
		spec.Azure = azureSpec{}
	}

	return spec
}

func (cs channelSpec) SourceURL() string {
	u, err := url.Parse(cs.BaseURL)
	if err != nil {
		panic(err)
	}
	u.Path = path.Join(u.Path, specBoard, specVersion)
	return u.String()
}

func (ss storageSpec) ParentPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		panic(err)
	}
	return []string{u.Path, path.Join(u.Path, specBoard)}
}

func (ss storageSpec) FinalPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		plog.Panic(err)
	}

	prefixes := []string{}
	if ss.VersionPath {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, specVersion))
	}
	if ss.NamedPath != "" {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, ss.NamedPath))
	}
	if len(prefixes) == 0 {
		plog.Panicf("Invalid destination: %#v", ss)
	}

	return prefixes
}
<commit_msg>cmd/plume: drop pre-publish from azure blob container list<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"net/url"
	"path"
	"strings"

	"github.com/spf13/pflag"

	"github.com/coreos/mantle/lang/maps"
	"github.com/coreos/mantle/sdk"
)

type storageSpec struct {
	BaseURL       string
	Title         string // Replace the bucket name in index page titles
	NamedPath     string // Copy to $BaseURL/$Board/$NamedPath
	VersionPath   bool   // Copy to $BaseURL/$Board/$Version
	DirectoryHTML bool
	IndexHTML     bool
}

type gceSpec struct {
	Project     string   // GCE project name
	Family      string   // A group name, also used as name prefix
	Description string   // Human readable-ish description
	Licenses    []string // Identifiers for tracking usage
	Image       string   // File name of image source
	Publish     string   // Write published image name to given file
	Limit       int      // Limit on # of old images to keep
}

type azureSpec struct {
	Image          string   // File name of image source
	StorageAccount string   // Storage account to use for image uploads
	Containers     []string // Containers to upload images to

	// Fields for azure.OSImage
	Label             string
	Description       string // Description of an image in this channel
	RecommendedVMSize string
	IconURI           string
	SmallIconURI      string
}

type channelSpec struct {
	BaseURL      string // Copy from $BaseURL/$Board/$Version
	Destinations []storageSpec
	GCE          gceSpec
	Azure        azureSpec
}

var (
	specBoard   string
	specChannel string
	specVersion string
	boards      = []string{"amd64-usr", "arm64-usr"}
	gceBoards   = []string{"amd64-usr"}
	azureBoards = []string{"amd64-usr"}
	specs       = map[string]channelSpec{
		"alpha": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/alpha/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://alpha.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-alpha",
				Title:         "alpha.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:     "gs://storage.core-os.net/coreos",
				NamedPath:   "alpha",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "alpha",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-alpha",
				Description: "CoreOS, CoreOS alpha",
				Licenses:    []string{"coreos-alpha"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
			Azure: azureSpec{
				Image:             "coreos_production_azure_image.vhd.bz2",
				StorageAccount:    "coreos",
				Containers:        []string{"publish"},
				Label:             "CoreOS Alpha",
				Description:       "The Alpha channel closely tracks current development work and is released frequently. The newest versions of docker, etcd and fleet will be available for testing.",
				RecommendedVMSize: "Medium",
				IconURI:           "coreos-globe-color-lg-100px.png",
				SmallIconURI:      "coreos-globe-color-lg-45px.png",
			},
		},
		"beta": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/beta/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://beta.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-beta",
				Title:         "beta.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:   "gs://storage.core-os.net/coreos",
				NamedPath: "beta",
				IndexHTML: true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "beta",
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-beta",
				Description: "CoreOS, CoreOS beta",
				Licenses:    []string{"coreos-beta"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
			Azure: azureSpec{
				Image:             "coreos_production_azure_image.vhd.bz2",
				StorageAccount:    "coreos",
				Containers:        []string{"publish"},
				Label:             "CoreOS Beta",
				Description:       "The Beta channel consists of promoted Alpha releases. Mix a few Beta machines into your production clusters to catch any bugs specific to your hardware or configuration.",
				RecommendedVMSize: "Medium",
				IconURI:           "coreos-globe-color-lg-100px.png",
				SmallIconURI:      "coreos-globe-color-lg-45px.png",
			},
		},
		"stable": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/stable/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://stable.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-stable",
				Title:         "stable.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-stable",
				Description: "CoreOS, CoreOS stable",
				Licenses:    []string{"coreos-stable"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
			Azure: azureSpec{
				Image:             "coreos_production_azure_image.vhd.bz2",
				StorageAccount:    "coreos",
				Containers:        []string{"publish"},
				Label:             "CoreOS Stable",
				Description:       "The Stable channel should be used by production clusters. Versions of CoreOS are battle-tested within the Beta and Alpha channels before being promoted.",
				RecommendedVMSize: "Medium",
				IconURI:           "coreos-globe-color-lg-100px.png",
				SmallIconURI:      "coreos-globe-color-lg-45px.png",
			},
		},
	}
)

func AddSpecFlags(flags *pflag.FlagSet) {
	board := sdk.DefaultBoard()
	channels := strings.Join(maps.SortedKeys(specs), " ")
	versions, _ := sdk.VersionsFromManifest()
	flags.StringVarP(&specBoard, "board", "B",
		board, "target board")
	flags.StringVarP(&specChannel, "channel", "C",
		"alpha", "channels: "+channels)
	flags.StringVarP(&specVersion, "version", "V",
		versions.VersionID, "release version")
}

func ChannelSpec() channelSpec {
	if specBoard == "" {
		plog.Fatal("--board is required")
	}
	if specChannel == "" {
		plog.Fatal("--channel is required")
	}
	if specVersion == "" {
		plog.Fatal("--version is required")
	}

	spec, ok := specs[specChannel]
	if !ok {
		plog.Fatalf("Unknown channel: %s", specChannel)
	}

	boardOk := false
	for _, board := range boards {
		if specBoard == board {
			boardOk = true
			break
		}
	}
	if !boardOk {
		plog.Fatalf("Unknown board: %s", specBoard)
	}

	gceOk := false
	for _, board := range gceBoards {
		if specBoard == board {
			gceOk = true
			break
		}
	}
	if !gceOk {
		spec.GCE = gceSpec{}
	}

	azureOk := false
	for _, board := range azureBoards {
		if specBoard == board {
			azureOk = true
			break
		}
	}
	if !azureOk {
		spec.Azure = azureSpec{}
	}

	return spec
}

func (cs channelSpec) SourceURL() string {
	u, err := url.Parse(cs.BaseURL)
	if err != nil {
		panic(err)
	}
	u.Path = path.Join(u.Path, specBoard, specVersion)
	return u.String()
}

func (ss storageSpec) ParentPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		panic(err)
	}
	return []string{u.Path, path.Join(u.Path, specBoard)}
}

func (ss storageSpec) FinalPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		plog.Panic(err)
	}

	prefixes := []string{}
	if ss.VersionPath {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, specVersion))
	}
	if ss.NamedPath != "" {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, ss.NamedPath))
	}
	if len(prefixes) == 0 {
		plog.Panicf("Invalid destination: %#v", ss)
	}

	return prefixes
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"os"

	"github.com/docopt/docopt-go"
	"github.com/howeyc/gopass"
	_ "github.com/lib/pq"
	"golang.org/x/crypto/bcrypt"
)

const (
	usage = `passwd-reset. A helper to reset password for rtfblog.

Usage:
  passwd-reset <env var>
  passwd-reset -h | --help
  passwd-reset --version

Options:
  It takes a single argument -- a name of environment variable to look up a
  connection string.
  -h --help     Show this screen.
  --version     Show version.`
)

func EncryptBcrypt(passwd string) (hash string, err error) {
	hashBytes, err := bcrypt.GenerateFromPassword([]byte(passwd), bcrypt.DefaultCost)
	hash = string(hashBytes)
	return
}

func updateAuthorRow(connString, uname, passwd, fullname, email, www string) {
	db, err := sql.Open("postgres", connString)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	defer db.Close()
	stmt, _ := db.Prepare(`update author set
	disp_name=$1, passwd=$2, full_name=$3, email=$4, www=$5
	where id=1`)
	defer stmt.Close()
	passwdHash, err := EncryptBcrypt(passwd)
	if err != nil {
		fmt.Printf("Error in Encrypt(): %s\n", err)
		return
	}
	fmt.Printf("Updating user ID=1...\n")
	fmt.Printf("dbstr: %q\nuname: %q\npasswd: %q\nhash: %q\nfullname: %q\nemail: %q\nwww: %q\n",
		connString, uname, "***", passwdHash, fullname, email, www)
	stmt.Exec(uname, passwdHash, fullname, email, www)
}

func main() {
	args, err := docopt.Parse(usage, nil, true, "1.0", false)
	if err != nil {
		panic("Can't docopt.Parse!")
	}
	envVar := args["<env var>"].(string)
	fmt.Printf("Looking up connstr in $%s...\n", envVar)
	dbFile := os.Getenv(envVar)
	uname := "rtfb"
	fmt.Printf("New password: ")
	passwd, err := gopass.GetPasswd()
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("Confirm: ")
	passwd2, err := gopass.GetPasswd()
	if err != nil {
		panic(err.Error())
	}
	if string(passwd2) != string(passwd) {
		panic("Passwords do not match")
	}
	fullname := "Vytautas Šaltenis"
	email := "vytas@rtfb.lt"
	www := "http://rtfb.lt/"
	updateAuthorRow(dbFile, uname, string(passwd), fullname, email, www)
}
<commit_msg>Pin down gopass to my fork<commit_after>package main

import (
	"database/sql"
	"fmt"
	"os"

	"github.com/docopt/docopt-go"
	_ "github.com/lib/pq"
	"github.com/rtfb/gopass"
	"golang.org/x/crypto/bcrypt"
)

const (
	usage = `passwd-reset. A helper to reset password for rtfblog.

Usage:
  passwd-reset <env var>
  passwd-reset -h | --help
  passwd-reset --version

Options:
  It takes a single argument -- a name of environment variable to look up a
  connection string.
  -h --help     Show this screen.
  --version     Show version.`
)

func EncryptBcrypt(passwd string) (hash string, err error) {
	hashBytes, err := bcrypt.GenerateFromPassword([]byte(passwd), bcrypt.DefaultCost)
	hash = string(hashBytes)
	return
}

func updateAuthorRow(connString, uname, passwd, fullname, email, www string) {
	db, err := sql.Open("postgres", connString)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	defer db.Close()
	stmt, _ := db.Prepare(`update author set
	disp_name=$1, passwd=$2, full_name=$3, email=$4, www=$5
	where id=1`)
	defer stmt.Close()
	passwdHash, err := EncryptBcrypt(passwd)
	if err != nil {
		fmt.Printf("Error in Encrypt(): %s\n", err)
		return
	}
	fmt.Printf("Updating user ID=1...\n")
	fmt.Printf("dbstr: %q\nuname: %q\npasswd: %q\nhash: %q\nfullname: %q\nemail: %q\nwww: %q\n",
		connString, uname, "***", passwdHash, fullname, email, www)
	stmt.Exec(uname, passwdHash, fullname, email, www)
}

func main() {
	args, err := docopt.Parse(usage, nil, true, "1.0", false)
	if err != nil {
		panic("Can't docopt.Parse!")
	}
	envVar := args["<env var>"].(string)
	fmt.Printf("Looking up connstr in $%s...\n", envVar)
	dbFile := os.Getenv(envVar)
	uname := "rtfb"
	fmt.Printf("New password: ")
	passwd, err := gopass.GetPasswd()
	if err != nil {
		panic(err.Error())
	}
	fmt.Printf("Confirm: ")
	passwd2, err := gopass.GetPasswd()
	if err != nil {
		panic(err.Error())
	}
	if string(passwd2) != string(passwd) {
		panic("Passwords do not match")
	}
	fullname := "Vytautas Šaltenis"
	email := "vytas@rtfb.lt"
	www := "http://rtfb.lt/"
	updateAuthorRow(dbFile, uname, string(passwd), fullname, email, www)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/containers/image/copy"
	"github.com/containers/image/docker/reference"
	"github.com/containers/image/manifest"
	"github.com/containers/image/transports"
	"github.com/containers/image/transports/alltransports"
	"github.com/containers/image/types"
	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
)

// contextsFromGlobalOptions returns source and destionation types.SystemContext depending on c.
func contextsFromGlobalOptions(c *cli.Context) (*types.SystemContext, *types.SystemContext, error) {
	sourceCtx, err := contextFromGlobalOptions(c, "src-")
	if err != nil {
		return nil, nil, err
	}

	destinationCtx, err := contextFromGlobalOptions(c, "dest-")
	if err != nil {
		return nil, nil, err
	}

	return sourceCtx, destinationCtx, nil
}

type copyOptions struct {
}

func copyCmd() cli.Command {
	opts := copyOptions{}
	return cli.Command{
		Name:  "copy",
		Usage: "Copy an IMAGE-NAME from one location to another",
		Description: fmt.Sprintf(`

	Container "IMAGE-NAME" uses a "transport":"details" format.

	Supported transports:
	%s

	See skopeo(1) section "IMAGE NAMES" for the expected format
	`, strings.Join(transports.ListNames(), ", ")),
		ArgsUsage: "SOURCE-IMAGE DESTINATION-IMAGE",
		Action:    opts.run,
		// FIXME: Do we need to namespace the GPG aspect?
		Flags: []cli.Flag{
			cli.StringSliceFlag{
				Name:  "additional-tag",
				Usage: "additional tags (supports docker-archive)",
			},
			cli.StringFlag{
				Name:  "authfile",
				Usage: "path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json",
			},
			cli.BoolFlag{
				Name:  "remove-signatures",
				Usage: "Do not copy signatures from SOURCE-IMAGE",
			},
			cli.StringFlag{
				Name:  "sign-by",
				Usage: "Sign the image using a GPG key with the specified `FINGERPRINT`",
			},
			cli.StringFlag{
				Name:  "src-creds, screds",
				Value: "",
				Usage: "Use `USERNAME[:PASSWORD]` for accessing the source registry",
			},
			cli.StringFlag{
				Name:  "dest-creds, dcreds",
				Value: "",
				Usage: "Use `USERNAME[:PASSWORD]` for accessing the destination registry",
			},
			cli.StringFlag{
				Name:  "src-cert-dir",
				Value: "",
				Usage: "use certificates at `PATH` (*.crt, *.cert, *.key) to connect to the source registry or daemon",
			},
			cli.BoolTFlag{
				Name:  "src-tls-verify",
				Usage: "require HTTPS and verify certificates when talking to the container source registry or daemon (defaults to true)",
			},
			cli.StringFlag{
				Name:  "dest-cert-dir",
				Value: "",
				Usage: "use certificates at `PATH` (*.crt, *.cert, *.key) to connect to the destination registry or daemon",
			},
			cli.BoolTFlag{
				Name:  "dest-tls-verify",
				Usage: "require HTTPS and verify certificates when talking to the container destination registry or daemon (defaults to true)",
			},
			cli.StringFlag{
				Name:  "dest-ostree-tmp-dir",
				Value: "",
				Usage: "`DIRECTORY` to use for OSTree temporary files",
			},
			cli.StringFlag{
				Name:  "src-shared-blob-dir",
				Value: "",
				Usage: "`DIRECTORY` to use to fetch retrieved blobs (OCI layout sources only)",
			},
			cli.StringFlag{
				Name:  "dest-shared-blob-dir",
				Value: "",
				Usage: "`DIRECTORY` to use to store retrieved blobs (OCI layout destinations only)",
			},
			cli.StringFlag{
				Name:  "format, f",
				Usage: "`MANIFEST TYPE` (oci, v2s1, or v2s2) to use when saving image to directory using the 'dir:' transport (default is manifest type of source)",
			},
			cli.BoolFlag{
				Name:  "dest-compress",
				Usage: "Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source)",
			},
			cli.StringFlag{
				Name:  "src-daemon-host",
				Value: "",
				Usage: "use docker daemon host at `HOST` (docker-daemon sources only)",
			},
			cli.StringFlag{
				Name:  "dest-daemon-host",
				Value: "",
				Usage: "use docker daemon host at `HOST` (docker-daemon destinations only)",
			},
		},
	}
}

func (opts *copyOptions) run(c *cli.Context) error {
	if len(c.Args()) != 2 {
		cli.ShowCommandHelp(c, "copy")
		return errors.New("Exactly two arguments expected")
	}

	policyContext, err := getPolicyContext(c)
	if err != nil {
		return fmt.Errorf("Error loading trust policy: %v", err)
	}
	defer policyContext.Destroy()

	srcRef, err := alltransports.ParseImageName(c.Args()[0])
	if err != nil {
		return fmt.Errorf("Invalid source name %s: %v", c.Args()[0], err)
	}
	destRef, err := alltransports.ParseImageName(c.Args()[1])
	if err != nil {
		return fmt.Errorf("Invalid destination name %s: %v", c.Args()[1], err)
	}
	signBy := c.String("sign-by")
	removeSignatures := c.Bool("remove-signatures")

	sourceCtx, destinationCtx, err := contextsFromGlobalOptions(c)
	if err != nil {
		return err
	}

	var manifestType string
	if c.IsSet("format") {
		switch c.String("format") {
		case "oci":
			manifestType = imgspecv1.MediaTypeImageManifest
		case "v2s1":
			manifestType = manifest.DockerV2Schema1SignedMediaType
		case "v2s2":
			manifestType = manifest.DockerV2Schema2MediaType
		default:
			return fmt.Errorf("unknown format %q. Choose on of the supported formats: 'oci', 'v2s1', or 'v2s2'", c.String("format"))
		}
	}

	if c.IsSet("additional-tag") {
		for _, image := range c.StringSlice("additional-tag") {
			ref, err := reference.ParseNormalizedNamed(image)
			if err != nil {
				return fmt.Errorf("error parsing additional-tag '%s': %v", image, err)
			}
			namedTagged, isNamedTagged := ref.(reference.NamedTagged)
			if !isNamedTagged {
				return fmt.Errorf("additional-tag '%s' must be a tagged reference", image)
			}
			destinationCtx.DockerArchiveAdditionalTags = append(destinationCtx.DockerArchiveAdditionalTags, namedTagged)
		}
	}

	ctx, cancel := commandTimeoutContextFromGlobalOptions(c)
	defer cancel()

	_, err = copy.Image(ctx, policyContext, destRef, srcRef, &copy.Options{
		RemoveSignatures:      removeSignatures,
		SignBy:                signBy,
		ReportWriter:          os.Stdout,
		SourceCtx:             sourceCtx,
		DestinationCtx:        destinationCtx,
		ForceManifestMIMEType: manifestType,
	})
	return err
}
<commit_msg>Fix a typo<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/containers/image/copy"
	"github.com/containers/image/docker/reference"
	"github.com/containers/image/manifest"
	"github.com/containers/image/transports"
	"github.com/containers/image/transports/alltransports"
	"github.com/containers/image/types"
	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/urfave/cli"
)

// contextsFromGlobalOptions returns source and destionation types.SystemContext depending on c.
func contextsFromGlobalOptions(c *cli.Context) (*types.SystemContext, *types.SystemContext, error) {
	sourceCtx, err := contextFromGlobalOptions(c, "src-")
	if err != nil {
		return nil, nil, err
	}

	destinationCtx, err := contextFromGlobalOptions(c, "dest-")
	if err != nil {
		return nil, nil, err
	}

	return sourceCtx, destinationCtx, nil
}

type copyOptions struct {
}

func copyCmd() cli.Command {
	opts := copyOptions{}
	return cli.Command{
		Name:  "copy",
		Usage: "Copy an IMAGE-NAME from one location to another",
		Description: fmt.Sprintf(`

	Container "IMAGE-NAME" uses a "transport":"details" format.

	Supported transports:
	%s

	See skopeo(1) section "IMAGE NAMES" for the expected format
	`, strings.Join(transports.ListNames(), ", ")),
		ArgsUsage: "SOURCE-IMAGE DESTINATION-IMAGE",
		Action:    opts.run,
		// FIXME: Do we need to namespace the GPG aspect?
		Flags: []cli.Flag{
			cli.StringSliceFlag{
				Name:  "additional-tag",
				Usage: "additional tags (supports docker-archive)",
			},
			cli.StringFlag{
				Name:  "authfile",
				Usage: "path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json",
			},
			cli.BoolFlag{
				Name:  "remove-signatures",
				Usage: "Do not copy signatures from SOURCE-IMAGE",
			},
			cli.StringFlag{
				Name:  "sign-by",
				Usage: "Sign the image using a GPG key with the specified `FINGERPRINT`",
			},
			cli.StringFlag{
				Name:  "src-creds, screds",
				Value: "",
				Usage: "Use `USERNAME[:PASSWORD]` for accessing the source registry",
			},
			cli.StringFlag{
				Name:  "dest-creds, dcreds",
				Value: "",
				Usage: "Use `USERNAME[:PASSWORD]` for accessing the destination registry",
			},
			cli.StringFlag{
				Name:  "src-cert-dir",
				Value: "",
				Usage: "use certificates at `PATH` (*.crt, *.cert, *.key) to connect to the source registry or daemon",
			},
			cli.BoolTFlag{
				Name:  "src-tls-verify",
				Usage: "require HTTPS and verify certificates when talking to the container source registry or daemon (defaults to true)",
			},
			cli.StringFlag{
				Name:  "dest-cert-dir",
				Value: "",
				Usage: "use certificates at `PATH` (*.crt, *.cert, *.key) to connect to the destination registry or daemon",
			},
			cli.BoolTFlag{
				Name:  "dest-tls-verify",
				Usage: "require HTTPS and verify certificates when talking to the container destination registry or daemon (defaults to true)",
			},
			cli.StringFlag{
				Name:  "dest-ostree-tmp-dir",
				Value: "",
				Usage: "`DIRECTORY` to use for OSTree temporary files",
			},
			cli.StringFlag{
				Name:  "src-shared-blob-dir",
				Value: "",
				Usage: "`DIRECTORY` to use to fetch retrieved blobs (OCI layout sources only)",
			},
			cli.StringFlag{
				Name:  "dest-shared-blob-dir",
				Value: "",
				Usage: "`DIRECTORY` to use to store retrieved blobs (OCI layout destinations only)",
			},
			cli.StringFlag{
				Name:  "format, f",
				Usage: "`MANIFEST TYPE` (oci, v2s1, or v2s2) to use when saving image to directory using the 'dir:' transport (default is manifest type of source)",
			},
			cli.BoolFlag{
				Name:  "dest-compress",
				Usage: "Compress tarball image layers when saving to directory using the 'dir' transport. (default is same compression type as source)",
			},
			cli.StringFlag{
				Name:  "src-daemon-host",
				Value: "",
				Usage: "use docker daemon host at `HOST` (docker-daemon sources only)",
			},
			cli.StringFlag{
				Name:  "dest-daemon-host",
				Value: "",
				Usage: "use docker daemon host at `HOST` (docker-daemon destinations only)",
			},
		},
	}
}

func (opts *copyOptions) run(c *cli.Context) error {
	if len(c.Args()) != 2 {
		cli.ShowCommandHelp(c, "copy")
		return errors.New("Exactly two arguments expected")
	}

	policyContext, err := getPolicyContext(c)
	if err != nil {
		return fmt.Errorf("Error loading trust policy: %v", err)
	}
	defer policyContext.Destroy()

	srcRef, err := alltransports.ParseImageName(c.Args()[0])
	if err != nil {
		return fmt.Errorf("Invalid source name %s: %v", c.Args()[0], err)
	}
	destRef, err := alltransports.ParseImageName(c.Args()[1])
	if err != nil {
		return fmt.Errorf("Invalid destination name %s: %v", c.Args()[1], err)
	}
	signBy := c.String("sign-by")
	removeSignatures := c.Bool("remove-signatures")

	sourceCtx, destinationCtx, err := contextsFromGlobalOptions(c)
	if err != nil {
		return err
	}

	var manifestType string
	if c.IsSet("format") {
		switch c.String("format") {
		case "oci":
			manifestType = imgspecv1.MediaTypeImageManifest
		case "v2s1":
			manifestType = manifest.DockerV2Schema1SignedMediaType
		case "v2s2":
			manifestType = manifest.DockerV2Schema2MediaType
		default:
			return fmt.Errorf("unknown format %q. Choose one of the supported formats: 'oci', 'v2s1', or 'v2s2'", c.String("format"))
		}
	}

	if c.IsSet("additional-tag") {
		for _, image := range c.StringSlice("additional-tag") {
			ref, err := reference.ParseNormalizedNamed(image)
			if err != nil {
				return fmt.Errorf("error parsing additional-tag '%s': %v", image, err)
			}
			namedTagged, isNamedTagged := ref.(reference.NamedTagged)
			if !isNamedTagged {
				return fmt.Errorf("additional-tag '%s' must be a tagged reference", image)
			}
			destinationCtx.DockerArchiveAdditionalTags = append(destinationCtx.DockerArchiveAdditionalTags, namedTagged)
		}
	}

	ctx, cancel := commandTimeoutContextFromGlobalOptions(c)
	defer cancel()

	_, err = copy.Image(ctx, policyContext, destRef, srcRef, &copy.Options{
		RemoveSignatures:      removeSignatures,
		SignBy:                signBy,
		ReportWriter:          os.Stdout,
		SourceCtx:             sourceCtx,
		DestinationCtx:        destinationCtx,
		ForceManifestMIMEType: manifestType,
	})
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/libopenstorage/stork/drivers/volume"
	_ "github.com/libopenstorage/stork/drivers/volume/aws"
	_ "github.com/libopenstorage/stork/drivers/volume/azure"
	_ "github.com/libopenstorage/stork/drivers/volume/gcp"
	_ "github.com/libopenstorage/stork/drivers/volume/portworx"
	"github.com/libopenstorage/stork/pkg/apis"
	"github.com/libopenstorage/stork/pkg/applicationmanager"
	"github.com/libopenstorage/stork/pkg/clusterdomains"
	"github.com/libopenstorage/stork/pkg/dbg"
	"github.com/libopenstorage/stork/pkg/extender"
	"github.com/libopenstorage/stork/pkg/groupsnapshot"
	"github.com/libopenstorage/stork/pkg/migration"
	"github.com/libopenstorage/stork/pkg/monitor"
	"github.com/libopenstorage/stork/pkg/pvcwatcher"
	"github.com/libopenstorage/stork/pkg/resourcecollector"
	"github.com/libopenstorage/stork/pkg/rule"
	"github.com/libopenstorage/stork/pkg/schedule"
	"github.com/libopenstorage/stork/pkg/snapshot"
	"github.com/libopenstorage/stork/pkg/version"
	"github.com/libopenstorage/stork/pkg/webhookadmission"
	log "github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	api_v1 "k8s.io/api/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	core_v1 "k8s.io/client-go/kubernetes/typed/core/v1"
	_ "k8s.io/client-go/plugin/pkg/client/auth"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/manager"
)

const (
	defaultLockObjectName      = "stork"
	defaultLockObjectNamespace = "kube-system"
	defaultAdminNamespace      = "kube-system"
	eventComponentName         = "stork"
	debugFilePath              = "/var/cores"
)

var ext *extender.Extender
var webhook *webhookadmission.Controller

func main() {
	// Parse empty flags to suppress warnings from the snapshotter which uses
	// glog
	err := flag.CommandLine.Parse([]string{})
	if err != nil {
		log.Warnf("Error parsing flag: %v", err)
	}
	err = flag.Set("logtostderr", "true")
	if err != nil {
		log.Fatalf("Error setting glog flag: %v", err)
	}

	app := cli.NewApp()
	app.Name = "stork"
	app.Usage = "STorage Orchestartor Runtime for Kubernetes (STORK)"
	app.Version = version.Version
	app.Action = run

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Enable verbose logging",
		},
		cli.StringFlag{
			Name:  "driver,d",
			Usage: "Storage driver name",
		},
		cli.BoolTFlag{
			Name:  "leader-elect",
			Usage: "Enable leader election (default: true)",
		},
		cli.StringFlag{
			Name:  "lock-object-name",
			Usage: "Name for the lock object",
			Value: defaultLockObjectName,
		},
		cli.StringFlag{
			Name:  "lock-object-namespace",
			Usage: "Namespace for the lock object",
			Value: defaultLockObjectNamespace,
		},
		cli.BoolTFlag{
			Name:  "snapshotter",
			Usage: "Enable snapshotter (default: true)",
		},
		cli.BoolTFlag{
			Name:  "extender",
			Usage: "Enable scheduler extender for hyperconvergence (default: true)",
		},
		cli.BoolTFlag{
			Name:  "health-monitor",
			Usage: "Enable health monitoring of the storage driver (default: true)",
		},
		cli.Int64Flag{
			Name:  "health-monitor-interval",
			Value: 120,
			Usage: "The interval in seconds to monitor the health of the storage driver (min: 30)",
		},
		cli.BoolTFlag{
			Name:  "migration-controller",
			Usage: "Start the migration controller (default: true)",
		},
		cli.BoolTFlag{
			Name:  "application-controller",
			Usage: "Start the controllers for managing applications (default: true)",
		},
		cli.StringFlag{
			Name:  "admin-namespace",
			Value: defaultAdminNamespace,
			Usage: "Namespace to be used by a cluster admin which can migrate and backup all other namespaces",
		},
		cli.StringFlag{
			Name:  "migration-admin-namespace",
			Value: defaultAdminNamespace,
			Usage: "Namespace to be used by a cluster admin which can migrate all other namespaces (Deprecated, please use admin-namespace)",
		},
		cli.BoolTFlag{
			Name:  "cluster-domain-controllers",
			Usage: "Start the cluster domain controllers (default: true)",
		},
		cli.BoolTFlag{
			Name:  "pvc-watcher",
			Usage: "Start the controller to monitor PVC creation and deletions (default: true)",
		},
	}

	if err := app.Run(os.Args); err != nil {
		log.Fatalf("Error starting stork: %v", err)
	}
}

func run(c *cli.Context) {
	dbg.Init(c.App.Name, debugFilePath)

	log.Infof("Starting stork version %v", version.Version)
	driverName := c.String("driver")

	verbose := c.Bool("verbose")
	if verbose {
		log.SetLevel(log.DebugLevel)
	}

	config, err := rest.InClusterConfig()
	if err != nil {
		log.Fatalf("Error getting cluster config: %v", err)
	}

	k8sClient, err := clientset.NewForConfig(config)
	if err != nil {
		log.Fatalf("Error getting client, %v", err)
	}

	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartRecordingToSink(&core_v1.EventSinkImpl{Interface: k8sClient.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, api_v1.EventSource{Component: eventComponentName})

	var d volume.Driver
	if driverName != "" {
		d, err = volume.Get(driverName)
		if err != nil {
			log.Fatalf("Error getting Stork Driver %v: %v", driverName, err)
		}

		if err = d.Init(nil); err != nil {
			log.Fatalf("Error initializing Stork Driver %v: %v", driverName, err)
		}

		if c.Bool("extender") {
			ext = &extender.Extender{
				Driver:   d,
				Recorder: recorder,
			}

			if err = ext.Start(); err != nil {
				log.Fatalf("Error starting scheduler extender: %v", err)
			}
		}
	}
	webhook = &webhookadmission.Controller{
		Driver:   d,
		Recorder: recorder,
	}
	if err := webhook.Start(); err != nil {
		log.Fatalf("error starting webhook controller: %v", err)
	}

	// Create operator-sdk manager that will manage all controllers.
	mgr, err := manager.New(config, manager.Options{})
	if err != nil {
		log.Fatalf("Setup controller manager: %v", err)
	}

	// Setup scheme for all stork resources
	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
		log.Fatalf("Setup scheme failed for stork resources: %v", err)
	}

	runFunc := func(context.Context) {
		runStork(mgr, d, recorder, c)
	}

	if c.BoolT("leader-elect") {
		lockObjectName := c.String("lock-object-name")
		lockObjectNamespace := c.String("lock-object-namespace")
		id, err := os.Hostname()
		if err != nil {
			log.Fatalf("Error getting hostname: %v", err)
		}
		lockConfig := resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: recorder,
		}
		resourceLock, err := resourcelock.New(
			resourcelock.ConfigMapsResourceLock,
			lockObjectNamespace,
			lockObjectName,
			k8sClient.CoreV1(),
			k8sClient.CoordinationV1(),
			lockConfig)
		if err != nil {
			log.Fatalf("Error creating resource lock: %v", err)
		}
		leaderElectionConfig := leaderelection.LeaderElectionConfig{
			Lock:          resourceLock,
			LeaseDuration: 15 * time.Second,
			RenewDeadline: 10 * time.Second,
			RetryPeriod:   2 * time.Second,
			Callbacks: leaderelection.LeaderCallbacks{
				OnStartedLeading: runFunc,
				OnStoppedLeading: func() {
					log.Fatalf("Stork lost master")
				},
			},
		}
		leaderElector, err := leaderelection.NewLeaderElector(leaderElectionConfig)
		if err != nil {
			log.Fatalf("Error creating leader elector: %v", err)
		}
		leaderElector.Run(context.Background())
	} else {
		runFunc(nil)
	}
}

func runStork(mgr manager.Manager, d volume.Driver, recorder record.EventRecorder, c *cli.Context) {
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	if err := rule.Init(); err != nil {
		log.Fatalf("Error initializing rule: %v", err)
	}

	resourceCollector := resourcecollector.ResourceCollector{
		Driver: d,
	}
	if err := resourceCollector.Init(nil); err != nil {
		log.Fatalf("Error initializing ResourceCollector: %v", err)
	}
	adminNamespace := c.String("admin-namespace")
	if adminNamespace == "" {
		adminNamespace = c.String("migration-admin-namespace")
	}

	monitor := &monitor.Monitor{
		Driver:      d,
		IntervalSec: c.Int64("health-monitor-interval"),
	}
	snapshot := &snapshot.Snapshot{
		Driver:   d,
		Recorder: recorder,
	}
	if err := schedule.Init(); err != nil {
		log.Fatalf("Error initializing schedule: %v", err)
	}
	if d != nil {
		if c.Bool("health-monitor") {
			if err := monitor.Start(); err != nil {
				log.Fatalf("Error starting storage monitor: %v", err)
			}
		}

		if c.Bool("snapshotter") {
			if err := snapshot.Start(mgr); err != nil {
				log.Fatalf("Error starting snapshot controller: %v", err)
			}

			groupsnapshotInst := groupsnapshot.GroupSnapshot{
				Driver:   d,
				Recorder: recorder,
			}
			if err := groupsnapshotInst.Init(mgr); err != nil {
				log.Fatalf("Error initializing groupsnapshot controller: %v", err)
			}
		}
		if c.Bool("pvc-watcher") {
			pvcWatcher := pvcwatcher.New(mgr, d, recorder)
			if err := pvcWatcher.Start(mgr); err != nil {
				log.Fatalf("Error starting pvc watcher: %v", err)
			}
		}

		if c.Bool("migration-controller") {
			migration := migration.Migration{
				Driver:            d,
				Recorder:          recorder,
				ResourceCollector: resourceCollector,
			}
			if err := migration.Init(mgr, adminNamespace); err != nil {
				log.Fatalf("Error initializing migration: %v", err)
			}
		}

		if c.Bool("cluster-domain-controllers") {
			clusterDomains := clusterdomains.ClusterDomains{
				Driver:   d,
				Recorder: recorder,
			}
			if err := clusterDomains.Init(mgr); err != nil {
				log.Fatalf("Error initializing cluster domain controllers: %v", err)
			}
		}
	}

	if c.Bool("application-controller") {
		appManager := applicationmanager.ApplicationManager{
			Driver:            d,
			Recorder:          recorder,
			ResourceCollector: resourceCollector,
		}
		if err := appManager.Init(mgr, adminNamespace, signalChan); err != nil {
			log.Fatalf("Error initializing application manager: %v", err)
		}
	}

	stopCh := make(chan struct{}, 1)

	go func() {
		for {
			<-signalChan
			log.Printf("Shutdown signal received, exiting...")
			if c.Bool("extender") {
				if err := ext.Stop(); err != nil {
					log.Warnf("Error stopping extender: %v", err)
				}
			}
			if c.Bool("health-monitor") {
				if err := monitor.Stop(); err != nil {
					log.Warnf("Error stopping monitor: %v", err)
				}
			}
			if c.Bool("snapshotter") {
				if err := snapshot.Stop(); err != nil {
					log.Warnf("Error stopping snapshot controllers: %v", err)
				}
			}
			if err := d.Stop(); err != nil {
				log.Warnf("Error stopping driver: %v", err)
			}
			if err := webhook.Stop(); err != nil {
				log.Warnf("error stopping webhook controller %v", err)
			}

			stopCh <- struct{}{}
		}
	}()

	if err := mgr.Start(stopCh); err != nil {
		log.Fatalf("Controller manager: %v", err)
	}
	os.Exit(0)
}
<commit_msg>Don't start webhook controller if driver is empty<commit_after>package main

import (
	"context"
	"flag"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/libopenstorage/stork/drivers/volume"
	_ "github.com/libopenstorage/stork/drivers/volume/aws"
	_ "github.com/libopenstorage/stork/drivers/volume/azure"
	_ "github.com/libopenstorage/stork/drivers/volume/gcp"
	_ "github.com/libopenstorage/stork/drivers/volume/portworx"
	"github.com/libopenstorage/stork/pkg/apis"
	"github.com/libopenstorage/stork/pkg/applicationmanager"
	"github.com/libopenstorage/stork/pkg/clusterdomains"
	"github.com/libopenstorage/stork/pkg/dbg"
	"github.com/libopenstorage/stork/pkg/extender"
	"github.com/libopenstorage/stork/pkg/groupsnapshot"
	"github.com/libopenstorage/stork/pkg/migration"
	"github.com/libopenstorage/stork/pkg/monitor"
	"github.com/libopenstorage/stork/pkg/pvcwatcher"
	"github.com/libopenstorage/stork/pkg/resourcecollector"
	"github.com/libopenstorage/stork/pkg/rule"
	"github.com/libopenstorage/stork/pkg/schedule"
	"github.com/libopenstorage/stork/pkg/snapshot"
	"github.com/libopenstorage/stork/pkg/version"
	"github.com/libopenstorage/stork/pkg/webhookadmission"
	log "github.com/sirupsen/logrus"
	"github.com/urfave/cli"
	api_v1 "k8s.io/api/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	core_v1 "k8s.io/client-go/kubernetes/typed/core/v1"
	_ "k8s.io/client-go/plugin/pkg/client/auth"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"sigs.k8s.io/controller-runtime/pkg/manager"
)

const (
	defaultLockObjectName      = "stork"
	defaultLockObjectNamespace = "kube-system"
	defaultAdminNamespace      = "kube-system"
	eventComponentName         = "stork"
	debugFilePath              = "/var/cores"
)

var ext *extender.Extender
var webhook *webhookadmission.Controller

func main() {
	// Parse empty flags to suppress warnings from the snapshotter which uses
	// glog
	err := flag.CommandLine.Parse([]string{})
	if err != nil {
		log.Warnf("Error parsing flag: %v", err)
	}
	err = flag.Set("logtostderr", "true")
	if err != nil {
		log.Fatalf("Error setting glog flag: %v", err)
	}

	app := cli.NewApp()
	app.Name = "stork"
	app.Usage = "STorage Orchestartor Runtime for Kubernetes (STORK)"
	app.Version = version.Version
	app.Action = run

	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Enable verbose logging",
		},
		cli.StringFlag{
			Name:  "driver,d",
			Usage: "Storage driver name",
		},
		cli.BoolTFlag{
			Name:  "leader-elect",
			Usage: "Enable leader election (default: true)",
		},
		cli.StringFlag{
			Name:  "lock-object-name",
			Usage: "Name for the lock object",
			Value: defaultLockObjectName,
		},
		cli.StringFlag{
			Name:  "lock-object-namespace",
			Usage: "Namespace for the lock object",
			Value: defaultLockObjectNamespace,
		},
		cli.BoolTFlag{
			Name:  "snapshotter",
			Usage: "Enable snapshotter (default: true)",
		},
		cli.BoolTFlag{
			Name:  "extender",
			Usage: "Enable scheduler extender for hyperconvergence (default: true)",
		},
		cli.BoolTFlag{
			Name:  "health-monitor",
			Usage: "Enable health monitoring of the storage driver (default: true)",
		},
		cli.Int64Flag{
			Name:  "health-monitor-interval",
			Value: 120,
			Usage: "The interval in seconds to monitor the health of the storage driver (min: 30)",
		},
		cli.BoolTFlag{
			Name:  "migration-controller",
			Usage: "Start the migration controller (default: true)",
		},
		cli.BoolTFlag{
			Name:  "application-controller",
			Usage: "Start the controllers for managing applications (default: true)",
		},
		cli.StringFlag{
			Name:  "admin-namespace",
			Value: defaultAdminNamespace,
			Usage: "Namespace to be used by a cluster admin which can migrate and backup all other namespaces",
		},
		cli.StringFlag{
			Name:  "migration-admin-namespace",
			Value: defaultAdminNamespace,
			Usage: "Namespace to be used by a cluster admin which can migrate all other namespaces (Deprecated, please use admin-namespace)",
		},
		cli.BoolTFlag{
			Name:  "cluster-domain-controllers",
			Usage: "Start the cluster domain controllers (default: true)",
		},
		cli.BoolTFlag{
			Name:  "pvc-watcher",
			Usage: "Start the controller to monitor PVC creation and deletions (default: true)",
		},
	}

	if err := app.Run(os.Args); err != nil {
		log.Fatalf("Error starting stork: %v", err)
	}
}

func run(c *cli.Context) {
	dbg.Init(c.App.Name, debugFilePath)

	log.Infof("Starting stork version %v", version.Version)
	driverName := c.String("driver")

	verbose := c.Bool("verbose")
	if verbose {
		log.SetLevel(log.DebugLevel)
	}

	config, err := rest.InClusterConfig()
	if err != nil {
		log.Fatalf("Error getting cluster config: %v", err)
	}

	k8sClient, err := clientset.NewForConfig(config)
	if err != nil {
		log.Fatalf("Error getting client, %v", err)
	}

	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartRecordingToSink(&core_v1.EventSinkImpl{Interface: k8sClient.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, api_v1.EventSource{Component: eventComponentName})

	var d volume.Driver
	if driverName != "" {
		log.Infof("Using driver %v", driverName)
		d, err = volume.Get(driverName)
		if err != nil {
			log.Fatalf("Error getting Stork Driver %v: %v", driverName, err)
		}

		if err = d.Init(nil); err != nil {
			log.Fatalf("Error initializing Stork Driver %v: %v", driverName, err)
		}

		if c.Bool("extender") {
			ext = &extender.Extender{
				Driver:   d,
				Recorder: recorder,
			}

			if err = ext.Start(); err != nil {
				log.Fatalf("Error starting scheduler extender: %v", err)
			}
		}
		webhook = &webhookadmission.Controller{
			Driver:   d,
			Recorder: recorder,
		}
		if err := webhook.Start(); err != nil {
			log.Fatalf("error starting webhook controller: %v", err)
		}
	}
	// Create operator-sdk manager that will manage all controllers.
	mgr, err := manager.New(config, manager.Options{})
	if err != nil {
		log.Fatalf("Setup controller manager: %v", err)
	}

	// Setup scheme for all stork resources
	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
		log.Fatalf("Setup scheme failed for stork resources: %v", err)
	}

	runFunc := func(context.Context) {
		runStork(mgr, d, recorder, c)
	}

	if c.BoolT("leader-elect") {
		lockObjectName := c.String("lock-object-name")
		lockObjectNamespace := c.String("lock-object-namespace")
		id, err := os.Hostname()
		if err != nil {
			log.Fatalf("Error getting hostname: %v", err)
		}
		lockConfig := resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: recorder,
		}
		resourceLock, err := resourcelock.New(
			resourcelock.ConfigMapsResourceLock,
			lockObjectNamespace,
			lockObjectName,
			k8sClient.CoreV1(),
			k8sClient.CoordinationV1(),
			lockConfig)
		if err != nil {
			log.Fatalf("Error creating resource lock: %v", err)
		}
		leaderElectionConfig := leaderelection.LeaderElectionConfig{
			Lock:          resourceLock,
			LeaseDuration: 15 * time.Second,
			RenewDeadline: 10 * time.Second,
			RetryPeriod:   2 * time.Second,
			Callbacks: leaderelection.LeaderCallbacks{
				OnStartedLeading: runFunc,
				OnStoppedLeading: func() {
					log.Fatalf("Stork lost master")
				},
			},
		}
		leaderElector, err := leaderelection.NewLeaderElector(leaderElectionConfig)
		if err != nil {
			log.Fatalf("Error creating leader elector: %v", err)
		}
		leaderElector.Run(context.Background())
	} else {
		runFunc(nil)
	}
}

func runStork(mgr manager.Manager, d volume.Driver, recorder record.EventRecorder, c *cli.Context) {
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	if err := rule.Init(); err != nil {
		log.Fatalf("Error initializing rule: %v", err)
	}

	resourceCollector := resourcecollector.ResourceCollector{
		Driver: d,
	}
	if err := resourceCollector.Init(nil); err != nil {
		log.Fatalf("Error initializing ResourceCollector: %v", err)
	}
	adminNamespace := c.String("admin-namespace")
	if adminNamespace == "" {
		adminNamespace = c.String("migration-admin-namespace")
	}

	monitor := &monitor.Monitor{
		Driver:      d,
		IntervalSec: c.Int64("health-monitor-interval"),
	}
	snapshot := &snapshot.Snapshot{
		Driver:   d,
		Recorder: recorder,
	}
	if err := schedule.Init(); err != nil {
		log.Fatalf("Error initializing schedule: %v", err)
	}
	if d != nil {
		if c.Bool("health-monitor") {
			if err := monitor.Start(); err != nil {
				log.Fatalf("Error starting storage monitor: %v", err)
			}
		}

		if c.Bool("snapshotter") {
			if err := snapshot.Start(mgr); err != nil {
				log.Fatalf("Error starting snapshot controller: %v", err)
			}

			groupsnapshotInst := groupsnapshot.GroupSnapshot{
				Driver:   d,
				Recorder: recorder,
			}
			if err := groupsnapshotInst.Init(mgr); err != nil {
				log.Fatalf("Error initializing groupsnapshot controller: %v", err)
			}
		}
		if c.Bool("pvc-watcher") {
			pvcWatcher := pvcwatcher.New(mgr, d, recorder)
			if err := pvcWatcher.Start(mgr); err != nil {
				log.Fatalf("Error starting pvc watcher: %v", err)
			}
		}

		if c.Bool("migration-controller") {
			migration := migration.Migration{
				Driver:            d,
				Recorder:          recorder,
				ResourceCollector: resourceCollector,
			}
			if err := migration.Init(mgr, adminNamespace); err != nil {
				log.Fatalf("Error initializing migration: %v", err)
			}
		}

		if c.Bool("cluster-domain-controllers") {
			clusterDomains := clusterdomains.ClusterDomains{
				Driver:   d,
				Recorder: recorder,
			}
			if err := clusterDomains.Init(mgr); err != nil {
				log.Fatalf("Error initializing cluster domain controllers: %v", err)
			}
		}
	}

	if c.Bool("application-controller") {
		appManager := applicationmanager.ApplicationManager{
			Driver:            d,
			Recorder:          recorder,
			ResourceCollector: resourceCollector,
		}
		if err := appManager.Init(mgr, adminNamespace, signalChan); err != nil {
			log.Fatalf("Error initializing application manager: %v", err)
		}
	}

	stopCh := make(chan struct{}, 1)

	go func() {
		for {
			<-signalChan
			log.Printf("Shutdown signal received, exiting...")
			if c.Bool("extender") {
				if err := ext.Stop(); err != nil {
					log.Warnf("Error stopping extender: %v", err)
				}
			}
			if c.Bool("health-monitor") {
				if err := monitor.Stop(); err != nil {
					log.Warnf("Error stopping monitor: %v", err)
				}
			}
			if c.Bool("snapshotter") {
				if err := snapshot.Stop(); err != nil {
					log.Warnf("Error stopping snapshot controllers: %v", err)
				}
			}
			if err := d.Stop(); err != nil {
				log.Warnf("Error stopping driver: %v", err)
			}
			if err := webhook.Stop(); err != nil {
				log.Warnf("error stopping webhook controller %v", err)
			}

			stopCh <- struct{}{}
		}
	}()

	if err := mgr.Start(stopCh); err != nil {
		log.Fatalf("Controller manager: %v", err)
	}
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"strings"

	"github.com/influxdb/tivan"
	_ "github.com/influxdb/tivan/plugins/all"
)

var fDebug = flag.Bool("debug", false, "show metrics as they're generated to stdout")
var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
var fConfig = flag.String("config", "", "configuration file to load")

func main() {
	flag.Parse()

	var (
		config *tivan.Config
		err    error
	)

	if *fConfig != "" {
		config, err = tivan.LoadConfig(*fConfig)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		config = tivan.DefaultConfig()
	}

	ag, err := tivan.NewAgent(config)
	if err != nil {
		log.Fatal(err)
	}

	if *fDebug {
		ag.Debug = true
	}

	plugins, err := ag.LoadPlugins()
	if err != nil {
		log.Fatal(err)
	}

	if *fTest {
		err = ag.Test()
		if err != nil {
			log.Fatal(err)
		}

		return
	}

	err = ag.Connect()
	if err != nil {
		log.Fatal(err)
	}

	shutdown := make(chan struct{})

	signals := make(chan os.Signal)

	signal.Notify(signals, os.Interrupt)

	go func() {
		<-signals
		close(shutdown)
	}()

	log.Print("InfluxDB Agent running")
	log.Printf("Loaded plugins: %s", strings.Join(plugins, " "))
	if ag.Debug {
		log.Printf("Debug: enabled")
		log.Printf("Agent Config: %#v", ag)
	}

	if config.URL != "" {
		log.Printf("Sending metrics to: %s", config.URL)
		log.Printf("Tags enabled: %v", config.ListTags())
	}

	ag.Run(shutdown)
}
<commit_msg>Add -version option<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strings"

	"github.com/influxdb/tivan"
	_ "github.com/influxdb/tivan/plugins/all"
)

var fDebug = flag.Bool("debug", false, "show metrics as they're generated to stdout")
var fTest = flag.Bool("test", false, "gather metrics, print them out, and exit")
var fConfig = flag.String("config", "", "configuration file to load")
var fVersion = flag.Bool("version", false, "display the version")

var Version = "unreleased"

func main() {
	flag.Parse()

	if *fVersion {
		fmt.Printf("InfluxDB Tivan agent - Version %s\n", Version)
		return
	}

	var (
		config *tivan.Config
		err    error
	)

	if *fConfig != "" {
		config, err = tivan.LoadConfig(*fConfig)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		config = tivan.DefaultConfig()
	}

	ag, err := tivan.NewAgent(config)
	if err != nil {
		log.Fatal(err)
	}

	if *fDebug {
		ag.Debug = true
	}

	plugins, err := ag.LoadPlugins()
	if err != nil {
		log.Fatal(err)
	}

	if *fTest {
		err = ag.Test()
		if err != nil {
			log.Fatal(err)
		}

		return
	}

	err = ag.Connect()
	if err != nil {
		log.Fatal(err)
	}

	shutdown := make(chan struct{})

	signals := make(chan os.Signal)

	signal.Notify(signals, os.Interrupt)

	go func() {
		<-signals
		close(shutdown)
	}()

	log.Print("InfluxDB Agent running")
	log.Printf("Loaded plugins: %s", strings.Join(plugins, " "))
	if ag.Debug {
		log.Printf("Debug: enabled")
		log.Printf("Agent Config: %#v", ag)
	}

	if config.URL != "" {
		log.Printf("Sending metrics to: %s", config.URL)
		log.Printf("Tags enabled: %v", config.ListTags())
	}

	ag.Run(shutdown)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"runtime"

	"github.com/appc/cni/pkg/skel"
	"github.com/appc/cni/pkg/types"
	"github.com/cloudfoundry-incubator/ducati-daemon/client"
	"github.com/cloudfoundry-incubator/ducati-daemon/models"
)

type NetConf struct {
	types.NetConf
	NetworkID     string `json:"network_id"`
	DaemonBaseURL string `json:"daemon_base_url"`
}

func init() {
	runtime.LockOSThread()
}

func loadConf(bytes []byte) (*NetConf, error) {
	n := &NetConf{}

	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %v", err)
	}

	if n.NetworkID == "" {
		return nil, fmt.Errorf(`"network_id" field is required. It identifies the network.`)
	}

	if n.DaemonBaseURL == "" {
		return nil, fmt.Errorf(`"daemon_base_url" field required.`)
	}

	return n, nil
}

func cmdAdd(args *skel.CmdArgs) error {
	if args.ContainerID == "" {
		return errors.New("CNI_CONTAINERID is required")
	}

	netConf, err := loadConf(args.StdinData)
	if err != nil {
		return fmt.Errorf("loading config: %s", err)
	}

	daemonClient := client.New(netConf.DaemonBaseURL, http.DefaultClient)

	ipamResult, err := daemonClient.ContainerUp(netConf.NetworkID, args.ContainerID, models.NetworksSetupContainerPayload{
		Args:               args.Args,
		ContainerNamespace: args.Netns,
		InterfaceName:      args.IfName,
	})
	if err != nil {
		return err
	}

	return ipamResult.Print()
}

func cmdDel(args *skel.CmdArgs) error {
	netConf, err := loadConf(args.StdinData)
	if err != nil {
		return fmt.Errorf("loading config: %s", err)
	}

	daemonClient := client.New(netConf.DaemonBaseURL, http.DefaultClient)

	err = daemonClient.ContainerDown(netConf.NetworkID, args.ContainerID, models.NetworksDeleteContainerPayload{
		ContainerNamespace: args.Netns,
		InterfaceName:      args.IfName,
	})
	if err != nil {
		return err
	}

	return nil
}

func main() {
	skel.PluginMain(cmdAdd, cmdDel)
}
<commit_msg>No longer lock the OS thread<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	"github.com/appc/cni/pkg/skel"
	"github.com/appc/cni/pkg/types"
	"github.com/cloudfoundry-incubator/ducati-daemon/client"
	"github.com/cloudfoundry-incubator/ducati-daemon/models"
)

type NetConf struct {
	types.NetConf
	NetworkID     string `json:"network_id"`
	DaemonBaseURL string `json:"daemon_base_url"`
}

func loadConf(bytes []byte) (*NetConf, error) {
	n := &NetConf{}

	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %v", err)
	}

	if n.NetworkID == "" {
		return nil, fmt.Errorf(`"network_id" field is required. It identifies the network.`)
	}

	if n.DaemonBaseURL == "" {
		return nil, fmt.Errorf(`"daemon_base_url" field required.`)
	}

	return n, nil
}

func cmdAdd(args *skel.CmdArgs) error {
	if args.ContainerID == "" {
		return errors.New("CNI_CONTAINERID is required")
	}

	netConf, err := loadConf(args.StdinData)
	if err != nil {
		return fmt.Errorf("loading config: %s", err)
	}

	daemonClient := client.New(netConf.DaemonBaseURL, http.DefaultClient)

	ipamResult, err := daemonClient.ContainerUp(netConf.NetworkID, args.ContainerID, models.NetworksSetupContainerPayload{
		Args:               args.Args,
		ContainerNamespace: args.Netns,
		InterfaceName:      args.IfName,
	})
	if err != nil {
		return err
	}

	return ipamResult.Print()
}

func cmdDel(args *skel.CmdArgs) error {
	netConf, err := loadConf(args.StdinData)
	if err != nil {
		return fmt.Errorf("loading config: %s", err)
	}

	daemonClient := client.New(netConf.DaemonBaseURL, http.DefaultClient)

	err = daemonClient.ContainerDown(netConf.NetworkID, args.ContainerID, models.NetworksDeleteContainerPayload{
		ContainerNamespace: args.Netns,
		InterfaceName:      args.IfName,
	})
	if err != nil {
		return err
	}

	return nil
}

func main() {
	skel.PluginMain(cmdAdd, cmdDel)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/drosseau/degob"
)

var (
	outFile     = flag.String("ofile", "", "Output file (defaults to stdout)")
	inFile      = flag.String("ifile", "", "Input file (defaults to stdin)")
	truncateOut = flag.Bool("trunc", false, "Truncate output file")
	base64d     = flag.Bool("b64", false, "base64 input")
	base64urld  = flag.Bool("b64url", false, "base64url input")
	noComments  = flag.Bool("nc", false, "don't print additional comments")
)

func errorf(format string, v ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format, v...)
	os.Exit(1)
}

func getWriter() io.WriteCloser {
	if *outFile != "" {
		opts := os.O_WRONLY | os.O_CREATE
		if *truncateOut {
			opts |= os.O_TRUNC
		} else {
			opts |= os.O_APPEND
		}
		f, err := os.OpenFile(*outFile, opts, 0644)
		if err != nil {
			errorf("failed to open %s: %v\n", *outFile, err)
		}
		return f
	}
	return os.Stdout
}

func getReader() io.ReadCloser {
	if *inFile != "" {
		f, err := os.Open(*outFile)
		if err != nil {
			errorf("failed to open %s: %v\n", *inFile, err)
		}
		return f
	}
	return ioutil.NopCloser(os.Stdin)
}

func main() {
	flag.Parse()
	out := getWriter()
	defer out.Close()
	in := getReader()
	defer in.Close()

	if *base64d {
		in = ioutil.NopCloser(base64.NewDecoder(base64.StdEncoding, in))
	} else if *base64urld {
		in = ioutil.NopCloser(base64.NewDecoder(base64.URLEncoding, in))
	}

	dec := degob.NewDecoder(in)
	gobs, err := dec.Decode()
	if err != nil {
		errorf("failed to decode gob: %s\n", err)
	}
	for i, g := range gobs {
		if !*noComments {
			_, err = fmt.Fprintf(out, "// Decoded gob #%d\n\n", i+1)
			if err != nil {
				errorf("error writing to output: %v\n", err)
			}
		}
		if !*noComments {
			_, err := fmt.Fprintln(out, "// Types:")
			if err != nil {
				errorf("%v", err)
			}
		}
		err := g.WriteTypes(out)
		if err != nil {
			errorf("failed to write types: %s\n", err)
		}

		if !*noComments {
			_, err = fmt.Fprintln(out, "// Values:")
			if err != nil {
				errorf("%v", err)
			}
		}
		err = g.WriteValue(out, degob.SingleLine)
		if err != nil {
			errorf("failed to write value: %s\n", err)
		}
		if !*noComments {
			_, err = fmt.Fprintf(out, "\n// End gob #%d\n\n", i+1)
			if err != nil {
				errorf("error writing to output: %v\n", err)
			}
		}
	}
}
<commit_msg>cleaned up main<commit_after>package main

import (
	"encoding/base64"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/drosseau/degob"
)

var (
	outFile     = flag.String("ofile", "", "Output file (defaults to stdout)")
	inFile      = flag.String("ifile", "", "Input file (defaults to stdin)")
	truncateOut = flag.Bool("trunc", false, "Truncate output file")
	base64d     = flag.Bool("b64", false, "base64 input")
	base64urld  = flag.Bool("b64url", false, "base64url input")
	noComments  = flag.Bool("nc", false, "don't print additional comments")
)

func errorf(s string, v ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, s, v...)
	os.Exit(1)
}

func getWriter() io.WriteCloser {
	if *outFile != "" {
		opts := os.O_WRONLY | os.O_CREATE
		if *truncateOut {
			opts |= os.O_TRUNC
		} else {
			opts |= os.O_APPEND
		}
		f, err := os.OpenFile(*outFile, opts, 0644)
		if err != nil {
			errorf("failed to open %s: %v\n", *outFile, err)
		}
		return f
	}
	return os.Stdout
}

func getReader() io.ReadCloser {
	if *inFile != "" {
		f, err := os.Open(*outFile)
		if err != nil {
			errorf("failed to open %s: %v\n", *inFile, err)
		}
		return f
	}
	return ioutil.NopCloser(os.Stdin)
}

type writer struct {
	w   io.Writer
	err error
}

func (w writer) writeStr(s string, v ...interface{}) {
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	_, w.err = fmt.Fprintf(w.w, s, v...)
}

func (w writer) Write(b []byte) (int, error) {
	if w.err != nil {
		errorf("error writing output: %v\n", w.err)
	}
	return w.w.Write(b)
}

func (w writer) writeComment(s string, v ...interface{}) {
	if *noComments {
		return
	}
	w.writeStr(s, v...)
}

func main() {
	flag.Parse()
	out := getWriter()
	defer out.Close()
	in := getReader()
	defer in.Close()

	if *base64d {
		in = ioutil.NopCloser(base64.NewDecoder(base64.StdEncoding, in))
	} else if *base64urld {
		in = ioutil.NopCloser(base64.NewDecoder(base64.URLEncoding, in))
	}

	w := writer{w: out}

	dec := degob.NewDecoder(in)
	gobs, err := dec.Decode()
	if err != nil {
		errorf("failed to decode gob: %s\n", err)
	}
	for i, g := range gobs {
		w.writeComment("// Decoded gob #d\n\n//Types\n", i+1)
		_ = g.WriteTypes(w)
		w.writeComment("// Values:")
		_ = g.WriteValue(w, degob.SingleLine)
		w.writeComment("\n// End gob %d\n\n", i+1)
	}
}
<|endoftext|>


<commit_before>package elasticsearch

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"reflect"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/malice-plugins/go-plugin-utils/utils"
	elastic "gopkg.in/olivere/elastic.v5"
)

const mapping = `
{
"settings":{
  "number_of_shards": 1,
  "number_of_replicas": 0,
	"analysis": {
		"filter": {
			"my_stop": {
				"type":       "stop",
				"stopwords":  "_english_"
				}
			}
		}
},
"mappings":{
  "image":{
    "properties":{
      "id":{
        "type":"keyword"
      },
      "name":{
        "type":"keyword"
      },
      "text":{
        "type":"text",
        "store": true,
				"stopwords": "_english_"
        "fielddata": true
      },
      "text":{
        "type":"text",
        "store": true,
        "fielddata": true
      },
      "path":{
        "type":"keyword"
      },
      "suggest_field":{
        "type":"completion"
      }
    }
  }
}
}`

// ImageMetaData image meta-data object
type ImageMetaData struct {
	ID          string                `json:"id,omitempty"`
	Name        string                `json:"name,omitempty"`
	Title       string                `json:"title,omitempty"`
	Text        string                `json:"text,omitempty"`
	Description string                `json:"description,omitempty"`
	Path        string                `json:"path,omitempty"`
	Suggest     *elastic.SuggestField `json:"suggest_field,omitempty"`
}

// StartElasticsearch starts the elasticsearch database
func StartElasticsearch() error {
	output, err := utils.RunCommand(context.Background(), "/elastic-entrypoint.sh")
	log.Info(output)
	return err
}

// TestConnection tests the ElasticSearch connection
func TestConnection() (bool, error) {

	var err error

	client, err := elastic.NewClient()
	if err != nil {
		return false, err
	}

	// Ping the Elasticsearch server to get e.g. the version number
	log.Debug("attempting to PING elasticsearch")
	info, code, err := client.Ping("http://127.0.0.1:9200").Do(context.Background())
	if err != nil {
		return false, err
	}

	log.WithFields(log.Fields{
		"code":    code,
		"cluster": info.ClusterName,
		"version": info.Version.Number,
	}).Debug("ElasticSearch connection successful.")

	if code == 200 {
		return true, err
	}
	return false, err
}

// WaitForConnection waits for connection to Elasticsearch to be ready
func WaitForConnection(ctx context.Context, timeout int) error {

	var ready bool
	var connErr error
	secondsWaited := 0

	connCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
	defer cancel()

	log.Debug("===> trying to connect to elasticsearch")
	for {
		// Try to connect to Elasticsearch
		select {
		case <-connCtx.Done():
			log.WithFields(log.Fields{"timeout": timeout}).Error("connecting to elasticsearch timed out")
			return connErr
		default:
			ready, connErr = TestConnection()
			if ready {
				log.Infof("Elasticsearch came online after %d seconds", secondsWaited)
				return connErr
			}
			secondsWaited++
			time.Sleep(1 * time.Second)
		}
	}
}

// SearchImages searches elasticsearch for images
func SearchImages(query string) error {
	ctx := context.Background()

	client, err := elastic.NewClient()
	if err != nil {
		return err
	}

	// Search with a term query
	termQuery := elastic.NewQueryStringQuery(query)
	searchResult, err := client.Search().
		Index("scifgif").    // search in index "twitter"
		Query(termQuery).    // specify the query
		Sort("title", true). // sort by "user" field, ascending
		From(0).Size(10).    // take documents 0-9
		Pretty(true).        // pretty print request and response JSON
		Do(ctx)              // execute
	if err != nil {
		return err
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// Each is a convenience function that iterates over hits in a search result.
	// It makes sure you don't need to check for nil values in the response.
	// However, it ignores errors in serialization. If you want full control
	// over iterating the hits, see below.
	var ityp ImageMetaData
	for _, item := range searchResult.Each(reflect.TypeOf(ityp)) {
		if i, ok := item.(ImageMetaData); ok {
			fmt.Printf("Image  %s: %s\n", i.Name, i.Path)
		}
	}
	// TotalHits is another convenience function that works even when something goes wrong.
	fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())

	// Here's how you iterate through results with full control over each step.
	if searchResult.Hits.TotalHits > 0 {
		fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var i ImageMetaData
			err := json.Unmarshal(*hit.Source, &i)
			if err != nil {
				return err
			}

			// Work with image
			fmt.Printf("Image  %s: %s\n", i.Name, i.Path)
		}
	} else {
		// No hits
		fmt.Print("Found no tweets\n")
	}
	return nil
}

// WriteImageToDatabase upserts image metadata into Database
func WriteImageToDatabase(image ImageMetaData) error {
	var err error
	ctx := context.Background()

	client, err := elastic.NewClient()
	if err != nil {
		return err
	}

	// Use the IndexExists service to check if a specified index exists.
	exists, err := client.IndexExists("scifgif").Do(ctx)
	if err != nil {
		// Handle error
		panic(err)
	}
	if !exists {
		// Create a new index.
		createIndex, err := client.CreateIndex("scifgif").BodyString(mapping).Do(ctx)
		if err != nil {
			// Handle error
			panic(err)
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
		}
	}

	put, err := client.Index().
		Index("scifgif").
		Type("image").
		OpType("index").
		BodyJson(image).
		Do(ctx)
	if err != nil {
		// Handle error
		panic(err)
	}

	log.WithFields(log.Fields{
		"id":    put.Id,
		"index": put.Index,
		"type":  put.Type,
	}).Debug("Indexed image.")

	// Flush to make sure the documents got written.
	_, err = client.Flush().Index("scifgif").Do(ctx)
	if err != nil {
		panic(err)
	}

	return err
}

// DownloadImage downloads image to filepath
func DownloadImage(url, filepath string) {
	// Create the file
	out, err := os.Create(filepath)
	if err != nil {
		log.Error(err)
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	log.WithFields(log.Fields{
		"status":   resp.Status,
		"size":     resp.ContentLength,
		"filepath": filepath,
	}).Debug("downloading file")

	// Writer the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		log.Error(err)
	}
}
<commit_msg>fix elasticsearch yet again<commit_after>package elasticsearch

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"reflect"
	"time"

	log "github.com/Sirupsen/logrus"
	elastic "gopkg.in/olivere/elastic.v5"
)

const mapping = `
{
"settings":{
  "number_of_shards": 1,
  "number_of_replicas": 0
},
"mappings":{
  "image":{
    "properties":{
      "id":{
        "type":"keyword"
      },
      "name":{
        "type":"keyword"
      },
      "text":{
        "type":"text",
        "store": true,
        "fielddata": true
      },
      "text":{
        "type":"text",
        "store": true,
        "fielddata": true
      },
      "path":{
        "type":"keyword"
      },
      "suggest_field":{
        "type":"completion"
      }
    }
  }
}
}`

// ImageMetaData image meta-data object
type ImageMetaData struct {
	ID          string                `json:"id,omitempty"`
	Name        string                `json:"name,omitempty"`
	Title       string                `json:"title,omitempty"`
	Text        string                `json:"text,omitempty"`
	Description string                `json:"description,omitempty"`
	Path        string                `json:"path,omitempty"`
	Suggest     *elastic.SuggestField `json:"suggest_field,omitempty"`
}

// StartElasticsearch starts the elasticsearch database
func StartElasticsearch() error {
	// _, err := utils.RunCommand(context.Background(), "/elastic-entrypoint.sh", "elasticsearch")
	// // log.Info(output)
	// return err
	cmd := exec.Command("/elastic-entrypoint.sh", "elasticsearch")
	cmd.Start()
	return nil
}

// TestConnection tests the ElasticSearch connection
func TestConnection() (bool, error) {

	var err error

	client, err := elastic.NewClient()
	if err != nil {
		return false, err
	}

	// Ping the Elasticsearch server to get e.g. the version number
	log.Debug("attempting to PING elasticsearch")
	info, code, err := client.Ping("http://127.0.0.1:9200").Do(context.Background())
	if err != nil {
		return false, err
	}

	log.WithFields(log.Fields{
		"code":    code,
		"cluster": info.ClusterName,
		"version": info.Version.Number,
	}).Debug("elasticSearch connection successful.")

	if code == 200 {
		return true, err
	}
	return false, err
}

// WaitForConnection waits for connection to Elasticsearch to be ready
func WaitForConnection(ctx context.Context, timeout int) error {

	var ready bool
	var connErr error
	secondsWaited := 0

	connCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
	defer cancel()

	log.Debug("===> trying to connect to elasticsearch")
	for {
		// Try to connect to Elasticsearch
		select {
		case <-connCtx.Done():
			log.WithFields(log.Fields{"timeout": timeout}).Error("connecting to elasticsearch timed out")
			return connErr
		default:
			ready, connErr = TestConnection()
			if ready {
				log.Infof("elasticsearch came online after %d seconds", secondsWaited)
				return connErr
			}
			secondsWaited++
			time.Sleep(1 * time.Second)
		}
	}
}

// SearchImages searches elasticsearch for images
func SearchImages(query string) error {
	ctx := context.Background()

	client, err := elastic.NewClient()
	if err != nil {
		return err
	}

	// Search with a term query
	termQuery := elastic.NewQueryStringQuery(query)
	searchResult, err := client.Search().
		Index("scifgif").    // search in index "twitter"
		Query(termQuery).    // specify the query
		Sort("title", true). // sort by "user" field, ascending
		From(0).Size(10).    // take documents 0-9
		Pretty(true).        // pretty print request and response JSON
		Do(ctx)              // execute
	if err != nil {
		return err
	}

	// searchResult is of type SearchResult and returns hits, suggestions,
	// and all kinds of other information from Elasticsearch.
	fmt.Printf("Query took %d milliseconds\n", searchResult.TookInMillis)

	// Each is a convenience function that iterates over hits in a search result.
	// It makes sure you don't need to check for nil values in the response.
	// However, it ignores errors in serialization. If you want full control
	// over iterating the hits, see below.
	var ityp ImageMetaData
	for _, item := range searchResult.Each(reflect.TypeOf(ityp)) {
		if i, ok := item.(ImageMetaData); ok {
			fmt.Printf("Image  %s: %s\n", i.Name, i.Path)
		}
	}
	// TotalHits is another convenience function that works even when something goes wrong.
	fmt.Printf("Found a total of %d tweets\n", searchResult.TotalHits())

	// Here's how you iterate through results with full control over each step.
	if searchResult.Hits.TotalHits > 0 {
		fmt.Printf("Found a total of %d tweets\n", searchResult.Hits.TotalHits)

		// Iterate through results
		for _, hit := range searchResult.Hits.Hits {
			// hit.Index contains the name of the index

			// Deserialize hit.Source into a Tweet (could also be just a map[string]interface{}).
			var i ImageMetaData
			err := json.Unmarshal(*hit.Source, &i)
			if err != nil {
				return err
			}

			// Work with image
			fmt.Printf("Image  %s: %s\n", i.Name, i.Path)
		}
	} else {
		// No hits
		fmt.Print("Found no tweets\n")
	}
	return nil
}

// WriteImageToDatabase upserts image metadata into Database
func WriteImageToDatabase(image ImageMetaData) error {
	var err error
	ctx := context.Background()

	client, err := elastic.NewClient()
	if err != nil {
		return err
	}

	// Use the IndexExists service to check if a specified index exists.
	exists, err := client.IndexExists("scifgif").Do(ctx)
	if err != nil {
		// Handle error
		panic(err)
	}
	if !exists {
		// Create a new index.
		createIndex, err := client.CreateIndex("scifgif").BodyString(mapping).Do(ctx)
		if err != nil {
			// Handle error
			panic(err)
		}
		if !createIndex.Acknowledged {
			// Not acknowledged
		}
	}

	put, err := client.Index().
		Index("scifgif").
		Type("image").
		OpType("index").
		BodyJson(image).
		Do(ctx)
	if err != nil {
		// Handle error
		panic(err)
	}

	log.WithFields(log.Fields{
		"id":    put.Id,
		"index": put.Index,
		"type":  put.Type,
	}).Debug("indexed image")

	// Flush to make sure the documents got written.
	_, err = client.Flush().Index("scifgif").Do(ctx)
	if err != nil {
		panic(err)
	}

	return err
}

// DownloadImage downloads image to filepath
func DownloadImage(url, filepath string) {
	// Create the file
	out, err := os.Create(filepath)
	if err != nil {
		log.Error(err)
	}
	defer out.Close()

	resp, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	log.WithFields(log.Fields{
		"status":   resp.Status,
		"size":     resp.ContentLength,
		"filepath": filepath,
	}).Debug("downloading file")

	// Writer the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		log.Error(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

// VerticalPodAutoscalerVersion is the version of VPA.
const VerticalPodAutoscalerVersion = "0.5.0"
<commit_msg>VPA version 0.6.0<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

// VerticalPodAutoscalerVersion is the version of VPA.
const VerticalPodAutoscalerVersion = "0.6.0"
<|endoftext|>


<commit_before>package testing

import (
	. "github.com/onsi/gomega"
	"github.com/satori/go.uuid"
	. "github.com/talbright/go-curator"
	"github.com/talbright/go-zookeeper/zk"

	"fmt"
	"time"
)

//TODO grab via dotenv/env
var zkHosts = []string{"127.0.0.1:2181"}
var zkSessionTimeout = time.Second * 20
var zkConnectionTimeout = time.Second * 20

/*
Collects go-zookeeper events.
*/
func ZkCollectEvents(count int, evntChn <-chan zk.Event) []zk.Event {
	var event zk.Event
	events := make([]zk.Event, 0)
	for i := 0; i < count; i++ {
		EventuallyWithOffset(1, evntChn).Should(Receive(&event))
		events = append(events, event)
	}
	return events
}

/*
Create paths (without children) in zk.
*/
func ZkCreatePaths(client *Client, paths ...string) {
	for _, path := range paths {
		if err := client.CreatePath(path, zk.NoData, zk.WorldACLPermAll); err != nil {
			panic(fmt.Sprintf("unable to create zk path: \"%s\"", path))
		}
	}
}

/*
Delete paths (without children) in zk.
*/
func ZkDeletePaths(client *Client, paths ...string) {
	for _, path := range paths {
		if err := client.Delete(path, -1); err != nil {
			panic(fmt.Sprintf("unable to delete zk path %s", path))
		}
	}
}

/*
Create client connection to zk with timeout.
*/
func ZkConnect() (client *Client) {
	client = NewClient()
	settings := &Settings{
		ZkServers:               zkHosts,
		ZkSessionTimeout:        zkSessionTimeout,
		ZkWaitForSessionTimeout: zkConnectionTimeout,
		ZkWaitForSession:        true,
	}
	if _, err := client.Connect(settings, zk.WithLogger(&NullLogger{})); err != nil {
		panic(fmt.Sprintf("Unable to connect to zk: %s", err))
	}
	return client
}

/*
Generates a unique string suitable for using as a node name.
*/
func ZkUniqueNodeName() string {
	return uuid.NewV4().String()
}
<commit_msg>Allow conn params to be passed to testing helper<commit_after>package testing

import (
	. "github.com/onsi/gomega"
	"github.com/satori/go.uuid"
	. "github.com/talbright/go-curator"
	"github.com/talbright/go-zookeeper/zk"

	"fmt"
	"time"
)

/*
Collects go-zookeeper events.
*/
func ZkCollectEvents(count int, evntChn <-chan zk.Event) []zk.Event {
	var event zk.Event
	events := make([]zk.Event, 0)
	for i := 0; i < count; i++ {
		EventuallyWithOffset(1, evntChn).Should(Receive(&event))
		events = append(events, event)
	}
	return events
}

/*
Create paths (without children) in zk.
*/
func ZkCreatePaths(client *Client, paths ...string) {
	for _, path := range paths {
		if err := client.CreatePath(path, zk.NoData, zk.WorldACLPermAll); err != nil {
			panic(fmt.Sprintf("unable to create zk path: \"%s\"", path))
		}
	}
}

/*
Delete paths (without children) in zk.
*/
func ZkDeletePaths(client *Client, paths ...string) {
	for _, path := range paths {
		if err := client.Delete(path, -1); err != nil {
			panic(fmt.Sprintf("unable to delete zk path %s", path))
		}
	}
}

/*
Create client connection to zk with timeout.
*/
func ZkConnect(zkHosts []string, zkSessionTimeout time.Duration, zkWaitForSessionTimeout time.Duration) (client *Client) {
	client = NewClient()
	settings := &Settings{
		ZkServers:               zkHosts,
		ZkSessionTimeout:        zkSessionTimeout,
		ZkWaitForSessionTimeout: zkWaitForSessionTimeout,
		ZkWaitForSession:        true,
	}
	if _, err := client.Connect(settings, zk.WithLogger(&NullLogger{})); err != nil {
		panic(fmt.Sprintf("Unable to connect to zk: %s", err))
	}
	return client
}

/*
Generates a unique string suitable for using as a node name.
*/
func ZkUniqueNodeName() string {
	return uuid.NewV4().String()
}
<|endoftext|>