<commit_before>package dao

import (
	"fmt"

	"labix.org/v2/mgo"
)

//------------------------------------------------------------
// DB Mongo connector
//------------------------------------------------------------

const (
	// Parameter keys
	KEY_DB_SERVER_ID = "id"
	KEY_DB_SERVER    = "server"
	//KEY_DB_PORT       = "port"
	KEY_DB_OPTIONS = "options"
	KEY_DB_MODE    = "mode"
	KEY_DB_LOG     = "log"
)

//------------------------------------------------------------
// Vars
//------------------------------------------------------------

var (
	_dbServers = map[string]*mgo.Session{}
)

//------------------------------------------------------------
// Implementation
//------------------------------------------------------------

// Session getter returns a copy of the master session.
func getSession(servername, dbname, collname string) (sess *mgo.Session, coll *mgo.Collection) {

	if masterSess, ok := _dbServers[servername]; ok {
		sess = masterSess.Copy()
		coll = sess.DB(dbname).C(collname)
	}
	return
}

// Creates master DB session to given server and stores it for future reference.
// Use getSession() to get a connection to a specific DB/collection.
func connectToServer(params map[string]string) (err error) {

	// Server ID, ensure at least default
	id := params[KEY_DB_SERVER_ID]
	if id == "" {
		id = "_"
	}

	// Is server with this ID already connected?
	if _, ok := _dbServers[id]; ok {
		err = fmt.Errorf("Mongo DB server already connected: %v", id)
		return
	}

	// Params
	server := params[KEY_DB_SERVER]
	options := params[KEY_DB_OPTIONS]
	mode := params[KEY_DB_MODE]
	log := params[KEY_DB_LOG]

	// Validate
	if server == "" {
		err = fmt.Errorf("Mongo DB init aborted due to empty 'server' key")
		return
	}

	// Build URL
	url := "mongodb://" + server
	if options != "" {
		url += "/?" + options
	}

	fmt.Println("MongoDB URL      :", url)

	// Enable logging ?
	if log != "" {
		mgo.SetDebug(true)
		mgo.SetLogger(&MongoLogger{})
	}

	// Dial server
	var sess *mgo.Session
	if sess, err = mgo.Dial(url); err != nil {
		err = fmt.Errorf("Error connecting to MongoDB: %v", err)
		return
	}

	// Set Mongo mode
	switch mode {
	case "eventual":
		sess.SetMode(mgo.Eventual, true)
		fmt.Println("MongoDB Mode     : Eventual")

	case "monotonic":
		sess.SetMode(mgo.Monotonic, true)
		fmt.Println("MongoDB Mode     : Monotonic")

	case "strong":
		sess.SetMode(mgo.Strong, true)
		fmt.Println("MongoDB Mode     : Strong")

	default:
		fmt.Println("MongoDB Mode     : Undefined")
	}

	// Remember connection
	_dbServers[id] = sess
	return
}

// Disconnects all servers.
func disconnectServers() {
	for _, sess := range _dbServers {
		sess.Close()
	}
	_dbServers = map[string]*mgo.Session{}
}

//------------------------------------------------------------
// Collections operations
//------------------------------------------------------------

// Lists all collections present.
func GetCollections(sess *mgo.Session, dbname string) (collnames []string) {
	var err error
	collnames, err = sess.DB(dbname).CollectionNames()
	if err != nil {
		panic(fmt.Sprintf("MongoDB cannot list collection names for '%v' due to error: %v", dbname, err.Error()))
	}
	return
}

// Checks if given collection exists.
func IsCollectionExists(sess *mgo.Session, dbname, collname string) bool {
	colls := GetCollections(sess, dbname)
	for _, name := range colls {
		if name == collname {
			return true
		}
	}
	return false
}

//------------------------------------------------------------
// Optional Mgo logger
//------------------------------------------------------------

// Logger for Mgo
type MongoLogger struct{}

// Mgo MongoLogger interface implementation.
func (l *MongoLogger) Output(calldepth int, s string) error {
	fmt.Println("[MGO]", calldepth, s)
	return nil
}
<commit_msg>updated GetCollections<commit_after>package dao

import (
	"errors"
	"fmt"

	"labix.org/v2/mgo"
)

//------------------------------------------------------------
// DB Mongo connector
//------------------------------------------------------------

const (
	// Parameter keys
	KEY_DB_SERVER_ID = "id"
	KEY_DB_SERVER    = "server"
	//KEY_DB_PORT       = "port"
	KEY_DB_OPTIONS = "options"
	KEY_DB_MODE    = "mode"
	KEY_DB_LOG     = "log"
)

//------------------------------------------------------------
// Vars
//------------------------------------------------------------

var (
	_dbServers = map[string]*mgo.Session{}
)

//------------------------------------------------------------
// Implementation
//------------------------------------------------------------

// Session getter returns a copy of the master session.
func getSession(servername, dbname, collname string) (sess *mgo.Session, coll *mgo.Collection) {

	if masterSess, ok := _dbServers[servername]; ok {
		sess = masterSess.Copy()
		coll = sess.DB(dbname).C(collname)
	}
	return
}

// Session getter returns a copy of the master session.
func getDb(servername, dbname string) (db *mgo.Database) {

	if masterSess, ok := _dbServers[servername]; ok {
		db = masterSess.Copy().DB(dbname)
	}
	return
}

// Creates master DB session to given server and stores it for future reference.
// Use getSession() to get a connection to a specific DB/collection.
func connectToServer(params map[string]string) (err error) {

	// Server ID, ensure at least default
	id := params[KEY_DB_SERVER_ID]
	if id == "" {
		id = "_"
	}

	// Is server with this ID already connected?
	if _, ok := _dbServers[id]; ok {
		err = fmt.Errorf("Mongo DB server already connected: %v", id)
		return
	}

	// Params
	server := params[KEY_DB_SERVER]
	options := params[KEY_DB_OPTIONS]
	mode := params[KEY_DB_MODE]
	log := params[KEY_DB_LOG]

	// Validate
	if server == "" {
		err = fmt.Errorf("Mongo DB init aborted due to empty 'server' key")
		return
	}

	// Build URL
	url := "mongodb://" + server
	if options != "" {
		url += "/?" + options
	}

	fmt.Println("[dao] MongoDB URL      :", url)

	// Enable logging ?
	if log != "" {
		mgo.SetDebug(true)
		mgo.SetLogger(&MongoLogger{})
	}

	// Dial server
	var sess *mgo.Session
	if sess, err = mgo.Dial(url); err != nil {
		err = fmt.Errorf("Error connecting to MongoDB: %v", err)
		return
	}

	// Set Mongo mode
	switch mode {
	case "eventual":
		sess.SetMode(mgo.Eventual, true)
		fmt.Println("[dao] MongoDB Mode     : Eventual")

	case "monotonic":
		sess.SetMode(mgo.Monotonic, true)
		fmt.Println("[dao] MongoDB Mode     : Monotonic")

	case "strong":
		sess.SetMode(mgo.Strong, true)
		fmt.Println("[dao] MongoDB Mode     : Strong")

	default:
		fmt.Println("[dao] MongoDB Mode     : Undefined")
	}

	// Remember connection
	_dbServers[id] = sess
	return
}

// Disconnects all servers.
func disconnectServers() {
	for _, sess := range _dbServers {
		sess.Close()
	}
	_dbServers = map[string]*mgo.Session{}
}

//------------------------------------------------------------
// Collections operations
//------------------------------------------------------------

// Lists all collections present.
func GetCollections(servername, dbname string) (collnames []string, err error) {
	if db := getDb(servername, dbname); db != nil {
		collnames, err = db.CollectionNames()
	} else {
		err = errors.New("Cannot acquire Mongo DB")
	}
	return
}

// Checks if given collection exists.
func IsCollectionExists(servername, dbname, collname string) (exists bool, err error) {
	var colls []string
	colls, err = GetCollections(servername, dbname)
	if err != nil {
		return
	}
	for _, name := range colls {
		if name == collname {
			exists = true
			return
		}
	}
	return
}

//------------------------------------------------------------
// Optional Mgo logger
//------------------------------------------------------------

// Logger for Mgo
type MongoLogger struct{}

// Mgo MongoLogger interface implementation.
func (l *MongoLogger) Output(calldepth int, s string) error {
	fmt.Println("[MGO]", calldepth, s)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"../cmd"
	"fmt"
	"github.com/spf13/cobra/doc"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

var version = "2.0.0"

func usage() {
	fmt.Println("usage: genman <output_path>")
}

func main() {
	destinationPath, _ := os.Getwd()
	if len(os.Args) > 2 {
		usage()
		os.Exit(1)
	}
	if len(os.Args) == 2 {
		destinationPath = os.Args[1]
	}

	header := &doc.GenManHeader{
		Title:   "GIT-USER",
		Section: "1",
		Manual:  "Git Manual",
		Source:  fmt.Sprintf("git-user %s", version),
	}
	cmd.RootCmd.DisableAutoGenTag = true
	fmt.Printf("Generating documentation for git-user\n")
	err := doc.GenManTree(cmd.RootCmd, header, destinationPath)

	if err != nil {
		fmt.Printf("%s\n", err)
		os.Exit(2)
	}

	//Remove all of the double blank lines from output docs
	err = filepath.Walk(destinationPath, func(path string, f os.FileInfo, err error) error {
		stripFile(path)
		return nil
	})

	if err != nil {
		fmt.Printf("Could not clean up all the files\n")
		fmt.Printf("%s", err)
	}
}

func stripFile(path string) error {
	input, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	regex, err := regexp.Compile("\n{2,}")
	if err != nil {
		return err
	}
	output := regex.ReplaceAllString(string(input), "\n")

	err = ioutil.WriteFile(path, []byte(output), 0644)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>fix genman app<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"

	"github.com/gesquive/git-user/cmd"
	"github.com/spf13/cobra/doc"
)

var version = "2.0.0"

func usage() {
	fmt.Println("usage: genman <output_path>")
}

func main() {
	destinationPath, _ := os.Getwd()
	if len(os.Args) > 2 {
		usage()
		os.Exit(1)
	}
	if len(os.Args) == 2 {
		destinationPath = os.Args[1]
	}

	header := &doc.GenManHeader{
		Title:   "GIT-USER",
		Section: "1",
		Manual:  "Git Manual",
		Source:  fmt.Sprintf("git-user %s", version),
	}
	cmd.RootCmd.DisableAutoGenTag = true
	fmt.Printf("Generating documentation for git-user\n")
	err := doc.GenManTree(cmd.RootCmd, header, destinationPath)

	if err != nil {
		fmt.Printf("%s\n", err)
		os.Exit(2)
	}

	//Remove all of the double blank lines from output docs
	err = filepath.Walk(destinationPath, func(path string, f os.FileInfo, err error) error {
		stripFile(path)
		return nil
	})

	if err != nil {
		fmt.Printf("Could not clean up all the files\n")
		fmt.Printf("%s", err)
	}
}

func stripFile(path string) error {
	input, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	regex, err := regexp.Compile("\n{2,}")
	if err != nil {
		return err
	}
	output := regex.ReplaceAllString(string(input), "\n")

	err = ioutil.WriteFile(path, []byte(output), 0644)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/client_golang/prometheus/promhttp"

	gocache "github.com/patrickmn/go-cache"
	macaron "gopkg.in/macaron.v1"

	"github.com/grafana/grafana/pkg/api/live"
	httpstatic "github.com/grafana/grafana/pkg/api/static"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/setting"
)

type HttpServer struct {
	log           log.Logger
	macaron       *macaron.Macaron
	context       context.Context
	streamManager *live.StreamManager
	cache         *gocache.Cache

	httpSrv *http.Server
}

func NewHttpServer() *HttpServer {
	return &HttpServer{
		log:   log.New("http.server"),
		cache: gocache.New(5*time.Minute, 10*time.Minute),
	}
}

func (hs *HttpServer) Start(ctx context.Context) error {
	var err error

	hs.context = ctx
	hs.streamManager = live.NewStreamManager()
	hs.macaron = hs.newMacaron()
	hs.registerRoutes()

	hs.streamManager.Run(ctx)

	listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
	hs.log.Info("Initializing HTTP Server", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath)

	hs.httpSrv = &http.Server{Addr: listenAddr, Handler: hs.macaron}
	switch setting.Protocol {
	case setting.HTTP:
		err = hs.httpSrv.ListenAndServe()
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.HTTPS:
		err = hs.listenAndServeTLS(setting.CertFile, setting.KeyFile)
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.SOCKET:
		ln, err := net.Listen("unix", setting.SocketPath)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}

		err = hs.httpSrv.Serve(ln)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	default:
		hs.log.Error("Invalid protocol", "protocol", setting.Protocol)
		err = errors.New("Invalid Protocol")
	}

	return err
}

func (hs *HttpServer) Shutdown(ctx context.Context) error {
	err := hs.httpSrv.Shutdown(ctx)
	hs.log.Info("stopped http server")
	return err
}

func (hs *HttpServer) listenAndServeTLS(certfile, keyfile string) error {
	if certfile == "" {
		return fmt.Errorf("cert_file cannot be empty when using HTTPS")
	}

	if keyfile == "" {
		return fmt.Errorf("cert_key cannot be empty when using HTTPS")
	}

	if _, err := os.Stat(setting.CertFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL cert_file at %v`, setting.CertFile)
	}

	if _, err := os.Stat(setting.KeyFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL key_file at %v`, setting.KeyFile)
	}

	tlsCfg := &tls.Config{
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true,
		CipherSuites: []uint16{
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}

	hs.httpSrv.TLSConfig = tlsCfg
	hs.httpSrv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)

	return hs.httpSrv.ListenAndServeTLS(setting.CertFile, setting.KeyFile)
}

func (hs *HttpServer) newMacaron() *macaron.Macaron {
	macaron.Env = setting.Env
	m := macaron.New()

	m.Use(middleware.Logger())

	if setting.EnableGzip {
		m.Use(middleware.Gziper())
	}

	m.Use(middleware.Recovery())

	for _, route := range plugins.StaticRoutes {
		pluginRoute := path.Join("/public/plugins/", route.PluginId)
		hs.log.Debug("Plugins: Adding route", "route", pluginRoute, "dir", route.Directory)
		hs.mapStatic(m, route.Directory, "", pluginRoute)
	}

	hs.mapStatic(m, setting.StaticRootPath, "", "public")
	hs.mapStatic(m, setting.StaticRootPath, "robots.txt", "robots.txt")

	m.Use(macaron.Renderer(macaron.RenderOptions{
		Directory:  path.Join(setting.StaticRootPath, "views"),
		IndentJSON: macaron.Env != macaron.PROD,
		Delims:     macaron.Delims{Left: "[[", Right: "]]"},
	}))

	m.Use(hs.healthHandler)
	m.Use(hs.metricsEndpoint)
	m.Use(middleware.GetContextHandler())
	m.Use(middleware.Sessioner(&setting.SessionOptions))
	m.Use(middleware.OrgRedirect())

	// needs to be after context handler
	if setting.EnforceDomain {
		m.Use(middleware.ValidateHostHeader(setting.Domain))
	}

	m.Use(middleware.AddDefaultResponseHeaders())

	return m
}

func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/metrics" {
		return
	}

	promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}).
		ServeHTTP(ctx.Resp, ctx.Req.Request)
}

func (hs *HttpServer) healthHandler(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/api/health" {
		return
	}

	data := simplejson.New()
	data.Set("database", "ok")
	data.Set("version", setting.BuildVersion)
	data.Set("commit", setting.BuildCommit)

	if err := bus.Dispatch(&models.GetDBHealthQuery{}); err != nil {
		data.Set("database", "failing")
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(503)
	} else {
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(200)
	}

	dataBytes, _ := data.EncodePretty()
	ctx.Resp.Write(dataBytes)
}

func (hs *HttpServer) mapStatic(m *macaron.Macaron, rootDir string, dir string, prefix string) {
	headers := func(c *macaron.Context) {
		c.Resp.Header().Set("Cache-Control", "public, max-age=3600")
	}

	if setting.Env == setting.DEV {
		headers = func(c *macaron.Context) {
			c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache")
		}
	}

	m.Use(httpstatic.Static(
		path.Join(rootDir, dir),
		httpstatic.StaticOptions{
			SkipLogging: true,
			Prefix:      prefix,
			AddHeaders:  headers,
		},
	))
}
<commit_msg>allows head requests for /api/health endpoint<commit_after>package api

import (
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/prometheus/client_golang/prometheus/promhttp"

	gocache "github.com/patrickmn/go-cache"
	macaron "gopkg.in/macaron.v1"

	"github.com/grafana/grafana/pkg/api/live"
	httpstatic "github.com/grafana/grafana/pkg/api/static"
	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/components/simplejson"
	"github.com/grafana/grafana/pkg/log"
	"github.com/grafana/grafana/pkg/middleware"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/setting"
)

type HttpServer struct {
	log           log.Logger
	macaron       *macaron.Macaron
	context       context.Context
	streamManager *live.StreamManager
	cache         *gocache.Cache

	httpSrv *http.Server
}

func NewHttpServer() *HttpServer {
	return &HttpServer{
		log:   log.New("http.server"),
		cache: gocache.New(5*time.Minute, 10*time.Minute),
	}
}

func (hs *HttpServer) Start(ctx context.Context) error {
	var err error

	hs.context = ctx
	hs.streamManager = live.NewStreamManager()
	hs.macaron = hs.newMacaron()
	hs.registerRoutes()

	hs.streamManager.Run(ctx)

	listenAddr := fmt.Sprintf("%s:%s", setting.HttpAddr, setting.HttpPort)
	hs.log.Info("Initializing HTTP Server", "address", listenAddr, "protocol", setting.Protocol, "subUrl", setting.AppSubUrl, "socket", setting.SocketPath)

	hs.httpSrv = &http.Server{Addr: listenAddr, Handler: hs.macaron}
	switch setting.Protocol {
	case setting.HTTP:
		err = hs.httpSrv.ListenAndServe()
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.HTTPS:
		err = hs.listenAndServeTLS(setting.CertFile, setting.KeyFile)
		if err == http.ErrServerClosed {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	case setting.SOCKET:
		ln, err := net.Listen("unix", setting.SocketPath)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}

		err = hs.httpSrv.Serve(ln)
		if err != nil {
			hs.log.Debug("server was shutdown gracefully")
			return nil
		}
	default:
		hs.log.Error("Invalid protocol", "protocol", setting.Protocol)
		err = errors.New("Invalid Protocol")
	}

	return err
}

func (hs *HttpServer) Shutdown(ctx context.Context) error {
	err := hs.httpSrv.Shutdown(ctx)
	hs.log.Info("stopped http server")
	return err
}

func (hs *HttpServer) listenAndServeTLS(certfile, keyfile string) error {
	if certfile == "" {
		return fmt.Errorf("cert_file cannot be empty when using HTTPS")
	}

	if keyfile == "" {
		return fmt.Errorf("cert_key cannot be empty when using HTTPS")
	}

	if _, err := os.Stat(setting.CertFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL cert_file at %v`, setting.CertFile)
	}

	if _, err := os.Stat(setting.KeyFile); os.IsNotExist(err) {
		return fmt.Errorf(`Cannot find SSL key_file at %v`, setting.KeyFile)
	}

	tlsCfg := &tls.Config{
		MinVersion:               tls.VersionTLS12,
		PreferServerCipherSuites: true,
		CipherSuites: []uint16{
			tls.TLS_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}

	hs.httpSrv.TLSConfig = tlsCfg
	hs.httpSrv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)

	return hs.httpSrv.ListenAndServeTLS(setting.CertFile, setting.KeyFile)
}

func (hs *HttpServer) newMacaron() *macaron.Macaron {
	macaron.Env = setting.Env
	m := macaron.New()

	m.Use(middleware.Logger())

	if setting.EnableGzip {
		m.Use(middleware.Gziper())
	}

	m.Use(middleware.Recovery())

	for _, route := range plugins.StaticRoutes {
		pluginRoute := path.Join("/public/plugins/", route.PluginId)
		hs.log.Debug("Plugins: Adding route", "route", pluginRoute, "dir", route.Directory)
		hs.mapStatic(m, route.Directory, "", pluginRoute)
	}

	hs.mapStatic(m, setting.StaticRootPath, "", "public")
	hs.mapStatic(m, setting.StaticRootPath, "robots.txt", "robots.txt")

	m.Use(macaron.Renderer(macaron.RenderOptions{
		Directory:  path.Join(setting.StaticRootPath, "views"),
		IndentJSON: macaron.Env != macaron.PROD,
		Delims:     macaron.Delims{Left: "[[", Right: "]]"},
	}))

	m.Use(hs.healthHandler)
	m.Use(hs.metricsEndpoint)
	m.Use(middleware.GetContextHandler())
	m.Use(middleware.Sessioner(&setting.SessionOptions))
	m.Use(middleware.OrgRedirect())

	// needs to be after context handler
	if setting.EnforceDomain {
		m.Use(middleware.ValidateHostHeader(setting.Domain))
	}

	m.Use(middleware.AddDefaultResponseHeaders())

	return m
}

func (hs *HttpServer) metricsEndpoint(ctx *macaron.Context) {
	if ctx.Req.Method != "GET" || ctx.Req.URL.Path != "/metrics" {
		return
	}

	promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}).
		ServeHTTP(ctx.Resp, ctx.Req.Request)
}

func (hs *HttpServer) healthHandler(ctx *macaron.Context) {
	notHeadOrGet := ctx.Req.Method != http.MethodGet && ctx.Req.Method != http.MethodHead
	if notHeadOrGet || ctx.Req.URL.Path != "/api/health" {
		return
	}

	data := simplejson.New()
	data.Set("database", "ok")
	data.Set("version", setting.BuildVersion)
	data.Set("commit", setting.BuildCommit)

	if err := bus.Dispatch(&models.GetDBHealthQuery{}); err != nil {
		data.Set("database", "failing")
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(503)
	} else {
		ctx.Resp.Header().Set("Content-Type", "application/json; charset=UTF-8")
		ctx.Resp.WriteHeader(200)
	}

	dataBytes, _ := data.EncodePretty()
	ctx.Resp.Write(dataBytes)
}

func (hs *HttpServer) mapStatic(m *macaron.Macaron, rootDir string, dir string, prefix string) {
	headers := func(c *macaron.Context) {
		c.Resp.Header().Set("Cache-Control", "public, max-age=3600")
	}

	if setting.Env == setting.DEV {
		headers = func(c *macaron.Context) {
			c.Resp.Header().Set("Cache-Control", "max-age=0, must-revalidate, no-cache")
		}
	}

	m.Use(httpstatic.Static(
		path.Join(rootDir, dir),
		httpstatic.StaticOptions{
			SkipLogging: true,
			Prefix:      prefix,
			AddHeaders:  headers,
		},
	))
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/kube-etcd-controller/pkg/util/etcdutil"
	"github.com/coreos/kube-etcd-controller/pkg/util/k8sutil"
	"golang.org/x/net/context"
	k8sapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"
)

type clusterEventType string

const (
	eventDeleteCluster clusterEventType = "Delete"
	eventReconcile     clusterEventType = "Reconcile"
	eventModifyCluster clusterEventType = "Modify"
)

type clusterEvent struct {
	typ  clusterEventType
	spec Spec
	// currently running pods in kubernetes
	running etcdutil.MemberSet
}

type Cluster struct {
	kclient *unversioned.Client

	spec *Spec

	name string

	idCounter int
	eventCh   chan *clusterEvent
	stopCh    chan struct{}

	// members repsersents the members in the etcd cluster.
	// the name of the member is the the name of the pod the member
	// process runs in.
	members etcdutil.MemberSet

	backupDir string
}

func New(c *unversioned.Client, name string, spec *Spec) *Cluster {
	return new(c, name, spec, true)
}

func Restore(c *unversioned.Client, name string, spec *Spec) *Cluster {
	return new(c, name, spec, false)
}

func new(kclient *unversioned.Client, name string, spec *Spec, isNewCluster bool) *Cluster {
	c := &Cluster{
		kclient: kclient,
		name:    name,
		eventCh: make(chan *clusterEvent, 100),
		stopCh:  make(chan struct{}),
		spec:    spec,
	}
	go c.run()

	if isNewCluster {
		if err := c.startSeedMember(spec); err != nil {
			panic(err)
		}
	}

	return c
}

func (c *Cluster) Delete() {
	c.send(&clusterEvent{typ: eventDeleteCluster})
}

func (c *Cluster) send(ev *clusterEvent) {
	select {
	case c.eventCh <- ev:
	case <-c.stopCh:
	default:
		panic("TODO: too many events queued...")
	}
}

func (c *Cluster) run() {
	go c.monitorPods()
	for {
		select {
		case event := <-c.eventCh:
			switch event.typ {
			case eventReconcile:
				if err := c.reconcile(event.running); err != nil {
					panic(err)
				}
			case eventModifyCluster:
				log.Printf("update: from: %#v, to: %#v", c.spec, event.spec)
				c.spec.Size = event.spec.Size
			case eventDeleteCluster:
				c.delete()
				close(c.stopCh)
				return
			}
		}
	}
}

func (c *Cluster) startSeedMember(spec *Spec) error {
	members := etcdutil.MemberSet{}
	c.spec = spec
	// we want to make use of member's utility methods.
	etcdName := fmt.Sprintf("%s-%04d", c.name, 0)
	members.Add(&etcdutil.Member{Name: etcdName})
	if err := c.createPodAndService(members, members[etcdName], "new"); err != nil {
		return fmt.Errorf("failed to create seed member: %v", err)
	}
	c.idCounter++
	log.Println("created cluster:", members)
	return nil
}

func (c *Cluster) Update(spec *Spec) {
	// Only handles size change now. TODO: handle other updates.
	if spec.Size == c.spec.Size {
		return
	}
	c.send(&clusterEvent{
		typ:  eventModifyCluster,
		spec: *spec,
	})
}

func (c *Cluster) updateMembers(etcdcli *clientv3.Client) {
	resp, err := etcdcli.MemberList(context.TODO())
	if err != nil {
		panic(err)
	}
	c.members = etcdutil.MemberSet{}
	for _, m := range resp.Members {
		id := findID(m.Name)
		if id+1 > c.idCounter {
			c.idCounter = id + 1
		}

		c.members[m.Name] = &etcdutil.Member{
			Name: m.Name,
			ID:   m.ID,
		}
	}
}

func findID(name string) int {
	i := strings.LastIndex(name, "-")
	id, err := strconv.Atoi(name[i+1:])
	if err != nil {
		panic(err)
	}
	return id
}

func (c *Cluster) delete() {
	option := k8sapi.ListOptions{
		LabelSelector: labels.SelectorFromSet(map[string]string{
			"etcd_cluster": c.name,
		}),
	}

	pods, err := c.kclient.Pods("default").List(option)
	if err != nil {
		panic(err)
	}
	for i := range pods.Items {
		if err := c.removePodAndService(pods.Items[i].Name); err != nil {
			panic(err)
		}
	}
}

// todo: use a struct to replace the huge arg list.
func (c *Cluster) createPodAndService(members etcdutil.MemberSet, m *etcdutil.Member, state string) error {
	if err := k8sutil.CreateEtcdService(c.kclient, m.Name, c.name); err != nil {
		return err
	}
	return k8sutil.CreateEtcdPod(c.kclient, members.PeerURLPairs(), m, c.name, state, c.spec.AntiAffinity)
}

func (c *Cluster) removePodAndService(name string) error {
	err := c.kclient.Pods("default").Delete(name, nil)
	if err != nil {
		if !k8sutil.IsKubernetesResourceNotFoundError(err) {
			return err
		}
	}
	err = c.kclient.Services("default").Delete(name)
	if err != nil {
		if !k8sutil.IsKubernetesResourceNotFoundError(err) {
			return err
		}
	}
	return nil
}

func (c *Cluster) monitorPods() {
	opts := k8sapi.ListOptions{
		LabelSelector: labels.SelectorFromSet(map[string]string{
			"app":          "etcd",
			"etcd_cluster": c.name,
		}),
	}
	// TODO: Select "etcd_node" to remove left service.
	for {
		select {
		case <-c.stopCh:
			return
		case <-time.After(5 * time.Second):
		}

		podList, err := c.kclient.Pods("default").List(opts)
		if err != nil {
			panic(err)
		}
		running := etcdutil.MemberSet{}
		for i := range podList.Items {
			running.Add(&etcdutil.Member{Name: podList.Items[i].Name})
		}

		c.send(&clusterEvent{
			typ:     eventReconcile,
			running: running,
		})
	}
}
<commit_msg>Only consider running pods into running set<commit_after>package cluster

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/coreos/etcd/clientv3"
	"github.com/coreos/kube-etcd-controller/pkg/util/etcdutil"
	"github.com/coreos/kube-etcd-controller/pkg/util/k8sutil"
	"golang.org/x/net/context"
	k8sapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"
)

type clusterEventType string

const (
	eventDeleteCluster clusterEventType = "Delete"
	eventReconcile     clusterEventType = "Reconcile"
	eventModifyCluster clusterEventType = "Modify"
)

type clusterEvent struct {
	typ  clusterEventType
	spec Spec
	// currently running pods in kubernetes
	running etcdutil.MemberSet
}

type Cluster struct {
	kclient *unversioned.Client

	spec *Spec

	name string

	idCounter int
	eventCh   chan *clusterEvent
	stopCh    chan struct{}

	// members repsersents the members in the etcd cluster.
	// the name of the member is the the name of the pod the member
	// process runs in.
	members etcdutil.MemberSet

	backupDir string
}

func New(c *unversioned.Client, name string, spec *Spec) *Cluster {
	return new(c, name, spec, true)
}

func Restore(c *unversioned.Client, name string, spec *Spec) *Cluster {
	return new(c, name, spec, false)
}

func new(kclient *unversioned.Client, name string, spec *Spec, isNewCluster bool) *Cluster {
	c := &Cluster{
		kclient: kclient,
		name:    name,
		eventCh: make(chan *clusterEvent, 100),
		stopCh:  make(chan struct{}),
		spec:    spec,
	}
	go c.run()

	if isNewCluster {
		if err := c.startSeedMember(spec); err != nil {
			panic(err)
		}
	}

	return c
}

func (c *Cluster) Delete() {
	c.send(&clusterEvent{typ: eventDeleteCluster})
}

func (c *Cluster) send(ev *clusterEvent) {
	select {
	case c.eventCh <- ev:
	case <-c.stopCh:
	default:
		panic("TODO: too many events queued...")
	}
}

func (c *Cluster) run() {
	go c.monitorPods()
	for {
		select {
		case event := <-c.eventCh:
			switch event.typ {
			case eventReconcile:
				if err := c.reconcile(event.running); err != nil {
					panic(err)
				}
			case eventModifyCluster:
				log.Printf("update: from: %#v, to: %#v", c.spec, event.spec)
				c.spec.Size = event.spec.Size
			case eventDeleteCluster:
				c.delete()
				close(c.stopCh)
				return
			}
		}
	}
}

func (c *Cluster) startSeedMember(spec *Spec) error {
	members := etcdutil.MemberSet{}
	c.spec = spec
	// we want to make use of member's utility methods.
	etcdName := fmt.Sprintf("%s-%04d", c.name, 0)
	members.Add(&etcdutil.Member{Name: etcdName})
	if err := c.createPodAndService(members, members[etcdName], "new"); err != nil {
		return fmt.Errorf("failed to create seed member: %v", err)
	}
	c.idCounter++
	log.Println("created cluster:", members)
	return nil
}

func (c *Cluster) Update(spec *Spec) {
	// Only handles size change now. TODO: handle other updates.
	if spec.Size == c.spec.Size {
		return
	}
	c.send(&clusterEvent{
		typ:  eventModifyCluster,
		spec: *spec,
	})
}

func (c *Cluster) updateMembers(etcdcli *clientv3.Client) {
	resp, err := etcdcli.MemberList(context.TODO())
	if err != nil {
		panic(err)
	}
	c.members = etcdutil.MemberSet{}
	for _, m := range resp.Members {
		id := findID(m.Name)
		if id+1 > c.idCounter {
			c.idCounter = id + 1
		}

		c.members[m.Name] = &etcdutil.Member{
			Name: m.Name,
			ID:   m.ID,
		}
	}
}

func findID(name string) int {
	i := strings.LastIndex(name, "-")
	id, err := strconv.Atoi(name[i+1:])
	if err != nil {
		panic(err)
	}
	return id
}

func (c *Cluster) delete() {
	option := k8sapi.ListOptions{
		LabelSelector: labels.SelectorFromSet(map[string]string{
			"etcd_cluster": c.name,
		}),
	}

	pods, err := c.kclient.Pods("default").List(option)
	if err != nil {
		panic(err)
	}
	for i := range pods.Items {
		if err := c.removePodAndService(pods.Items[i].Name); err != nil {
			panic(err)
		}
	}
}

// todo: use a struct to replace the huge arg list.
func (c *Cluster) createPodAndService(members etcdutil.MemberSet, m *etcdutil.Member, state string) error {
	if err := k8sutil.CreateEtcdService(c.kclient, m.Name, c.name); err != nil {
		return err
	}
	return k8sutil.CreateEtcdPod(c.kclient, members.PeerURLPairs(), m, c.name, state, c.spec.AntiAffinity)
}

func (c *Cluster) removePodAndService(name string) error {
	err := c.kclient.Pods("default").Delete(name, nil)
	if err != nil {
		if !k8sutil.IsKubernetesResourceNotFoundError(err) {
			return err
		}
	}
	err = c.kclient.Services("default").Delete(name)
	if err != nil {
		if !k8sutil.IsKubernetesResourceNotFoundError(err) {
			return err
		}
	}
	return nil
}

func (c *Cluster) monitorPods() {
	opts := k8sapi.ListOptions{
		LabelSelector: labels.SelectorFromSet(map[string]string{
			"app":          "etcd",
			"etcd_cluster": c.name,
		}),
	}
	for {
		select {
		case <-c.stopCh:
			return
		case <-time.After(5 * time.Second):
		}

		podList, err := c.kclient.Pods("default").List(opts)
		if err != nil {
			panic(err)
		}
		running := etcdutil.MemberSet{}
		for i := range podList.Items {
			pod := podList.Items[i]
			// TODO: use liveness probe to do checking
			if pod.Status.Phase != k8sapi.PodRunning {
				log.Debugf("skipping non-running pod: %s", pod.Name)
				continue
			}
			running.Add(&etcdutil.Member{Name: pod.Name})
		}
		if running.Size() == 0 {
			log.Infof("cluster (%s) not ready yet", c.name)
			continue
		}

		c.send(&clusterEvent{
			typ:     eventReconcile,
			running: running,
		})
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"

	"github.com/docker/docker/pkg/namesgenerator"
	"github.com/juju/errors"
	"github.com/online-net/c14-cli/pkg/api"
)

type create struct {
	Base
	createFlags
}

type createFlags struct {
	flName   string
	flDesc   string
	flSafe   string
	flQuiet  bool
	flParity string
	flLarge  bool
	flCrypto string
}

// Create returns a new command "create"
func Create() Command {
	ret := &create{}
	ret.Init(Config{
		UsageLine:   "create [OPTIONS]",
		Description: "Create a new archive",
		Help:        "Create a new archive, by default with a random name, standard storage (0.0002€/GB/month), automatic locked in 7 days and your datas will be stored at DC2.",
		Examples: `
        $ c14 create
        $ c14 create --name "MyBooks" --description "hardware books"
        $ c14 create --name "MyBooks" --description "hardware books" --safe "Bookshelf"
`,
	})
	ret.Flags.StringVar(&ret.flName, []string{"n", "-name"}, "", "Assigns a name")
	ret.Flags.StringVar(&ret.flDesc, []string{"d", "-description"}, "", "Assigns a description")
	ret.Flags.BoolVar(&ret.flQuiet, []string{"q", "-quiet"}, false, "Don't display the waiting loop")
	ret.Flags.StringVar(&ret.flSafe, []string{"s", "-safe"}, "", "Name of the safe to use. If it doesn't exists it will be created.")
	ret.Flags.StringVar(&ret.flParity, []string{"p", "-parity"}, "standard", "Specify a parity to use")
	ret.Flags.BoolVar(&ret.flLarge, []string{"l", "-large"}, false, "Ask for a large bucket")
	ret.Flags.StringVar(&ret.flCrypto, []string{"c", "-crypto"}, "aes-256-cbc", "Which cryptography to use: aes-256-cbc or none")
	return ret
}

func (c *create) GetName() string {
	return "create"
}

func (c *create) CheckFlags(args []string) (err error) {
	if len(args) != 0 {
		c.PrintUsage()
		os.Exit(1)
	}

	if c.flName == "" {
		c.flName = namesgenerator.GetRandomName(0)
	}
	if c.flDesc == "" {
		c.flDesc = " "
	}
	return
}

func (c *create) Run(args []string) (err error) {
	if err = c.InitAPI(); err != nil {
		return
	}
	var (
		uuidArchive string
		safeName    string
		keys        []api.OnlineGetSSHKey
	)

	if keys, err = c.OnlineAPI.GetSSHKeys(); err != nil {
		err = errors.Annotate(err, "Run:GetSSHKey")
		return
	}
	if len(keys) == 0 {
		err = errors.New("Please add an SSH Key here: https://console.online.net/en/account/ssh-keys")
		return
	}

	safeName = c.flSafe

	if safeName == "" {
		safeName = fmt.Sprintf("%s_safe", c.flName)
	}

	if _, uuidArchive, _, err = c.OnlineAPI.CreateSSHBucketFromScratch(api.ConfigCreateSSHBucketFromScratch{
		SafeName:    safeName,
		ArchiveName: c.flName,
		Desc:        c.flDesc,
		UUIDSSHKeys: []string{keys[0].UUIDRef},
		Platforms:   []string{"1"},
		Days:        7,
		Quiet:       c.flQuiet,
		Parity:      c.flParity,
		LargeBucket: c.flLarge,
		Crypto:      c.flCrypto,
	}); err != nil {
		err = errors.Annotate(err, "Run:CreateSSHBucketFromScratch")
		return
	}
	fmt.Printf("%s\n", uuidArchive)
	return
}
<commit_msg>from string to bool to bypass bad API argument<commit_after>package commands

import (
	"fmt"
	"os"

	"github.com/docker/docker/pkg/namesgenerator"
	"github.com/juju/errors"
	"github.com/online-net/c14-cli/pkg/api"
)

type create struct {
	Base
	createFlags
}

type createFlags struct {
	flName   string
	flDesc   string
	flSafe   string
	flQuiet  bool
	flParity string
	flLarge  bool
	flCrypto bool
}

// Create returns a new command "create"
func Create() Command {
	ret := &create{}
	ret.Init(Config{
		UsageLine:   "create [OPTIONS]",
		Description: "Create a new archive",
		Help:        "Create a new archive, by default with a random name, standard storage (0.0002€/GB/month), automatic locked in 7 days and your datas will be stored at DC2.",
		Examples: `
        $ c14 create
        $ c14 create --name "MyBooks" --description "hardware books"
        $ c14 create --name "MyBooks" --description "hardware books" --safe "Bookshelf"
`,
	})
	ret.Flags.StringVar(&ret.flName, []string{"n", "-name"}, "", "Assigns a name")
	ret.Flags.StringVar(&ret.flDesc, []string{"d", "-description"}, "", "Assigns a description")
	ret.Flags.BoolVar(&ret.flQuiet, []string{"q", "-quiet"}, false, "Don't display the waiting loop")
	ret.Flags.StringVar(&ret.flSafe, []string{"s", "-safe"}, "", "Name of the safe to use. If it doesn't exists it will be created.")
	ret.Flags.StringVar(&ret.flParity, []string{"p", "-parity"}, "standard", "Specify a parity to use")
	ret.Flags.BoolVar(&ret.flLarge, []string{"l", "-large"}, false, "Ask for a large bucket")
	ret.Flags.BoolVar(&ret.flCrypto, []string{"c", "-crypto"}, true, "Enable aes-256-bc cryptography, enabled by default.")
	return ret
}

func (c *create) GetName() string {
	return "create"
}

func (c *create) CheckFlags(args []string) (err error) {
	if len(args) != 0 {
		c.PrintUsage()
		os.Exit(1)
	}

	if c.flName == "" {
		c.flName = namesgenerator.GetRandomName(0)
	}
	if c.flDesc == "" {
		c.flDesc = " "
	}
	return
}

func (c *create) Run(args []string) (err error) {
	if err = c.InitAPI(); err != nil {
		return
	}
	var (
		uuidArchive string
		safeName    string
		keys        []api.OnlineGetSSHKey
		crypto      string
	)

	if keys, err = c.OnlineAPI.GetSSHKeys(); err != nil {
		err = errors.Annotate(err, "Run:GetSSHKey")
		return
	}
	if len(keys) == 0 {
		err = errors.New("Please add an SSH Key here: https://console.online.net/en/account/ssh-keys")
		return
	}

	safeName = c.flSafe

	if safeName == "" {
		safeName = fmt.Sprintf("%s_safe", c.flName)
	}

	if c.flCrypto == false {
		crypto = "none"
	} else {
		crypto = "aes-256-cbc"
	}

	if _, uuidArchive, _, err = c.OnlineAPI.CreateSSHBucketFromScratch(api.ConfigCreateSSHBucketFromScratch{
		SafeName:    safeName,
		ArchiveName: c.flName,
		Desc:        c.flDesc,
		UUIDSSHKeys: []string{keys[0].UUIDRef},
		Platforms:   []string{"1"},
		Days:        7,
		Quiet:       c.flQuiet,
		Parity:      c.flParity,
		LargeBucket: c.flLarge,
		Crypto:      crypto,
	}); err != nil {
		err = errors.Annotate(err, "Run:CreateSSHBucketFromScratch")
		return
	}
	fmt.Printf("%s\n", uuidArchive)
	return
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
)

type Controller interface {
	Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup)
}

type BaseController interface {
	Controller
	reconcile(key string) error
}

type Base struct {
	Clients
	queue      workqueue.RateLimitingInterface
	informer   cache.SharedIndexInformer
	Controller BaseController
}

func NewBaseController(clients Clients, informer cache.SharedIndexInformer) Base {
	base := Base{
		Clients:  clients,
		queue:    workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(5*time.Second, 300*time.Second)),
		informer: informer,
	}

	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				base.queue.Add(key)
			}
		},
		UpdateFunc: func(old interface{}, new interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(new)
			if err == nil {
				base.queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				base.queue.Add(key)
			}
		},
	})

	return base
}

func (base *Base) Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer base.queue.ShutDown()
	defer wg.Done()
	wg.Add(1)
	glog.Infof("Starting %v with %d workers", base.getName(), threadiness)

	for i := 0; i < threadiness; i++ {
		go wait.Until(base.runWorker, time.Second, stopCh)
	}

	ticker := time.NewTicker(KLUSTER_RECHECK_INTERVAL)
	go func() {
		for {
			select {
			case <-ticker.C:
				for key := range base.informer.GetStore().ListKeys() {
					base.queue.Add(key)
				}
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	<-stopCh
}

func (base *Base) runWorker() {
	for base.processNextWorkItem() {
	}
}

func (base *Base) processNextWorkItem() bool {
	key, quit := base.queue.Get()
	if quit {
		return false
	}
	defer base.queue.Done(key)

	// Invoke the method containing the business logic
	err := base.reconciliation(key.(string))
	base.handleErr(err, key)
	return true
}

func (base *Base) reconcile(key string) error {
	return fmt.Errorf("NotImplemented")
}

func (base *Base) handleErr(err error, key interface{}) {
	if err == nil {
		// Forget about the #AddRateLimited history of the key on every successful synchronization.
		// This ensures that future processing of updates for this key is not delayed because of
		// an outdated error history.
		base.queue.Forget(key)
		return
	}

	glog.Infof("[%v] Error while processing %v: %v", base.getName(), key, err)
	// This controller retries 5 times if something goes wrong. After that, it stops trying.
	if base.queue.NumRequeues(key) < 5 {
		// Re-enqueue the key rate limited. Based on the rate limiter on the
		// queue and the re-enqueue history, the key will be processed later again.
		base.queue.AddRateLimited(key)
		return
	}

	glog.V(5).Infof("[%v] Dropping %v from queue because of too many errors...", base.getName(), key)
	base.queue.Forget(key)
}

func getControllerName(c Controller) string {
	return reflect.TypeOf(c).Elem().Name()

}

func (base *Base) getName() string {
	return getControllerName(base.Controller)
}

func (base *Base) reconciliation(key string) error {
	glog.V(5).Infof("[%v] Reconciling %v", base.getName(), key)
	return base.Controller.reconcile(key)
}
<commit_msg>actually queue keys and not their index<commit_after>package controller

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/golang/glog"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
)

type Controller interface {
	Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup)
}

type BaseController interface {
	Controller
	reconcile(key string) error
}

type Base struct {
	Clients
	queue      workqueue.RateLimitingInterface
	informer   cache.SharedIndexInformer
	Controller BaseController
}

func NewBaseController(clients Clients, informer cache.SharedIndexInformer) Base {
	base := Base{
		Clients:  clients,
		queue:    workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(5*time.Second, 300*time.Second)),
		informer: informer,
	}

	informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(obj)
			if err == nil {
				base.queue.Add(key)
			}
		},
		UpdateFunc: func(old interface{}, new interface{}) {
			key, err := cache.MetaNamespaceKeyFunc(new)
			if err == nil {
				base.queue.Add(key)
			}
		},
		DeleteFunc: func(obj interface{}) {
			key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
			if err == nil {
				base.queue.Add(key)
			}
		},
	})

	return base
}

func (base *Base) Run(threadiness int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer base.queue.ShutDown()
	defer wg.Done()
	wg.Add(1)
	glog.Infof("Starting %v with %d workers", base.getName(), threadiness)

	for i := 0; i < threadiness; i++ {
		go wait.Until(base.runWorker, time.Second, stopCh)
	}

	ticker := time.NewTicker(KLUSTER_RECHECK_INTERVAL)
	go func() {
		for {
			select {
			case <-ticker.C:
				for _, key := range base.informer.GetStore().ListKeys() {
					base.queue.Add(key)
				}
			case <-stopCh:
				ticker.Stop()
				return
			}
		}
	}()

	<-stopCh
}

func (base *Base) runWorker() {
	for base.processNextWorkItem() {
	}
}

func (base *Base) processNextWorkItem() bool {
	key, quit := base.queue.Get()
	if quit {
		return false
	}
	defer base.queue.Done(key)

	// Invoke the method containing the business logic
	err := base.reconciliation(key.(string))
	base.handleErr(err, key)
	return true
}

func (base *Base) reconcile(key string) error {
	return fmt.Errorf("NotImplemented")
}

func (base *Base) handleErr(err error, key interface{}) {
	if err == nil {
		// Forget about the #AddRateLimited history of the key on every successful synchronization.
		// This ensures that future processing of updates for this key is not delayed because of
		// an outdated error history.
		base.queue.Forget(key)
		return
	}

	glog.Infof("[%v] Error while processing %v: %v", base.getName(), key, err)
	// This controller retries 5 times if something goes wrong. After that, it stops trying.
	if base.queue.NumRequeues(key) < 5 {
		// Re-enqueue the key rate limited. Based on the rate limiter on the
		// queue and the re-enqueue history, the key will be processed later again.
		base.queue.AddRateLimited(key)
		return
	}

	glog.V(5).Infof("[%v] Dropping %v from queue because of too many errors...", base.getName(), key)
	base.queue.Forget(key)
}

func getControllerName(c Controller) string {
	return reflect.TypeOf(c).Elem().Name()

}

func (base *Base) getName() string {
	return getControllerName(base.Controller)
}

func (base *Base) reconciliation(key string) error {
	glog.V(5).Infof("[%v] Reconciling %v", base.getName(), key)
	return base.Controller.reconcile(key)
}
<|endoftext|>


<commit_before>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"sort"

	envoyAPI "github.com/cilium/cilium/pkg/envoy/cilium"
	"github.com/cilium/cilium/pkg/envoy/xds"
	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ipcache"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/sirupsen/logrus"
)

const (
	// ListenerTypeURL is the type URL of Listener resources.
	ListenerTypeURL = "type.googleapis.com/envoy.api.v2.Listener"

	// NetworkPolicyTypeURL is the type URL of NetworkPolicy resources.
	NetworkPolicyTypeURL = "type.googleapis.com/cilium.NetworkPolicy"

	// NetworkPolicyHostsTypeURL is the type URL of NetworkPolicyHosts resources.
	NetworkPolicyHostsTypeURL = "type.googleapis.com/cilium.NetworkPolicyHosts"
)

// NPHDSCache is a cache of resources in the Network Policy Hosts Discovery
// Service.
type NPHDSCache struct {
	*xds.Cache
}

func newNPHDSCache() NPHDSCache {
	return NPHDSCache{Cache: xds.NewCache()}
}

var (
	// NetworkPolicyHostsCache is the global cache of resources of type
	// NetworkPolicyHosts. Resources in this cache must have the
	// NetworkPolicyHostsTypeURL type URL.
	NetworkPolicyHostsCache = newNPHDSCache()
)

// OnIPIdentityCacheGC is required to implement IPIdentityMappingListener.
func (cache *NPHDSCache) OnIPIdentityCacheGC() {
	// We don't have anything to synchronize in this case.
}

// OnIPIdentityCacheChange pushes modifications to the IP<->Identity mapping
// into the Network Policy Host Discovery Service (NPHDS).
func (cache *NPHDSCache) OnIPIdentityCacheChange(modType ipcache.CacheModification,
	oldIPIDPair *identity.IPIdentityPair, newIPIDPair identity.IPIdentityPair) {

	// An upsert where an existing pair exists should translate into a
	// delete (for the old Identity) followed by an upsert (for the new).
	if oldIPIDPair != nil && modType == ipcache.Upsert {
		// Skip update if identity is identical
		if oldIPIDPair.ID == newIPIDPair.ID {
			return
		}

		cache.OnIPIdentityCacheChange(ipcache.Delete, nil, *oldIPIDPair)
	}

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       newIPIDPair.IP,
		logfields.Identity:     newIPIDPair.ID,
		logfields.Modification: modType,
	})
	// Look up the current resources for the specified Identity.
	resourceName := newIPIDPair.ID.StringID()
	msg, err := cache.Lookup(NetworkPolicyHostsTypeURL, resourceName)
	if err != nil {
		scopedLog.WithError(err).Warning("Can't lookup NPHDS cache")
		return
	}

	switch modType {
	case ipcache.Upsert:
		var hostAddresses []string
		if msg == nil {
			hostAddresses = make([]string, 0, 1)
		} else {
			// If the resource already exists, create a copy of it and insert
			// the new IP address into its HostAddresses list.
			npHost := msg.(*envoyAPI.NetworkPolicyHosts)
			hostAddresses = make([]string, 0, len(npHost.HostAddresses)+1)
			hostAddresses = append(hostAddresses, npHost.HostAddresses...)
		}
		hostAddresses = append(hostAddresses, newIPIDPair.PrefixString())
		sort.Strings(hostAddresses)

		newNpHost := envoyAPI.NetworkPolicyHosts{
			Policy:        uint64(newIPIDPair.ID),
			HostAddresses: hostAddresses,
		}
		if err := newNpHost.Validate(); err != nil {
			scopedLog.WithError(err).WithFields(logrus.Fields{
				logfields.XDSResource: newNpHost,
			}).Warning("Could not validate NPHDS resource update on upsert")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, resourceName, &newNpHost, false)
	case ipcache.Delete:
		if msg == nil {
			// Doesn't exist; already deleted.
			return
		}
		cache.handleIPDelete(msg.(*envoyAPI.NetworkPolicyHosts), resourceName, newIPIDPair.PrefixString())
	}
}

// handleIPUpsert deletes elements from the NPHDS cache with the specified peer IP->ID mapping.
func (cache *NPHDSCache) handleIPDelete(npHost *envoyAPI.NetworkPolicyHosts, peerIdentity, peerIP string) {
	targetIndex := -1

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       peerIP,
		logfields.Identity:     peerIdentity,
		logfields.Modification: ipcache.Delete,
	})
	for i, endpointIP := range npHost.HostAddresses {
		if endpointIP == peerIP {
			targetIndex = i
			break
		}
	}
	if targetIndex < 0 {
		scopedLog.Fatal("Can't find IP in NPHDS cache")
		return
	}

	// If removing this host would result in empty list, delete it.
	// Otherwise, update to a list that doesn't contain the target IP
	if len(npHost.HostAddresses) <= 1 {
		cache.Delete(NetworkPolicyHostsTypeURL, peerIdentity, false)
	} else {
		// If the resource is to be updated, create a copy of it before
		// removing the IP address from its HostAddresses list.
		hostAddresses := make([]string, 0, len(npHost.HostAddresses)-1)
		if len(npHost.HostAddresses) == targetIndex {
			hostAddresses = append(hostAddresses, npHost.HostAddresses[0:targetIndex]...)
		} else {
			hostAddresses = append(hostAddresses, npHost.HostAddresses[0:targetIndex]...)
			hostAddresses = append(hostAddresses, npHost.HostAddresses[targetIndex+1:]...)
		}

		newNpHost := envoyAPI.NetworkPolicyHosts{
			Policy:        uint64(npHost.Policy),
			HostAddresses: hostAddresses,
		}
		if err := newNpHost.Validate(); err != nil {
			scopedLog.WithError(err).Warning("Could not validate NPHDS resource update on delete")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, peerIdentity, &newNpHost, false)
	}
}
<commit_msg>envoy: Do not exit agent if NPHDS entry cannot be found<commit_after>// Copyright 2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"sort"

	envoyAPI "github.com/cilium/cilium/pkg/envoy/cilium"
	"github.com/cilium/cilium/pkg/envoy/xds"
	"github.com/cilium/cilium/pkg/identity"
	"github.com/cilium/cilium/pkg/ipcache"
	"github.com/cilium/cilium/pkg/logging/logfields"

	"github.com/sirupsen/logrus"
)

const (
	// ListenerTypeURL is the type URL of Listener resources.
	ListenerTypeURL = "type.googleapis.com/envoy.api.v2.Listener"

	// NetworkPolicyTypeURL is the type URL of NetworkPolicy resources.
	NetworkPolicyTypeURL = "type.googleapis.com/cilium.NetworkPolicy"

	// NetworkPolicyHostsTypeURL is the type URL of NetworkPolicyHosts resources.
	NetworkPolicyHostsTypeURL = "type.googleapis.com/cilium.NetworkPolicyHosts"
)

// NPHDSCache is a cache of resources in the Network Policy Hosts Discovery
// Service.
type NPHDSCache struct {
	*xds.Cache
}

func newNPHDSCache() NPHDSCache {
	return NPHDSCache{Cache: xds.NewCache()}
}

var (
	// NetworkPolicyHostsCache is the global cache of resources of type
	// NetworkPolicyHosts. Resources in this cache must have the
	// NetworkPolicyHostsTypeURL type URL.
	NetworkPolicyHostsCache = newNPHDSCache()
)

// OnIPIdentityCacheGC is required to implement IPIdentityMappingListener.
func (cache *NPHDSCache) OnIPIdentityCacheGC() {
	// We don't have anything to synchronize in this case.
}

// OnIPIdentityCacheChange pushes modifications to the IP<->Identity mapping
// into the Network Policy Host Discovery Service (NPHDS).
func (cache *NPHDSCache) OnIPIdentityCacheChange(modType ipcache.CacheModification,
	oldIPIDPair *identity.IPIdentityPair, newIPIDPair identity.IPIdentityPair) {

	// An upsert where an existing pair exists should translate into a
	// delete (for the old Identity) followed by an upsert (for the new).
	if oldIPIDPair != nil && modType == ipcache.Upsert {
		// Skip update if identity is identical
		if oldIPIDPair.ID == newIPIDPair.ID {
			return
		}

		cache.OnIPIdentityCacheChange(ipcache.Delete, nil, *oldIPIDPair)
	}

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       newIPIDPair.IP,
		logfields.Identity:     newIPIDPair.ID,
		logfields.Modification: modType,
	})
	// Look up the current resources for the specified Identity.
	resourceName := newIPIDPair.ID.StringID()
	msg, err := cache.Lookup(NetworkPolicyHostsTypeURL, resourceName)
	if err != nil {
		scopedLog.WithError(err).Warning("Can't lookup NPHDS cache")
		return
	}

	switch modType {
	case ipcache.Upsert:
		var hostAddresses []string
		if msg == nil {
			hostAddresses = make([]string, 0, 1)
		} else {
			// If the resource already exists, create a copy of it and insert
			// the new IP address into its HostAddresses list.
			npHost := msg.(*envoyAPI.NetworkPolicyHosts)
			hostAddresses = make([]string, 0, len(npHost.HostAddresses)+1)
			hostAddresses = append(hostAddresses, npHost.HostAddresses...)
		}
		hostAddresses = append(hostAddresses, newIPIDPair.PrefixString())
		sort.Strings(hostAddresses)

		newNpHost := envoyAPI.NetworkPolicyHosts{
			Policy:        uint64(newIPIDPair.ID),
			HostAddresses: hostAddresses,
		}
		if err := newNpHost.Validate(); err != nil {
			scopedLog.WithError(err).WithFields(logrus.Fields{
				logfields.XDSResource: newNpHost,
			}).Warning("Could not validate NPHDS resource update on upsert")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, resourceName, &newNpHost, false)
	case ipcache.Delete:
		if msg == nil {
			// Doesn't exist; already deleted.
			return
		}
		cache.handleIPDelete(msg.(*envoyAPI.NetworkPolicyHosts), resourceName, newIPIDPair.PrefixString())
	}
}

// handleIPUpsert deletes elements from the NPHDS cache with the specified peer IP->ID mapping.
func (cache *NPHDSCache) handleIPDelete(npHost *envoyAPI.NetworkPolicyHosts, peerIdentity, peerIP string) {
	targetIndex := -1

	scopedLog := log.WithFields(logrus.Fields{
		logfields.IPAddr:       peerIP,
		logfields.Identity:     peerIdentity,
		logfields.Modification: ipcache.Delete,
	})
	for i, endpointIP := range npHost.HostAddresses {
		if endpointIP == peerIP {
			targetIndex = i
			break
		}
	}
	if targetIndex < 0 {
		scopedLog.Warning("Can't find IP in NPHDS cache")
		return
	}

	// If removing this host would result in empty list, delete it.
	// Otherwise, update to a list that doesn't contain the target IP
	if len(npHost.HostAddresses) <= 1 {
		cache.Delete(NetworkPolicyHostsTypeURL, peerIdentity, false)
	} else {
		// If the resource is to be updated, create a copy of it before
		// removing the IP address from its HostAddresses list.
		hostAddresses := make([]string, 0, len(npHost.HostAddresses)-1)
		if len(npHost.HostAddresses) == targetIndex {
			hostAddresses = append(hostAddresses, npHost.HostAddresses[0:targetIndex]...)
		} else {
			hostAddresses = append(hostAddresses, npHost.HostAddresses[0:targetIndex]...)
			hostAddresses = append(hostAddresses, npHost.HostAddresses[targetIndex+1:]...)
		}

		newNpHost := envoyAPI.NetworkPolicyHosts{
			Policy:        uint64(npHost.Policy),
			HostAddresses: hostAddresses,
		}
		if err := newNpHost.Validate(); err != nil {
			scopedLog.WithError(err).Warning("Could not validate NPHDS resource update on delete")
			return
		}
		cache.Upsert(NetworkPolicyHostsTypeURL, peerIdentity, &newNpHost, false)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package build

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/google/uuid"

	"sigs.k8s.io/yaml"
)

const (
	gcsSourceDir = "/source"
	gcsLogsDir   = "/logs"
)

// TODO: Pull some of these options in cmd/gcbuilder, so they don't have to be public.
type Options struct {
	BuildDir       string
	ConfigDir      string
	CloudbuildFile string
	LogDir         string
	ScratchBucket  string
	Project        string
	AllowDirty     bool
	NoSource       bool
	Variant        string
	EnvPassthrough string
}

func runCmd(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func getVersion() (string, error) {
	cmd := exec.Command("git", "describe", "--tags", "--always", "--dirty")
	output, err := cmd.Output()
	if err != nil {
		return "", err
	}
	t := time.Now().Format("20060102")
	return fmt.Sprintf("v%s-%s", t, strings.TrimSpace(string(output))), nil
}

func (o *Options) ValidateConfigDir() error {
	configDir := o.ConfigDir
	dirInfo, err := os.Stat(o.ConfigDir)
	if os.IsNotExist(err) {
		log.Fatalf("Config directory (%s) does not exist", configDir)
	}

	if !dirInfo.IsDir() {
		log.Fatalf("Config directory (%s) is not actually a directory", configDir)
	}

	_, err = os.Stat(o.CloudbuildFile)
	if os.IsNotExist(err) {
		log.Fatalf("%s does not exist", o.CloudbuildFile)
	}

	return nil
}

func (o *Options) uploadBuildDir(targetBucket string) (string, error) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %v", err)
	}
	name := f.Name()
	_ = f.Close()
	defer os.Remove(name)

	log.Printf("Creating source tarball at %s...\n", name)
	if err := runCmd("tar", "--exclude", ".git", "-czf", name, "."); err != nil {
		return "", fmt.Errorf("failed to tar files: %s", err)
	}

	u := uuid.New()
	uploaded := fmt.Sprintf("%s/%s.tgz", targetBucket, u.String())
	log.Printf("Uploading %s to %s...\n", name, uploaded)
	if err := runCmd("gsutil", "cp", name, uploaded); err != nil {
		return "", fmt.Errorf("failed to upload files: %s", err)
	}

	return uploaded, nil
}

func getExtraSubs(o Options) map[string]string {
	envs := strings.Split(o.EnvPassthrough, ",")
	subs := map[string]string{}
	for _, e := range envs {
		e = strings.TrimSpace(e)
		if e != "" {
			subs[e] = os.Getenv(e)
		}
	}
	return subs
}

func runSingleJob(o Options, jobName, uploaded, version string, subs map[string]string) error {
	s := make([]string, 0, len(subs)+1)
	for k, v := range subs {
		s = append(s, fmt.Sprintf("_%s=%s", k, v))
	}

	s = append(s, "_GIT_TAG="+version)
	args := []string{
		"builds", "submit",
		"--verbosity", "info",
		"--config", o.CloudbuildFile,
		"--substitutions", strings.Join(s, ","),
	}

	if o.Project != "" {
		args = append(args, "--project", o.Project)
	}

	if o.ScratchBucket != "" {
		args = append(args, "--gcs-log-dir", o.ScratchBucket+gcsLogsDir)
		args = append(args, "--gcs-source-staging-dir", o.ScratchBucket+gcsSourceDir)
	}

	if uploaded != "" {
		args = append(args, uploaded)
	} else {
		if o.NoSource {
			args = append(args, "--no-source")
		} else {
			args = append(args, ".")
		}
	}

	cmd := exec.Command("gcloud", args...)

	if o.LogDir != "" {
		p := path.Join(o.LogDir, strings.Replace(jobName, "/", "-", -1)+".log")
		f, err := os.Create(p)

		if err != nil {
			return fmt.Errorf("couldn't create %s: %v", p, err)
		}

		defer f.Close()

		cmd.Stdout = f
		cmd.Stderr = f
	} else {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running %s: %v", cmd.Args, err)
	}

	return nil
}

type variants map[string]map[string]string

func getVariants(o Options) (variants, error) {
	content, err := ioutil.ReadFile(path.Join(o.ConfigDir, "variants.yaml"))
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("failed to load variants.yaml: %v", err)
		}
		if o.Variant != "" {
			return nil, fmt.Errorf("no variants.yaml found, but a build variant (%q) was specified", o.Variant)
		}
		return nil, nil
	}
	v := struct {
		Variants variants `json:"variants"`
	}{}
	if err := yaml.UnmarshalStrict(content, &v); err != nil {
		return nil, fmt.Errorf("failed to read variants.yaml: %v", err)
	}
	if o.Variant != "" {
		va, ok := v.Variants[o.Variant]
		if !ok {
			return nil, fmt.Errorf("requested variant %q, which is not present in variants.yaml", o.Variant)
		}
		return variants{o.Variant: va}, nil
	}
	return v.Variants, nil
}

func RunBuildJobs(o Options) []error {
	var uploaded string
	if o.ScratchBucket != "" {
		if !o.NoSource {
			var err error
			uploaded, err = o.uploadBuildDir(o.ScratchBucket + gcsSourceDir)
			if err != nil {
				return []error{fmt.Errorf("failed to upload source: %v", err)}
			}
		}
	} else {
		log.Println("Skipping advance upload and relying on gcloud...")
	}

	log.Println("Running build jobs...")
	tag, err := getVersion()
	if err != nil {
		return []error{fmt.Errorf("failed to get current tag: %v", err)}
	}

	if !o.AllowDirty && strings.HasSuffix(tag, "-dirty") {
		return []error{fmt.Errorf("the working copy is dirty")}
	}

	vs, err := getVariants(o)
	if err != nil {
		return []error{err}
	}
	if len(vs) == 0 {
		log.Println("No variants.yaml, starting single build job...")
		if err := runSingleJob(o, "build", uploaded, tag, getExtraSubs(o)); err != nil {
			return []error{err}
		}
		return nil
	}

	log.Printf("Found variants.yaml, starting %d build jobs...\n", len(vs))

	w := sync.WaitGroup{}
	w.Add(len(vs))
	var errors []error
	extraSubs := getExtraSubs(o)
	for k, v := range vs {
		go func(job string, vc map[string]string) {
			defer w.Done()
			log.Printf("Starting job %q...\n", job)
			if err := runSingleJob(o, job, uploaded, tag, mergeMaps(extraSubs, vc)); err != nil {
				errors = append(errors, fmt.Errorf("job %q failed: %v", job, err))
				log.Printf("Job %q failed: %v\n", job, err)
			} else {
				log.Printf("Job %q completed.\n", job)
			}
		}(k, v)
	}
	w.Wait()
	return errors
}

func mergeMaps(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			out[k] = v
		}
	}
	return out
}
<commit_msg>pkg/gcp/build: Fix gocritic lint warnings<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package build

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/google/uuid"

	"sigs.k8s.io/yaml"
)

const (
	gcsSourceDir = "/source"
	gcsLogsDir   = "/logs"
)

// TODO: Pull some of these options in cmd/gcbuilder, so they don't have to be public.
type Options struct {
	BuildDir       string
	ConfigDir      string
	CloudbuildFile string
	LogDir         string
	ScratchBucket  string
	Project        string
	AllowDirty     bool
	NoSource       bool
	Variant        string
	EnvPassthrough string
}

func runCmd(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func getVersion() (string, error) {
	cmd := exec.Command("git", "describe", "--tags", "--always", "--dirty")
	output, err := cmd.Output()
	if err != nil {
		return "", err
	}
	t := time.Now().Format("20060102")
	return fmt.Sprintf("v%s-%s", t, strings.TrimSpace(string(output))), nil
}

func (o *Options) ValidateConfigDir() error {
	configDir := o.ConfigDir
	dirInfo, err := os.Stat(o.ConfigDir)
	if os.IsNotExist(err) {
		log.Fatalf("Config directory (%s) does not exist", configDir)
	}

	if !dirInfo.IsDir() {
		log.Fatalf("Config directory (%s) is not actually a directory", configDir)
	}

	_, err = os.Stat(o.CloudbuildFile)
	if os.IsNotExist(err) {
		log.Fatalf("%s does not exist", o.CloudbuildFile)
	}

	return nil
}

func (o *Options) uploadBuildDir(targetBucket string) (string, error) {
	f, err := ioutil.TempFile("", "")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %v", err)
	}
	name := f.Name()
	_ = f.Close()
	defer os.Remove(name)

	log.Printf("Creating source tarball at %s...\n", name)
	if err := runCmd("tar", "--exclude", ".git", "-czf", name, "."); err != nil {
		return "", fmt.Errorf("failed to tar files: %s", err)
	}

	u := uuid.New()
	uploaded := fmt.Sprintf("%s/%s.tgz", targetBucket, u.String())
	log.Printf("Uploading %s to %s...\n", name, uploaded)
	if err := runCmd("gsutil", "cp", name, uploaded); err != nil {
		return "", fmt.Errorf("failed to upload files: %s", err)
	}

	return uploaded, nil
}

func getExtraSubs(o Options) map[string]string {
	envs := strings.Split(o.EnvPassthrough, ",")
	subs := map[string]string{}
	for _, e := range envs {
		e = strings.TrimSpace(e)
		if e != "" {
			subs[e] = os.Getenv(e)
		}
	}
	return subs
}

func runSingleJob(o Options, jobName, uploaded, version string, subs map[string]string) error {
	s := make([]string, 0, len(subs)+1)
	for k, v := range subs {
		s = append(s, fmt.Sprintf("_%s=%s", k, v))
	}

	s = append(s, "_GIT_TAG="+version)
	args := []string{
		"builds", "submit",
		"--verbosity", "info",
		"--config", o.CloudbuildFile,
		"--substitutions", strings.Join(s, ","),
	}

	if o.Project != "" {
		args = append(args, "--project", o.Project)
	}

	if o.ScratchBucket != "" {
		args = append(
			args,
			"--gcs-log-dir",
			o.ScratchBucket+gcsLogsDir,
			"--gcs-source-staging-dir",
			o.ScratchBucket+gcsSourceDir,
		)
	}

	if uploaded != "" {
		args = append(args, uploaded)
	} else {
		if o.NoSource {
			args = append(args, "--no-source")
		} else {
			args = append(args, ".")
		}
	}

	cmd := exec.Command("gcloud", args...)

	if o.LogDir != "" {
		p := path.Join(o.LogDir, strings.ReplaceAll(jobName, "/", "-")+".log")
		f, err := os.Create(p)

		if err != nil {
			return fmt.Errorf("couldn't create %s: %v", p, err)
		}

		defer f.Close()

		cmd.Stdout = f
		cmd.Stderr = f
	} else {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}

	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running %s: %v", cmd.Args, err)
	}

	return nil
}

type variants map[string]map[string]string

func getVariants(o Options) (variants, error) {
	content, err := ioutil.ReadFile(path.Join(o.ConfigDir, "variants.yaml"))
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("failed to load variants.yaml: %v", err)
		}
		if o.Variant != "" {
			return nil, fmt.Errorf("no variants.yaml found, but a build variant (%q) was specified", o.Variant)
		}
		return nil, nil
	}
	v := struct {
		Variants variants `json:"variants"`
	}{}
	if err := yaml.UnmarshalStrict(content, &v); err != nil {
		return nil, fmt.Errorf("failed to read variants.yaml: %v", err)
	}
	if o.Variant != "" {
		va, ok := v.Variants[o.Variant]
		if !ok {
			return nil, fmt.Errorf("requested variant %q, which is not present in variants.yaml", o.Variant)
		}
		return variants{o.Variant: va}, nil
	}
	return v.Variants, nil
}

func RunBuildJobs(o Options) []error {
	var uploaded string
	if o.ScratchBucket != "" {
		if !o.NoSource {
			var err error
			uploaded, err = o.uploadBuildDir(o.ScratchBucket + gcsSourceDir)
			if err != nil {
				return []error{fmt.Errorf("failed to upload source: %v", err)}
			}
		}
	} else {
		log.Println("Skipping advance upload and relying on gcloud...")
	}

	log.Println("Running build jobs...")
	tag, err := getVersion()
	if err != nil {
		return []error{fmt.Errorf("failed to get current tag: %v", err)}
	}

	if !o.AllowDirty && strings.HasSuffix(tag, "-dirty") {
		return []error{fmt.Errorf("the working copy is dirty")}
	}

	vs, err := getVariants(o)
	if err != nil {
		return []error{err}
	}
	if len(vs) == 0 {
		log.Println("No variants.yaml, starting single build job...")
		if err := runSingleJob(o, "build", uploaded, tag, getExtraSubs(o)); err != nil {
			return []error{err}
		}
		return nil
	}

	log.Printf("Found variants.yaml, starting %d build jobs...\n", len(vs))

	w := sync.WaitGroup{}
	w.Add(len(vs))
	var errors []error
	extraSubs := getExtraSubs(o)
	for k, v := range vs {
		go func(job string, vc map[string]string) {
			defer w.Done()
			log.Printf("Starting job %q...\n", job)
			if err := runSingleJob(o, job, uploaded, tag, mergeMaps(extraSubs, vc)); err != nil {
				errors = append(errors, fmt.Errorf("job %q failed: %v", job, err))
				log.Printf("Job %q failed: %v\n", job, err)
			} else {
				log.Printf("Job %q completed.\n", job)
			}
		}(k, v)
	}
	w.Wait()
	return errors
}

func mergeMaps(maps ...map[string]string) map[string]string {
	out := map[string]string{}
	for _, m := range maps {
		for k, v := range m {
			out[k] = v
		}
	}
	return out
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl"
	cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/resource"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	"github.com/spf13/cobra"
)

const (
	get_long = `Display one or many resources.

Possible resources include pods (po), replication controllers (rc), services
(svc), nodes, events (ev), component statuses (cs), limit ranges (limits),
nodes (no), persistent volumes (pv), persistent volume claims (pvc)
or resource quotas (quota).

By specifying the output as 'template' and providing a Go template as the value
of the --template flag, you can filter the attributes of the fetched resource(s).`
	get_example = `// List all pods in ps output format.
$ kubectl get pods

// List a single replication controller with specified NAME in ps output format.
$ kubectl get replicationcontroller web

// List a single pod in JSON output format.
$ kubectl get -o json pod web-pod-13je7

// Return only the phase value of the specified pod.
$ kubectl get -o template web-pod-13je7 --template={{.status.phase}} --api-version=v1

// List all replication controllers and services together in ps output format.
$ kubectl get rc,services

// List one or more resources by their type and names
$ kubectl get rc/web service/frontend pods/web-pod-13je7`
)

// NewCmdGet creates a command object for the generic "get" action, which
// retrieves one or more resources from a server.
func NewCmdGet(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	p := kubectl.NewHumanReadablePrinter(false, false)
	validArgs := p.HandledResources()

	cmd := &cobra.Command{
		Use:     "get [(-o|--output=)json|yaml|template|...] (RESOURCE [NAME] | RESOURCE/NAME ...)",
		Short:   "Display one or many resources",
		Long:    get_long,
		Example: get_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunGet(f, out, cmd, args)
			cmdutil.CheckErr(err)
		},
		ValidArgs: validArgs,
	}
	cmdutil.AddPrinterFlags(cmd)
	cmd.Flags().StringP("selector", "l", "", "Selector (label query) to filter on")
	cmd.Flags().BoolP("watch", "w", false, "After listing/getting the requested object, watch for changes.")
	cmd.Flags().Bool("watch-only", false, "Watch for changes to the requested object(s), without listing/getting first.")
	cmd.Flags().Bool("all-namespaces", false, "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.")
	return cmd
}

// RunGet implements the generic Get command
// TODO: convert all direct flag accessors to a struct and pass that instead of cmd
func RunGet(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error {
	selector := cmdutil.GetFlagString(cmd, "selector")
	allNamespaces := cmdutil.GetFlagBool(cmd, "all-namespaces")
	mapper, typer := f.Object()

	cmdNamespace, err := f.DefaultNamespace()
	if err != nil {
		return err
	}

	// handle watch separately since we cannot watch multiple resource types
	isWatch, isWatchOnly := cmdutil.GetFlagBool(cmd, "watch"), cmdutil.GetFlagBool(cmd, "watch-only")
	if isWatch || isWatchOnly {
		r := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand()).
			NamespaceParam(cmdNamespace).DefaultNamespace().AllNamespaces(allNamespaces).
			SelectorParam(selector).
			ResourceTypeOrNameArgs(true, args...).
			SingleResourceType().
			Do()
		if err != nil {
			return err
		}

		mapping, err := r.ResourceMapping()
		if err != nil {
			return err
		}

		printer, err := f.PrinterForMapping(cmd, mapping, allNamespaces)
		if err != nil {
			return err
		}

		obj, err := r.Object()
		if err != nil {
			return err
		}

		rv, err := mapping.MetadataAccessor.ResourceVersion(obj)
		if err != nil {
			return err
		}

		// print the current object
		if !isWatchOnly {
			if err := printer.PrintObj(obj, out); err != nil {
				return fmt.Errorf("unable to output the provided object: %v", err)
			}
		}

		// print watched changes
		w, err := r.Watch(rv)
		if err != nil {
			return err
		}

		kubectl.WatchLoop(w, func(e watch.Event) error {
			return printer.PrintObj(e.Object, out)
		})
		return nil
	}

	b := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand()).
		NamespaceParam(cmdNamespace).DefaultNamespace().AllNamespaces(allNamespaces).
		SelectorParam(selector).
		ResourceTypeOrNameArgs(true, args...).
		ContinueOnError().
		Latest()
	printer, generic, err := cmdutil.PrinterForCommand(cmd)
	if err != nil {
		return err
	}

	if generic {
		clientConfig, err := f.ClientConfig()
		if err != nil {
			return err
		}
		defaultVersion := clientConfig.Version

		singular := false
		r := b.Flatten().Do()
		infos, err := r.IntoSingular(&singular).Infos()
		if err != nil {
			return err
		}

		// the outermost object will be converted to the output-version, but inner
		// objects can use their mappings
		version := cmdutil.OutputVersion(cmd, defaultVersion)
		obj, err := resource.AsVersionedObject(infos, !singular, version)
		if err != nil {
			return err
		}

		return printer.PrintObj(obj, out)
	}

	// use the default printer for each object
	return b.Do().Visit(func(r *resource.Info) error {
		printer, err := f.PrinterForMapping(cmd, r.Mapping, allNamespaces)
		if err != nil {
			return err
		}
		return printer.PrintObj(r.Object, out)
	})
}
<commit_msg>Add a better message if the user doesn't specify a resource type.<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"errors"
	"fmt"
	"io"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl"
	cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/resource"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/watch"

	"github.com/spf13/cobra"
)

const (
	get_long = `Display one or many resources.

Possible resources include pods (po), replication controllers (rc), services
(svc), nodes, events (ev), component statuses (cs), limit ranges (limits),
nodes (no), persistent volumes (pv), persistent volume claims (pvc)
or resource quotas (quota).

By specifying the output as 'template' and providing a Go template as the value
of the --template flag, you can filter the attributes of the fetched resource(s).`
	get_example = `// List all pods in ps output format.
$ kubectl get pods

// List a single replication controller with specified NAME in ps output format.
$ kubectl get replicationcontroller web

// List a single pod in JSON output format.
$ kubectl get -o json pod web-pod-13je7

// Return only the phase value of the specified pod.
$ kubectl get -o template web-pod-13je7 --template={{.status.phase}} --api-version=v1

// List all replication controllers and services together in ps output format.
$ kubectl get rc,services

// List one or more resources by their type and names
$ kubectl get rc/web service/frontend pods/web-pod-13je7`
)

// NewCmdGet creates a command object for the generic "get" action, which
// retrieves one or more resources from a server.
func NewCmdGet(f *cmdutil.Factory, out io.Writer) *cobra.Command {
	p := kubectl.NewHumanReadablePrinter(false, false)
	validArgs := p.HandledResources()

	cmd := &cobra.Command{
		Use:     "get [(-o|--output=)json|yaml|template|...] (RESOURCE [NAME] | RESOURCE/NAME ...)",
		Short:   "Display one or many resources",
		Long:    get_long,
		Example: get_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunGet(f, out, cmd, args)
			cmdutil.CheckErr(err)
		},
		ValidArgs: validArgs,
	}
	cmdutil.AddPrinterFlags(cmd)
	cmd.Flags().StringP("selector", "l", "", "Selector (label query) to filter on")
	cmd.Flags().BoolP("watch", "w", false, "After listing/getting the requested object, watch for changes.")
	cmd.Flags().Bool("watch-only", false, "Watch for changes to the requested object(s), without listing/getting first.")
	cmd.Flags().Bool("all-namespaces", false, "If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace.")
	return cmd
}

// RunGet implements the generic Get command
// TODO: convert all direct flag accessors to a struct and pass that instead of cmd
func RunGet(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error {
	selector := cmdutil.GetFlagString(cmd, "selector")
	allNamespaces := cmdutil.GetFlagBool(cmd, "all-namespaces")
	mapper, typer := f.Object()

	cmdNamespace, err := f.DefaultNamespace()
	if err != nil {
		return err
	}

	if len(args) == 0 {
		fmt.Fprint(out, `
You must specify the type of resource to get. Valid resource types include:
   * pods (aka 'po')
   * replicationcontrollers (aka 'rc')
   * services
   * nodes (aka 'no')
   * events (aka 'ev')
   * secrets
   * limits
   * persistentVolumes (aka 'pv')
   * persistentVolumeClaims (aka 'pvc')
   * quota
`)
		return errors.New("Required resource not specified.")
	}

	// handle watch separately since we cannot watch multiple resource types
	isWatch, isWatchOnly := cmdutil.GetFlagBool(cmd, "watch"), cmdutil.GetFlagBool(cmd, "watch-only")
	if isWatch || isWatchOnly {
		r := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand()).
			NamespaceParam(cmdNamespace).DefaultNamespace().AllNamespaces(allNamespaces).
			SelectorParam(selector).
			ResourceTypeOrNameArgs(true, args...).
			SingleResourceType().
			Do()
		if err != nil {
			return err
		}

		mapping, err := r.ResourceMapping()
		if err != nil {
			return err
		}

		printer, err := f.PrinterForMapping(cmd, mapping, allNamespaces)
		if err != nil {
			return err
		}

		obj, err := r.Object()
		if err != nil {
			return err
		}

		rv, err := mapping.MetadataAccessor.ResourceVersion(obj)
		if err != nil {
			return err
		}

		// print the current object
		if !isWatchOnly {
			if err := printer.PrintObj(obj, out); err != nil {
				return fmt.Errorf("unable to output the provided object: %v", err)
			}
		}

		// print watched changes
		w, err := r.Watch(rv)
		if err != nil {
			return err
		}

		kubectl.WatchLoop(w, func(e watch.Event) error {
			return printer.PrintObj(e.Object, out)
		})
		return nil
	}

	b := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand()).
		NamespaceParam(cmdNamespace).DefaultNamespace().AllNamespaces(allNamespaces).
		SelectorParam(selector).
		ResourceTypeOrNameArgs(true, args...).
		ContinueOnError().
		Latest()
	printer, generic, err := cmdutil.PrinterForCommand(cmd)
	if err != nil {
		return err
	}

	if generic {
		clientConfig, err := f.ClientConfig()
		if err != nil {
			return err
		}
		defaultVersion := clientConfig.Version

		singular := false
		r := b.Flatten().Do()
		infos, err := r.IntoSingular(&singular).Infos()
		if err != nil {
			return err
		}

		// the outermost object will be converted to the output-version, but inner
		// objects can use their mappings
		version := cmdutil.OutputVersion(cmd, defaultVersion)
		obj, err := resource.AsVersionedObject(infos, !singular, version)
		if err != nil {
			return err
		}

		return printer.PrintObj(obj, out)
	}

	// use the default printer for each object
	return b.Do().Visit(func(r *resource.Info) error {
		printer, err := f.PrinterForMapping(cmd, r.Mapping, allNamespaces)
		if err != nil {
			return err
		}
		return printer.PrintObj(r.Object, out)
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rkt

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
)

const (
	// The layout of the time format that satisfies the `--since` option for journalctl.
	// See man journalctl for more details.
	journalSinceLayout = "2006-01-02 15:04:05"
)

// pipeLog reads and parses the journal json object from r,
// and writes the logs line by line to w.
func pipeLog(wg *sync.WaitGroup, logOptions *api.PodLogOptions, r io.ReadCloser, w io.Writer) {
	defer func() {
		r.Close()
		wg.Done()
	}()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		var data interface{}
		b := scanner.Bytes()

		if err := json.Unmarshal(b, &data); err != nil {
			glog.Warningf("rkt: Cannot unmarshal journal log, skipping line: %v", err)
			continue
		}

		// Decode the json object as a map so we don't have to create a struct for it.
		m := data.(map[string]interface{})
		msg := m["MESSAGE"].(string)
		tss := m["__REALTIME_TIMESTAMP"].(string)

		ts, err := strconv.ParseInt(tss, 0, 64)
		if err != nil {
			glog.Warningf("rkt: Cannot parse timestamp of journal log, skipping line: %v", err)
			continue
		}

		// '_REALTIME_TIMESTAMP' is the microseconds since epoch.
		// http://www.freedesktop.org/software/systemd/man/sd_journal_get_realtime_usec.html#Description
		micros := time.Duration(ts) * time.Microsecond
		t := time.Unix(int64(micros.Seconds()), 0)

		var result string
		if logOptions.Timestamps {
			// Use the same time format as docker.
			result = fmt.Sprintf("%s %s\n", t.Format(time.RFC3339), msg)
		} else {
			result = fmt.Sprintf("%s\n", msg)
		}

		// When user interrupts the 'kubectl logs $POD -f' with 'Ctrl-C', this write will fail.
		// We need to close the reader to force the journalctl process to exit, also we need to
		// return here to avoid goroutine leak.
		if _, err := io.WriteString(w, result); err != nil {
			glog.Warningf("rkt: Cannot write log to output: %v, data: %s", err, result)
			return
		}
	}

	if err := scanner.Err(); err != nil {
		glog.Warningf("rkt: Cannot read journal logs", err)
	}
}

// GetContainerLogs uses journalctl to get the logs of the container.
// By default, it returns a snapshot of the container log. Set |follow| to true to
// stream the log. Set |follow| to false and specify the number of lines (e.g.
// "100" or "all") to tail the log.
//
// In rkt runtime's implementation, per container log is get via 'journalctl -m _HOSTNAME=[rkt-$UUID] -u [APP_NAME]'.
// See https://github.com/coreos/rkt/blob/master/Documentation/commands.md#logging for more details.
//
// TODO(yifan): If the rkt is using lkvm as the stage1 image, then this function will fail.
func (r *Runtime) GetContainerLogs(pod *api.Pod, containerID kubecontainer.ContainerID, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error {
	id, err := parseContainerID(containerID)
	if err != nil {
		return err
	}

	cmd := exec.Command("journalctl", "-m", fmt.Sprintf("_HOSTNAME=rkt-%s", id.uuid), "-u", id.appName, "-a")

	// Get the json structured logs.
	cmd.Args = append(cmd.Args, "-o", "json")

	if logOptions.Follow {
		cmd.Args = append(cmd.Args, "-f")
	}

	if logOptions.TailLines != nil {
		cmd.Args = append(cmd.Args, "-n", strconv.FormatInt(*logOptions.TailLines, 10))
	}

	var since int64
	if logOptions.SinceSeconds != nil {
		t := unversioned.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
		since = t.Unix()
	}
	if logOptions.SinceTime != nil {
		since = logOptions.SinceTime.Unix()
	}

	if since > 0 {
		// Need to add '-r' flag if we include '--since' and '-n' at the both time,
		// see https://github.com/systemd/systemd/issues/1477
		cmd.Args = append(cmd.Args, "--since", time.Unix(since, 0).Format(journalSinceLayout))
		if logOptions.TailLines != nil {
			cmd.Args = append(cmd.Args, "-r")
		}
	}

	outPipe, err := cmd.StdoutPipe()
	if err != nil {
		glog.Errorf("rkt: cannot create pipe for journalctl's stdout", err)
		return err
	}
	errPipe, err := cmd.StderrPipe()
	if err != nil {
		glog.Errorf("rkt: cannot create pipe for journalctl's stderr", err)
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	var wg sync.WaitGroup

	wg.Add(2)

	go pipeLog(&wg, logOptions, outPipe, stdout)
	go pipeLog(&wg, logOptions, errPipe, stderr)

	// Wait until the logs are fed to stdout, stderr.
	wg.Wait()
	cmd.Wait()

	return nil
}
<commit_msg>Use `-M` flag for journalctl<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rkt

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"strconv"
	"sync"
	"time"

	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
)

const (
	// The layout of the time format that satisfies the `--since` option for journalctl.
	// See man journalctl for more details.
	journalSinceLayout = "2006-01-02 15:04:05"
)

// pipeLog reads and parses the journal json object from r,
// and writes the logs line by line to w.
func pipeLog(wg *sync.WaitGroup, logOptions *api.PodLogOptions, r io.ReadCloser, w io.Writer) {
	defer func() {
		r.Close()
		wg.Done()
	}()

	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		var data interface{}
		b := scanner.Bytes()

		if err := json.Unmarshal(b, &data); err != nil {
			glog.Warningf("rkt: Cannot unmarshal journal log, skipping line: %v", err)
			continue
		}

		// Decode the json object as a map so we don't have to create a struct for it.
		m := data.(map[string]interface{})
		msg := m["MESSAGE"].(string)
		tss := m["__REALTIME_TIMESTAMP"].(string)

		ts, err := strconv.ParseInt(tss, 0, 64)
		if err != nil {
			glog.Warningf("rkt: Cannot parse timestamp of journal log, skipping line: %v", err)
			continue
		}

		// '_REALTIME_TIMESTAMP' is the microseconds since epoch.
		// http://www.freedesktop.org/software/systemd/man/sd_journal_get_realtime_usec.html#Description
		micros := time.Duration(ts) * time.Microsecond
		t := time.Unix(int64(micros.Seconds()), 0)

		var result string
		if logOptions.Timestamps {
			// Use the same time format as docker.
			result = fmt.Sprintf("%s %s\n", t.Format(time.RFC3339), msg)
		} else {
			result = fmt.Sprintf("%s\n", msg)
		}

		// When user interrupts the 'kubectl logs $POD -f' with 'Ctrl-C', this write will fail.
		// We need to close the reader to force the journalctl process to exit, also we need to
		// return here to avoid goroutine leak.
		if _, err := io.WriteString(w, result); err != nil {
			glog.Warningf("rkt: Cannot write log to output: %v, data: %s", err, result)
			return
		}
	}

	if err := scanner.Err(); err != nil {
		glog.Warningf("rkt: Cannot read journal logs", err)
	}
}

// GetContainerLogs uses journalctl to get the logs of the container.
// By default, it returns a snapshot of the container log. Set |follow| to true to
// stream the log. Set |follow| to false and specify the number of lines (e.g.
// "100" or "all") to tail the log.
//
// In rkt runtime's implementation, per container log is get via 'journalctl -m _HOSTNAME=[rkt-$UUID] -u [APP_NAME]'.
// See https://github.com/coreos/rkt/blob/master/Documentation/commands.md#logging for more details.
//
// TODO(yifan): If the rkt is using lkvm as the stage1 image, then this function will fail.
func (r *Runtime) GetContainerLogs(pod *api.Pod, containerID kubecontainer.ContainerID, logOptions *api.PodLogOptions, stdout, stderr io.Writer) error {
	id, err := parseContainerID(containerID)
	if err != nil {
		return err
	}

	cmd := exec.Command("journalctl", "-m", "-M", fmt.Sprintf("rkt-%s", id.uuid), "-u", id.appName, "-a")

	// Get the json structured logs.
	cmd.Args = append(cmd.Args, "-o", "json")

	if logOptions.Follow {
		cmd.Args = append(cmd.Args, "-f")
	}

	if logOptions.TailLines != nil {
		cmd.Args = append(cmd.Args, "-n", strconv.FormatInt(*logOptions.TailLines, 10))
	}

	var since int64
	if logOptions.SinceSeconds != nil {
		t := unversioned.Now().Add(-time.Duration(*logOptions.SinceSeconds) * time.Second)
		since = t.Unix()
	}
	if logOptions.SinceTime != nil {
		since = logOptions.SinceTime.Unix()
	}

	if since > 0 {
		// Need to add '-r' flag if we include '--since' and '-n' at the both time,
		// see https://github.com/systemd/systemd/issues/1477
		cmd.Args = append(cmd.Args, "--since", time.Unix(since, 0).Format(journalSinceLayout))
		if logOptions.TailLines != nil {
			cmd.Args = append(cmd.Args, "-r")
		}
	}

	outPipe, err := cmd.StdoutPipe()
	if err != nil {
		glog.Errorf("rkt: cannot create pipe for journalctl's stdout", err)
		return err
	}
	errPipe, err := cmd.StderrPipe()
	if err != nil {
		glog.Errorf("rkt: cannot create pipe for journalctl's stderr", err)
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	var wg sync.WaitGroup

	wg.Add(2)

	go pipeLog(&wg, logOptions, outPipe, stdout)
	go pipeLog(&wg, logOptions, errPipe, stderr)

	// Wait until the logs are fed to stdout, stderr.
	wg.Wait()
	cmd.Wait()

	return nil
}
<|endoftext|>