<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "blocks-gcs-proxy"
	app.Usage = "github.com/groovenauts/blocks-gcs-proxy"
	app.Version = VERSION

	configFlag := cli.StringFlag{
		Name:  "config, c",
		Usage: "Load configuration from `FILE`",
	}
	app.Flags = []cli.Flag{
		configFlag,
	}

	app.Commands = []cli.Command{
		{
			Name:  "check",
			Usage: "Check config file is valid",
			Action: func(c *cli.Context) error {
				LoadAndSetupProcessConfig(c)
				fmt.Println("OK")
				return nil
			},
			Flags: []cli.Flag{
				configFlag,
			},
		},
		{
			Name:  "upload",
			Usage: "Upload the files under uploads directory",
			Action: func(c *cli.Context) error {
				config := &ProcessConfig{}
				config.Log = &LogConfig{Level: "debug"}
				config.setup([]string{})
				config.Command.Uploaders = c.Int("uploaders")
				p := setupProcess(config)
				p.setup()
				job := &Job{
					config:      config.Command,
					uploads_dir: c.String("uploads_dir"),
					storage:     p.storage,
				}
				err := job.uploadFiles()
				return err
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "uploads_dir, d",
					Usage: "Path to the directory which has bucket_name/path/to/file",
				},
				cli.IntFlag{
					Name:  "uploaders, n",
					Usage: "Number of uploaders",
					Value: 6,
				},
			},
		},
	}

	app.Action = run

	app.Run(os.Args)
}

func run(c *cli.Context) error {
	config := LoadAndSetupProcessConfig(c)
	p := setupProcess(config)

	err := p.run()
	if err != nil {
		fmt.Printf("Error to run cause of %v\n", err)
		os.Exit(1)
	}
	return nil
}

func setupProcess(config *ProcessConfig) *Process {
	p := &Process{config: config}
	err := p.setup()
	if err != nil {
		fmt.Printf("Error to setup Process cause of %v\n", err)
		os.Exit(1)
	}
	return p
}

func LoadAndSetupProcessConfig(c *cli.Context) *ProcessConfig {
	path := configPath(c)
	config, err := LoadProcessConfig(path)
	if err != nil {
		fmt.Printf("Error to load %v cause of %v\n", path, err)
		os.Exit(1)
	}
	err = config.setup(os.Args[1:])
	if err != nil {
		fmt.Printf("Error to setup %v cause of %v\n", path, err)
		os.Exit(1)
	}
	return config
}

func configPath(c *cli.Context) string {
	r := c.String("config")
	if r == "" {
		r = "./config.json"
	}
	return r
}
<commit_msg>:+1: Add download subcommand<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "blocks-gcs-proxy"
	app.Usage = "github.com/groovenauts/blocks-gcs-proxy"
	app.Version = VERSION

	configFlag := cli.StringFlag{
		Name:  "config, c",
		Usage: "Load configuration from `FILE`",
	}
	app.Flags = []cli.Flag{
		configFlag,
	}

	app.Commands = []cli.Command{
		{
			Name:  "check",
			Usage: "Check config file is valid",
			Action: func(c *cli.Context) error {
				LoadAndSetupProcessConfig(c)
				fmt.Println("OK")
				return nil
			},
			Flags: []cli.Flag{
				configFlag,
			},
		},
		{
			Name:  "download",
			Usage: "Download the files under downloads directory",
			Action: func(c *cli.Context) error {
				config := &ProcessConfig{}
				config.Log = &LogConfig{Level: "debug"}
				config.setup([]string{})
				config.Command.Downloaders = c.Int("downloaders")
				p := setupProcess(config)
				p.setup()
				files := []interface{}{}
				for _, arg := range c.Args() {
					files = append(files, arg)
				}
				job := &Job{
					config:      config.Command,
					downloads_dir: c.String("downloads_dir"),
					remoteDownloadFiles: files,
					storage:     p.storage,
				}
				err := job.setupDownloadFiles()
				if err != nil {
					return err
				}
				err = job.downloadFiles()
				return err
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "downloads_dir, d",
					Usage: "Path to the directory which has bucket_name/path/to/file",
				},
				cli.IntFlag{
					Name:  "downloaders, n",
					Usage: "Number of downloaders",
					Value: 6,
				},
			},
		},
		{
			Name:  "upload",
			Usage: "Upload the files under uploads directory",
			Action: func(c *cli.Context) error {
				config := &ProcessConfig{}
				config.Log = &LogConfig{Level: "debug"}
				config.setup([]string{})
				config.Command.Uploaders = c.Int("uploaders")
				p := setupProcess(config)
				p.setup()
				job := &Job{
					config:      config.Command,
					uploads_dir: c.String("uploads_dir"),
					storage:     p.storage,
				}
				err := job.uploadFiles()
				return err
			},
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "uploads_dir, d",
					Usage: "Path to the directory which has bucket_name/path/to/file",
				},
				cli.IntFlag{
					Name:  "uploaders, n",
					Usage: "Number of uploaders",
					Value: 6,
				},
			},
		},
	}

	app.Action = run

	app.Run(os.Args)
}

func run(c *cli.Context) error {
	config := LoadAndSetupProcessConfig(c)
	p := setupProcess(config)

	err := p.run()
	if err != nil {
		fmt.Printf("Error to run cause of %v\n", err)
		os.Exit(1)
	}
	return nil
}

func setupProcess(config *ProcessConfig) *Process {
	p := &Process{config: config}
	err := p.setup()
	if err != nil {
		fmt.Printf("Error to setup Process cause of %v\n", err)
		os.Exit(1)
	}
	return p
}

func LoadAndSetupProcessConfig(c *cli.Context) *ProcessConfig {
	path := configPath(c)
	config, err := LoadProcessConfig(path)
	if err != nil {
		fmt.Printf("Error to load %v cause of %v\n", path, err)
		os.Exit(1)
	}
	err = config.setup(os.Args[1:])
	if err != nil {
		fmt.Printf("Error to setup %v cause of %v\n", path, err)
		os.Exit(1)
	}
	return config
}

func configPath(c *cli.Context) string {
	r := c.String("config")
	if r == "" {
		r = "./config.json"
	}
	return r
}
<|endoftext|>


<commit_before>package cluster

import (
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/gansoi/gansoi/ca"
	"github.com/gansoi/gansoi/logger"
	"github.com/gin-gonic/gin"
)

type (
	// Core describes a Gansoi core node.
	Core struct {
		info        *Info
		ca          *ca.CA
		pair        []tls.Certificate
		raftHandler http.Handler
		peerAdder   PeerAdder
	}
)

const (
	// CorePrefix is the http prefix core should use.
	CorePrefix = "/core"
)

// NewCore initializes a new core.
func NewCore(info *Info) *Core {
	c := &Core{
		info: info,
	}

	return c
}

// CA returns the CA.
func (c *Core) CA() *ca.CA {
	return c.ca
}

func nodeInit(info *Info, coreCA *ca.CA) ([]tls.Certificate, error) {
	// Genmerate new key for current node.
	nodeKey, err := ca.GenerateKey()
	if err != nil {
		return nil, err
	}

	// Generate CSR for current node.
	hostID := ca.RandomString(16)

	ips := localIps()
	ip := info.IP()
	if ip != nil {
		ips = append(ips, ip)
	}

	csr, err := ca.GenerateCSR(nodeKey, hostID, ips)
	if err != nil {
		return nil, err
	}

	// Sign said CSR.
	nodeCert, err := coreCA.SignCSR(csr)
	if err != nil {
		return nil, err
	}

	info.NodeCert, err = ca.EncodeCert(nodeCert)
	if err != nil {
		return nil, err
	}

	info.NodeKey, err = ca.EncodeKey(nodeKey)
	if err != nil {
		return nil, err
	}

	err = info.Save()
	if err != nil {
		return nil, err
	}

	return nodeSetup(info, coreCA)
}

func nodeSetup(info *Info, coreCA *ca.CA) ([]tls.Certificate, error) {
	var err error
	info.CAKey, err = coreCA.CertificatePEM()
	if err != nil {
		return nil, err
	}

	info.CAKey, err = coreCA.KeyPEM()
	if err != nil {
		return nil, err
	}

	pair, err := tls.X509KeyPair(info.NodeCert, info.NodeKey)
	if err != nil {
		return nil, err
	}

	return []tls.Certificate{pair}, nil
}

// Bootstrap a new cluster.
func (c *Core) Bootstrap() error {
	var err error

	if c.info.CACert != nil {
		logger.Info("main", "Cluster seem to already be initialized")
		os.Exit(1)
	}

	c.ca, err = ca.InitCA()
	if err != nil {
		return err
	}
	c.info.CACert, err = c.ca.CertificatePEM()
	if err != nil {
		return err
	}

	c.info.CAKey, err = c.ca.KeyPEM()
	if err != nil {
		return err
	}

	_, err = nodeInit(c.info, c.ca)
	if err != nil {
		return err
	}

	// Compute join token
	c.info.ClusterToken = ca.RandomString(40)
	c.info.Save()

	return nil
}

// client will return a client filled with everything we know.
func (c *Core) client() *http.Client {
	tlsConfig := &tls.Config{}

	if c.info.CACert != nil {
		// If we have establish a root, use this for verifying peer.
		pool := x509.NewCertPool()
		pool.AppendCertsFromPEM(c.info.CACert)
		tlsConfig.RootCAs = pool
	} else {
		// If we don't have any root yet, use no verification.
		tlsConfig.InsecureSkipVerify = true
	}

	// If we have client certifcates, use them :)
	if c.pair != nil {
		tlsConfig.Certificates = c.pair
	}

	transport := &http.Transport{
		TLSClientConfig: tlsConfig,
	}

	return &http.Client{Transport: transport}
}

// Join an existing cluster.
func (c *Core) Join(address string, hash string, token string, bindPrivate string) error {
	// Get the certifcate - ignore TLS errors. We verify the cert based on the
	// hash provided in the join-token.
	resp, err := c.client().Get("https://" + DefaultPort(address) + CorePrefix + "/cert")
	if err != nil {
		return err
	}

	c.info.CACert, _ = ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if fmt.Sprintf("%x", sha256.Sum256(c.info.CACert)) != hash {
		return errors.New("Remote certifcate hash doesn't match")
	}

	_, err = ca.DecodeCert(c.info.CACert)
	if err != nil {
		return err
	}

	logger.Info("join", "Got cluster root certificate")

	// Get the root key authenticating with our cluster token.
	req, _ := http.NewRequest("GET", "https://"+DefaultPort(address)+CorePrefix+"/key", nil)
	req.Header.Add("X-Gansoi-Token", token)

	resp, err = c.client().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	c.info.CAKey, _ = ioutil.ReadAll(resp.Body)
	logger.Info("join", "Got cluster root key")

	// Set up local CA
	c.ca, err = ca.OpenCA(c.info.CAKey, c.info.CACert)
	if err != nil {
		return err
	}

	c.pair, err = nodeInit(c.info, c.ca)
	if err != nil {
		return err
	}

	logger.Info("join", "Local core initialized")

	c.info.ClusterToken = token

	// Request to join cluster.
	logger.Info("join", "Requesting raft join")
	req, _ = http.NewRequest("GET", "https://"+DefaultPort(address)+CorePrefix+"/join", nil)
	if bindPrivate != "" {
		req.Header.Add("X-Gansoi-Announce", DefaultPort(bindPrivate))
	}

	resp, err = c.client().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		return errors.New(string(b))
	}

	c.info.SetPeers([]string{
		DefaultPort(bindPrivate),
		DefaultPort(address),
	})

	logger.Info("join", "Joined.")

	return nil
}

// Start a Gansoi core node.
func (c *Core) Start() ([]tls.Certificate, error) {
	var err error

	// Set up CA.
	c.ca, err = ca.OpenCA(c.info.CAKey, c.info.CACert)
	if err != nil {
		return nil, err
	}

	return nodeSetup(c.info, c.ca)
}

// ClientCertificatePair provides a certificates used to identify this core
// node to other nodes.
func (c *Core) ClientCertificatePair() []tls.Certificate {
	// FIXME: stub
	return nil
}

func (c *Core) handleCert(context *gin.Context) {
	cert, err := c.CA().CertificatePEM()
	if err != nil {
		context.AbortWithError(500, err)
		return
	}

	context.Writer.Write(cert)
}

func (c *Core) handleKey(context *gin.Context) {
	token := context.Request.Header.Get("X-Gansoi-Token")

	if token != string(c.info.ClusterToken) {
		context.Data(401, "text/plain", []byte("token mismatch"))
		return
	}

	key, _ := c.CA().KeyPEM()
	context.Writer.Write(key)
}

func (c *Core) handleJoin(context *gin.Context) {
	name, err := c.CA().VerifyHTTPRequest(context.Request)
	if err != nil {
		context.Data(http.StatusUnauthorized, "text/plain", []byte(err.Error()))
		return
	}

	announce := context.Request.Header.Get("X-Gansoi-Announce")
	if announce == "" {
		context.Data(http.StatusBadRequest, "text/plain", []byte("Set X-Gansoi-Announce header"))
		return
	}

	logger.Debug("cluster", "%s at %s requesting to join", name, context.Request.RemoteAddr)

	err = c.peerAdder.AddPeer(announce)
	if err != nil {
		logger.Info("join", err.Error())
	}
}

func (c *Core) handleRaft(context *gin.Context) {
	name, err := c.CA().VerifyHTTPRequest(context.Request)
	if err != nil {
		context.Data(http.StatusUnauthorized, "text/plain", []byte(err.Error()))
		return
	}

	logger.Debug("internal-comm", "%s connected", name)

	c.raftHandler.ServeHTTP(context.Writer, context.Request)
}

// Router can be used to assign a Gin routergroup.
func (c *Core) Router(router *gin.RouterGroup, raftHandler http.Handler, peerAdder PeerAdder) {
	c.raftHandler = raftHandler
	c.peerAdder = peerAdder

	router.GET("/cert", c.handleCert)
	router.GET("/key", c.handleKey)
	router.GET("/join", c.handleJoin)
	router.GET("/raft", c.handleRaft)
}
<commit_msg>Peer list should be empty on new nodes. The leader will dial the new contact member.<commit_after>package cluster

import (
	"crypto/sha256"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"

	"github.com/gansoi/gansoi/ca"
	"github.com/gansoi/gansoi/logger"
	"github.com/gin-gonic/gin"
)

type (
	// Core describes a Gansoi core node.
	Core struct {
		info        *Info
		ca          *ca.CA
		pair        []tls.Certificate
		raftHandler http.Handler
		peerAdder   PeerAdder
	}
)

const (
	// CorePrefix is the http prefix core should use.
	CorePrefix = "/core"
)

// NewCore initializes a new core.
func NewCore(info *Info) *Core {
	c := &Core{
		info: info,
	}

	return c
}

// CA returns the CA.
func (c *Core) CA() *ca.CA {
	return c.ca
}

func nodeInit(info *Info, coreCA *ca.CA) ([]tls.Certificate, error) {
	// Genmerate new key for current node.
	nodeKey, err := ca.GenerateKey()
	if err != nil {
		return nil, err
	}

	// Generate CSR for current node.
	hostID := ca.RandomString(16)

	ips := localIps()
	ip := info.IP()
	if ip != nil {
		ips = append(ips, ip)
	}

	csr, err := ca.GenerateCSR(nodeKey, hostID, ips)
	if err != nil {
		return nil, err
	}

	// Sign said CSR.
	nodeCert, err := coreCA.SignCSR(csr)
	if err != nil {
		return nil, err
	}

	info.NodeCert, err = ca.EncodeCert(nodeCert)
	if err != nil {
		return nil, err
	}

	info.NodeKey, err = ca.EncodeKey(nodeKey)
	if err != nil {
		return nil, err
	}

	err = info.Save()
	if err != nil {
		return nil, err
	}

	return nodeSetup(info, coreCA)
}

func nodeSetup(info *Info, coreCA *ca.CA) ([]tls.Certificate, error) {
	var err error
	info.CAKey, err = coreCA.CertificatePEM()
	if err != nil {
		return nil, err
	}

	info.CAKey, err = coreCA.KeyPEM()
	if err != nil {
		return nil, err
	}

	pair, err := tls.X509KeyPair(info.NodeCert, info.NodeKey)
	if err != nil {
		return nil, err
	}

	return []tls.Certificate{pair}, nil
}

// Bootstrap a new cluster.
func (c *Core) Bootstrap() error {
	var err error

	if c.info.CACert != nil {
		logger.Info("main", "Cluster seem to already be initialized")
		os.Exit(1)
	}

	c.ca, err = ca.InitCA()
	if err != nil {
		return err
	}
	c.info.CACert, err = c.ca.CertificatePEM()
	if err != nil {
		return err
	}

	c.info.CAKey, err = c.ca.KeyPEM()
	if err != nil {
		return err
	}

	_, err = nodeInit(c.info, c.ca)
	if err != nil {
		return err
	}

	// Compute join token
	c.info.ClusterToken = ca.RandomString(40)
	c.info.Save()

	return nil
}

// client will return a client filled with everything we know.
func (c *Core) client() *http.Client {
	tlsConfig := &tls.Config{}

	if c.info.CACert != nil {
		// If we have establish a root, use this for verifying peer.
		pool := x509.NewCertPool()
		pool.AppendCertsFromPEM(c.info.CACert)
		tlsConfig.RootCAs = pool
	} else {
		// If we don't have any root yet, use no verification.
		tlsConfig.InsecureSkipVerify = true
	}

	// If we have client certifcates, use them :)
	if c.pair != nil {
		tlsConfig.Certificates = c.pair
	}

	transport := &http.Transport{
		TLSClientConfig: tlsConfig,
	}

	return &http.Client{Transport: transport}
}

// Join an existing cluster.
func (c *Core) Join(address string, hash string, token string, bindPrivate string) error {
	// Get the certifcate - ignore TLS errors. We verify the cert based on the
	// hash provided in the join-token.
	resp, err := c.client().Get("https://" + DefaultPort(address) + CorePrefix + "/cert")
	if err != nil {
		return err
	}

	c.info.CACert, _ = ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	if fmt.Sprintf("%x", sha256.Sum256(c.info.CACert)) != hash {
		return errors.New("Remote certifcate hash doesn't match")
	}

	_, err = ca.DecodeCert(c.info.CACert)
	if err != nil {
		return err
	}

	logger.Info("join", "Got cluster root certificate")

	// Get the root key authenticating with our cluster token.
	req, _ := http.NewRequest("GET", "https://"+DefaultPort(address)+CorePrefix+"/key", nil)
	req.Header.Add("X-Gansoi-Token", token)

	resp, err = c.client().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	c.info.CAKey, _ = ioutil.ReadAll(resp.Body)
	logger.Info("join", "Got cluster root key")

	// Set up local CA
	c.ca, err = ca.OpenCA(c.info.CAKey, c.info.CACert)
	if err != nil {
		return err
	}

	c.pair, err = nodeInit(c.info, c.ca)
	if err != nil {
		return err
	}

	logger.Info("join", "Local core initialized")

	c.info.ClusterToken = token

	// Request to join cluster.
	logger.Info("join", "Requesting raft join")
	req, _ = http.NewRequest("GET", "https://"+DefaultPort(address)+CorePrefix+"/join", nil)
	if bindPrivate != "" {
		req.Header.Add("X-Gansoi-Announce", DefaultPort(bindPrivate))
	}

	resp, err = c.client().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		b, _ := ioutil.ReadAll(resp.Body)
		return errors.New(string(b))
	}

	logger.Info("join", "Joined.")

	return nil
}

// Start a Gansoi core node.
func (c *Core) Start() ([]tls.Certificate, error) {
	var err error

	// Set up CA.
	c.ca, err = ca.OpenCA(c.info.CAKey, c.info.CACert)
	if err != nil {
		return nil, err
	}

	return nodeSetup(c.info, c.ca)
}

// ClientCertificatePair provides a certificates used to identify this core
// node to other nodes.
func (c *Core) ClientCertificatePair() []tls.Certificate {
	// FIXME: stub
	return nil
}

func (c *Core) handleCert(context *gin.Context) {
	cert, err := c.CA().CertificatePEM()
	if err != nil {
		context.AbortWithError(500, err)
		return
	}

	context.Writer.Write(cert)
}

func (c *Core) handleKey(context *gin.Context) {
	token := context.Request.Header.Get("X-Gansoi-Token")

	if token != string(c.info.ClusterToken) {
		context.Data(401, "text/plain", []byte("token mismatch"))
		return
	}

	key, _ := c.CA().KeyPEM()
	context.Writer.Write(key)
}

func (c *Core) handleJoin(context *gin.Context) {
	name, err := c.CA().VerifyHTTPRequest(context.Request)
	if err != nil {
		context.Data(http.StatusUnauthorized, "text/plain", []byte(err.Error()))
		return
	}

	announce := context.Request.Header.Get("X-Gansoi-Announce")
	if announce == "" {
		context.Data(http.StatusBadRequest, "text/plain", []byte("Set X-Gansoi-Announce header"))
		return
	}

	logger.Debug("cluster", "%s at %s requesting to join", name, context.Request.RemoteAddr)

	err = c.peerAdder.AddPeer(announce)
	if err != nil {
		logger.Info("join", err.Error())
	}
}

func (c *Core) handleRaft(context *gin.Context) {
	name, err := c.CA().VerifyHTTPRequest(context.Request)
	if err != nil {
		context.Data(http.StatusUnauthorized, "text/plain", []byte(err.Error()))
		return
	}

	logger.Debug("internal-comm", "%s connected", name)

	c.raftHandler.ServeHTTP(context.Writer, context.Request)
}

// Router can be used to assign a Gin routergroup.
func (c *Core) Router(router *gin.RouterGroup, raftHandler http.Handler, peerAdder PeerAdder) {
	c.raftHandler = raftHandler
	c.peerAdder = peerAdder

	router.GET("/cert", c.handleCert)
	router.GET("/key", c.handleKey)
	router.GET("/join", c.handleJoin)
	router.GET("/raft", c.handleRaft)
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/Shopify/sarama"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	testGroup     = "sarama-cluster-group"
	testKafkaData = "/tmp/sarama-cluster-test"
)

var (
	testKafkaRoot  = "kafka_2.12-1.1.0"
	testKafkaAddrs = []string{"127.0.0.1:29092"}
	testTopics     = []string{"topic-a", "topic-b"}

	testClient              sarama.Client
	testKafkaCmd, testZkCmd *exec.Cmd
)

func init() {
	if dir := os.Getenv("KAFKA_DIR"); dir != "" {
		testKafkaRoot = dir
	}
}

var _ = Describe("offsetInfo", func() {

	It("should calculate next offset", func() {
		Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
		Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))
		Expect(offsetInfo{-1, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
		Expect(offsetInfo{-1, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))
		Expect(offsetInfo{0, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(int64(0)))
		Expect(offsetInfo{100, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(int64(100)))
	})

})

var _ = Describe("int32Slice", func() {

	It("should diff", func() {
		Expect(((int32Slice)(nil)).Diff(int32Slice{1, 3, 5})).To(BeNil())
		Expect(int32Slice{1, 3, 5}.Diff((int32Slice)(nil))).To(Equal([]int32{1, 3, 5}))
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 3, 5})).To(BeNil())
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 2, 3, 4, 5})).To(BeNil())
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{2, 3, 4})).To(Equal([]int32{1, 5}))
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 4})).To(Equal([]int32{3, 5}))
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{2, 5})).To(Equal([]int32{1, 3}))
	})

})

// --------------------------------------------------------------------

var _ = BeforeSuite(func() {
	testZkCmd = testCmd(
		testDataDir(testKafkaRoot, "bin", "kafka-run-class.sh"),
		"org.apache.zookeeper.server.quorum.QuorumPeerMain",
		testDataDir("zookeeper.properties"),
	)

	testKafkaCmd = testCmd(
		testDataDir(testKafkaRoot, "bin", "kafka-run-class.sh"),
		"-name", "kafkaServer", "kafka.Kafka",
		testDataDir("server.properties"),
	)

	// Remove old test data before starting
	Expect(os.RemoveAll(testKafkaData)).NotTo(HaveOccurred())

	Expect(os.MkdirAll(testKafkaData, 0777)).To(Succeed())
	Expect(testZkCmd.Start()).To(Succeed())
	Expect(testKafkaCmd.Start()).To(Succeed())

	// Wait for client
	Eventually(func() error {
		var err error

		// sync-producer requires Return.Successes set to true
		testConf := sarama.NewConfig()
		testConf.Producer.Return.Successes = true
		testClient, err = sarama.NewClient(testKafkaAddrs, testConf)
		return err
	}, "30s", "1s").Should(Succeed())

	// Ensure we can retrieve partition info
	Eventually(func() error {
		_, err := testClient.Partitions(testTopics[0])
		return err
	}, "30s", "1s").Should(Succeed())

	// Seed a few messages
	Expect(testSeed(1000, testTopics)).To(Succeed())
})

var _ = AfterSuite(func() {
	if testClient != nil {
		_ = testClient.Close()
	}

	_ = testKafkaCmd.Process.Kill()
	_ = testZkCmd.Process.Kill()
	_ = testKafkaCmd.Wait()
	_ = testZkCmd.Wait()
	_ = os.RemoveAll(testKafkaData)
})

// --------------------------------------------------------------------

func TestSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "sarama/cluster")
}

func testDataDir(tokens ...string) string {
	tokens = append([]string{"testdata"}, tokens...)
	return filepath.Join(tokens...)
}

func testSeed(n int, testTopics []string) error {
	producer, err := sarama.NewSyncProducerFromClient(testClient)
	if err != nil {
		return err
	}
	defer producer.Close()

	for i := 0; i < n; i++ {
		kv := sarama.StringEncoder(fmt.Sprintf("PLAINDATA-%08d", i))
		for _, t := range testTopics {
			msg := &sarama.ProducerMessage{Topic: t, Key: kv, Value: kv}
			if _, _, err := producer.SendMessage(msg); err != nil {
				return err
			}
		}
	}
	return nil
}

func testCmd(name string, arg ...string) *exec.Cmd {
	cmd := exec.Command(name, arg...)
	if testing.Verbose() || os.Getenv("CI") != "" {
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
	}
	cmd.Env = []string{"KAFKA_HEAP_OPTS=-Xmx1G -Xms1G"}
	return cmd
}

type testConsumerMessage struct {
	sarama.ConsumerMessage
	ConsumerID string
}

// --------------------------------------------------------------------

var _ sarama.Consumer = &mockConsumer{}
var _ sarama.PartitionConsumer = &mockPartitionConsumer{}

type mockClient struct {
	sarama.Client

	topics map[string][]int32
}
type mockConsumer struct{ sarama.Consumer }
type mockPartitionConsumer struct {
	sarama.PartitionConsumer

	Topic     string
	Partition int32
	Offset    int64
}

func (m *mockClient) Partitions(t string) ([]int32, error) {
	pts, ok := m.topics[t]
	if !ok {
		return nil, sarama.ErrInvalidTopic
	}
	return pts, nil
}

func (*mockConsumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) {
	if offset > -1 && offset < 1000 {
		return nil, sarama.ErrOffsetOutOfRange
	}
	return &mockPartitionConsumer{
		Topic:     topic,
		Partition: partition,
		Offset:    offset,
	}, nil
}

func (*mockPartitionConsumer) AsyncClose() {}
<commit_msg>Revert "Fix tests"<commit_after>package cluster

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/Shopify/sarama"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	testGroup     = "sarama-cluster-group"
	testKafkaData = "/tmp/sarama-cluster-test"
)

var (
	testKafkaRoot  = "kafka_2.12-1.1.0"
	testKafkaAddrs = []string{"127.0.0.1:29092"}
	testTopics     = []string{"topic-a", "topic-b"}

	testClient              sarama.Client
	testKafkaCmd, testZkCmd *exec.Cmd
)

func init() {
	if dir := os.Getenv("KAFKA_DIR"); dir != "" {
		testKafkaRoot = dir
	}
}

var _ = Describe("offsetInfo", func() {

	It("should calculate next offset", func() {
		Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
		Expect(offsetInfo{-2, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))
		Expect(offsetInfo{-1, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(sarama.OffsetOldest))
		Expect(offsetInfo{-1, ""}.NextOffset(sarama.OffsetNewest)).To(Equal(sarama.OffsetNewest))
		Expect(offsetInfo{0, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(int64(0)))
		Expect(offsetInfo{100, ""}.NextOffset(sarama.OffsetOldest)).To(Equal(int64(100)))
	})

})

var _ = Describe("int32Slice", func() {

	It("should diff", func() {
		Expect(((int32Slice)(nil)).Diff(int32Slice{1, 3, 5})).To(BeNil())
		Expect(int32Slice{1, 3, 5}.Diff((int32Slice)(nil))).To(Equal([]int32{1, 3, 5}))
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 3, 5})).To(BeNil())
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 2, 3, 4, 5})).To(BeNil())
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{2, 3, 4})).To(Equal([]int32{1, 5}))
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{1, 4})).To(Equal([]int32{3, 5}))
		Expect(int32Slice{1, 3, 5}.Diff(int32Slice{2, 5})).To(Equal([]int32{1, 3}))
	})

})

// --------------------------------------------------------------------

var _ = BeforeSuite(func() {
	testZkCmd = testCmd(
		testDataDir(testKafkaRoot, "bin", "kafka-run-class.sh"),
		"org.apache.zookeeper.server.quorum.QuorumPeerMain",
		testDataDir("zookeeper.properties"),
	)

	testKafkaCmd = testCmd(
		testDataDir(testKafkaRoot, "bin", "kafka-run-class.sh"),
		"-name", "kafkaServer", "kafka.Kafka",
		testDataDir("server.properties"),
	)

	// Remove old test data before starting
	Expect(os.RemoveAll(testKafkaData)).NotTo(HaveOccurred())

	Expect(os.MkdirAll(testKafkaData, 0777)).To(Succeed())
	Expect(testZkCmd.Start()).To(Succeed())
	Expect(testKafkaCmd.Start()).To(Succeed())

	// Wait for client
	Eventually(func() error {
		var err error

		// sync-producer requires Return.Successes set to true
		testConf := sarama.NewConfig()
		testConf.Producer.Return.Successes = true
		testClient, err = sarama.NewClient(testKafkaAddrs, testConf)
		return err
	}, "30s", "1s").Should(Succeed())

	// Ensure we can retrieve partition info
	Eventually(func() error {
		_, err := testClient.Partitions(testTopics[0])
		return err
	}, "30s", "1s").Should(Succeed())

	// Seed a few messages
	Expect(testSeed(1000, testTopics)).To(Succeed())
})

var _ = AfterSuite(func() {
	if testClient != nil {
		_ = testClient.Close()
	}

	_ = testKafkaCmd.Process.Kill()
	_ = testZkCmd.Process.Kill()
	_ = testKafkaCmd.Wait()
	_ = testZkCmd.Wait()
	_ = os.RemoveAll(testKafkaData)
})

// --------------------------------------------------------------------

func TestSuite(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "sarama/cluster")
}

func testDataDir(tokens ...string) string {
	tokens = append([]string{"testdata"}, tokens...)
	return filepath.Join(tokens...)
}

func testSeed(n int, testTopics []string) error {
	producer, err := sarama.NewSyncProducerFromClient(testClient)
	if err != nil {
		return err
	}
	defer producer.Close()

	for i := 0; i < n; i++ {
		kv := sarama.StringEncoder(fmt.Sprintf("PLAINDATA-%08d", i))
		for _, t := range testTopics {
			msg := &sarama.ProducerMessage{Topic: t, Key: kv, Value: kv}
			if _, _, err := producer.SendMessage(msg); err != nil {
				return err
			}
		}
	}
	return nil
}

func testCmd(name string, arg ...string) *exec.Cmd {
	cmd := exec.Command(name, arg...)
	if testing.Verbose() || os.Getenv("CI") != "" {
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
	}
	cmd.Env = []string{"KAFKA_HEAP_OPTS=-Xmx1G -Xms1G"}
	return cmd
}

type testConsumerMessage struct {
	sarama.ConsumerMessage
	ConsumerID string
}

// --------------------------------------------------------------------

var _ sarama.Consumer = &mockConsumer{}
var _ sarama.PartitionConsumer = &mockPartitionConsumer{}

type mockClient struct {
	sarama.Client

	topics map[string][]int32
}
type mockConsumer struct{ sarama.Consumer }
type mockPartitionConsumer struct {
	sarama.PartitionConsumer

	Topic     string
	Partition int32
	Offset    int64
}

func (m *mockClient) Partitions(t string) ([]int32, error) {
	pts, ok := m.topics[t]
	if !ok {
		return nil, sarama.ErrInvalidTopic
	}
	return pts, nil
}

func (*mockConsumer) ConsumePartition(topic string, partition int32, offset int64) (sarama.PartitionConsumer, error) {
	if offset > -1 && offset < 1000 {
		return nil, sarama.ErrOffsetOutOfRange
	}
	return &mockPartitionConsumer{
		Topic:     topic,
		Partition: partition,
		Offset:    offset,
	}, nil
}

func (*mockPartitionConsumer) Close() error { return nil }
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-lambda-go/lambda"
	"github.com/buildkite/buildkite-agent-metrics/backend"
	"github.com/buildkite/buildkite-agent-metrics/collector"
	"github.com/buildkite/buildkite-agent-metrics/version"
)

func main() {
	if os.Getenv(`DEBUG`) != "" {
		_, err := Handler(context.Background(), json.RawMessage([]byte{}))
		if err != nil {
			log.Fatal(err)
		}
	} else {
		lambda.Start(Handler)
	}
}

func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
	token := os.Getenv("BUILDKITE_AGENT_TOKEN")
	backendOpt := os.Getenv("BUILDKITE_BACKEND")
	queue := os.Getenv("BUILDKITE_QUEUE")
	clwDimensions := os.Getenv("BUILDKITE_CLOUDWATCH_DIMENSIONS")
	quietString := os.Getenv("BUILDKITE_QUIET")
	quiet := quietString == "1" || quietString == "true"

	if quiet {
		log.SetOutput(ioutil.Discard)
	}

	t := time.Now()

	userAgent := fmt.Sprintf("buildkite-metrics/%s buildkite-metrics-lambda", version.Version)

	c := collector.Collector{
		UserAgent: userAgent,
		Endpoint:  "https://agent.buildkite.com/v3",
		Token:     token,
		Queue:     queue,
		Quiet:     quiet,
		Debug:     false,
		DebugHttp: false,
	}

	var b backend.Backend
	var err error
	if backendOpt == "statsd" {
		statsdHost := os.Getenv("STATSD_HOST")
		statsdTags := strings.ToLower(os.Getenv("STATSD_TAGS")) == "true"
		b, err = backend.NewStatsDBackend(statsdHost, statsdTags)
		if err != nil {
			return nil, err
		}
	} else {
		dimensions, err := backend.ParseCloudWatchDimensions(clwDimensions)
		if err != nil {
			return nil, err
		}
		b = backend.NewCloudWatchBackend(dimensions)
	}

	res, err := c.Collect()
	if err != nil {
		return nil, err
	}

	res.Dump()

	err = b.Collect(res)
	if err != nil {
		return nil, err
	}

	log.Printf("Finished in %s", time.Now().Sub(t))
	return "", nil
}
<commit_msg>Fix return types in lambda<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-lambda-go/lambda"
	"github.com/buildkite/buildkite-agent-metrics/backend"
	"github.com/buildkite/buildkite-agent-metrics/collector"
	"github.com/buildkite/buildkite-agent-metrics/version"
)

func main() {
	if os.Getenv(`DEBUG`) != "" {
		_, err := Handler(context.Background(), json.RawMessage([]byte{}))
		if err != nil {
			log.Fatal(err)
		}
	} else {
		lambda.Start(Handler)
	}
}

func Handler(ctx context.Context, evt json.RawMessage) (string, error) {
	token := os.Getenv("BUILDKITE_AGENT_TOKEN")
	backendOpt := os.Getenv("BUILDKITE_BACKEND")
	queue := os.Getenv("BUILDKITE_QUEUE")
	clwDimensions := os.Getenv("BUILDKITE_CLOUDWATCH_DIMENSIONS")
	quietString := os.Getenv("BUILDKITE_QUIET")
	quiet := quietString == "1" || quietString == "true"

	if quiet {
		log.SetOutput(ioutil.Discard)
	}

	t := time.Now()

	userAgent := fmt.Sprintf("buildkite-metrics/%s buildkite-metrics-lambda", version.Version)

	c := collector.Collector{
		UserAgent: userAgent,
		Endpoint:  "https://agent.buildkite.com/v3",
		Token:     token,
		Queue:     queue,
		Quiet:     quiet,
		Debug:     false,
		DebugHttp: false,
	}

	var b backend.Backend
	var err error
	if backendOpt == "statsd" {
		statsdHost := os.Getenv("STATSD_HOST")
		statsdTags := strings.ToLower(os.Getenv("STATSD_TAGS")) == "true"
		b, err = backend.NewStatsDBackend(statsdHost, statsdTags)
		if err != nil {
			return "", err
		}
	} else {
		dimensions, err := backend.ParseCloudWatchDimensions(clwDimensions)
		if err != nil {
			return "", err
		}
		b = backend.NewCloudWatchBackend(dimensions)
	}

	res, err := c.Collect()
	if err != nil {
		return "", err
	}

	res.Dump()

	err = b.Collect(res)
	if err != nil {
		return "", err
	}

	log.Printf("Finished in %s", time.Now().Sub(t))
	return "", nil
}
<|endoftext|>


<commit_before>package caicloudsource

import (
	"fmt"
	"net/url"
	"time"

	"github.com/golang/glog"
	. "k8s.io/heapster/metrics/core"
	"k8s.io/heapster/metrics/core/caicloud"
	"k8s.io/heapster/metrics/sources/kubelet"
	"k8s.io/heapster/metrics/sources/summary"
	kubeapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kubeclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

type caicloudMetricsSource struct {
	node          NodeInfo
	summary       MetricsSource
	kubeletClient *kubelet.KubeletClient
}

func NewCaicloudMetricsSource(node NodeInfo, client *kubelet.KubeletClient, summary MetricsSource) MetricsSource {
	return &caicloudMetricsSource{
		node:          node,
		summary:       summary,
		kubeletClient: client,
	}
}

type NodeInfo struct {
	summary.NodeInfo
	Unschedulable bool
}

func (s *caicloudMetricsSource) Name() string {
	return "caicloud_source"
}

func (s *caicloudMetricsSource) ScrapeMetrics(start, end time.Time) *DataBatch {
	dataBatch := s.summary.ScrapeMetrics(start, end)
	machineInfo, err := s.kubeletClient.GetMachineInfo(s.node.Host)
	if err != nil {
		glog.Errorf("can't get machine info from kubelet")
		return dataBatch
	}
	key := NodeKey(s.node.NodeName)
	if metricSet, found := dataBatch.MetricSets[key]; found {
		var cpuLimit int64 = int64(machineInfo.NumCores) * 1000

		s.addIntMetric(metricSet, &MetricCpuLimit, cpuLimit)
		s.addIntMetric(metricSet, &MetricMemoryLimit, machineInfo.MemoryCapacity)
		if s.node.Unschedulable {
			s.addIntMetric(metricSet, &caicloudcore.MetricCpuAvailable, 0)
			s.addIntMetric(metricSet, &caicloudcore.MetricMemoryAvailable, 0)
			flag := false
			for idx, labeledMetric := range metricSet.LabeledMetrics {
				if labeledMetric.Name == MetricFilesystemAvailable.Name {
					labeledMetric.MetricValue.IntValue = 0
					metricSet.LabeledMetrics[idx] = labeledMetric
					flag = true
					break
				}
			}
			if !flag {
				metricSet.LabeledMetrics = append(metricSet.LabeledMetrics,
					LabeledMetric{
						Name:   MetricFilesystemAvailable.Name,
						Labels: map[string]string{LabelResourceID.Key: "/"},
						MetricValue: MetricValue{
							ValueType:  ValueInt64,
							MetricType: MetricFilesystemAvailable.Type,
							IntValue:   0,
						},
					})
			}
		} else {
			cpuUsageRate := metricSet.MetricValues[MetricCpuUsageRate.Name]
			cpuLimit := metricSet.MetricValues[MetricCpuLimit.Name]
			s.addIntMetric(metricSet, &caicloudcore.MetricCpuAvailable, cpuLimit.IntValue-cpuUsageRate.IntValue)
			memoryUsage := metricSet.MetricValues[MetricMemoryUsage.Name]
			memoryLimit := metricSet.MetricValues[MetricMemoryLimit.Name]
			s.addIntMetric(metricSet, &caicloudcore.MetricMemoryAvailable, memoryLimit.IntValue-memoryUsage.IntValue)
		}
	}
	return dataBatch
}

func (s *caicloudMetricsSource) addIntMetric(metrics *MetricSet, metric *Metric, value int64) {
	val := MetricValue{
		ValueType:  ValueInt64,
		MetricType: metric.Type,
		IntValue:   int64(value),
	}
	metrics.MetricValues[metric.Name] = val
}

type caicloudProvider struct {
	nodeLister    *cache.StoreToNodeLister
	reflector     *cache.Reflector
	kubeletClient *kubelet.KubeletClient
}

func (p *caicloudProvider) getNodeInfo(node *kubeapi.Node) (NodeInfo, error) {
	for _, c := range node.Status.Conditions {
		if c.Type == kubeapi.NodeReady && c.Status != kubeapi.ConditionTrue {
			return NodeInfo{}, fmt.Errorf("Node %v is not ready", node.Name)
		}
	}
	info := NodeInfo{
		summary.NodeInfo{
			NodeName: node.Name,
			HostName: node.Name,
			HostID:   node.Spec.ExternalID,
			Host: kubelet.Host{
				Port: p.kubeletClient.GetPort(),
			},
			// hack for enable summay api
			KubeletVersion: "v1.2.4",
		},
		node.Spec.Unschedulable,
	}

	for _, addr := range node.Status.Addresses {
		if addr.Type == kubeapi.NodeHostName && addr.Address != "" {
			info.HostName = addr.Address
		}
		if addr.Type == kubeapi.NodeInternalIP && addr.Address != "" {
			info.IP = addr.Address
		}
		if addr.Type == kubeapi.NodeLegacyHostIP && addr.Address != "" && info.IP == "" {
			info.IP = addr.Address
		}
	}

	if info.IP == "" {
		return info, fmt.Errorf("Node %v has no valid hostname and/or IP address: %v %v", node.Name, info.HostName, info.IP)
	}

	return info, nil
}

func (p *caicloudProvider) GetMetricsSources() []MetricsSource {
	sources := []MetricsSource{}
	nodes, err := p.nodeLister.List()
	if err != nil {
		glog.Errorf("error while listing nodes: %v", err)
		return sources
	}
	if len(nodes.Items) == 0 {
		glog.Error("No nodes received from APIserver.")
		return sources
	}

	for _, node := range nodes.Items {
		info, err := p.getNodeInfo(&node)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		fallback := kubelet.NewKubeletMetricsSource(
			info.Host,
			p.kubeletClient,
			info.NodeName,
			info.HostName,
			info.HostID,
		)
		summary := summary.NewSummaryMetricsSource(info.NodeInfo, p.kubeletClient, fallback)
		sources = append(sources, NewCaicloudMetricsSource(info, p.kubeletClient, summary))
	}
	return sources
}

func NewCaicloudProvider(uri *url.URL) (MetricsSourceProvider, error) {
	// create clients
	kubeConfig, kubeletConfig, err := kubelet.GetKubeConfigs(uri)
	if err != nil {
		return nil, err
	}
	kubeClient := kubeclient.NewOrDie(kubeConfig)
	kubeletClient, err := kubelet.NewKubeletClient(kubeletConfig)
	if err != nil {
		return nil, err
	}
	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "nodes", kubeapi.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &kubeapi.Node{}, nodeLister.Store, time.Hour)
	reflector.Run()

	return &caicloudProvider{
		nodeLister:    nodeLister,
		reflector:     reflector,
		kubeletClient: kubeletClient,
	}, nil
}
<commit_msg>rebase to v1.2.0<commit_after>package caicloudsource

import (
	"fmt"
	"net/url"
	"time"

	"github.com/golang/glog"
	. "k8s.io/heapster/metrics/core"
	"k8s.io/heapster/metrics/core/caicloud"
	"k8s.io/heapster/metrics/sources/kubelet"
	"k8s.io/heapster/metrics/sources/summary"
	kubeapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	kubeclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
)

type caicloudMetricsSource struct {
	node          NodeInfo
	summary       MetricsSource
	kubeletClient *kubelet.KubeletClient
}

func NewCaicloudMetricsSource(node NodeInfo, client *kubelet.KubeletClient, summary MetricsSource) MetricsSource {
	return &caicloudMetricsSource{
		node:          node,
		summary:       summary,
		kubeletClient: client,
	}
}

type NodeInfo struct {
	summary.NodeInfo
	Unschedulable bool
}

func (s *caicloudMetricsSource) Name() string {
	return "caicloud_source"
}

func (s *caicloudMetricsSource) ScrapeMetrics(start, end time.Time) *DataBatch {
	dataBatch := s.summary.ScrapeMetrics(start, end)
	machineInfo, err := s.kubeletClient.GetMachineInfo(s.node.Host)
	if err != nil {
		glog.Errorf("can't get machine info from kubelet")
		return dataBatch
	}
	key := NodeKey(s.node.NodeName)
	if metricSet, found := dataBatch.MetricSets[key]; found {
		var cpuLimit int64 = int64(machineInfo.NumCores) * 1000

		s.addIntMetric(metricSet, &MetricCpuLimit, cpuLimit)
		s.addIntMetric(metricSet, &MetricMemoryLimit, int64(machineInfo.MemoryCapacity))
		if s.node.Unschedulable {
			s.addIntMetric(metricSet, &caicloudcore.MetricCpuAvailable, 0)
			s.addIntMetric(metricSet, &caicloudcore.MetricMemoryAvailable, 0)
			flag := false
			for idx, labeledMetric := range metricSet.LabeledMetrics {
				if labeledMetric.Name == MetricFilesystemAvailable.Name {
					labeledMetric.MetricValue.IntValue = 0
					metricSet.LabeledMetrics[idx] = labeledMetric
					flag = true
					break
				}
			}
			if !flag {
				metricSet.LabeledMetrics = append(metricSet.LabeledMetrics,
					LabeledMetric{
						Name:   MetricFilesystemAvailable.Name,
						Labels: map[string]string{LabelResourceID.Key: "/"},
						MetricValue: MetricValue{
							ValueType:  ValueInt64,
							MetricType: MetricFilesystemAvailable.Type,
							IntValue:   0,
						},
					})
			}
		} else {
			cpuUsageRate := metricSet.MetricValues[MetricCpuUsageRate.Name]
			cpuLimit := metricSet.MetricValues[MetricCpuLimit.Name]
			s.addIntMetric(metricSet, &caicloudcore.MetricCpuAvailable, cpuLimit.IntValue-cpuUsageRate.IntValue)
			memoryUsage := metricSet.MetricValues[MetricMemoryUsage.Name]
			memoryLimit := metricSet.MetricValues[MetricMemoryLimit.Name]
			s.addIntMetric(metricSet, &caicloudcore.MetricMemoryAvailable, memoryLimit.IntValue-memoryUsage.IntValue)
		}
	}
	return dataBatch
}

func (s *caicloudMetricsSource) addIntMetric(metrics *MetricSet, metric *Metric, value int64) {
	val := MetricValue{
		ValueType:  ValueInt64,
		MetricType: metric.Type,
		IntValue:   int64(value),
	}
	metrics.MetricValues[metric.Name] = val
}

type caicloudProvider struct {
	nodeLister    *cache.StoreToNodeLister
	reflector     *cache.Reflector
	kubeletClient *kubelet.KubeletClient
}

func (p *caicloudProvider) getNodeInfo(node *kubeapi.Node) (NodeInfo, error) {
	for _, c := range node.Status.Conditions {
		if c.Type == kubeapi.NodeReady && c.Status != kubeapi.ConditionTrue {
			return NodeInfo{}, fmt.Errorf("Node %v is not ready", node.Name)
		}
	}
	info := NodeInfo{
		summary.NodeInfo{
			NodeName: node.Name,
			HostName: node.Name,
			HostID:   node.Spec.ExternalID,
			Host: kubelet.Host{
				Port: p.kubeletClient.GetPort(),
			},
			// hack for enable summay api
			KubeletVersion: "v1.2.4",
		},
		node.Spec.Unschedulable,
	}

	for _, addr := range node.Status.Addresses {
		if addr.Type == kubeapi.NodeHostName && addr.Address != "" {
			info.HostName = addr.Address
		}
		if addr.Type == kubeapi.NodeInternalIP && addr.Address != "" {
			info.IP = addr.Address
		}
		if addr.Type == kubeapi.NodeLegacyHostIP && addr.Address != "" && info.IP == "" {
			info.IP = addr.Address
		}
	}

	if info.IP == "" {
		return info, fmt.Errorf("Node %v has no valid hostname and/or IP address: %v %v", node.Name, info.HostName, info.IP)
	}

	return info, nil
}

func (p *caicloudProvider) GetMetricsSources() []MetricsSource {
	sources := []MetricsSource{}
	nodes, err := p.nodeLister.List()
	if err != nil {
		glog.Errorf("error while listing nodes: %v", err)
		return sources
	}
	if len(nodes.Items) == 0 {
		glog.Error("No nodes received from APIserver.")
		return sources
	}

	for _, node := range nodes.Items {
		info, err := p.getNodeInfo(&node)
		if err != nil {
			glog.Errorf("%v", err)
			continue
		}
		fallback := kubelet.NewKubeletMetricsSource(
			info.Host,
			p.kubeletClient,
			info.NodeName,
			info.HostName,
			info.HostID,
		)
		summary := summary.NewSummaryMetricsSource(info.NodeInfo, p.kubeletClient, fallback)
		sources = append(sources, NewCaicloudMetricsSource(info, p.kubeletClient, summary))
	}
	return sources
}

func NewCaicloudProvider(uri *url.URL) (MetricsSourceProvider, error) {
	// create clients
	kubeConfig, kubeletConfig, err := kubelet.GetKubeConfigs(uri)
	if err != nil {
		return nil, err
	}
	kubeClient := kubeclient.NewOrDie(kubeConfig)
	kubeletClient, err := kubelet.NewKubeletClient(kubeletConfig)
	if err != nil {
		return nil, err
	}
	// watch nodes
	lw := cache.NewListWatchFromClient(kubeClient, "nodes", kubeapi.NamespaceAll, fields.Everything())
	nodeLister := &cache.StoreToNodeLister{Store: cache.NewStore(cache.MetaNamespaceKeyFunc)}
	reflector := cache.NewReflector(lw, &kubeapi.Node{}, nodeLister.Store, time.Hour)
	reflector.Run()

	return &caicloudProvider{
		nodeLister:    nodeLister,
		reflector:     reflector,
		kubeletClient: kubeletClient,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the i/o primitive functions.

package golisp

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"unicode"
)

func RegisterIOPrimitives() {
	MakeRestrictedPrimitiveFunction("open-input-file", "1", OpenInputFileImpl)
	MakeRestrictedPrimitiveFunction("open-output-file", "1|2", OpenOutputFileImpl)
	MakeRestrictedPrimitiveFunction("close-port", "1", ClosePortImpl)
	MakeRestrictedPrimitiveFunction("write-bytes", "2", WriteBytesImpl)

	MakePrimitiveFunction("write-string", "1|2", WriteStringImpl)
	MakePrimitiveFunction("newline", "0|1", NewlineImpl)
	MakePrimitiveFunction("write", "1|2", WriteImpl)
	MakePrimitiveFunction("read", "1", ReadImpl)
	MakePrimitiveFunction("eof-object?", "1", EofObjectImpl)

	MakePrimitiveFunction("format", ">=3", FormatImpl)
}

func OpenOutputFileImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	filename := Car(args)
	if !StringP(filename) {
		err = ProcessError("open-output-port expects its argument to be a string", env)
		return
	}

	var openFlag = os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	if Length(args) == 2 && BooleanValue(Cadr(args)) {
		openFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND
	}

	f, err := os.OpenFile(StringValue(filename), openFlag, 0666)
	if err != nil {
		return
	}
	return PortWithValue(f), nil
}

func OpenInputFileImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	filename := Car(args)
	if !StringP(filename) {
		err = ProcessError("open-input-port expects its argument to be a string", env)
		return
	}

	f, err := os.Open(StringValue(filename))
	if err != nil {
		return
	}
	return PortWithValue(f), nil
}

func ClosePortImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	p := Car(args)
	if !PortP(p) {
		err = ProcessError("close-port expects its argument be a port", env)
		return
	}

	(*os.File)(PortValue(p)).Close()
	return

}

func WriteBytesImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	bytes := Car(args)
	if !ObjectP(bytes) || ObjectType(bytes) != "[]byte" {
		err = ProcessError("write expects its first argument to be a bytearray", env)
		return
	}

	p := Cadr(args)
	if !PortP(p) {
		err = ProcessError("write expects its second argument be a port", env)
		return
	}

	_, err = (*os.File)(PortValue(p)).Write(*(*[]byte)(ObjectValue(bytes)))
	return
}

func WriteStringImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	str := Car(args)
	if !StringP(str) {
		err = ProcessError("write-string expects its first argument to be a string", env)
		return
	}

	var port *os.File
	if Length(args) == 1 {
		port = os.Stdout
	} else {
		p := Cadr(args)
		if !PortP(p) {
			err = ProcessError("write-string expects its second argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	_, err = port.WriteString(StringValue(str))
	return
}

func WriteImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	var port *os.File

	if Length(args) == 1 {
		port = os.Stdout
	} else {
		p := Cadr(args)
		if !PortP(p) {
			err = ProcessError("write expects its second argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	_, err = port.WriteString(String(Car(args)))
	return
}

func NewlineImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	var port *os.File

	if Length(args) == 0 {
		port = os.Stdout
	} else {
		p := Car(args)
		if !PortP(p) {
			err = ProcessError("newline expects its argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	_, err = port.WriteString("\n")
	return
}

func ReadImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	var port *os.File

	if Length(args) == 0 {
		port = os.Stdin
	} else {
		p := Car(args)
		if !PortP(p) {
			err = ProcessError("read expects its argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	result, err = ParseObjectFromFileInEnv(port, env)
	return
}

func EofObjectImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	return BooleanWithValue(IsEqual(Car(args), EofObject)), nil
}

func FormatImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	destination := Car(args)
	if !BooleanP(destination) && !PortP(destination) {
		err = ProcessError(fmt.Sprintf("format expects its second argument be a boolean or port, but was %s", String(destination)), env)
		return
	}

	controlStringObj := Cadr(args)
	if !StringP(controlStringObj) {
		err = ProcessError("format expects its second argument be a string", env)
		return
	}
	controlString := StringValue(controlStringObj)

	arguments := Cddr(args)

	numberOfSubstitutions := strings.Count(controlString, "~")
	parts := make([]string, 0, numberOfSubstitutions*2+1)
	start := 0
	var i int
	var numericArg int
	var atModifier bool
	var substitution string
	var padding string
	var n int64

	for i < len(controlString) {
		if controlString[i] == '~' { // start of a substitution
			parts = append(parts, controlString[start:i])
			i++
			start = i
			for unicode.IsDigit(rune(controlString[i])) {
				i++
			}
			if i == start {
				if controlString[i] == '#' {
					numericArg = Length(arguments)
					i++
				} else if controlString[i] == 'V' || controlString[i] == 'v' {
					if IntegerP(Car(arguments)) {
						numericArg = int(IntegerValue(Car(arguments)))
						arguments = Cdr(arguments)
					} else {
						err = ProcessError(fmt.Sprintf("format encountered a size argument mismatch at index %d", i), env)
						return
					}
					i++
				} else {
					numericArg = 0
				}
			} else {
				n, err = strconv.ParseInt(string(controlString[start:i]), 10, 64)
				if err != nil {
					return
				}
				numericArg = int(n)
			}
			if controlString[i] == '@' {
				atModifier = true
				i++
			}
			switch controlString[i] {
			case 'A', 'a':
				substitution = PrintString(Car(arguments))
				if len(substitution) < numericArg {
					padding = strings.Repeat(" ", numericArg-len(substitution))
				} else {
					padding = ""
				}
				if atModifier {
					parts = append(parts, padding)
				}
				parts = append(parts, substitution)
				if !atModifier {
					parts = append(parts, padding)
				}
				arguments = Cdr(arguments)
				start = i + 1

			case 'S', 's':
				substitution = String(Car(arguments))
				if len(substitution) < numericArg {
					padding = strings.Repeat(" ", numericArg-len(substitution))
				} else {
					padding = ""
				}
				if atModifier {
					parts = append(parts, padding)
				}
				parts = append(parts, substitution)
				if !atModifier {
					parts = append(parts, padding)
				}
				arguments = Cdr(arguments)
				start = i + 1

			case '%':
				if numericArg > 0 {
					parts = append(parts, strings.Repeat("\n", numericArg))
				} else {
					parts = append(parts, "\n")
				}
				start = i + 1

			case '~':
				if numericArg > 0 {
					parts = append(parts, strings.Repeat("~", numericArg))
				} else {
					parts = append(parts, "~")
				}
				start = i + 1

			case '\n':
				for i < len(controlString) && unicode.IsSpace(rune(controlString[i])) {
					i++
				}
				if atModifier {
					parts = append(parts, "\n")
				}
				start = i
				i--

			default:
				err = ProcessError(fmt.Sprintf("format encountered an unsupported substitution at index %d", i), env)
				return
			}
		}
		i++
	}

	if start < len(controlString) {
		parts = append(parts, controlString[start:i])
	}

	if i < len(controlString) || !NilP(arguments) {
		err = ProcessError("number of replacements in the control string and number of arguments must be equal", env)
		return
	}

	combinedString := strings.Join(parts, "")

	if PortP(destination) {
		port := PortValue(destination)
		_, err = port.WriteString(combinedString)
	} else if BooleanValue(destination) {
		_, err = os.Stdout.WriteString(combinedString)
	} else {
		result = StringWithValue(combinedString)
	}

	return
}
<commit_msg>Allow format to just output a string for consistency.<commit_after>// Copyright 2014 SteelSeries ApS.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package implements a basic LISP interpretor for embedding in a go program for scripting.
// This file contains the i/o primitive functions.

package golisp

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"unicode"
)

func RegisterIOPrimitives() {
	MakeRestrictedPrimitiveFunction("open-input-file", "1", OpenInputFileImpl)
	MakeRestrictedPrimitiveFunction("open-output-file", "1|2", OpenOutputFileImpl)
	MakeRestrictedPrimitiveFunction("close-port", "1", ClosePortImpl)
	MakeRestrictedPrimitiveFunction("write-bytes", "2", WriteBytesImpl)

	MakePrimitiveFunction("write-string", "1|2", WriteStringImpl)
	MakePrimitiveFunction("newline", "0|1", NewlineImpl)
	MakePrimitiveFunction("write", "1|2", WriteImpl)
	MakePrimitiveFunction("read", "1", ReadImpl)
	MakePrimitiveFunction("eof-object?", "1", EofObjectImpl)

	MakePrimitiveFunction("format", ">=2", FormatImpl)
}

func OpenOutputFileImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	filename := Car(args)
	if !StringP(filename) {
		err = ProcessError("open-output-port expects its argument to be a string", env)
		return
	}

	var openFlag = os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	if Length(args) == 2 && BooleanValue(Cadr(args)) {
		openFlag = os.O_WRONLY | os.O_CREATE | os.O_APPEND
	}

	f, err := os.OpenFile(StringValue(filename), openFlag, 0666)
	if err != nil {
		return
	}
	return PortWithValue(f), nil
}

func OpenInputFileImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	filename := Car(args)
	if !StringP(filename) {
		err = ProcessError("open-input-port expects its argument to be a string", env)
		return
	}

	f, err := os.Open(StringValue(filename))
	if err != nil {
		return
	}
	return PortWithValue(f), nil
}

func ClosePortImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	p := Car(args)
	if !PortP(p) {
		err = ProcessError("close-port expects its argument be a port", env)
		return
	}

	(*os.File)(PortValue(p)).Close()
	return

}

func WriteBytesImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	bytes := Car(args)
	if !ObjectP(bytes) || ObjectType(bytes) != "[]byte" {
		err = ProcessError("write expects its first argument to be a bytearray", env)
		return
	}

	p := Cadr(args)
	if !PortP(p) {
		err = ProcessError("write expects its second argument be a port", env)
		return
	}

	_, err = (*os.File)(PortValue(p)).Write(*(*[]byte)(ObjectValue(bytes)))
	return
}

func WriteStringImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	str := Car(args)
	if !StringP(str) {
		err = ProcessError("write-string expects its first argument to be a string", env)
		return
	}

	var port *os.File
	if Length(args) == 1 {
		port = os.Stdout
	} else {
		p := Cadr(args)
		if !PortP(p) {
			err = ProcessError("write-string expects its second argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	_, err = port.WriteString(StringValue(str))
	return
}

func WriteImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	var port *os.File

	if Length(args) == 1 {
		port = os.Stdout
	} else {
		p := Cadr(args)
		if !PortP(p) {
			err = ProcessError("write expects its second argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	_, err = port.WriteString(String(Car(args)))
	return
}

func NewlineImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	var port *os.File

	if Length(args) == 0 {
		port = os.Stdout
	} else {
		p := Car(args)
		if !PortP(p) {
			err = ProcessError("newline expects its argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	_, err = port.WriteString("\n")
	return
}

func ReadImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	var port *os.File

	if Length(args) == 0 {
		port = os.Stdin
	} else {
		p := Car(args)
		if !PortP(p) {
			err = ProcessError("read expects its argument be a port", env)
			return
		}
		port = PortValue(p)
	}

	result, err = ParseObjectFromFileInEnv(port, env)
	return
}

func EofObjectImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	return BooleanWithValue(IsEqual(Car(args), EofObject)), nil
}

func FormatImpl(args *Data, env *SymbolTableFrame) (result *Data, err error) {
	destination := Car(args)
	if !BooleanP(destination) && !PortP(destination) {
		err = ProcessError(fmt.Sprintf("format expects its second argument be a boolean or port, but was %s", String(destination)), env)
		return
	}

	controlStringObj := Cadr(args)
	if !StringP(controlStringObj) {
		err = ProcessError("format expects its second argument be a string", env)
		return
	}
	controlString := StringValue(controlStringObj)

	arguments := Cddr(args)

	numberOfSubstitutions := strings.Count(controlString, "~")
	parts := make([]string, 0, numberOfSubstitutions*2+1)
	start := 0
	var i int
	var numericArg int
	var atModifier bool
	var substitution string
	var padding string
	var n int64

	for i < len(controlString) {
		if controlString[i] == '~' { // start of a substitution
			parts = append(parts, controlString[start:i])
			i++
			start = i
			for unicode.IsDigit(rune(controlString[i])) {
				i++
			}
			if i == start {
				if controlString[i] == '#' {
					numericArg = Length(arguments)
					i++
				} else if controlString[i] == 'V' || controlString[i] == 'v' {
					if IntegerP(Car(arguments)) {
						numericArg = int(IntegerValue(Car(arguments)))
						arguments = Cdr(arguments)
					} else {
						err = ProcessError(fmt.Sprintf("format encountered a size argument mismatch at index %d", i), env)
						return
					}
					i++
				} else {
					numericArg = 0
				}
			} else {
				n, err = strconv.ParseInt(string(controlString[start:i]), 10, 64)
				if err != nil {
					return
				}
				numericArg = int(n)
			}
			if controlString[i] == '@' {
				atModifier = true
				i++
			}
			switch controlString[i] {
			case 'A', 'a':
				substitution = PrintString(Car(arguments))
				if len(substitution) < numericArg {
					padding = strings.Repeat(" ", numericArg-len(substitution))
				} else {
					padding = ""
				}
				if atModifier {
					parts = append(parts, padding)
				}
				parts = append(parts, substitution)
				if !atModifier {
					parts = append(parts, padding)
				}
				arguments = Cdr(arguments)
				start = i + 1

			case 'S', 's':
				substitution = String(Car(arguments))
				if len(substitution) < numericArg {
					padding = strings.Repeat(" ", numericArg-len(substitution))
				} else {
					padding = ""
				}
				if atModifier {
					parts = append(parts, padding)
				}
				parts = append(parts, substitution)
				if !atModifier {
					parts = append(parts, padding)
				}
				arguments = Cdr(arguments)
				start = i + 1

			case '%':
				if numericArg > 0 {
					parts = append(parts, strings.Repeat("\n", numericArg))
				} else {
					parts = append(parts, "\n")
				}
				start = i + 1

			case '~':
				if numericArg > 0 {
					parts = append(parts, strings.Repeat("~", numericArg))
				} else {
					parts = append(parts, "~")
				}
				start = i + 1

			case '\n':
				for i < len(controlString) && unicode.IsSpace(rune(controlString[i])) {
					i++
				}
				if atModifier {
					parts = append(parts, "\n")
				}
				start = i
				i--

			default:
				err = ProcessError(fmt.Sprintf("format encountered an unsupported substitution at index %d", i), env)
				return
			}
		}
		i++
	}

	if start < len(controlString) {
		parts = append(parts, controlString[start:i])
	}

	if i < len(controlString) || !NilP(arguments) {
		err = ProcessError("number of replacements in the control string and number of arguments must be equal", env)
		return
	}

	combinedString := strings.Join(parts, "")

	if PortP(destination) {
		port := PortValue(destination)
		_, err = port.WriteString(combinedString)
	} else if BooleanValue(destination) {
		_, err = os.Stdout.WriteString(combinedString)
	} else {
		result = StringWithValue(combinedString)
	}

	return
}
<|endoftext|>


<commit_before>package oz

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path"

	"github.com/subgraph/oz/network"
)

type Profile struct {
	// Name of this profile
	Name string
	// Path to binary to launch
	Path string
	// Optional path of binary to watch for watchdog purposes if different than Path
	Watchdog string
	// Optional wrapper binary to use when launching command (ex: tsocks)
	Wrapper string
	// If true launch one sandbox per instance, otherwise run all instances in same sandbox
	Multi bool
	// Disable mounting of sys and proc inside the sandbox
	NoSysProc bool
	// Disable bind mounting of default directories (etc,usr,bin,lib,lib64)
	// Also disables default blacklist items (/sbin, /usr/sbin, /usr/bin/sudo)
	// Normally not used
	NoDefaults bool
	// Allow bind mounting of files passed as arguments inside the sandbox
	AllowFiles bool `json:"allow_files"`
	// List of paths to bind mount inside jail
	Whitelist []WhitelistItem
	// List of paths to blacklist inside jail
	Blacklist []BlacklistItem
	// Optional XServer config
	XServer XServerConf
	// List of environment variables
	Environment []EnvVar
	// Networking
	Networking NetworkProfile
}

type XServerConf struct {
	Enabled          bool
	TrayIcon         string `json:"tray_icon"`
	WindowIcon       string `json:"window_icon"`
	EnableTray       bool   `json:"enable_tray"`
	UseDBUS          bool   `json:"use_dbus"`
	UsePulseAudio    bool   `json:"use_pulse_audio"`
	DisableClipboard bool   `json:"disable_clipboard"`
	DisableAudio     bool   `json:"disable_audio"`
}

type WhitelistItem struct {
	Path     string
	ReadOnly bool
}

type BlacklistItem struct {
	Path string
}

type EnvVar struct {
	Name  string
	Value string
}

// Sandbox network definition
type NetworkProfile struct {
	// One of empty, host, bridge
	Nettype string `json:"type"`

	// Name of the bridge to attach to
	//Bridge string

	// List of Sockets we want to attach to the jail
	//  Applies to Nettype: bridge and empty only
	Sockets []network.ProxyConfig
}

const defaultProfileDirectory = "/var/lib/oz/cells.d"

var loadedProfiles []*Profile

type Profiles []*Profile

func (ps Profiles) GetProfileByName(name string) (*Profile, error) {
	if loadedProfiles == nil {
		ps, err := LoadProfiles(defaultProfileDirectory)
		if err != nil {
			return nil, err
		}
		loadedProfiles = ps
	}

	for _, p := range loadedProfiles {
		if p.Name == name {
			return p, nil
		}
	}
	return nil, nil
}

func (ps Profiles) GetProfileByPath(bpath string) (*Profile, error) {
	if loadedProfiles == nil {
		ps, err := LoadProfiles(defaultProfileDirectory)
		if err != nil {
			return nil, err
		}
		loadedProfiles = ps
	}

	for _, p := range loadedProfiles {
		if p.Path == bpath {
			return p, nil
		}
	}
	return nil, nil
}

func LoadProfiles(dir string) (Profiles, error) {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	ps := []*Profile{}
	for _, f := range fs {
		if !f.IsDir() {
			name := path.Join(dir, f.Name())
			p, err := loadProfileFile(name)
			if err != nil {
				return nil, fmt.Errorf("error loading '%s': %v", f.Name(), err)
			}
			ps = append(ps, p)
		}
	}

	loadedProfiles = ps
	return ps, nil
}

func loadProfileFile(file string) (*Profile, error) {
	bs, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	p := new(Profile)
	if err := json.Unmarshal(bs, p); err != nil {
		return nil, err
	}
	if p.Name == "" {
		p.Name = path.Base(p.Path)
	}
	return p, nil
}
<commit_msg>Added config file path to profile<commit_after>package oz

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path"

	"github.com/subgraph/oz/network"
)

type Profile struct {
	// Name of this profile
	Name string
	// Path to binary to launch
	Path string
	// Path of the config file
	ConfigPath string `json:"-"`
	// Optional path of binary to watch for watchdog purposes if different than Path
	Watchdog string
	// Optional wrapper binary to use when launching command (ex: tsocks)
	Wrapper string
	// If true launch one sandbox per instance, otherwise run all instances in same sandbox
	Multi bool
	// Disable mounting of sys and proc inside the sandbox
	NoSysProc bool
	// Disable bind mounting of default directories (etc,usr,bin,lib,lib64)
	// Also disables default blacklist items (/sbin, /usr/sbin, /usr/bin/sudo)
	// Normally not used
	NoDefaults bool
	// Allow bind mounting of files passed as arguments inside the sandbox
	AllowFiles bool `json:"allow_files"`
	// List of paths to bind mount inside jail
	Whitelist []WhitelistItem
	// List of paths to blacklist inside jail
	Blacklist []BlacklistItem
	// Optional XServer config
	XServer XServerConf
	// List of environment variables
	Environment []EnvVar
	// Networking
	Networking NetworkProfile
}

type XServerConf struct {
	Enabled          bool
	TrayIcon         string `json:"tray_icon"`
	WindowIcon       string `json:"window_icon"`
	EnableTray       bool   `json:"enable_tray"`
	UseDBUS          bool   `json:"use_dbus"`
	UsePulseAudio    bool   `json:"use_pulse_audio"`
	DisableClipboard bool   `json:"disable_clipboard"`
	DisableAudio     bool   `json:"disable_audio"`
}

type WhitelistItem struct {
	Path     string
	ReadOnly bool
}

type BlacklistItem struct {
	Path string
}

type EnvVar struct {
	Name  string
	Value string
}

// Sandbox network definition
type NetworkProfile struct {
	// One of empty, host, bridge
	Nettype string `json:"type"`

	// Name of the bridge to attach to
	//Bridge string

	// List of Sockets we want to attach to the jail
	//  Applies to Nettype: bridge and empty only
	Sockets []network.ProxyConfig
}

const defaultProfileDirectory = "/var/lib/oz/cells.d"

var loadedProfiles []*Profile

type Profiles []*Profile

func (ps Profiles) GetProfileByName(name string) (*Profile, error) {
	if loadedProfiles == nil {
		ps, err := LoadProfiles(defaultProfileDirectory)
		if err != nil {
			return nil, err
		}
		loadedProfiles = ps
	}

	for _, p := range loadedProfiles {
		if p.Name == name {
			return p, nil
		}
	}
	return nil, nil
}

func (ps Profiles) GetProfileByPath(bpath string) (*Profile, error) {
	if loadedProfiles == nil {
		ps, err := LoadProfiles(defaultProfileDirectory)
		if err != nil {
			return nil, err
		}
		loadedProfiles = ps
	}

	for _, p := range loadedProfiles {
		if p.Path == bpath {
			return p, nil
		}
	}
	return nil, nil
}

func LoadProfiles(dir string) (Profiles, error) {
	fs, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	ps := []*Profile{}
	for _, f := range fs {
		if !f.IsDir() {
			name := path.Join(dir, f.Name())
			p, err := loadProfileFile(name)
			if err != nil {
				return nil, fmt.Errorf("error loading '%s': %v", f.Name(), err)
			}
			ps = append(ps, p)
		}
	}

	loadedProfiles = ps
	return ps, nil
}

func loadProfileFile(file string) (*Profile, error) {
	bs, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	p := new(Profile)
	if err := json.Unmarshal(bs, p); err != nil {
		return nil, err
	}
	if p.Name == "" {
		p.Name = path.Base(p.Path)
	}
	p.ConfigPath = file
	return p, nil
}
<|endoftext|>


<commit_before>package action

import (
	"encoding/gob"
	"encoding/json"
	"fmt"
	"github.com/lib/pq"
	"strings"
	"time"
)

type InsertRow struct {
	SchemaName     string
	TableName      string
	PrimaryKeyName string
	Rows           Rows
	BulkInsert     bool
}

// Register type for gob
func init() {
	gob.Register(&InsertRow{})
	gob.Register(&time.Time{})
}

func (a *InsertRow) Execute(c *Context) error {
	if a.BulkInsert {
		escapedCols := make([]string, 0)
		values := make([]interface{}, 0)

		for _, row := range a.Rows {
			escapedCols = append(escapedCols, row.Column.Name)
			values = append(values, row.GetValue())
		}

		stmt, err := c.GetPreparedStatement(
			pq.CopyInSchema(a.SchemaName, a.TableName, escapedCols...),
		)

		if err != nil {
			return err
		}

		_, err = stmt.Exec(values...)

		if err != nil {
			return err
		}
	} else {
		// Perform a single insert (upsert)
		escapedCols := make([]string, 0)
		escapedRows := make([]string, 0)
		values := make([]interface{}, 0)

		var primaryKeyRow *Row

		for i, row := range a.Rows {
			escapedCols = append(escapedCols, fmt.Sprintf("\"%s\"", row.Column.Name))
			escapedRows = append(escapedRows, fmt.Sprintf("$%d::%s\"%s\"", i+1, row.Column.GetTypeSchemaStr(a.SchemaName), row.Column.Type))

			// Marshall JSON objects as pg driver does not support it
			if obj, ok := row.Value.(*map[string]interface{}); ok {
				jsonStr, err := json.Marshal(obj)

				if err != nil {
					return err
				}

				values = append(values, string(jsonStr))
			} else {
				values = append(values, row.Value)
			}

			if row.Column.Name == a.PrimaryKeyName {
				primaryKeyRow = &row
			}
		}

		// Save transaction prior to inserting to rollback
		// if INSERT fails, so a UPDATE can be tried
		_, err := c.Tx.Exec(fmt.Sprintf(
			`SAVEPOINT "%s%s";`,
			a.SchemaName,
			a.TableName,
		))

		if err != nil {
			return err
		}

		_, err = c.Tx.Exec(
			fmt.Sprintf(
				`
					INSERT INTO "%s"."%s" (%s) VALUES (%s);
				`,
				a.SchemaName,
				a.TableName,
				strings.Join(escapedCols, ","),
				strings.Join(escapedRows, ","),
			),
			values...,
		)

		// Try to UPDATE (upsert) if INSERT fails...
		if err != nil {
			// Rollback to SAVEPOINT
			_, err = c.Tx.Exec(fmt.Sprintf(
				`ROLLBACK TO SAVEPOINT "%s%s";`,
				a.SchemaName,
				a.TableName,
			))

			if err != nil {
				return err
			}

			updateAction := &UpdateRow{
				a.SchemaName,
				a.TableName,
				*primaryKeyRow,
				a.Rows,
			}

			err = updateAction.Execute(c)

			if err != nil {
				return err
			}
		} else {
			// Release SAVEPOINT to avoid "out of shared memory"
			_, err := c.Tx.Exec(fmt.Sprintf(
				`RELEASE SAVEPOINT "%s%s";`,
				a.SchemaName,
				a.TableName,
			))

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (a *InsertRow) Filter(targetExpression string) bool {
	return IsInTargetExpression(&targetExpression, &a.SchemaName, &a.TableName)
}

func (a *InsertRow) NeedsSeparatedBatch() bool {
	return false
}
<commit_msg>Fix fetching primary key.<commit_after>package action

import (
	"encoding/gob"
	"encoding/json"
	"fmt"
	"github.com/lib/pq"
	"strings"
	"time"
)

type InsertRow struct {
	SchemaName     string
	TableName      string
	PrimaryKeyName string
	Rows           Rows
	BulkInsert     bool
}

// Register type for gob
func init() {
	gob.Register(&InsertRow{})
	gob.Register(&time.Time{})
}

func (a *InsertRow) Execute(c *Context) error {
	if a.BulkInsert {
		escapedCols := make([]string, 0)
		values := make([]interface{}, 0)

		for _, row := range a.Rows {
			escapedCols = append(escapedCols, row.Column.Name)
			values = append(values, row.GetValue())
		}

		stmt, err := c.GetPreparedStatement(
			pq.CopyInSchema(a.SchemaName, a.TableName, escapedCols...),
		)

		if err != nil {
			return err
		}

		_, err = stmt.Exec(values...)

		if err != nil {
			return err
		}
	} else {
		// Perform a single insert (upsert)
		escapedCols := make([]string, 0)
		escapedRows := make([]string, 0)
		values := make([]interface{}, 0)

		var primaryKeyRow Row

		for i, row := range a.Rows {
			escapedCols = append(escapedCols, fmt.Sprintf("\"%s\"", row.Column.Name))
			escapedRows = append(escapedRows, fmt.Sprintf("$%d::%s\"%s\"", i+1, row.Column.GetTypeSchemaStr(a.SchemaName), row.Column.Type))

			// Marshall JSON objects as pg driver does not support it
			if obj, ok := row.Value.(*map[string]interface{}); ok {
				jsonStr, err := json.Marshal(obj)

				if err != nil {
					return err
				}

				values = append(values, string(jsonStr))
			} else {
				values = append(values, row.Value)
			}

			if row.Column.Name == a.PrimaryKeyName {
				primaryKeyRow = row
			}
		}

		// Save transaction prior to inserting to rollback
		// if INSERT fails, so a UPDATE can be tried
		_, err := c.Tx.Exec(fmt.Sprintf(
			`SAVEPOINT "%s%s";`,
			a.SchemaName,
			a.TableName,
		))

		if err != nil {
			return err
		}

		_, err = c.Tx.Exec(
			fmt.Sprintf(
				`
					INSERT INTO "%s"."%s" (%s) VALUES (%s);
				`,
				a.SchemaName,
				a.TableName,
				strings.Join(escapedCols, ","),
				strings.Join(escapedRows, ","),
			),
			values...,
		)

		// Try to UPDATE (upsert) if INSERT fails...
		if err != nil {
			// Rollback to SAVEPOINT
			_, err = c.Tx.Exec(fmt.Sprintf(
				`ROLLBACK TO SAVEPOINT "%s%s";`,
				a.SchemaName,
				a.TableName,
			))

			if err != nil {
				return err
			}

			updateAction := &UpdateRow{
				a.SchemaName,
				a.TableName,
				primaryKeyRow,
				a.Rows,
			}

			err = updateAction.Execute(c)

			if err != nil {
				return err
			}
		} else {
			// Release SAVEPOINT to avoid "out of shared memory"
			_, err := c.Tx.Exec(fmt.Sprintf(
				`RELEASE SAVEPOINT "%s%s";`,
				a.SchemaName,
				a.TableName,
			))

			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (a *InsertRow) Filter(targetExpression string) bool {
	return IsInTargetExpression(&targetExpression, &a.SchemaName, &a.TableName)
}

func (a *InsertRow) NeedsSeparatedBatch() bool {
	return false
}
<|endoftext|>


<commit_before>package elk

import (
	"errors"
	"fmt"
	"io"
	"log/syslog"
	"net"
	"os"
	"strings"
	"time"

	"github.com/delectable/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}
	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &ElkAdapter{
		route: route,
		conn:  conn,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
}

func (adapter *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		string = fmt.Sprintf("{entry: {time: %d, message: %s}}", message.Time.Unix, message.Data)
		io.WriteString(adapter.conn, message.Data)
		// err := a.tmpl.Execute(a.conn, &ElkMessage{message, a})
		// if err != nil {
		// 	log.Println("syslog:", err)
		// 	a.route.Close()
		// 	return
		// }
	}
}

// func (m *ElkMessage) Priority() syslog.Priority {
// 	switch m.Message.Source {
// 	case "stdout":
// 		return syslog.LOG_USER | syslog.LOG_INFO
// 	case "stderr":
// 		return syslog.LOG_USER | syslog.LOG_ERR
// 	default:
// 		return syslog.LOG_DAEMON | syslog.LOG_INFO
// 	}
// }

// func (m *ElkMessage) ContainerName() string {
// 	return m.Message.Container.Name[1:]
// }
<commit_msg>Testing ELK Adapter<commit_after>package elk

import (
	"errors"
	"fmt"
	"io"
	// "log/syslog"
	"net"
	"os"
	// "strings"
	// "time"

	"github.com/delectable/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewElkAdapter, "elk")
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

func NewElkAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}
	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &ElkAdapter{
		route: route,
		conn:  conn,
	}, nil
}

type ElkAdapter struct {
	conn  net.Conn
	route *router.Route
}

func (adapter *ElkAdapter) Stream(logstream chan *router.Message) {
	for message := range logstream {
		output_string := fmt.Sprintf("{entry: {time: %d, message: %s}}", message.Time.Unix, message.Data)
		io.WriteString(adapter.conn, output_string)
		// err := a.tmpl.Execute(a.conn, &ElkMessage{message, a})
		// if err != nil {
		// 	log.Println("syslog:", err)
		// 	a.route.Close()
		// 	return
		// }
	}
}

// func (m *ElkMessage) Priority() syslog.Priority {
// 	switch m.Message.Source {
// 	case "stdout":
// 		return syslog.LOG_USER | syslog.LOG_INFO
// 	case "stderr":
// 		return syslog.LOG_USER | syslog.LOG_ERR
// 	default:
// 		return syslog.LOG_DAEMON | syslog.LOG_INFO
// 	}
// }

// func (m *ElkMessage) ContainerName() string {
// 	return m.Message.Container.Name[1:]
// }
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/MG-RAST/AWE/lib/auth"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/controller"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/db"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/AWE/vendor/github.com/MG-RAST/golib/goweb"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
)

func launchSite(control chan int, port int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}

	site_directory := conf.SITE_PATH
	fileinfo, err := os.Stat(site_directory)
	if err != nil {
		message := fmt.Sprintf("ERROR: site, path %s does not exist: %s", site_directory, err.Error())
		if os.IsNotExist(err) {
			message += " IsNotExist"
		}

		fmt.Fprintf(os.Stderr, message, "\n")
		logger.Error(message)

		os.Exit(1)
	} else {
		if !fileinfo.IsDir() {
			message := fmt.Sprintf("ERROR: site, path %s exists, but is not a directory", site_directory)
			fmt.Fprintf(os.Stderr, message, "\n")
			logger.Error(message)
			os.Exit(1)
		}

	}

	template_conf_filename := path.Join(conf.SITE_PATH, "js/config.js.tt")
	target_conf_filename := path.Join(conf.SITE_PATH, "js/config.js")
	buf, err := ioutil.ReadFile(template_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not read config template: %v\n", err)
		logger.Error("ERROR: could not read config template: " + err.Error())
	}
	template_conf_string := string(buf)

	// add / replace AWE API url
	if conf.API_URL == "" {
        fmt.Fprintf(os.Stderr, "ERROR: API_URL is not defined. \n")
        logger.Error("ERROR: API_URL is not defined.")
    }
	template_conf_string = strings.Replace(template_conf_string, "[% api_url %]", conf.API_URL, -1)

	// add auth
	auth_on := "false"
	auth_resources := ""
	if conf.GLOBUS_OAUTH || conf.MGRAST_OAUTH {
		auth_on = "true"
		b, _ := json.Marshal(conf.AUTH_RESOURCES)
		b = bytes.TrimPrefix(b, []byte("{"))
		b = bytes.TrimSuffix(b, []byte("}"))
		auth_resources = "," + string(b)
	}

	// replace auth
	template_conf_string = strings.Replace(template_conf_string, "[% auth_on %]", auth_on, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_default %]", conf.AUTH_DEFAULT, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_resources %]", auth_resources, -1)

	target_conf_file, err := os.Create(target_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	_, err = io.WriteString(target_conf_file, template_conf_string)

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	target_conf_file.Close()

	r.MapFunc("*", controller.SiteDir)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.SITE_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.SITE_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int, port int) {
	c := controller.NewServerController()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.Map("/job/{jid}/acl/{type}", c.JobAcl["typed"])
	r.Map("/job/{jid}/acl", c.JobAcl["base"])
	r.Map("/cgroup/{cgid}/acl/{type}", c.ClientGroupAcl["typed"])
	r.Map("/cgroup/{cgid}/acl", c.ClientGroupAcl["base"])
	r.Map("/cgroup/{cgid}/token", c.ClientGroupToken)
	r.MapRest("/job", c.Job)
	r.MapRest("/work", c.Work)
	r.MapRest("/cgroup", c.ClientGroup)
	r.MapRest("/client", c.Client)
	r.MapRest("/queue", c.Queue)
	r.MapRest("/awf", c.Awf)
	r.MapFunc("*", controller.ResourceDescription, goweb.GetMethod)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.API_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.API_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {

	err := conf.Init_conf("server")

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: "+err.Error())
		os.Exit(1)
	}

	//if !conf.INIT_SUCCESS {
	//	conf.PrintServerUsage()
	//	os.Exit(1)
	//}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("DEBUG_LEVEL > 0")
	}
	if _, err := os.Stat(conf.DATA_PATH); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(conf.DATA_PATH, 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR in creating data_path \"%s\", %s\n", conf.DATA_PATH, err.Error())
			os.Exit(1)
		}
	}

	if _, err := os.Stat(conf.LOGS_PATH); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(conf.LOGS_PATH, 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR in creating log_path \"%s\" %s\n", conf.LOGS_PATH, err.Error())
			os.Exit(1)
		}
	}

	if _, err := os.Stat(conf.DATA_PATH + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.DATA_PATH+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			os.Exit(1)
		}
	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("logger.Initialize...")
	}
	//init logger
	logger.Initialize("server")

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init db...")
	}
	//init db
	if err := db.Initialize(); err != nil {
		fmt.Printf("failed to initialize job db: %s\n", err.Error())
		os.Exit(1)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init db collection for user...")
	}
	//init db collection for user
	if err := user.Initialize(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR initializing user database: %s\n", err.Error())
		os.Exit(1)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init resource manager...")
	}
	//init resource manager
	core.InitResMgr("server")
	core.InitAwfMgr()
	core.InitJobDB()
	core.InitClientGroupDB()

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init auth...")
	}
	//init auth
	auth.Initialize()

	controller.PrintLogo()
	conf.Print("server")

	// reload job directory
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		if err := reload(conf.RELOAD); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init max job number (jid)...")
	}
	//init max job number (jid)
	if err := core.QMgr.InitMaxJid(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR from InitMaxJid : %v\n", err)
		os.Exit(1)
	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("launching server...")
	}
	//launch server
	control := make(chan int)
	go core.QMgr.Handle()
	go core.QMgr.Timer()
	go core.QMgr.ClientChecker()
	go launchSite(control, conf.SITE_PORT)
	go launchAPI(control, conf.API_PORT)

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("API launched...")
	}
	if err := core.AwfMgr.LoadWorkflows(); err != nil {
		logger.Error("LoadWorkflows: " + err.Error())
	}

	var host string
	if hostname, err := os.Hostname(); err == nil {
		host = fmt.Sprintf("%s:%d", hostname, conf.API_PORT)
	}

	//recover unfinished jobs before server went down last time
	if conf.RECOVER {
		fmt.Println("####### Recovering unfinished jobs #######")
		if err := core.QMgr.RecoverJobs(); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
		logger.Event(event.SERVER_RECOVER, "host="+host)
	} else {
		logger.Event(event.SERVER_START, "host="+host)
	}

	if conf.PID_FILE_PATH != "" {
		f, err := os.Create(conf.PID_FILE_PATH)
		if err != nil {
			err_msg := "Could not create pid file: " + conf.PID_FILE_PATH + "\n"
			fmt.Fprintf(os.Stderr, err_msg)
			logger.Error("ERROR: " + err_msg)
			os.Exit(1)
		}
		defer f.Close()

		pid := os.Getpid()
		fmt.Fprintln(f, pid)

		fmt.Println("##### pidfile #####")
		fmt.Printf("pid: %d saved to file: %s\n\n", pid, conf.PID_FILE_PATH)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("setting GOMAXPROCS...")
	}
	// setting GOMAXPROCS
	var procs int
	avail := runtime.NumCPU()
	if avail <= 2 {
		procs = 1
	} else if avail == 3 {
		procs = 2
	} else {
		procs = avail - 2
	}
	fmt.Println("##### Procs #####")
	fmt.Printf("Number of available CPUs = %d\n", avail)
	if conf.GOMAXPROCS > 0 {
		procs = conf.GOMAXPROCS
	}
	if procs <= avail {
		fmt.Printf("Running AWE server with GOMAXPROCS = %d\n\n", procs)
		runtime.GOMAXPROCS(procs)
	} else {
		fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.")
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", avail)
		runtime.GOMAXPROCS(avail)
	}

	<-control //block till something dies
}
<commit_msg>go fmt<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/MG-RAST/AWE/lib/auth"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/controller"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/db"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/AWE/vendor/github.com/MG-RAST/golib/goweb"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
)

func launchSite(control chan int, port int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}

	site_directory := conf.SITE_PATH
	fileinfo, err := os.Stat(site_directory)
	if err != nil {
		message := fmt.Sprintf("ERROR: site, path %s does not exist: %s", site_directory, err.Error())
		if os.IsNotExist(err) {
			message += " IsNotExist"
		}

		fmt.Fprintf(os.Stderr, message, "\n")
		logger.Error(message)

		os.Exit(1)
	} else {
		if !fileinfo.IsDir() {
			message := fmt.Sprintf("ERROR: site, path %s exists, but is not a directory", site_directory)
			fmt.Fprintf(os.Stderr, message, "\n")
			logger.Error(message)
			os.Exit(1)
		}

	}

	template_conf_filename := path.Join(conf.SITE_PATH, "js/config.js.tt")
	target_conf_filename := path.Join(conf.SITE_PATH, "js/config.js")
	buf, err := ioutil.ReadFile(template_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not read config template: %v\n", err)
		logger.Error("ERROR: could not read config template: " + err.Error())
	}
	template_conf_string := string(buf)

	// add / replace AWE API url
	if conf.API_URL == "" {
		fmt.Fprintf(os.Stderr, "ERROR: API_URL is not defined. \n")
		logger.Error("ERROR: API_URL is not defined.")
	}
	template_conf_string = strings.Replace(template_conf_string, "[% api_url %]", conf.API_URL, -1)

	// add auth
	auth_on := "false"
	auth_resources := ""
	if conf.GLOBUS_OAUTH || conf.MGRAST_OAUTH {
		auth_on = "true"
		b, _ := json.Marshal(conf.AUTH_RESOURCES)
		b = bytes.TrimPrefix(b, []byte("{"))
		b = bytes.TrimSuffix(b, []byte("}"))
		auth_resources = "," + string(b)
	}

	// replace auth
	template_conf_string = strings.Replace(template_conf_string, "[% auth_on %]", auth_on, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_default %]", conf.AUTH_DEFAULT, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_resources %]", auth_resources, -1)

	target_conf_file, err := os.Create(target_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	_, err = io.WriteString(target_conf_file, template_conf_string)

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	target_conf_file.Close()

	r.MapFunc("*", controller.SiteDir)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.SITE_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.SITE_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int, port int) {
	c := controller.NewServerController()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.Map("/job/{jid}/acl/{type}", c.JobAcl["typed"])
	r.Map("/job/{jid}/acl", c.JobAcl["base"])
	r.Map("/cgroup/{cgid}/acl/{type}", c.ClientGroupAcl["typed"])
	r.Map("/cgroup/{cgid}/acl", c.ClientGroupAcl["base"])
	r.Map("/cgroup/{cgid}/token", c.ClientGroupToken)
	r.MapRest("/job", c.Job)
	r.MapRest("/work", c.Work)
	r.MapRest("/cgroup", c.ClientGroup)
	r.MapRest("/client", c.Client)
	r.MapRest("/queue", c.Queue)
	r.MapRest("/awf", c.Awf)
	r.MapFunc("*", controller.ResourceDescription, goweb.GetMethod)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.API_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.API_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {

	err := conf.Init_conf("server")

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: "+err.Error())
		os.Exit(1)
	}

	//if !conf.INIT_SUCCESS {
	//	conf.PrintServerUsage()
	//	os.Exit(1)
	//}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("DEBUG_LEVEL > 0")
	}
	if _, err := os.Stat(conf.DATA_PATH); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(conf.DATA_PATH, 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR in creating data_path \"%s\", %s\n", conf.DATA_PATH, err.Error())
			os.Exit(1)
		}
	}

	if _, err := os.Stat(conf.LOGS_PATH); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(conf.LOGS_PATH, 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR in creating log_path \"%s\" %s\n", conf.LOGS_PATH, err.Error())
			os.Exit(1)
		}
	}

	if _, err := os.Stat(conf.DATA_PATH + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.DATA_PATH+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			os.Exit(1)
		}
	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("logger.Initialize...")
	}
	//init logger
	logger.Initialize("server")

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init db...")
	}
	//init db
	if err := db.Initialize(); err != nil {
		fmt.Printf("failed to initialize job db: %s\n", err.Error())
		os.Exit(1)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init db collection for user...")
	}
	//init db collection for user
	if err := user.Initialize(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR initializing user database: %s\n", err.Error())
		os.Exit(1)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init resource manager...")
	}
	//init resource manager
	core.InitResMgr("server")
	core.InitAwfMgr()
	core.InitJobDB()
	core.InitClientGroupDB()

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init auth...")
	}
	//init auth
	auth.Initialize()

	controller.PrintLogo()
	conf.Print("server")

	// reload job directory
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		if err := reload(conf.RELOAD); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init max job number (jid)...")
	}
	//init max job number (jid)
	if err := core.QMgr.InitMaxJid(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR from InitMaxJid : %v\n", err)
		os.Exit(1)
	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("launching server...")
	}
	//launch server
	control := make(chan int)
	go core.QMgr.Handle()
	go core.QMgr.Timer()
	go core.QMgr.ClientChecker()
	go launchSite(control, conf.SITE_PORT)
	go launchAPI(control, conf.API_PORT)

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("API launched...")
	}
	if err := core.AwfMgr.LoadWorkflows(); err != nil {
		logger.Error("LoadWorkflows: " + err.Error())
	}

	var host string
	if hostname, err := os.Hostname(); err == nil {
		host = fmt.Sprintf("%s:%d", hostname, conf.API_PORT)
	}

	//recover unfinished jobs before server went down last time
	if conf.RECOVER {
		fmt.Println("####### Recovering unfinished jobs #######")
		if err := core.QMgr.RecoverJobs(); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
		logger.Event(event.SERVER_RECOVER, "host="+host)
	} else {
		logger.Event(event.SERVER_START, "host="+host)
	}

	if conf.PID_FILE_PATH != "" {
		f, err := os.Create(conf.PID_FILE_PATH)
		if err != nil {
			err_msg := "Could not create pid file: " + conf.PID_FILE_PATH + "\n"
			fmt.Fprintf(os.Stderr, err_msg)
			logger.Error("ERROR: " + err_msg)
			os.Exit(1)
		}
		defer f.Close()

		pid := os.Getpid()
		fmt.Fprintln(f, pid)

		fmt.Println("##### pidfile #####")
		fmt.Printf("pid: %d saved to file: %s\n\n", pid, conf.PID_FILE_PATH)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("setting GOMAXPROCS...")
	}
	// setting GOMAXPROCS
	var procs int
	avail := runtime.NumCPU()
	if avail <= 2 {
		procs = 1
	} else if avail == 3 {
		procs = 2
	} else {
		procs = avail - 2
	}
	fmt.Println("##### Procs #####")
	fmt.Printf("Number of available CPUs = %d\n", avail)
	if conf.GOMAXPROCS > 0 {
		procs = conf.GOMAXPROCS
	}
	if procs <= avail {
		fmt.Printf("Running AWE server with GOMAXPROCS = %d\n\n", procs)
		runtime.GOMAXPROCS(procs)
	} else {
		fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.")
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", avail)
		runtime.GOMAXPROCS(avail)
	}

	<-control //block till something dies
}
<|endoftext|>


<commit_before>package adaptors

import (
	"net/http"

	"github.com/codegangsta/negroni"
)

func FromNegroni(handler negroni.Handler) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		n := negroni.New()
		n.Use(handler)
		n.UseHandler(next)
		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
			n.ServeHTTP(rw, req)
		})
	}
}

func HandlerFromNegroni(handler negroni.Handler) http.Handler {
	n := negroni.New()
	n.Use(handler)
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		n.ServeHTTP(rw, req)
	})
}
<commit_msg>Update broken negroni dependancy<commit_after>package adaptors

import (
	"net/http"

	"github.com/urfave/negroni"
)

func FromNegroni(handler negroni.Handler) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		n := negroni.New()
		n.Use(handler)
		n.UseHandler(next)
		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
			n.ServeHTTP(rw, req)
		})
	}
}

func HandlerFromNegroni(handler negroni.Handler) http.Handler {
	n := negroni.New()
	n.Use(handler)
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		n.ServeHTTP(rw, req)
	})
}
<|endoftext|>