<commit_before>// Copyright 2015 The Numgrad Authors. All rights reserved.
// See the LICENSE file for rights to use this source code.

package frame

import (
	"errors"
	"fmt"
	"io"

	"numgrad.io/eval"
	"numgrad.io/parser"
)

// A Frame is a two-dimensional data set.
//
// The typical Frame has a small number of named columns and a
// potentially large number of rows. It is inspired by (and indeed
// may be implemented by) an SQL table. However, it is possible to
// have a Frame without column names, and some implementations make
// large columns as cheap as large rows.
//
// Transforming a frame (slicing, aggregating, arithmetic, etc) is
// intended to be cheap. In some sense, transformations are lazily
// evaluated.
//
// As a performance optimization, a Frame may implement any of the
// following methods to provide implementation-specific versions of
// the common frame package functions. Users should not use these
// directly, instead preferring the frame.Fn(f) version for any Fn:
//
//	ColumnNames() []string
//	ColumnType(x int) Type
//	Permute(cols []int) Frame
//	Slice(x, xlen, y, ylen int) Frame
//	Set(x, y int, vals ...interface{}) error
//	Transpose() Frame
//	CopyFrom(src Frame) (n int, err error)
//	CopyTo(dst Frame) (n int, err error)
//	Accumulate(g Grouping) (Frame, error)
//	Len() (int, error)
//
// Maybe TODO:
//	Slice(Rectangle) Frame
//	Read(dst interface{}, col, off int) error // dst is []T.
type Frame interface {
	Cols() []string

	// TODO io.EOF for y out of range, io.ErrUnexpectedEOF for x out of range?
	Get(x, y int, dst ...interface{}) error
}

func Copy(dst, src Frame) (n int, err error) {
	dstf, ok := dst.(interface {
		CopyFrom(src Frame) (n int, err error)
	})
	if ok {
		return dstf.CopyFrom(src)
	}
	srcf, ok := src.(interface {
		CopyTo(dst Frame) (n int, err error)
	})
	if ok {
		return srcf.CopyTo(dst)
	}

	set, ok := dst.(interface {
		Set(x, y int, vals ...interface{}) error
	})
	if !ok {
		return 0, errors.New("frame.Copy: dst Frame does not implement Set")
	}

	cols := len(src.Cols())
	if dstCols := len(dst.Cols()); dstCols != cols {
		return 0, fmt.Errorf("frame.Copy: dst has %d columns, src has %d", dstCols, cols)
	}

	row := make([]interface{}, cols)
	rowp := make([]interface{}, len(row))
	for i := range row {
		rowp[i] = &row[i]
	}
	// TODO if src provides a Len, set backwards to allow impls to create space efficiently
	y := 0
	for {
		err := src.Get(0, y, rowp...)
		if err == io.EOF {
			break
		}
		if err != nil {
			return y, err
		}
		if err := set.Set(0, y, row...); err != nil {
			return y, err
		}
		y++
	}
	return y, nil
}

// Slice slices the Frame.
//
// A ylen of -1 means maximize the length. That is, on a 2x2 Frame,
// Slice(f2x2, 0, 2, 1, -1) produces a 2x1 Frame. This is not intended
// to be an implementation of Python's negative indexes, it is simply
// necessary as it can be expensive or impossible to determine the Len
// of a frame.
func Slice(f Frame, x, xlen, y, ylen int) Frame {
	fr, ok := f.(interface {
		Slice(x, xlen, y, ylen int) Frame
	})
	if ok {
		return fr.Slice(x, xlen, y, ylen)
	}
	panic("TODO Slice")
}

func Transpose(f Frame) Frame {
	fr, ok := f.(interface {
		Transpose() Frame
	})
	if ok {
		return fr.Transpose()
	}
	panic("TODO Transpose")
}

// TODO: consider baking the By field directly into the Frame.
// Then: x.groupby("Col0", "Col2").fold(sum) could work.
type Grouping struct {
	By   []int               // column index to group by, in order
	Func map[int]parser.Expr // column index -> binary operator
}

func Accumulate(f Frame, g Grouping) (Frame, error) {
	fr, ok := f.(interface {
		Accumulate(g Grouping) (Frame, error)
	})
	if ok {
		return fr.Accumulate(g)
	}
	panic("TODO Aggregate")
}

func Len(f Frame) (int, error) {
	fr, ok := f.(interface {
		Len() (int, error)
	})
	if ok {
		return fr.Len()
	}
	y := 0
	for {
		var v interface{}
		err := f.Get(0, y, &v)
		if err == io.EOF {
			break
		}
		if err != nil {
			return 0, err
		}
		y++
	}
	return y, nil
}

func Filter(f Frame, s *eval.Scope, e *parser.Expr) (Frame, error) {
	fr, ok := f.(interface {
		Filter(s *eval.Scope, e *parser.Expr) (Frame, error)
	})
	if ok {
		return fr.Filter(s, e)
	}
	panic("TODO Filter")
}

// Want:
// SQL-like powers over frames.
// Filtering by expression.
// Copyable
// Materializable
<commit_msg>frame: fix build<commit_after>// Copyright 2015 The Numgrad Authors. All rights reserved.
// See the LICENSE file for rights to use this source code.

package frame

import (
	"errors"
	"fmt"
	"io"

	"numgrad.io/eval"
	"numgrad.io/lang/expr"
)

// A Frame is a two-dimensional data set.
//
// The typical Frame has a small number of named columns and a
// potentially large number of rows. It is inspired by (and indeed
// may be implemented by) an SQL table. However, it is possible to
// have a Frame without column names, and some implementations make
// large columns as cheap as large rows.
//
// Transforming a frame (slicing, aggregating, arithmetic, etc) is
// intended to be cheap. In some sense, transformations are lazily
// evaluated.
//
// As a performance optimization, a Frame may implement any of the
// following methods to provide implementation-specific versions of
// the common frame package functions. Users should not use these
// directly, instead preferring the frame.Fn(f) version for any Fn:
//
//	ColumnNames() []string
//	ColumnType(x int) Type
//	Permute(cols []int) Frame
//	Slice(x, xlen, y, ylen int) Frame
//	Set(x, y int, vals ...interface{}) error
//	Transpose() Frame
//	CopyFrom(src Frame) (n int, err error)
//	CopyTo(dst Frame) (n int, err error)
//	Accumulate(g Grouping) (Frame, error)
//	Len() (int, error)
//
// Maybe TODO:
//	Slice(Rectangle) Frame
//	Read(dst interface{}, col, off int) error // dst is []T.
type Frame interface {
	Cols() []string

	// TODO io.EOF for y out of range, io.ErrUnexpectedEOF for x out of range?
	Get(x, y int, dst ...interface{}) error
}

func Copy(dst, src Frame) (n int, err error) {
	dstf, ok := dst.(interface {
		CopyFrom(src Frame) (n int, err error)
	})
	if ok {
		return dstf.CopyFrom(src)
	}
	srcf, ok := src.(interface {
		CopyTo(dst Frame) (n int, err error)
	})
	if ok {
		return srcf.CopyTo(dst)
	}

	set, ok := dst.(interface {
		Set(x, y int, vals ...interface{}) error
	})
	if !ok {
		return 0, errors.New("frame.Copy: dst Frame does not implement Set")
	}

	cols := len(src.Cols())
	if dstCols := len(dst.Cols()); dstCols != cols {
		return 0, fmt.Errorf("frame.Copy: dst has %d columns, src has %d", dstCols, cols)
	}

	row := make([]interface{}, cols)
	rowp := make([]interface{}, len(row))
	for i := range row {
		rowp[i] = &row[i]
	}
	// TODO if src provides a Len, set backwards to allow impls to create space efficiently
	y := 0
	for {
		err := src.Get(0, y, rowp...)
		if err == io.EOF {
			break
		}
		if err != nil {
			return y, err
		}
		if err := set.Set(0, y, row...); err != nil {
			return y, err
		}
		y++
	}
	return y, nil
}

// Slice slices the Frame.
//
// A ylen of -1 means maximize the length. That is, on a 2x2 Frame,
// Slice(f2x2, 0, 2, 1, -1) produces a 2x1 Frame. This is not intended
// to be an implementation of Python's negative indexes, it is simply
// necessary as it can be expensive or impossible to determine the Len
// of a frame.
func Slice(f Frame, x, xlen, y, ylen int) Frame {
	fr, ok := f.(interface {
		Slice(x, xlen, y, ylen int) Frame
	})
	if ok {
		return fr.Slice(x, xlen, y, ylen)
	}
	panic("TODO Slice")
}

func Transpose(f Frame) Frame {
	fr, ok := f.(interface {
		Transpose() Frame
	})
	if ok {
		return fr.Transpose()
	}
	panic("TODO Transpose")
}

// TODO: consider baking the By field directly into the Frame.
// Then: x.groupby("Col0", "Col2").fold(sum) could work.
type Grouping struct {
	By   []int             // column index to group by, in order
	Func map[int]expr.Expr // column index -> binary operator
}

func Accumulate(f Frame, g Grouping) (Frame, error) {
	fr, ok := f.(interface {
		Accumulate(g Grouping) (Frame, error)
	})
	if ok {
		return fr.Accumulate(g)
	}
	panic("TODO Aggregate")
}

func Len(f Frame) (int, error) {
	fr, ok := f.(interface {
		Len() (int, error)
	})
	if ok {
		return fr.Len()
	}
	y := 0
	for {
		var v interface{}
		err := f.Get(0, y, &v)
		if err == io.EOF {
			break
		}
		if err != nil {
			return 0, err
		}
		y++
	}
	return y, nil
}

func Filter(f Frame, s *eval.Scope, e expr.Expr) (Frame, error) {
	fr, ok := f.(interface {
		Filter(s *eval.Scope, e expr.Expr) (Frame, error)
	})
	if ok {
		return fr.Filter(s, e)
	}
	panic("TODO Filter")
}

// Want:
// SQL-like powers over frames.
// Filtering by expression.
// Copyable
// Materializable
<|endoftext|>


<commit_before>package cluster

import (
	"crypto/tls"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"
)

const (
	// Force-refresh the state of the node this often.
	stateRefreshPeriod = 30 * time.Second
)

func NewNode(id string, addr string) *Node {
	e := &Node{
		ID:     id,
		Addr:   addr,
		Labels: make(map[string]string),
		ch:     make(chan bool),
	}
	return e
}

type Node struct {
	sync.Mutex

	ID     string
	IP     string
	Addr   string
	Cpus   int
	Memory int64
	Labels map[string]string

	ch           chan bool
	containers   map[string]*Container
	client       dockerclient.Client
	eventHandler EventHandler
}

// Connect will initialize a connection to the Docker daemon running on the
// host, gather machine specs (memory, cpu, ...) and monitor state changes.
func (n *Node) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClient(n.Addr, config)
	if err != nil {
		return err
	}

	addr, err := net.ResolveIPAddr("ip4", strings.Split(c.URL.Host, ":")[0])
	if err != nil {
		return err
	}
	n.IP = addr.IP.String()

	return n.connectClient(c)
}

func (n *Node) connectClient(client dockerclient.Client) error {
	n.client = client

	// Fetch the engine labels.
	if err := n.updateSpecs(); err != nil {
		n.client = nil
		return err
	}

	// Force a state update before returning.
	if err := n.updateContainers(); err != nil {
		n.client = nil
		return err
	}

	// Start the update loop.
	go n.updateLoop()

	// Start monitoring events from the Node.
	n.client.StartMonitorEvents(n.handler)

	return nil
}

// IsConnected returns true if the engine is connected to a remote docker API
func (e *Node) IsConnected() bool {
	return e.client != nil
}

// Gather node specs (CPU, memory, constraints, ...).
func (n *Node) updateSpecs() error {
	info, err := n.client.Info()
	if err != nil {
		return err
	}
	n.Cpus = info.NCPU
	n.Memory = info.MemTotal
	n.Labels = map[string]string{
		"graphdriver":     info.Driver,
		"executiondriver": info.ExecutionDriver,
		"kernelversion":   info.KernelVersion,
		"operatingsystem": info.OperatingSystem,
	}
	return nil
}

// Refresh the list and status of containers running on the node.
func (n *Node) updateContainers() error {
	containers, err := n.client.ListContainers(true, false, "")
	if err != nil {
		return err
	}

	n.Lock()
	defer n.Unlock()

	merged := make(map[string]*Container)
	for _, c := range containers {
		if current, exists := n.containers[c.Id]; exists {
			// The container exists. Update its state.
			current.Container = c
			merged[current.Id] = current
		} else {
			// This is a brand new container.
			container := &Container{}
			container.Container = c
			container.node = n

			info, err := n.client.InspectContainer(c.Id)
			if err != nil {
				log.Errorf("[%s] Unable to update state of %s", n.ID, c.Id)
				continue
			}
			container.Info = *info

			merged[container.Id] = container
		}
	}

	n.containers = merged

	log.Debugf("[%s] Updated state", n.ID)
	return nil
}

func (n *Node) updateContainersAsync() {
	n.ch <- true
}

func (n *Node) updateLoop() {
	for {
		var err error
		select {
		case <-n.ch:
			err = n.updateContainers()
		case <-time.After(stateRefreshPeriod):
			err = n.updateContainers()
		}
		if err != nil {
			log.Errorf("[%s] Updated state failed: %v", n.ID, err)
		}
	}
}

// Return the sum of memory reserved by containers.
func (n *Node) ReservedMemory() int64 {
	var r int64 = 0
	for _, c := range n.containers {
		r += int64(c.Info.Config.Memory)
	}
	return r
}

// Return the sum of CPUs reserved by containers.
func (n *Node) ReservedCpus() float64 {
	r := 0.0
	for _, c := range n.containers {
		r += float64(c.Info.Config.CpuShares) / 100.0 * float64(n.Cpus)
	}
	return r
}

func (n *Node) Create(config *dockerclient.ContainerConfig, name string, pullImage bool) (*Container, error) {
	var (
		err    error
		id     string
		client = n.client
	)

	if id, err = client.CreateContainer(config, name); err != nil {
		// If the error is other than not found, abort immediately.
		if err != dockerclient.ErrNotFound {
			return nil, err
		}
		// Otherwise, try to pull the image...
		if err = n.Pull(config.Image); err != nil {
			return nil, err
		}
		// ...And try again.
		if id, err = client.CreateContainer(config, name); err != nil {
			return nil, err
		}
	}

	// Register the container immediately while waiting for a state refresh.
	// Force a state refresh to pick up the newly created container.
	log.Debug("Updating containers after create")
	n.updateContainers()

	return n.containers[id], nil
}

func (n *Node) ListImages() ([]string, error) {
	images, err := n.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

func (n *Node) Remove(container *Container, force bool) error {
	if err := n.client.RemoveContainer(container.Id, force); err != nil {
		return err
	}

	// Remove the container from the state. Eventually, the state refresh loop
	// will rewrite this.
	n.Lock()
	defer n.Unlock()
	delete(n.containers, container.Id)

	return nil
}

func (n *Node) Pull(image string) error {
	if err := n.client.PullImage(image); err != nil {
		return err
	}
	return nil
}

// Register an event handler.
func (n *Node) Events(h EventHandler) error {
	if n.eventHandler != nil {
		return fmt.Errorf("event handler already set")
	}
	n.eventHandler = h
	return nil
}

func (n *Node) Containers() map[string]*Container {
	return n.containers
}

func (n *Node) String() string {
	return fmt.Sprintf("node %s addr %s", n.ID, n.Addr)
}

func (n *Node) handler(ev *dockerclient.Event, args ...interface{}) {
	// Something changed - refresh our internal state.
	n.updateContainers()

	// If there is no event handler registered, abort right now.
	if n.eventHandler == nil {
		return
	}

	event := &Event{
		Node: n,
		Type: ev.Status,
		Time: time.Unix(int64(ev.Time), 0),
	}

	if container, ok := n.containers[ev.Id]; ok {
		event.Container = container
	} else {
		event.Container = &Container{
			node: n,
			Container: dockerclient.Container{
				Id:    ev.Id,
				Image: ev.From,
			},
		}
	}

	n.eventHandler.Handle(event)
}
<commit_msg>Removed debug log when refreshing containers after creation<commit_after>package cluster

import (
	"crypto/tls"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"
)

const (
	// Force-refresh the state of the node this often.
	stateRefreshPeriod = 30 * time.Second
)

func NewNode(id string, addr string) *Node {
	e := &Node{
		ID:     id,
		Addr:   addr,
		Labels: make(map[string]string),
		ch:     make(chan bool),
	}
	return e
}

type Node struct {
	sync.Mutex

	ID     string
	IP     string
	Addr   string
	Cpus   int
	Memory int64
	Labels map[string]string

	ch           chan bool
	containers   map[string]*Container
	client       dockerclient.Client
	eventHandler EventHandler
}

// Connect will initialize a connection to the Docker daemon running on the
// host, gather machine specs (memory, cpu, ...) and monitor state changes.
func (n *Node) Connect(config *tls.Config) error {
	c, err := dockerclient.NewDockerClient(n.Addr, config)
	if err != nil {
		return err
	}

	addr, err := net.ResolveIPAddr("ip4", strings.Split(c.URL.Host, ":")[0])
	if err != nil {
		return err
	}
	n.IP = addr.IP.String()

	return n.connectClient(c)
}

func (n *Node) connectClient(client dockerclient.Client) error {
	n.client = client

	// Fetch the engine labels.
	if err := n.updateSpecs(); err != nil {
		n.client = nil
		return err
	}

	// Force a state update before returning.
	if err := n.updateContainers(); err != nil {
		n.client = nil
		return err
	}

	// Start the update loop.
	go n.updateLoop()

	// Start monitoring events from the Node.
	n.client.StartMonitorEvents(n.handler)

	return nil
}

// IsConnected returns true if the engine is connected to a remote docker API
func (e *Node) IsConnected() bool {
	return e.client != nil
}

// Gather node specs (CPU, memory, constraints, ...).
func (n *Node) updateSpecs() error {
	info, err := n.client.Info()
	if err != nil {
		return err
	}
	n.Cpus = info.NCPU
	n.Memory = info.MemTotal
	n.Labels = map[string]string{
		"graphdriver":     info.Driver,
		"executiondriver": info.ExecutionDriver,
		"kernelversion":   info.KernelVersion,
		"operatingsystem": info.OperatingSystem,
	}
	return nil
}

// Refresh the list and status of containers running on the node.
func (n *Node) updateContainers() error {
	containers, err := n.client.ListContainers(true, false, "")
	if err != nil {
		return err
	}

	n.Lock()
	defer n.Unlock()

	merged := make(map[string]*Container)
	for _, c := range containers {
		if current, exists := n.containers[c.Id]; exists {
			// The container exists. Update its state.
			current.Container = c
			merged[current.Id] = current
		} else {
			// This is a brand new container.
			container := &Container{}
			container.Container = c
			container.node = n

			info, err := n.client.InspectContainer(c.Id)
			if err != nil {
				log.Errorf("[%s] Unable to update state of %s", n.ID, c.Id)
				continue
			}
			container.Info = *info

			merged[container.Id] = container
		}
	}

	n.containers = merged

	log.Debugf("[%s] Updated state", n.ID)
	return nil
}

func (n *Node) updateContainersAsync() {
	n.ch <- true
}

func (n *Node) updateLoop() {
	for {
		var err error
		select {
		case <-n.ch:
			err = n.updateContainers()
		case <-time.After(stateRefreshPeriod):
			err = n.updateContainers()
		}
		if err != nil {
			log.Errorf("[%s] Updated state failed: %v", n.ID, err)
		}
	}
}

// Return the sum of memory reserved by containers.
func (n *Node) ReservedMemory() int64 {
	var r int64 = 0
	for _, c := range n.containers {
		r += int64(c.Info.Config.Memory)
	}
	return r
}

// Return the sum of CPUs reserved by containers.
func (n *Node) ReservedCpus() float64 {
	r := 0.0
	for _, c := range n.containers {
		r += float64(c.Info.Config.CpuShares) / 100.0 * float64(n.Cpus)
	}
	return r
}

func (n *Node) Create(config *dockerclient.ContainerConfig, name string, pullImage bool) (*Container, error) {
	var (
		err    error
		id     string
		client = n.client
	)

	if id, err = client.CreateContainer(config, name); err != nil {
		// If the error is other than not found, abort immediately.
		if err != dockerclient.ErrNotFound {
			return nil, err
		}
		// Otherwise, try to pull the image...
		if err = n.Pull(config.Image); err != nil {
			return nil, err
		}
		// ...And try again.
		if id, err = client.CreateContainer(config, name); err != nil {
			return nil, err
		}
	}

	// Register the container immediately while waiting for a state refresh.
	// Force a state refresh to pick up the newly created container.
	n.updateContainers()

	return n.containers[id], nil
}

func (n *Node) ListImages() ([]string, error) {
	images, err := n.client.ListImages()
	if err != nil {
		return nil, err
	}

	out := []string{}

	for _, i := range images {
		for _, t := range i.RepoTags {
			out = append(out, t)
		}
	}

	return out, nil
}

func (n *Node) Remove(container *Container, force bool) error {
	if err := n.client.RemoveContainer(container.Id, force); err != nil {
		return err
	}

	// Remove the container from the state. Eventually, the state refresh loop
	// will rewrite this.
	n.Lock()
	defer n.Unlock()
	delete(n.containers, container.Id)

	return nil
}

func (n *Node) Pull(image string) error {
	if err := n.client.PullImage(image); err != nil {
		return err
	}
	return nil
}

// Register an event handler.
func (n *Node) Events(h EventHandler) error {
	if n.eventHandler != nil {
		return fmt.Errorf("event handler already set")
	}
	n.eventHandler = h
	return nil
}

func (n *Node) Containers() map[string]*Container {
	return n.containers
}

func (n *Node) String() string {
	return fmt.Sprintf("node %s addr %s", n.ID, n.Addr)
}

func (n *Node) handler(ev *dockerclient.Event, args ...interface{}) {
	// Something changed - refresh our internal state.
	n.updateContainers()

	// If there is no event handler registered, abort right now.
	if n.eventHandler == nil {
		return
	}

	event := &Event{
		Node: n,
		Type: ev.Status,
		Time: time.Unix(int64(ev.Time), 0),
	}

	if container, ok := n.containers[ev.Id]; ok {
		event.Container = container
	} else {
		event.Container = &Container{
			node: n,
			Container: dockerclient.Container{
				Id:    ev.Id,
				Image: ev.From,
			},
		}
	}

	n.eventHandler.Handle(event)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apiextensionsinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	jsonserializer "k8s.io/apimachinery/pkg/runtime/serializer/json"
	"k8s.io/apimachinery/pkg/runtime/serializer/versioning"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/rest"
	"sigs.k8s.io/controller-runtime/pkg/envtest"

	webhooktesting "github.com/jetstack/cert-manager/cmd/webhook/app/testing"
	apitesting "github.com/jetstack/cert-manager/pkg/api/testing"
)

func init() {
	// Set environment variables for controller-runtime's envtest package.
	// This is done once as we cannot scope environment variables to a single
	// invocation of RunControlPlane due to envtest's design.
	setUpEnvTestEnv()
}

type StopFunc func()

func RunControlPlane(t *testing.T) (*rest.Config, StopFunc) {
	webhookOpts, stopWebhook := webhooktesting.StartWebhookServer(t, []string{})
	crdsDir := apitesting.CRDDirectory(t)
	crds := readCustomResourcesAtPath(t, crdsDir)
	for _, crd := range crds {
		t.Logf("Found CRD with name %q", crd.Name)
	}
	patchCRDConversion(crds, webhookOpts.URL, webhookOpts.CAPEM)
	// environment variables
	env := &envtest.Environment{
		AttachControlPlaneOutput: true,
		CRDs:                     crdsToRuntimeObjects(crds),
	}
	config, err := env.Start()
	if err != nil {
		t.Fatalf("failed to start control plane: %v", err)
	}
	// TODO: configure Validating and Mutating webhook
	return config, func() {
		defer stopWebhook()
		if err := env.Stop(); err != nil {
			t.Logf("failed to shut down control plane, not failing test: %v", err)
		}
	}
}

var (
	internalScheme = runtime.NewScheme()
)

func init() {
	utilruntime.Must(metav1.AddMetaToScheme(internalScheme))
	apiextensionsinstall.Install(internalScheme)
}

func patchCRDConversion(crds []*v1beta1.CustomResourceDefinition, url string, caPEM []byte) {
	for _, crd := range crds {
		if crd.Spec.Conversion == nil {
			continue
		}
		if crd.Spec.Conversion.WebhookClientConfig == nil {
			continue
		}
		if crd.Spec.Conversion.WebhookClientConfig.Service == nil {
			continue
		}
		path := ""
		if crd.Spec.Conversion.WebhookClientConfig.Service.Path != nil {
			path = *crd.Spec.Conversion.WebhookClientConfig.Service.Path
		}
		url := fmt.Sprintf("%s%s", url, path)
		crd.Spec.Conversion.WebhookClientConfig.URL = &url
		crd.Spec.Conversion.WebhookClientConfig.CABundle = caPEM
		crd.Spec.Conversion.WebhookClientConfig.Service = nil
	}
}

func readCustomResourcesAtPath(t *testing.T, path string) []*v1beta1.CustomResourceDefinition {
	serializer := jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, internalScheme, internalScheme, jsonserializer.SerializerOptions{
		Yaml: true,
	})
	converter := runtime.UnsafeObjectConvertor(internalScheme)
	codec := versioning.NewCodec(serializer, serializer, converter, internalScheme, internalScheme, internalScheme, runtime.InternalGroupVersioner, runtime.InternalGroupVersioner, internalScheme.Name())

	var crds []*v1beta1.CustomResourceDefinition
	if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if filepath.Ext(path) != ".yaml" {
			return nil
		}
		crd, err := readCRDsAtPath(codec, converter, path)
		if err != nil {
			return err
		}
		crds = append(crds, crd...)
		return nil
	}); err != nil {
		t.Fatal(err)
	}
	return crds
}

func readCRDsAtPath(codec runtime.Codec, converter runtime.ObjectConvertor, path string) ([]*v1beta1.CustomResourceDefinition, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	dataStr := string(data)
	datas := strings.Split(dataStr, "---")
	var crds []*v1beta1.CustomResourceDefinition
	for _, d := range datas {
		// skip empty YAML documents
		if strings.TrimSpace(d) == "" {
			continue
		}

		internalCRD := &apiextensions.CustomResourceDefinition{}
		if _, _, err := codec.Decode([]byte(d), nil, internalCRD); err != nil {
			return nil, err
		}

		out := &v1beta1.CustomResourceDefinition{}
		if err := converter.Convert(internalCRD, out, nil); err != nil {
			return nil, err
		}

		crds = append(crds, out)
	}

	return crds, nil
}

func crdsToRuntimeObjects(in []*v1beta1.CustomResourceDefinition) []runtime.Object {
	out := make([]runtime.Object, len(in))

	for i, crd := range in {
		out[i] = runtime.Object(crd)
	}

	return out
}
<commit_msg>Split with newlines<commit_after>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	apiextensionsinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	jsonserializer "k8s.io/apimachinery/pkg/runtime/serializer/json"
	"k8s.io/apimachinery/pkg/runtime/serializer/versioning"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/rest"
	"sigs.k8s.io/controller-runtime/pkg/envtest"

	webhooktesting "github.com/jetstack/cert-manager/cmd/webhook/app/testing"
	apitesting "github.com/jetstack/cert-manager/pkg/api/testing"
)

func init() {
	// Set environment variables for controller-runtime's envtest package.
	// This is done once as we cannot scope environment variables to a single
	// invocation of RunControlPlane due to envtest's design.
	setUpEnvTestEnv()
}

type StopFunc func()

func RunControlPlane(t *testing.T) (*rest.Config, StopFunc) {
	webhookOpts, stopWebhook := webhooktesting.StartWebhookServer(t, []string{})
	crdsDir := apitesting.CRDDirectory(t)
	crds := readCustomResourcesAtPath(t, crdsDir)
	for _, crd := range crds {
		t.Logf("Found CRD with name %q", crd.Name)
	}
	patchCRDConversion(crds, webhookOpts.URL, webhookOpts.CAPEM)
	// environment variables
	env := &envtest.Environment{
		AttachControlPlaneOutput: true,
		CRDs:                     crdsToRuntimeObjects(crds),
	}
	config, err := env.Start()
	if err != nil {
		t.Fatalf("failed to start control plane: %v", err)
	}
	// TODO: configure Validating and Mutating webhook
	return config, func() {
		defer stopWebhook()
		if err := env.Stop(); err != nil {
			t.Logf("failed to shut down control plane, not failing test: %v", err)
		}
	}
}

var (
	internalScheme = runtime.NewScheme()
)

func init() {
	utilruntime.Must(metav1.AddMetaToScheme(internalScheme))
	apiextensionsinstall.Install(internalScheme)
}

func patchCRDConversion(crds []*v1beta1.CustomResourceDefinition, url string, caPEM []byte) {
	for _, crd := range crds {
		if crd.Spec.Conversion == nil {
			continue
		}
		if crd.Spec.Conversion.WebhookClientConfig == nil {
			continue
		}
		if crd.Spec.Conversion.WebhookClientConfig.Service == nil {
			continue
		}
		path := ""
		if crd.Spec.Conversion.WebhookClientConfig.Service.Path != nil {
			path = *crd.Spec.Conversion.WebhookClientConfig.Service.Path
		}
		url := fmt.Sprintf("%s%s", url, path)
		crd.Spec.Conversion.WebhookClientConfig.URL = &url
		crd.Spec.Conversion.WebhookClientConfig.CABundle = caPEM
		crd.Spec.Conversion.WebhookClientConfig.Service = nil
	}
}

func readCustomResourcesAtPath(t *testing.T, path string) []*v1beta1.CustomResourceDefinition {
	serializer := jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, internalScheme, internalScheme, jsonserializer.SerializerOptions{
		Yaml: true,
	})
	converter := runtime.UnsafeObjectConvertor(internalScheme)
	codec := versioning.NewCodec(serializer, serializer, converter, internalScheme, internalScheme, internalScheme, runtime.InternalGroupVersioner, runtime.InternalGroupVersioner, internalScheme.Name())

	var crds []*v1beta1.CustomResourceDefinition
	if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if filepath.Ext(path) != ".yaml" {
			return nil
		}
		crd, err := readCRDsAtPath(codec, converter, path)
		if err != nil {
			return err
		}
		crds = append(crds, crd...)
		return nil
	}); err != nil {
		t.Fatal(err)
	}
	return crds
}

func readCRDsAtPath(codec runtime.Codec, converter runtime.ObjectConvertor, path string) ([]*v1beta1.CustomResourceDefinition, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	dataStr := string(data)
	datas := strings.Split(dataStr, "\n---\n")
	var crds []*v1beta1.CustomResourceDefinition
	for _, d := range datas {
		// skip empty YAML documents
		if strings.TrimSpace(d) == "" {
			continue
		}

		internalCRD := &apiextensions.CustomResourceDefinition{}
		if _, _, err := codec.Decode([]byte(d), nil, internalCRD); err != nil {
			return nil, err
		}

		out := &v1beta1.CustomResourceDefinition{}
		if err := converter.Convert(internalCRD, out, nil); err != nil {
			return nil, err
		}

		crds = append(crds, out)
	}

	return crds, nil
}

func crdsToRuntimeObjects(in []*v1beta1.CustomResourceDefinition) []runtime.Object {
	out := make([]runtime.Object, len(in))

	for i, crd := range in {
		out[i] = runtime.Object(crd)
	}

	return out
}
<|endoftext|>


<commit_before>package cmd

import (
	"log"
	"time"

	"github.com/influxdata/influxdb/client/v2"
	"github.com/spf13/cobra"
)

var initOffset int

var initCmd = &cobra.Command{
	Use:   "init",
	Short: "Prepares the InfluxDB to be used with Ruminant",
	Long: `Creates the InfluxDB as configured at sets an initial marker
timestamp with a given offset in relation to the current time.`,
	Run: func(cmd *cobra.Command, args []string) {
		c, err := Conf(true)
		if err != nil {
			log.Fatal(err)
		}

		l.Infow("Going to create InfluxDB client")
		i, err := NewInflux(c.Gulp.Host, c.Gulp.Proto, c.Gulp.Db, c.Gulp.User, c.Gulp.Pass, c.Gulp.Series, c.Gulp.Indicator, c.Gulp.Port)
		if err != nil {
			l.Fatal("Could net create InfluxDB client", "error", err.Error())
		}

		//l.Infof("Create InfluxDB %s", c.Gulp.Db)
		//_, err = i.Query(fmt.Sprintf("CREATE DATABASE %s", c.Gulp.Db))
		//if err != nil {
		//	l.Fatal("Could not create database", "error", err.Error())
		//}

		l.Infof("Creating initial timestamp with an offset of %d hours", initOffset)
		bp, err := client.NewBatchPoints(client.BatchPointsConfig{Database: i.DB, Precision: "s"})
		if err != nil {
			l.Fatal("Could create initial timestamp", "error", err.Error())
		}

		timestamp := time.Now().Add(-(time.Hour * time.Duration(initOffset)))
		bp.AddPoint(i.LatestMarker(timestamp, "init"))

		if err := i.Client.Write(bp); err != nil {
			l.Fatal("Could save initial timestamp", "error", err.Error())
		}
	},
}

func init() {
	RootCmd.AddCommand(initCmd)
	initCmd.PersistentFlags().IntVarP(&initOffset, "offset", "o", 24, "Offset of the initial timestamp in hours")
}
<commit_msg>try to create database in init only if not exists<commit_after>package cmd

import (
	"fmt"
	"log"
	"time"

	"github.com/influxdata/influxdb/client/v2"
	"github.com/spf13/cobra"
)

var initOffset int

var initCmd = &cobra.Command{
	Use:   "init",
	Short: "Prepares the InfluxDB to be used with Ruminant",
	Long: `Creates the InfluxDB as configured at sets an initial marker
timestamp with a given offset in relation to the current time.`,
	Run: func(cmd *cobra.Command, args []string) {
		c, err := Conf(true)
		if err != nil {
			log.Fatal(err)
		}

		l.Infow("Going to create InfluxDB client")
		i, err := NewInflux(c.Gulp.Host, c.Gulp.Proto, c.Gulp.Db, c.Gulp.User, c.Gulp.Pass, c.Gulp.Series, c.Gulp.Indicator, c.Gulp.Port)
		if err != nil {
			l.Fatal("Could net create InfluxDB client", "error", err.Error())
		}

		l.Infof("Create InfluxDB %s", c.Gulp.Db)
		_, err = i.Query(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", c.Gulp.Db))
		if err != nil {
			l.Fatal("Could not create database", "error", err.Error())
		}

		l.Infof("Creating initial timestamp with an offset of %d hours", initOffset)
		bp, err := client.NewBatchPoints(client.BatchPointsConfig{Database: i.DB, Precision: "s"})
		if err != nil {
			l.Fatal("Could create initial timestamp", "error", err.Error())
		}

		timestamp := time.Now().Add(-(time.Hour * time.Duration(initOffset)))
		bp.AddPoint(i.LatestMarker(timestamp, "init"))

		if err := i.Client.Write(bp); err != nil {
			l.Fatal("Could save initial timestamp", "error", err.Error())
		}
	},
}

func init() {
	RootCmd.AddCommand(initCmd)
	initCmd.PersistentFlags().IntVarP(&initOffset, "offset", "o", 24, "Offset of the initial timestamp in hours")
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/mbtproject/mbt/lib"
	"gopkg.in/spf13/cobra.v0"
)

var (
	formatAsJson bool
)

type applicationView struct {
	Name    string
	Path    string
	Version string
}

func init() {
	describePrCmd.Flags().StringVar(&src, "src", "", "source branch")
	describePrCmd.Flags().StringVar(&dst, "dst", "", "destination branch")

	describeIntersectionCmd.Flags().StringVar(&kind, "kind", "", "kind of input for first and second args (available options are 'branch' and 'commit')")
	describeIntersectionCmd.Flags().StringVar(&first, "first", "", "first item")
	describeIntersectionCmd.Flags().StringVar(&second, "second", "", "second item")

	describeCmd.PersistentFlags().BoolVar(&formatAsJson, "json", false, "format output as json")
	describeCmd.AddCommand(describeCommitCmd)
	describeCmd.AddCommand(describeBranchCmd)
	describeCmd.AddCommand(describePrCmd)
	describeCmd.AddCommand(describeIntersectionCmd)

	RootCmd.AddCommand(describeCmd)
}

var describeCmd = &cobra.Command{
	Use:   "describe",
	Short: "Describes the manifest of a repo",
}

var describeBranchCmd = &cobra.Command{
	Use:   "branch <branch>",
	Short: "Describes the manifest for the given branch",
	RunE: func(cmd *cobra.Command, args []string) error {
		branch := "master"
		if len(args) > 0 {
			branch = args[0]
		}
		m, err := lib.ManifestByBranch(in, branch)
		if err != nil {
			return handle(err)
		}

		output(m.Applications)
		return nil
	},
}

var describePrCmd = &cobra.Command{
	Use:   "pr --src <branch> --dst <branch>",
	Short: "Describes the manifest for a given pr",
	RunE: func(cmd *cobra.Command, args []string) error {
		if src == "" {
			return errors.New("requires source")
		}

		if dst == "" {
			return errors.New("requires dest")
		}

		m, err := lib.ManifestByPr(in, src, dst)
		if err != nil {
			return handle(err)
		}

		output(m.Applications)

		return nil
	},
}

var describeCommitCmd = &cobra.Command{
	Use:   "commit <sha>",
	Short: "Describes the manifest for a given commit",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("requires the commit sha")
		}

		commit := args[0]

		m, err := lib.ManifestBySha(in, commit)
		if err != nil {
			return handle(err)
		}

		output(m.Applications)

		return nil
	},
}

var describeIntersectionCmd = &cobra.Command{
	Use:   "intersection --kind <branch|commit> --first <first> --second <second>",
	Short: "Describes the intersection between two trees",
	RunE: func(cmd *cobra.Command, args []string) error {
		if kind == "" {
			return errors.New("requires the kind argument")
		}

		if first == "" {
			return errors.New("requires the first argument")
		}

		if second == "" {
			return errors.New("requires the second argument")
		}

		var apps lib.Applications
		var err error

		switch kind {
		case "branch":
			apps, err = lib.IntersectionByBranch(in, first, second)
		case "commit":
			apps, err = lib.IntersectionByCommit(in, first, second)
		default:
			err = errors.New("not a valid kind - available options are 'branch' and 'commit'")
		}

		if err != nil {
			return handle(err)
		}

		return handle(output(apps))
	},
}

const columnWidth = 30

func formatRow(args ...interface{}) string {
	padded := make([]interface{}, len(args))
	for i, a := range args {
		requiredPadding := columnWidth - len(a.(string))
		if requiredPadding > 0 {
			padded[i] = fmt.Sprintf("%s%s", a, strings.Join(make([]string, requiredPadding), " "))
		} else {
			padded[i] = a
		}
	}
	return fmt.Sprintf("%s\t\t%s\t\t%s\n", padded...)
}

func output(apps lib.Applications) error {
	if formatAsJson {
		v := make([]applicationView, len(apps))
		for i, a := range apps {
			v[i] = applicationView{
				Name:    a.Name(),
				Path:    a.Path(),
				Version: a.Version(),
			}
		}
		buff, err := json.MarshalIndent(v, "", "  ")
		if err != nil {
			return err
		}
		fmt.Println(string(buff))
	} else {
		fmt.Print(formatRow("NAME", "PATH", "VERSION"))
		for _, a := range apps {
			fmt.Printf(formatRow(a.Name(), a.Path(), a.Version()))
		}
	}

	return nil
}
<commit_msg>Added properties to json output<commit_after>package cmd

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/mbtproject/mbt/lib"
	"gopkg.in/spf13/cobra.v0"
)

var (
	formatAsJson bool
)

func init() {
	describePrCmd.Flags().StringVar(&src, "src", "", "source branch")
	describePrCmd.Flags().StringVar(&dst, "dst", "", "destination branch")

	describeIntersectionCmd.Flags().StringVar(&kind, "kind", "", "kind of input for first and second args (available options are 'branch' and 'commit')")
	describeIntersectionCmd.Flags().StringVar(&first, "first", "", "first item")
	describeIntersectionCmd.Flags().StringVar(&second, "second", "", "second item")

	describeCmd.PersistentFlags().BoolVar(&formatAsJson, "json", false, "format output as json")
	describeCmd.AddCommand(describeCommitCmd)
	describeCmd.AddCommand(describeBranchCmd)
	describeCmd.AddCommand(describePrCmd)
	describeCmd.AddCommand(describeIntersectionCmd)

	RootCmd.AddCommand(describeCmd)
}

var describeCmd = &cobra.Command{
	Use:   "describe",
	Short: "Describes the manifest of a repo",
}

var describeBranchCmd = &cobra.Command{
	Use:   "branch <branch>",
	Short: "Describes the manifest for the given branch",
	RunE: func(cmd *cobra.Command, args []string) error {
		branch := "master"
		if len(args) > 0 {
			branch = args[0]
		}
		m, err := lib.ManifestByBranch(in, branch)
		if err != nil {
			return handle(err)
		}

		output(m.Applications)
		return nil
	},
}

var describePrCmd = &cobra.Command{
	Use:   "pr --src <branch> --dst <branch>",
	Short: "Describes the manifest for a given pr",
	RunE: func(cmd *cobra.Command, args []string) error {
		if src == "" {
			return errors.New("requires source")
		}

		if dst == "" {
			return errors.New("requires dest")
		}

		m, err := lib.ManifestByPr(in, src, dst)
		if err != nil {
			return handle(err)
		}

		output(m.Applications)

		return nil
	},
}

var describeCommitCmd = &cobra.Command{
	Use:   "commit <sha>",
	Short: "Describes the manifest for a given commit",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) == 0 {
			return errors.New("requires the commit sha")
		}

		commit := args[0]

		m, err := lib.ManifestBySha(in, commit)
		if err != nil {
			return handle(err)
		}

		output(m.Applications)

		return nil
	},
}

var describeIntersectionCmd = &cobra.Command{
	Use:   "intersection --kind <branch|commit> --first <first> --second <second>",
	Short: "Describes the intersection between two trees",
	RunE: func(cmd *cobra.Command, args []string) error {
		if kind == "" {
			return errors.New("requires the kind argument")
		}

		if first == "" {
			return errors.New("requires the first argument")
		}

		if second == "" {
			return errors.New("requires the second argument")
		}

		var apps lib.Applications
		var err error

		switch kind {
		case "branch":
			apps, err = lib.IntersectionByBranch(in, first, second)
		case "commit":
			apps, err = lib.IntersectionByCommit(in, first, second)
		default:
			err = errors.New("not a valid kind - available options are 'branch' and 'commit'")
		}

		if err != nil {
			return handle(err)
		}

		return handle(output(apps))
	},
}

const columnWidth = 30

func formatRow(args ...interface{}) string {
	padded := make([]interface{}, len(args))
	for i, a := range args {
		requiredPadding := columnWidth - len(a.(string))
		if requiredPadding > 0 {
			padded[i] = fmt.Sprintf("%s%s", a, strings.Join(make([]string, requiredPadding), " "))
		} else {
			padded[i] = a
		}
	}
	return fmt.Sprintf("%s\t\t%s\t\t%s\n", padded...)
}

func output(apps lib.Applications) error {
	if formatAsJson {
		v := make(map[string]interface{})
		for _, a := range apps {
			v["Name"] = a.Name()
			v["Path"] = a.Path()
			v["Version"] = a.Version()
			v["Properties"] = a.Properties()
		}
		buff, err := json.MarshalIndent(v, "", "  ")
		if err != nil {
			return err
		}
		fmt.Println(string(buff))
	} else {
		fmt.Print(formatRow("NAME", "PATH", "VERSION"))
		for _, a := range apps {
			fmt.Printf(formatRow(a.Name(), a.Path(), a.Version()))
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

// BUG(ssw): JAP does not support TLS. To access the service with TLS (which you
//           really should be doing), use a reverse proxy such as Nginx.

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/rpc"
	"os"
	"path/filepath"

	"github.com/jitsi/jap"
	"golang.org/x/net/context"
	"golang.org/x/net/netutil"
	"golang.org/x/net/trace"
	"golang.org/x/text/language"
)

var (
	addr, pubDir, tmplDir, keyPath     string
	googleClientSecret, googleClientID string
	originURL                          string
	maxConns                           int
	rpcAddr, rpcMethod                 string

	tmpl    *template.Template
	devMode bool
)

// BUG(ssw): Most applications of Jitsi won't care about the RPC functionality
//           of JAP, but those that do may want to use a different codec, eg.
//           JSON-RPC so that they don't have to write a separate Go service.

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s\n\nUsage of %s:\n", help, os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&addr, "http", ":http-alt", "The address to listen on.")
	flag.StringVar(&pubDir, "public", "public/", "A directory containing static files to serve.")
	flag.StringVar(&tmplDir, "templates", "templates/", "A directory containing templates to render.")
	flag.StringVar(&keyPath, "key", os.Getenv("JAP_PRIVATE_KEY"), "An RSA private key in PEM format to use for signing tokens. Defaults to $JAP_PRIVATE_KEY.")
	flag.StringVar(&originURL, "origin", "", "A domain that the /login endpoint will send a postMessage too (eg. https://meet.jit.si).")
	flag.StringVar(&rpcAddr, "rpcaddr", "", "An address that can be used to make RPC calls to verify permissions for a user.")
	flag.StringVar(&rpcMethod, "rpc", "Permissions.Check", "The RPC call to make to rcpaddr. This should be a function that takes a string (the token) and replies with a boolean. It should be compatible with Go's net/rpc package.")
	flag.IntVar(&maxConns, "maxconns", 0, "The maximum number of connections to service at once or 0 for unlimited.")
	flag.BoolVar(&devMode, "dev", false, "Run in dev mode (reload templates on page refresh).")
	flag.Parse()

	googleClientID = os.Getenv("GOOGLE_CLIENT_ID")
	googleClientSecret = os.Getenv("GOOGLE_CLIENT_SECRET")

	loadTemplates()
}

// Load all templates found in the tmplDir directory; if any of them contain
// errors, panic.
func loadTemplates() {
	files, err := filepath.Glob(filepath.Join(tmplDir, "*.tmpl"))
	switch {
	case err != nil:
		log.Fatal(err)
	case len(files) < 1:
		log.Fatalf("No templates found in %s", tmplDir)
	}
	tmpl = template.Must(template.New("jap").ParseFiles(files...))
}

func loadRSAKeyFromPEM(keyPath string) (*rsa.PrivateKey, error) {
	pembytes, err := ioutil.ReadFile(keyPath)
	if err != nil {
		return nil, err
	}
	var blk *pem.Block
	for {
		blk, pembytes = pem.Decode(pembytes)
		if blk.Type == "RSA PRIVATE KEY" {
			return x509.ParsePKCS1PrivateKey(blk.Bytes)
		}
	}
	return nil, fmt.Errorf("No RSA private key found in pem file %s", keyPath)
}

func main() {
	if keyPath == "" {
		log.Fatalf("No private key specified. Try: %s -help", os.Args[0])
	}
	key, err := loadRSAKeyFromPEM(keyPath)
	if err != nil {
		log.Fatal(err)
	}

	var rpcClient *rpc.Client
	if rpcAddr != "" && rpcMethod != "" {
		log.Printf("Dialing RPC server at %s…", rpcAddr)
		if rpcClient, err = rpc.DialHTTP("tcp", rpcAddr); err != nil {
			log.Fatal("Failed to dial RPC server at %s: %v", rpcAddr, err)
		}
	}

	log.Printf("Starting server on %s…\n", addr)

	var permCheck jap.PermissionChecker
	if rpcClient != nil {
		permCheck = func(tok string) (b bool, err error) {
			err = rpcClient.Call(rpcMethod, tok, &b)
			return b, err
		}
	}
	http.HandleFunc("/googlelogin", jap.GoogleLogin(
		jap.NewCIDContext(context.Background(), googleClientID), key, permCheck))
	http.HandleFunc("/login", loginHandler(context.Background()))
	if pubDir != "" {
		http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(pubDir))))
	}

	l, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal(err)
	}
	if maxConns > 0 {
		l = netutil.LimitListener(l, maxConns)
	}
	log.Fatal(http.Serve(l, nil))
}

func loginHandler(ctx context.Context) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if devMode {
			loadTemplates()
		}

		tr := trace.New("jap.login", r.URL.Path)
		defer tr.Finish()

		tr.LazyPrintf("Executing login.tmpl…")
		err := tmpl.ExecuteTemplate(w, "login.tmpl", Login{
			Lang:           language.English,
			GoogleClientID: googleClientID,
			TargetOrigin:   originURL,
		})
		if err != nil {
			tr.LazyPrintf("Error exeuting login.tmpl:", err.Error())
			tr.SetError()
			return
		}
		tr.LazyPrintf("Done executing login.tmpl…")
	}
}

// Login represents all the information we need to show the login window.
type Login struct {
	Lang           language.Tag
	GoogleClientID string
	TargetOrigin   string
}
<commit_msg>Add JSON-RPC support<commit_after>package main

// BUG(ssw): JAP does not support TLS. To access the service with TLS (which you
//           really should be doing), use a reverse proxy such as Nginx.

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/rpc"
	"net/rpc/jsonrpc"
	"os"
	"path/filepath"

	"github.com/jitsi/jap"
	"golang.org/x/net/context"
	"golang.org/x/net/netutil"
	"golang.org/x/net/trace"
	"golang.org/x/text/language"
)

var (
	addr, pubDir, tmplDir, keyPath     string
	googleClientSecret, googleClientID string
	originURL                          string
	maxConns                           int
	rpcAddr, rpcMethod, rpcCodec       string

	tmpl    *template.Template
	devMode bool
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s\n\nUsage of %s:\n", help, os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&addr, "http", ":http-alt", "The address to listen on.")
	flag.StringVar(&pubDir, "public", "public/", "A directory containing static files to serve.")
	flag.StringVar(&tmplDir, "templates", "templates/", "A directory containing templates to render.")
	flag.StringVar(&keyPath, "key", os.Getenv("JAP_PRIVATE_KEY"), "An RSA private key in PEM format to use for signing tokens. Defaults to $JAP_PRIVATE_KEY.")
	flag.StringVar(&originURL, "origin", "", "A domain that the /login endpoint will send a postMessage too (eg. https://meet.jit.si).")
	flag.StringVar(&rpcAddr, "rpcaddr", "", "An address that can be used to make RPC calls to verify permissions for a user.")
	flag.StringVar(&rpcMethod, "rpc", "Permissions.Check", "The RPC call to make to rcpaddr. This should be a function that takes a string (the token) and replies with a boolean. It should be compatible with Go's net/rpc package.")
	flag.StringVar(&rpcCodec, "rpccodec", "gob", `The type of RPC call to make (either "gob" for Go gobs or "json" for JSON-RPC).`)
	flag.IntVar(&maxConns, "maxconns", 0, "The maximum number of connections to service at once or 0 for unlimited.")
	flag.BoolVar(&devMode, "dev", false, "Run in dev mode (reload templates on page refresh).")
	flag.Parse()

	googleClientID = os.Getenv("GOOGLE_CLIENT_ID")
	googleClientSecret = os.Getenv("GOOGLE_CLIENT_SECRET")

	loadTemplates()
}

// Load all templates found in the tmplDir directory; if any of them contain
// errors, panic.
func loadTemplates() {
	files, err := filepath.Glob(filepath.Join(tmplDir, "*.tmpl"))
	switch {
	case err != nil:
		log.Fatal(err)
	case len(files) < 1:
		log.Fatalf("No templates found in %s", tmplDir)
	}
	tmpl = template.Must(template.New("jap").ParseFiles(files...))
}

func loadRSAKeyFromPEM(keyPath string) (*rsa.PrivateKey, error) {
	pembytes, err := ioutil.ReadFile(keyPath)
	if err != nil {
		return nil, err
	}
	var blk *pem.Block
	for {
		blk, pembytes = pem.Decode(pembytes)
		if blk.Type == "RSA PRIVATE KEY" {
			return x509.ParsePKCS1PrivateKey(blk.Bytes)
		}
	}
	return nil, fmt.Errorf("No RSA private key found in pem file %s", keyPath)
}

func main() {
	if keyPath == "" {
		log.Fatalf("No private key specified. Try: %s -help", os.Args[0])
	}
	key, err := loadRSAKeyFromPEM(keyPath)
	if err != nil {
		log.Fatal(err)
	}

	var rpcClient *rpc.Client
	if rpcAddr != "" && rpcMethod != "" {
		log.Printf("Dialing RPC server at %s…", rpcAddr)
		switch rpcCodec {
		case "json":
			if rpcClient, err = jsonrpc.Dial("tcp", rpcAddr); err != nil {
				log.Fatal("Failed to dial JSON-RPC server at %s: %v", rpcAddr, err)
			}
		case "gob":
			if rpcClient, err = rpc.DialHTTP("tcp", rpcAddr); err != nil {
				log.Fatal("Failed to dial RPC server at %s: %v", rpcAddr, err)
			}
		default:
			log.Fatal("No such RPC codec %s", rpcCodec)
		}
	}

	log.Printf("Starting server on %s…\n", addr)

	var permCheck jap.PermissionChecker
	if rpcClient != nil {
		permCheck = func(tok string) (b bool, err error) {
			err = rpcClient.Call(rpcMethod, tok, &b)
			return b, err
		}
	}
	http.HandleFunc("/googlelogin", jap.GoogleLogin(
		jap.NewCIDContext(context.Background(), googleClientID), key, permCheck))
	http.HandleFunc("/login", loginHandler(context.Background()))
	if pubDir != "" {
		http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(pubDir))))
	}

	l, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal(err)
	}
	if maxConns > 0 {
		l = netutil.LimitListener(l, maxConns)
	}
	log.Fatal(http.Serve(l, nil))
}

func loginHandler(ctx context.Context) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if devMode {
			loadTemplates()
		}

		tr := trace.New("jap.login", r.URL.Path)
		defer tr.Finish()

		tr.LazyPrintf("Executing login.tmpl…")
		err := tmpl.ExecuteTemplate(w, "login.tmpl", Login{
			Lang:           language.English,
			GoogleClientID: googleClientID,
			TargetOrigin:   originURL,
		})
		if err != nil {
			tr.LazyPrintf("Error exeuting login.tmpl:", err.Error())
			tr.SetError()
			return
		}
		tr.LazyPrintf("Done executing login.tmpl…")
	}
}

// Login represents all the information we need to show the login window.
type Login struct {
	Lang           language.Tag
	GoogleClientID string
	TargetOrigin   string
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"time"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/cflager"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/executor"
	executorinit "code.cloudfoundry.org/executor/initializer"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/rep"
	"code.cloudfoundry.org/rep/auction_cell_rep"
	"code.cloudfoundry.org/rep/evacuation"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context"
	"code.cloudfoundry.org/rep/generator"
	"code.cloudfoundry.org/rep/handlers"
	"code.cloudfoundry.org/rep/harmonizer"
	"code.cloudfoundry.org/rep/maintain"
	"github.com/cloudfoundry/dropsonde"
	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/localip"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/rata"
)

var sessionName = flag.String(
	"sessionName",
	"rep",
	"consul session name",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server URLs (scheme://ip:port)",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:1800",
	"host:port to serve auction and LRP stop requests on",
)

var cellID = flag.String(
	"cellID",
	"",
	"the ID used by the rep to identify itself to external systems - must be specified",
)

var zone = flag.String(
	"zone",
	"",
	"the availability zone associated with the rep",
)

var pollingInterval = flag.Duration(
	"pollingInterval",
	30*time.Second,
	"the interval on which to scan the executor",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var evacuationTimeout = flag.Duration(
	"evacuationTimeout",
	10*time.Minute,
	"Timeout to wait for evacuation to complete",
)

var evacuationPollingInterval = flag.Duration(
	"evacuationPollingInterval",
	10*time.Second,
	"the interval on which to scan the executor during evacuation",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

type stackPathMap rep.StackPathMap

func (s *stackPathMap) String() string {
	return fmt.Sprintf("%v", *s)
}

func (s *stackPathMap) Set(value string) error {
	parts := strings.SplitN(value, ":", 2)
	if len(parts) != 2 {
		return errors.New("Invalid preloaded RootFS value: not of the form 'stack-name:path'")
	}

	if parts[0] == "" {
		return errors.New("Invalid preloaded RootFS value: blank stack")
	}

	if parts[1] == "" {
		return errors.New("Invalid preloaded RootFS value: blank path")
	}

	(*s)[parts[0]] = parts[1]
	return nil
}

type providers []string

func (p *providers) String() string {
	return fmt.Sprintf("%v", *p)
}

func (p *providers) Set(value string) error {
	if value == "" {
		return errors.New("Cannot set blank value for RootFS provider")
	}

	*p = append(*p, value)
	return nil
}

type argList []string

func (a *argList) String() string {
	return fmt.Sprintf("%v", *a)
}

func (a *argList) Set(value string) error {
	*a = strings.Split(value, ",")
	return nil
}

const (
	dropsondeOrigin = "rep"

	bbsPingTimeout = 5 * time.Minute
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cflager.AddFlags(flag.CommandLine)

	stackMap := stackPathMap{}
	supportedProviders := providers{}
	gardenHealthcheckEnv := argList{}
	gardenHealthcheckArgs := argList{}
	flag.Var(&stackMap, "preloadedRootFS", "List of preloaded RootFSes")
	flag.Var(&supportedProviders, "rootFSProvider", "List of RootFS providers")
	flag.Var(&gardenHealthcheckArgs, "gardenHealthcheckProcessArgs", "List of command line args to pass to the garden health check process")
	flag.Var(&gardenHealthcheckEnv, "gardenHealthcheckProcessEnv", "Environment variables to use when running the garden health check")
	flag.Parse()

	preloadedRootFSes := []string{}
	for k := range stackMap {
		preloadedRootFSes = append(preloadedRootFSes, k)
	}

	cfhttp.Initialize(*communicationTimeout)

	clock := clock.NewClock()
	logger, reconfigurableSink := cflager.New(*sessionName)

	var (
		executorConfiguration   executorinit.Configuration
		gardenHealthcheckRootFS string
		certBytes               []byte
		err                     error
	)

	if len(preloadedRootFSes) == 0 {
		gardenHealthcheckRootFS = ""
	} else {
		gardenHealthcheckRootFS = stackMap[preloadedRootFSes[0]]
	}

	if *pathToCACertsForDownloads != "" {
		certBytes, err = ioutil.ReadFile(*pathToCACertsForDownloads)
		if err != nil {
			logger.Error("failed-to-open-ca-cert-file", err)
			os.Exit(1)
		}

		certBytes = bytes.TrimSpace(certBytes)
	}

	executorConfiguration = executorConfig(certBytes, gardenHealthcheckRootFS, gardenHealthcheckArgs, gardenHealthcheckEnv)
	if !executorConfiguration.Validate(logger) {
		logger.Fatal("", errors.New("failed-to-configure-executor"))
	}

	initializeDropsonde(logger)

	if *cellID == "" {
		logger.Error("invalid-cell-id", errors.New("-cellID must be specified"))
		os.Exit(1)
	}

	executorClient, executorMembers, err := executorinit.Initialize(logger, executorConfiguration, clock)
	if err != nil {
		logger.Error("failed-to-initialize-executor", err)
		os.Exit(1)
	}
	defer executorClient.Cleanup(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Error("invalid-bbs-address", err)
		os.Exit(1)
	}

	serviceClient := initializeServiceClient(logger)

	evacuatable, evacuationReporter, evacuationNotifier := evacuation_context.New()

	// only one outstanding operation per container is necessary
	queue := operationq.NewSlidingQueue(1)

	evacuator := evacuation.NewEvacuator(
		logger,
		clock,
		executorClient,
		evacuationNotifier,
		*cellID,
		*evacuationTimeout,
		*evacuationPollingInterval,
	)

	bbsClient := initializeBBSClient(logger)
	httpServer, address := initializeServer(bbsClient, executorClient, evacuatable, evacuationReporter, logger, rep.StackPathMap(stackMap), supportedProviders)
	opGenerator := generator.New(*cellID, bbsClient, executorClient, evacuationReporter, uint64(evacuationTimeout.Seconds()))
	cleanup := evacuation.NewEvacuationCleanup(logger, *cellID, bbsClient)

	members := grouper.Members{
		{"presence", initializeCellPresence(address, serviceClient, executorClient, logger, supportedProviders, preloadedRootFSes)},
		{"http_server", httpServer},
		{"evacuation-cleanup", cleanup},
		{"bulker", harmonizer.NewBulker(logger, *pollingInterval, *evacuationPollingInterval, evacuationNotifier, clock, opGenerator, queue)},
		{"event-consumer", harmonizer.NewEventConsumer(logger, opGenerator, queue)},
		{"evacuator", evacuator},
	}

	members = append(executorMembers, members...)

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started", lager.Data{"cell-id": *cellID})

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeCellPresence(address string, serviceClient bbs.ServiceClient, executorClient executor.Client, logger lager.Logger, rootFSProviders, preloadedRootFSes []string) ifrit.Runner {
	config := maintain.Config{
		CellID:            *cellID,
		RepAddress:        address,
		Zone:              *zone,
		RetryInterval:     *lockRetryInterval,
		RootFSProviders:   rootFSProviders,
		PreloadedRootFSes: preloadedRootFSes,
	}
	return maintain.New(logger, config, executorClient, serviceClient, *lockTTL, clock.NewClock())
}

func initializeServiceClient(logger lager.Logger) bbs.ServiceClient {
	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	return bbs.NewServiceClient(consulClient, clock.NewClock())
}

func initializeServer(
	bbsClient bbs.InternalClient,
	executorClient executor.Client,
	evacuatable evacuation_context.Evacuatable,
	evacuationReporter evacuation_context.EvacuationReporter,
	logger lager.Logger,
	stackMap rep.StackPathMap,
	supportedProviders []string,
) (ifrit.Runner, string) {

	auctionCellRep := auction_cell_rep.New(*cellID, stackMap, supportedProviders, *zone, generateGuid, executorClient, evacuationReporter, logger)
	handlers := handlers.New(auctionCellRep, executorClient, evacuatable, logger)

	router, err := rata.NewRouter(rep.Routes, handlers)
	if err != nil {
		logger.Fatal("failed-to-construct-router", err)
	}

	ip, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("failed-to-fetch-ip", err)
	}

	port := strings.Split(*listenAddr, ":")[1]
	address := fmt.Sprintf("http://%s:%s", ip, port)

	return http_server.New(*listenAddr, router), address
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func generateGuid() (string, error) {
	guid, err := uuid.NewV4()
	if err != nil {
		return "", err
	}
	return guid.String(), nil
}

func initializeBBSClient(logger lager.Logger) bbs.InternalClient {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<commit_msg>Move localip to code.cloudfoundry.org<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"time"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/cflager"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	"code.cloudfoundry.org/executor"
	executorinit "code.cloudfoundry.org/executor/initializer"
	"code.cloudfoundry.org/localip"
	"code.cloudfoundry.org/locket"
	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/rep"
	"code.cloudfoundry.org/rep/auction_cell_rep"
	"code.cloudfoundry.org/rep/evacuation"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context"
	"code.cloudfoundry.org/rep/generator"
	"code.cloudfoundry.org/rep/handlers"
	"code.cloudfoundry.org/rep/harmonizer"
	"code.cloudfoundry.org/rep/maintain"
	"github.com/cloudfoundry/dropsonde"
	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/rata"
)

var sessionName = flag.String(
	"sessionName",
	"rep",
	"consul session name",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server URLs (scheme://ip:port)",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:1800",
	"host:port to serve auction and LRP stop requests on",
)

var cellID = flag.String(
	"cellID",
	"",
	"the ID used by the rep to identify itself to external systems - must be specified",
)

var zone = flag.String(
	"zone",
	"",
	"the availability zone associated with the rep",
)

var pollingInterval = flag.Duration(
	"pollingInterval",
	30*time.Second,
	"the interval on which to scan the executor",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var evacuationTimeout = flag.Duration(
	"evacuationTimeout",
	10*time.Minute,
	"Timeout to wait for evacuation to complete",
)

var evacuationPollingInterval = flag.Duration(
	"evacuationPollingInterval",
	10*time.Second,
	"the interval on which to scan the executor during evacuation",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

type stackPathMap rep.StackPathMap

func (s *stackPathMap) String() string {
	return fmt.Sprintf("%v", *s)
}

func (s *stackPathMap) Set(value string) error {
	parts := strings.SplitN(value, ":", 2)
	if len(parts) != 2 {
		return errors.New("Invalid preloaded RootFS value: not of the form 'stack-name:path'")
	}

	if parts[0] == "" {
		return errors.New("Invalid preloaded RootFS value: blank stack")
	}

	if parts[1] == "" {
		return errors.New("Invalid preloaded RootFS value: blank path")
	}

	(*s)[parts[0]] = parts[1]
	return nil
}

type providers []string

func (p *providers) String() string {
	return fmt.Sprintf("%v", *p)
}

func (p *providers) Set(value string) error {
	if value == "" {
		return errors.New("Cannot set blank value for RootFS provider")
	}

	*p = append(*p, value)
	return nil
}

type argList []string

func (a *argList) String() string {
	return fmt.Sprintf("%v", *a)
}

func (a *argList) Set(value string) error {
	*a = strings.Split(value, ",")
	return nil
}

const (
	dropsondeOrigin = "rep"

	bbsPingTimeout = 5 * time.Minute
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	cflager.AddFlags(flag.CommandLine)

	stackMap := stackPathMap{}
	supportedProviders := providers{}
	gardenHealthcheckEnv := argList{}
	gardenHealthcheckArgs := argList{}
	flag.Var(&stackMap, "preloadedRootFS", "List of preloaded RootFSes")
	flag.Var(&supportedProviders, "rootFSProvider", "List of RootFS providers")
	flag.Var(&gardenHealthcheckArgs, "gardenHealthcheckProcessArgs", "List of command line args to pass to the garden health check process")
	flag.Var(&gardenHealthcheckEnv, "gardenHealthcheckProcessEnv", "Environment variables to use when running the garden health check")
	flag.Parse()

	preloadedRootFSes := []string{}
	for k := range stackMap {
		preloadedRootFSes = append(preloadedRootFSes, k)
	}

	cfhttp.Initialize(*communicationTimeout)

	clock := clock.NewClock()
	logger, reconfigurableSink := cflager.New(*sessionName)

	var (
		executorConfiguration   executorinit.Configuration
		gardenHealthcheckRootFS string
		certBytes               []byte
		err                     error
	)

	if len(preloadedRootFSes) == 0 {
		gardenHealthcheckRootFS = ""
	} else {
		gardenHealthcheckRootFS = stackMap[preloadedRootFSes[0]]
	}

	if *pathToCACertsForDownloads != "" {
		certBytes, err = ioutil.ReadFile(*pathToCACertsForDownloads)
		if err != nil {
			logger.Error("failed-to-open-ca-cert-file", err)
			os.Exit(1)
		}

		certBytes = bytes.TrimSpace(certBytes)
	}

	executorConfiguration = executorConfig(certBytes, gardenHealthcheckRootFS, gardenHealthcheckArgs, gardenHealthcheckEnv)
	if !executorConfiguration.Validate(logger) {
		logger.Fatal("", errors.New("failed-to-configure-executor"))
	}

	initializeDropsonde(logger)

	if *cellID == "" {
		logger.Error("invalid-cell-id", errors.New("-cellID must be specified"))
		os.Exit(1)
	}

	executorClient, executorMembers, err := executorinit.Initialize(logger, executorConfiguration, clock)
	if err != nil {
		logger.Error("failed-to-initialize-executor", err)
		os.Exit(1)
	}
	defer executorClient.Cleanup(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Error("invalid-bbs-address", err)
		os.Exit(1)
	}

	serviceClient := initializeServiceClient(logger)

	evacuatable, evacuationReporter, evacuationNotifier := evacuation_context.New()

	// only one outstanding operation per container is necessary
	queue := operationq.NewSlidingQueue(1)

	evacuator := evacuation.NewEvacuator(
		logger,
		clock,
		executorClient,
		evacuationNotifier,
		*cellID,
		*evacuationTimeout,
		*evacuationPollingInterval,
	)

	bbsClient := initializeBBSClient(logger)
	httpServer, address := initializeServer(bbsClient, executorClient, evacuatable, evacuationReporter, logger, rep.StackPathMap(stackMap), supportedProviders)
	opGenerator := generator.New(*cellID, bbsClient, executorClient, evacuationReporter, uint64(evacuationTimeout.Seconds()))
	cleanup := evacuation.NewEvacuationCleanup(logger, *cellID, bbsClient)

	members := grouper.Members{
		{"presence", initializeCellPresence(address, serviceClient, executorClient, logger, supportedProviders, preloadedRootFSes)},
		{"http_server", httpServer},
		{"evacuation-cleanup", cleanup},
		{"bulker", harmonizer.NewBulker(logger, *pollingInterval, *evacuationPollingInterval, evacuationNotifier, clock, opGenerator, queue)},
		{"event-consumer", harmonizer.NewEventConsumer(logger, opGenerator, queue)},
		{"evacuator", evacuator},
	}

	members = append(executorMembers, members...)

	if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started", lager.Data{"cell-id": *cellID})

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeCellPresence(address string, serviceClient bbs.ServiceClient, executorClient executor.Client, logger lager.Logger, rootFSProviders, preloadedRootFSes []string) ifrit.Runner {
	config := maintain.Config{
		CellID:            *cellID,
		RepAddress:        address,
		Zone:              *zone,
		RetryInterval:     *lockRetryInterval,
		RootFSProviders:   rootFSProviders,
		PreloadedRootFSes: preloadedRootFSes,
	}
	return maintain.New(logger, config, executorClient, serviceClient, *lockTTL, clock.NewClock())
}

func initializeServiceClient(logger lager.Logger) bbs.ServiceClient {
	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	return bbs.NewServiceClient(consulClient, clock.NewClock())
}

func initializeServer(
	bbsClient bbs.InternalClient,
	executorClient executor.Client,
	evacuatable evacuation_context.Evacuatable,
	evacuationReporter evacuation_context.EvacuationReporter,
	logger lager.Logger,
	stackMap rep.StackPathMap,
	supportedProviders []string,
) (ifrit.Runner, string) {

	auctionCellRep := auction_cell_rep.New(*cellID, stackMap, supportedProviders, *zone, generateGuid, executorClient, evacuationReporter, logger)
	handlers := handlers.New(auctionCellRep, executorClient, evacuatable, logger)

	router, err := rata.NewRouter(rep.Routes, handlers)
	if err != nil {
		logger.Fatal("failed-to-construct-router", err)
	}

	ip, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("failed-to-fetch-ip", err)
	}

	port := strings.Split(*listenAddr, ":")[1]
	address := fmt.Sprintf("http://%s:%s", ip, port)

	return http_server.New(*listenAddr, router), address
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func generateGuid() (string, error) {
	guid, err := uuid.NewV4()
	if err != nil {
		return "", err
	}
	return guid.String(), nil
}

func initializeBBSClient(logger lager.Logger) bbs.InternalClient {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"github.com/openshift/source-to-image/pkg/sti"
	"github.com/openshift/source-to-image/pkg/sti/errors"
	"github.com/openshift/source-to-image/pkg/sti/version"
)

func parseEnvs(cmd *cobra.Command, name string) (map[string]string, error) {
	env := cmd.Flags().Lookup(name)
	if env == nil || len(env.Value.String()) == 0 {
		return nil, nil
	}

	envs := make(map[string]string)
	pairs := strings.Split(env.Value.String(), ",")
	for _, pair := range pairs {
		atoms := strings.Split(pair, "=")
		if len(atoms) != 2 {
			return nil, fmt.Errorf("malformed env string: %s", pair)
		}
		envs[atoms[0]] = atoms[1]
	}

	return envs, nil
}

func dockerSocket() string {
	if host := os.Getenv("DOCKER_HOST"); host != "" {
		return host
	}
	return "unix:///var/run/docker.sock"
}

func newCmdVersion() *cobra.Command {
	return &cobra.Command{
		Use:   "version",
		Short: "Display version",
		Long:  "Display version",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("sti %v\n", version.Get())
		},
	}
}

func newCmdBuild(req *sti.Request) *cobra.Command {
	buildCmd := &cobra.Command{
		Use:   "build <source> <image> <tag>",
		Short: "Build a new image",
		Long:  "Build a new Docker image named <tag> from a source repository and base image.",
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) < 3 {
				cmd.Help()
				os.Exit(1)
			}

			req.Source = args[0]
			req.BaseImage = args[1]
			req.Tag = args[2]
			envs, err := parseEnvs(cmd, "env")
			checkErr(err)
			req.Environment = envs

			b, err := sti.NewBuilder(req)
			checkErr(err)
			res, err := b.Build()
			checkErr(err)
			for _, message := range res.Messages {
				glog.V(1).Infof(message)
			}
		},
	}
	buildCmd.Flags().BoolVar(&(req.Clean), "clean", false, "Perform a clean build")
	buildCmd.Flags().BoolVar(&(req.RemovePreviousImage), "rm", false, "Remove the previous image during incremental builds")
	buildCmd.Flags().StringP("env", "e", "", "Specify an environment var NAME=VALUE,NAME2=VALUE2,...")
	buildCmd.Flags().StringVarP(&(req.Ref), "ref", "r", "", "Specify a ref to check-out")
	buildCmd.Flags().StringVar(&(req.CallbackURL), "callbackUrl", "", "Specify a URL to invoke via HTTP POST upon build completion")
	buildCmd.Flags().StringVarP(&(req.ScriptsURL), "scripts", "s", "", "Specify a URL for the assemble and run scripts")
	buildCmd.Flags().BoolVar(&(req.ForcePull), "forcePull", true, "Always pull the builder image even if it is present locally")
	return buildCmd
}

func newCmdUsage(req *sti.Request) *cobra.Command {
	usageCmd := &cobra.Command{
		Use:   "usage <image>",
		Short: "Print usage of the assemble script associated with the image",
		Long:  "Create and start a container from the image and invoke its usage script.",
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) == 0 {
				cmd.Help()
				os.Exit(1)
			}

			req.BaseImage = args[0]
			envs, err := parseEnvs(cmd, "env")
			checkErr(err)
			req.Environment = envs

			uh, err := sti.NewUsage(req)
			checkErr(err)
			err = uh.Show()
			checkErr(err)
		},
	}
	usageCmd.Flags().StringP("env", "e", "", "Specify an environment var NAME=VALUE,NAME2=VALUE2,...")
	usageCmd.Flags().StringVarP(&(req.ScriptsURL), "scripts", "s", "", "Specify a URL for the assemble and run scripts")
	return usageCmd
}

func setupGlog(flags *pflag.FlagSet) {
	from := flag.CommandLine
	if fflag := from.Lookup("v"); fflag != nil {
		level := fflag.Value.(*glog.Level)
		levelPtr := (*int32)(level)
		flags.Int32Var(levelPtr, "loglevel", 0, "Set the level of log output (0-3)")
	}
	// FIXME currently glog has only option to redirect output to stderr
	// the preferred for STI would be to redirect to stdout
	flag.CommandLine.Set("logtostderr", "true")
}

func checkErr(err error) {
	if err == nil {
		return
	}
	if e, ok := err.(errors.Error); ok {
		glog.Errorf("An error occurred: %v", e)
		glog.Errorf("Suggested solution: %v", e.Suggestion)
		if e.Details != nil {
			glog.V(1).Infof("Details: %v", e.Details)
		}
		glog.Error("If the problem persists reach us on freenode #openshift " +
			"or fill an issue at https://github.com/openshift/source-to-image/issues " +
			"providing us with a log from your build using --loglevel=3")
		os.Exit(e.ErrorCode)
	} else {
		glog.Fatalf("An error occurred: %v", err)
	}
}

func main() {
	req := &sti.Request{}
	stiCmd := &cobra.Command{
		Use: "sti",
		Long: "Source-to-image (STI) is a tool for building repeatable docker images.\n\n" +
			"A command line interface that injects and assembles source code into a docker image.\n" +
			"Complete documentation is available at http://github.com/openshift/source-to-image",
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Help()
		},
	}
	stiCmd.PersistentFlags().StringVarP(&(req.DockerSocket), "url", "U", dockerSocket(), "Set the url of the docker socket to use")
	stiCmd.PersistentFlags().BoolVar(&(req.PreserveWorkingDir), "savetempdir", false, "Save the temporary directory used by STI instead of deleting it")

	stiCmd.AddCommand(newCmdVersion())
	stiCmd.AddCommand(newCmdBuild(req))
	stiCmd.AddCommand(newCmdUsage(req))
	setupGlog(stiCmd.PersistentFlags())

	err := stiCmd.Execute()
	checkErr(err)
}
<commit_msg>Fixing output from cobra, when error occured.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"github.com/openshift/source-to-image/pkg/sti"
	"github.com/openshift/source-to-image/pkg/sti/errors"
	"github.com/openshift/source-to-image/pkg/sti/version"
)

func parseEnvs(cmd *cobra.Command, name string) (map[string]string, error) {
	env := cmd.Flags().Lookup(name)
	if env == nil || len(env.Value.String()) == 0 {
		return nil, nil
	}

	envs := make(map[string]string)
	pairs := strings.Split(env.Value.String(), ",")
	for _, pair := range pairs {
		atoms := strings.Split(pair, "=")
		if len(atoms) != 2 {
			return nil, fmt.Errorf("malformed env string: %s", pair)
		}
		envs[atoms[0]] = atoms[1]
	}

	return envs, nil
}

func dockerSocket() string {
	if host := os.Getenv("DOCKER_HOST"); host != "" {
		return host
	}
	return "unix:///var/run/docker.sock"
}

func newCmdVersion() *cobra.Command {
	return &cobra.Command{
		Use:   "version",
		Short: "Display version",
		Long:  "Display version",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("sti %v\n", version.Get())
		},
	}
}

func newCmdBuild(req *sti.Request) *cobra.Command {
	buildCmd := &cobra.Command{
		Use:   "build <source> <image> <tag>",
		Short: "Build a new image",
		Long:  "Build a new Docker image named <tag> from a source repository and base image.",
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) < 3 {
				cmd.Help()
				os.Exit(1)
			}

			req.Source = args[0]
			req.BaseImage = args[1]
			req.Tag = args[2]
			envs, err := parseEnvs(cmd, "env")
			checkErr(err)
			req.Environment = envs

			b, err := sti.NewBuilder(req)
			checkErr(err)
			res, err := b.Build()
			checkErr(err)
			for _, message := range res.Messages {
				glog.V(1).Infof(message)
			}
		},
	}
	buildCmd.Flags().BoolVar(&(req.Clean), "clean", false, "Perform a clean build")
	buildCmd.Flags().BoolVar(&(req.RemovePreviousImage), "rm", false, "Remove the previous image during incremental builds")
	buildCmd.Flags().StringP("env", "e", "", "Specify an environment var NAME=VALUE,NAME2=VALUE2,...")
	buildCmd.Flags().StringVarP(&(req.Ref), "ref", "r", "", "Specify a ref to check-out")
	buildCmd.Flags().StringVar(&(req.CallbackURL), "callbackUrl", "", "Specify a URL to invoke via HTTP POST upon build completion")
	buildCmd.Flags().StringVarP(&(req.ScriptsURL), "scripts", "s", "", "Specify a URL for the assemble and run scripts")
	buildCmd.Flags().BoolVar(&(req.ForcePull), "forcePull", true, "Always pull the builder image even if it is present locally")
	return buildCmd
}

func newCmdUsage(req *sti.Request) *cobra.Command {
	usageCmd := &cobra.Command{
		Use:   "usage <image>",
		Short: "Print usage of the assemble script associated with the image",
		Long:  "Create and start a container from the image and invoke its usage script.",
		Run: func(cmd *cobra.Command, args []string) {
			if len(args) == 0 {
				cmd.Help()
				os.Exit(1)
			}

			req.BaseImage = args[0]
			envs, err := parseEnvs(cmd, "env")
			checkErr(err)
			req.Environment = envs

			uh, err := sti.NewUsage(req)
			checkErr(err)
			err = uh.Show()
			checkErr(err)
		},
	}
	usageCmd.Flags().StringP("env", "e", "", "Specify an environment var NAME=VALUE,NAME2=VALUE2,...")
	usageCmd.Flags().StringVarP(&(req.ScriptsURL), "scripts", "s", "", "Specify a URL for the assemble and run scripts")
	return usageCmd
}

func setupGlog(flags *pflag.FlagSet) {
	from := flag.CommandLine
	if fflag := from.Lookup("v"); fflag != nil {
		level := fflag.Value.(*glog.Level)
		levelPtr := (*int32)(level)
		flags.Int32Var(levelPtr, "loglevel", 0, "Set the level of log output (0-3)")
	}
	// FIXME currently glog has only option to redirect output to stderr
	// the preferred for STI would be to redirect to stdout
	flag.CommandLine.Set("logtostderr", "true")
}

func checkErr(err error) {
	if err == nil {
		return
	}
	if e, ok := err.(errors.Error); ok {
		glog.Errorf("An error occurred: %v", e)
		glog.Errorf("Suggested solution: %v", e.Suggestion)
		if e.Details != nil {
			glog.V(1).Infof("Details: %v", e.Details)
		}
		glog.Error("If the problem persists reach us on freenode #openshift " +
			"or fill an issue at https://github.com/openshift/source-to-image/issues " +
			"providing us with a log from your build using --loglevel=3")
		os.Exit(e.ErrorCode)
	} else {
		glog.Fatalf("An error occurred: %v", err)
	}
}

func main() {
	req := &sti.Request{}
	stiCmd := &cobra.Command{
		Use: "sti",
		Long: "Source-to-image (STI) is a tool for building repeatable docker images.\n\n" +
			"A command line interface that injects and assembles source code into a docker image.\n" +
			"Complete documentation is available at http://github.com/openshift/source-to-image",
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Help()
		},
	}
	stiCmd.PersistentFlags().StringVarP(&(req.DockerSocket), "url", "U", dockerSocket(), "Set the url of the docker socket to use")
	stiCmd.PersistentFlags().BoolVar(&(req.PreserveWorkingDir), "savetempdir", false, "Save the temporary directory used by STI instead of deleting it")

	stiCmd.AddCommand(newCmdVersion())
	stiCmd.AddCommand(newCmdBuild(req))
	stiCmd.AddCommand(newCmdUsage(req))
	setupGlog(stiCmd.PersistentFlags())

	err := stiCmd.Execute()
	if err != nil {
		os.Exit(1)
	}
}
<|endoftext|>