<commit_before>// +build !windows

package main

import (
	gocontext "context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"syscall"
	"time"

	"github.com/containerd/fifo"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

func prepareStdio(stdin, stdout, stderr string, console bool) (*sync.WaitGroup, error) {
	var wg sync.WaitGroup
	ctx := gocontext.Background()

	f, err := fifo.OpenFifo(ctx, stdin, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700)
	if err != nil {
		return nil, err
	}
	defer func(c io.Closer) {
		if err != nil {
			c.Close()
		}
	}(f)
	go func(w io.WriteCloser) {
		io.Copy(w, os.Stdin)
		w.Close()
	}(f)

	f, err = fifo.OpenFifo(ctx, stdout, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700)
	if err != nil {
		return nil, err
	}
	defer func(c io.Closer) {
		if err != nil {
			c.Close()
		}
	}(f)
	wg.Add(1)
	go func(r io.ReadCloser) {
		io.Copy(os.Stdout, r)
		r.Close()
		wg.Done()
	}(f)

	f, err = fifo.OpenFifo(ctx, stderr, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700)
	if err != nil {
		return nil, err
	}
	defer func(c io.Closer) {
		if err != nil {
			c.Close()
		}
	}(f)
	if !console {
		wg.Add(1)
		go func(r io.ReadCloser) {
			io.Copy(os.Stderr, r)
			r.Close()
			wg.Done()
		}(f)
	}

	return &wg, nil
}

func getGRPCConnection(context *cli.Context) (*grpc.ClientConn, error) {
	if grpcConn != nil {
		return grpcConn, nil
	}

	bindSocket := context.GlobalString("address")
	// reset the logger for grpc to log to dev/null so that it does not mess with our stdio
	grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
	dialOpts := []grpc.DialOption{grpc.WithInsecure(), grpc.WithTimeout(100 * time.Second)}
	dialOpts = append(dialOpts,
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", bindSocket, timeout)
		},
		))

	conn, err := grpc.Dial(fmt.Sprintf("unix://%s", bindSocket), dialOpts...)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to dial %q", bindSocket)
	}

	grpcConn = conn
	return grpcConn, nil
}
<commit_msg>prepareStdio: use named return for err<commit_after>// +build !windows

package main

import (
	gocontext "context"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"syscall"
	"time"

	"github.com/containerd/fifo"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

func prepareStdio(stdin, stdout, stderr string, console bool) (wg *sync.WaitGroup, err error) {
	wg = &sync.WaitGroup{}
	ctx := gocontext.Background()

	f, err := fifo.OpenFifo(ctx, stdin, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700)
	if err != nil {
		return nil, err
	}
	defer func(c io.Closer) {
		if err != nil {
			c.Close()
		}
	}(f)
	go func(w io.WriteCloser) {
		io.Copy(w, os.Stdin)
		w.Close()
	}(f)

	f, err = fifo.OpenFifo(ctx, stdout, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700)
	if err != nil {
		return nil, err
	}
	defer func(c io.Closer) {
		if err != nil {
			c.Close()
		}
	}(f)
	wg.Add(1)
	go func(r io.ReadCloser) {
		io.Copy(os.Stdout, r)
		r.Close()
		wg.Done()
	}(f)

	f, err = fifo.OpenFifo(ctx, stderr, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700)
	if err != nil {
		return nil, err
	}
	defer func(c io.Closer) {
		if err != nil {
			c.Close()
		}
	}(f)
	if !console {
		wg.Add(1)
		go func(r io.ReadCloser) {
			io.Copy(os.Stderr, r)
			r.Close()
			wg.Done()
		}(f)
	}

	return wg, nil
}

func getGRPCConnection(context *cli.Context) (*grpc.ClientConn, error) {
	if grpcConn != nil {
		return grpcConn, nil
	}

	bindSocket := context.GlobalString("address")
	// reset the logger for grpc to log to dev/null so that it does not mess with our stdio
	grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
	dialOpts := []grpc.DialOption{grpc.WithInsecure(), grpc.WithTimeout(100 * time.Second)}
	dialOpts = append(dialOpts,
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", bindSocket, timeout)
		},
		))

	conn, err := grpc.Dial(fmt.Sprintf("unix://%s", bindSocket), dialOpts...)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to dial %q", bindSocket)
	}

	grpcConn = conn
	return grpcConn, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"context"

	"github.com/jessevdk/go-flags"
	uuid "github.com/satori/go.uuid"

	"github.com/mitchellh/cli"
	"github.com/nerdalize/nerd/pkg/transfer"
	"github.com/nerdalize/nerd/svc"
	"github.com/pkg/errors"
)

//DatasetUpload command
type DatasetUpload struct {
	KubeOpts
	TransferOpts
	Name string `long:"name" short:"n" description:"assign a name to the dataset"`

	*command
}

//DatasetUploadFactory creates the command
func DatasetUploadFactory(ui cli.Ui) cli.CommandFactory {
	cmd := &DatasetUpload{}
	cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, flags.PassAfterNonOption)
	return func() (cli.Command, error) {
		return cmd, nil
	}
}

//@TODO needs a better place instead of the cmd package
func uploadToDataset(ctx context.Context, trans transfer.Transfer, bucket string, kube *svc.Kube, path, datasetName string) (ref *transfer.Ref, name string, err error) {
	uid, err := uuid.NewV4()
	if err != nil {
		return nil, "", err
	}

	ref = &transfer.Ref{
		Bucket: bucket,
		Key:    uid.String() + ".zip", //@TODO move this to a library
	}

	var n int
	if path != "" { //path is optional
		n, err = trans.Upload(ctx, ref, path)
		if err != nil {
			return nil, "", errors.Wrap(err, "failed to perform upload")
		}
	}

	in := &svc.CreateDatasetInput{
		Name:   datasetName,
		Bucket: ref.Bucket,
		Key:    ref.Key,
		Size:   uint64(n),
	}

	out, err := kube.CreateDataset(ctx, in)
	if err != nil {
		return nil, "", renderServiceError(err, "failed to upload dataset")
	}

	return ref, out.Name, nil
}

//Execute runs the command
func (cmd *DatasetUpload) Execute(args []string) (err error) {
	if len(args) < 1 {
		return errShowUsage(MessageNotEnoughArguments)
	}

	deps, err := NewDeps(cmd.Logger(), cmd.KubeOpts)
	if err != nil {
		return renderConfigError(err, "failed to configure")
	}

	trans, err := cmd.TransferOpts.Transfer()
	if err != nil {
		return errors.Wrap(err, "failed configure transfer")
	}

	ctx := context.Background()
	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
	defer cancel()

	kube := svc.NewKube(deps)

	_, name, err := uploadToDataset(ctx, trans, cmd.AWSS3Bucket, kube, args[0], cmd.Name)
	if err != nil {
		return err
	}

	cmd.out.Infof("Uploaded dataset: '%s'", name)
	cmd.out.Infof("To run a job with a dataset, use: 'nerd job run'")
	return nil
}

// Description returns long-form help text
func (cmd *DatasetUpload) Description() string { return cmd.Synopsis() }

// Synopsis returns a one-line
func (cmd *DatasetUpload) Synopsis() string { return "Upload a dataset to your compute cluster." }

// Usage shows usage
func (cmd *DatasetUpload) Usage() string {
	return "nerd dataset upload [--name=] ~/my-project/my-input-1"
}
<commit_msg>fix type mismatch<commit_after>package cmd

import (
	"context"

	"github.com/jessevdk/go-flags"
	uuid "github.com/satori/go.uuid"

	"github.com/mitchellh/cli"
	"github.com/nerdalize/nerd/pkg/transfer"
	"github.com/nerdalize/nerd/svc"
	"github.com/pkg/errors"
)

//DatasetUpload command
type DatasetUpload struct {
	KubeOpts
	TransferOpts
	Name string `long:"name" short:"n" description:"assign a name to the dataset"`

	*command
}

//DatasetUploadFactory creates the command
func DatasetUploadFactory(ui cli.Ui) cli.CommandFactory {
	cmd := &DatasetUpload{}
	cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, flags.PassAfterNonOption)
	return func() (cli.Command, error) {
		return cmd, nil
	}
}

//@TODO needs a better place instead of the cmd package
func uploadToDataset(ctx context.Context, trans transfer.Transfer, bucket string, kube *svc.Kube, path, datasetName string) (ref *transfer.Ref, name string, err error) {
	uid, err := uuid.NewV4()
	if err != nil {
		return nil, "", err
	}

	ref = &transfer.Ref{
		Bucket: bucket,
		Key:    uid.String() + ".zip", //@TODO move this to a library
	}

	var n uint64
	if path != "" { //path is optional
		n, err = trans.Upload(ctx, ref, path)
		if err != nil {
			return nil, "", errors.Wrap(err, "failed to perform upload")
		}
	}

	in := &svc.CreateDatasetInput{
		Name:   datasetName,
		Bucket: ref.Bucket,
		Key:    ref.Key,
		Size:   uint64(n),
	}

	out, err := kube.CreateDataset(ctx, in)
	if err != nil {
		return nil, "", renderServiceError(err, "failed to upload dataset")
	}

	return ref, out.Name, nil
}

//Execute runs the command
func (cmd *DatasetUpload) Execute(args []string) (err error) {
	if len(args) < 1 {
		return errShowUsage(MessageNotEnoughArguments)
	}

	deps, err := NewDeps(cmd.Logger(), cmd.KubeOpts)
	if err != nil {
		return renderConfigError(err, "failed to configure")
	}

	trans, err := cmd.TransferOpts.Transfer()
	if err != nil {
		return errors.Wrap(err, "failed configure transfer")
	}

	ctx := context.Background()
	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
	defer cancel()

	kube := svc.NewKube(deps)

	_, name, err := uploadToDataset(ctx, trans, cmd.AWSS3Bucket, kube, args[0], cmd.Name)
	if err != nil {
		return err
	}

	cmd.out.Infof("Uploaded dataset: '%s'", name)
	cmd.out.Infof("To run a job with a dataset, use: 'nerd job run'")
	return nil
}

// Description returns long-form help text
func (cmd *DatasetUpload) Description() string { return cmd.Synopsis() }

// Synopsis returns a one-line
func (cmd *DatasetUpload) Synopsis() string { return "Upload a dataset to your compute cluster." }

// Usage shows usage
func (cmd *DatasetUpload) Usage() string {
	return "nerd dataset upload [--name=] ~/my-project/my-input-1"
}
<|endoftext|>


<commit_before>// Copyright 2015 The DER ASCII Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

var inPath = flag.String("i", "", "input file to use (defaults to stdin)")
var outPath = flag.String("o", "", "output file to use (defaults to stdout)")
var isPem = flag.Bool("pem", false, "treat the input as a PEM file")

func main() {
	flag.Parse()

	if flag.NArg() > 0 {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION...]\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	inFile := os.Stdin
	if *inPath != "" {
		var err error
		inFile, err = os.Open(*inPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening %s: %s\n", *inPath, err)
			os.Exit(1)
		}
		defer inFile.Close()
	}

	inBytes, err := ioutil.ReadAll(inFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading input: %s\n", err)
		os.Exit(1)
	}

	if *isPem {
		pemBlock, _ := pem.Decode(inBytes)
		if pemBlock == nil {
			fmt.Fprintf(os.Stderr, "-pem provided, but input could not be parsed as a PEM\n")
			os.Exit(1)
		}
		inBytes = pemBlock.Bytes
	}

	outFile := os.Stdout
	if *outPath != "" {
		outFile, err = os.Create(*outPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening %s: %s\n", *outPath, err)
			os.Exit(1)
		}
		defer outFile.Close()
	}
	_, err = outFile.Write([]byte(derToASCII(inBytes)))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error writing output: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>Remove unnecessary byte/string conversion.<commit_after>// Copyright 2015 The DER ASCII Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
)

var inPath = flag.String("i", "", "input file to use (defaults to stdin)")
var outPath = flag.String("o", "", "output file to use (defaults to stdout)")
var isPem = flag.Bool("pem", false, "treat the input as a PEM file")

func main() {
	flag.Parse()

	if flag.NArg() > 0 {
		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION...]\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	inFile := os.Stdin
	if *inPath != "" {
		var err error
		inFile, err = os.Open(*inPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening %s: %s\n", *inPath, err)
			os.Exit(1)
		}
		defer inFile.Close()
	}

	inBytes, err := ioutil.ReadAll(inFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error reading input: %s\n", err)
		os.Exit(1)
	}

	if *isPem {
		pemBlock, _ := pem.Decode(inBytes)
		if pemBlock == nil {
			fmt.Fprintf(os.Stderr, "-pem provided, but input could not be parsed as a PEM\n")
			os.Exit(1)
		}
		inBytes = pemBlock.Bytes
	}

	outFile := os.Stdout
	if *outPath != "" {
		outFile, err = os.Create(*outPath)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening %s: %s\n", *outPath, err)
			os.Exit(1)
		}
		defer outFile.Close()
	}
	_, err = outFile.WriteString(derToASCII(inBytes))
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error writing output: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/heroku/log-iss/Godeps/_workspace/src/github.com/heroku/authenticater"
	"github.com/heroku/log-iss/Godeps/_workspace/src/github.com/heroku/slog"
)

type ShutdownCh chan int

var Config *IssConfig

func Logf(format string, a ...interface{}) {
	orig := fmt.Sprintf(format, a...)
	fmt.Printf("app=log-iss source=%s %s\n", Config.Deploy, orig)
}

func LogContext(ctx slog.Context) {
	ctx["app"] = "log-iss"
	ctx["source"] = Config.Deploy
	fmt.Println(ctx)
}

func awaitShutdownSignals(chs []ShutdownCh) {
	sigCh := make(chan os.Signal)
	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
	sig := <-sigCh
	Logf("ns=main at=shutdown-signal signal=%q", sig)
	for _, ch := range chs {
		ch <- 1
	}
}

func main() {
	config, err := NewIssConfig()
	if err != nil {
		log.Fatalln(err)
	}

	auth, err := authenticater.NewBasicAuthFromString(config.Tokens)
	if err != nil {
		log.Fatalln(err)
	}

	forwarderSet := NewForwarderSet(Config)
	forwarderSet.Start()

	shutdownCh := make(ShutdownCh)

	httpServer := NewHttpServer(Config, auth, Fix, forwarderSet.Inbox)

	go awaitShutdownSignals([]ShutdownCh{httpServer.ShutdownCh, shutdownCh})

	go func() {
		if err := httpServer.Run(); err != nil {
			log.Fatalln("Unable to start HTTP server:", err)
		}
	}()

	Logf("ns=main at=start")
	<-shutdownCh
	Logf("ns=main at=drain")
	httpServer.InFlightWg.Wait()
	Logf("ns=main at=exit")
}
<commit_msg>fix typo<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/heroku/log-iss/Godeps/_workspace/src/github.com/heroku/authenticater"
	"github.com/heroku/log-iss/Godeps/_workspace/src/github.com/heroku/slog"
)

type ShutdownCh chan int

var Config *IssConfig

func Logf(format string, a ...interface{}) {
	orig := fmt.Sprintf(format, a...)
	fmt.Printf("app=log-iss source=%s %s\n", Config.Deploy, orig)
}

func LogContext(ctx slog.Context) {
	ctx["app"] = "log-iss"
	ctx["source"] = Config.Deploy
	fmt.Println(ctx)
}

func awaitShutdownSignals(chs []ShutdownCh) {
	sigCh := make(chan os.Signal)
	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
	sig := <-sigCh
	Logf("ns=main at=shutdown-signal signal=%q", sig)
	for _, ch := range chs {
		ch <- 1
	}
}

func main() {
	Config, err := NewIssConfig()
	if err != nil {
		log.Fatalln(err)
	}

	auth, err := authenticater.NewBasicAuthFromString(Config.Tokens)
	if err != nil {
		log.Fatalln(err)
	}

	forwarderSet := NewForwarderSet(Config)
	forwarderSet.Start()

	shutdownCh := make(ShutdownCh)

	httpServer := NewHttpServer(Config, auth, Fix, forwarderSet.Inbox)

	go awaitShutdownSignals([]ShutdownCh{httpServer.ShutdownCh, shutdownCh})

	go func() {
		if err := httpServer.Run(); err != nil {
			log.Fatalln("Unable to start HTTP server:", err)
		}
	}()

	Logf("ns=main at=start")
	<-shutdownCh
	Logf("ns=main at=drain")
	httpServer.InFlightWg.Wait()
	Logf("ns=main at=exit")
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/horgh/gorse"
)

// DBItem holds the information about an input/entry that is in the database.
// TODO(will@summercat.com): Refactor to combine with gorse.DBItem. I think we
// should have one that is the generic item. This one includes a field related
// to a single user.
type DBItem struct {
	gorse.DBItem

	// Name from the rss_feed table.
	FeedName string

	// Read state from rss_item_state table
	ReadState string
}

// connectToDB opens a new connection to the database.
func connectToDB(settings *Config) (*sql.DB, error) {
	dsn := fmt.Sprintf("user=%s password=%s dbname=%s host=%s connect_timeout=10",
		settings.DBUser, settings.DBPass, settings.DBName, settings.DBHost)

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	log.Print("Opened new connection to the database.")
	return db, nil
}

// getDB connects us to the database if necessary, and returns an active
// database connection.
//
// We use the global DB variable to try to ensure we use a single connection.
func getDB(settings *Config) (*sql.DB, error) {
	// If we have a db connection, ensure that it is still available so that we
	// reconnect if it is not.
	if DB != nil {
		err := DB.Ping()
		if err == nil {
			return DB, nil
		}

		log.Printf("Database ping failed: %s", err)

		// Continue on, but set us so that we attempt to reconnect.

		DBLock.Lock()
		if DB != nil {
			_ = DB.Close()
			DB = nil
		}
		DBLock.Unlock()
	}

	DBLock.Lock()
	defer DBLock.Unlock()

	if DB != nil {
		return DB, nil
	}

	db, err := connectToDB(settings)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	// Set global
	DB = db

	return DB, nil
}

// dbCountItems retrieves a count of items.
//
// This is for pagination.
func dbCountItems(db *sql.DB, userID int, state gorse.ReadState) (int, error) {
	query := `
		SELECT COUNT(*)
		FROM rss_item ri
		LEFT JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE rf.active = true AND
			COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $3
`

	rows, err := db.Query(query, state.String(), userID, userID)
	if err != nil {
		return -1, err
	}

	if !rows.Next() {
		return -1, errors.New("count not found")
	}

	var count int
	if err := rows.Scan(&count); err != nil {
		_ = rows.Close()
		return -1, err
	}

	if err := rows.Close(); err != nil {
		return -1, fmt.Errorf("problem closing rows: %s", err)
	}

	return count, nil
}

// dbRetrieveFeedItems retrieves feed items from the database which are marked
// a given state.
func dbRetrieveFeedItems(db *sql.DB, settings *Config, order sortOrder,
	page, userID int, state gorse.ReadState) ([]DBItem, error) {

	if page < 1 {
		return nil, errors.New("invalid page number")
	}

	query := `
		SELECT
			rf.name,
			ri.id,
			ri.title,
			ri.link,
			ri.description,
			ri.publication_date,
			ri.guid
		FROM rss_item ri
		LEFT JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE rf.active = true AND
			COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $3
`

	if order == sortAscending {
		query += "ORDER BY ri.publication_date ASC, rf.name, ri.title"
	} else {
		query += "ORDER BY ri.publication_date DESC, rf.name, ri.title"
	}

	query += " LIMIT $4 OFFSET $5"

	offset := (page - 1) * pageSize

	rows, err := db.Query(query, state.String(), userID, userID, pageSize, offset)
	if err != nil {
		return nil, err
	}

	var items []DBItem
	for rows.Next() {
		item := DBItem{}

		err := rows.Scan(&item.FeedName, &item.ID, &item.Title, &item.Link,
			&item.Description, &item.PublicationDate, &item.GUID)
		if err != nil {
			log.Printf("Failed to scan row information: %s", err)
			_ = rows.Close()
			return nil, err
		}

		items = append(items, item)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("failure fetching rows: %s", err)
	}

	return items, nil
}

// Retrieve an item's information from the database. This includes the item's
// state for the given user.
func dbGetItem(db *sql.DB, itemID int64, userID int) (DBItem, error) {
	query := `
		SELECT
			ri.id,
			ri.title,
			ri.description,
			ri.link,
			ri.publication_date,
			ri.guid,
			ri.rss_feed_id,
			rf.name,
			COALESCE(ris.state, 'unread')
		FROM rss_item ri
		JOIN rss_feed rf ON ri.rss_feed_id = rf.id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE ri.id = $1 AND
			COALESCE(ris.user_id, $2) = $3
`
	rows, err := db.Query(query, itemID, userID, userID)
	if err != nil {
		return DBItem{}, err
	}

	for rows.Next() {
		item := DBItem{}

		if err := rows.Scan(&item.ID, &item.Title, &item.Description, &item.Link,
			&item.PublicationDate, &item.GUID, &item.RSSFeedID, &item.FeedName,
			&item.ReadState); err != nil {
			_ = rows.Close()
			return DBItem{}, fmt.Errorf("failed to scan row: %s", err)
		}

		if err := rows.Close(); err != nil {
			return DBItem{}, fmt.Errorf("error closing rows: %s", err)
		}

		return item, nil
	}

	if err := rows.Err(); err != nil {
		return DBItem{}, fmt.Errorf("failure fetching rows: %s", err)
	}

	return DBItem{}, fmt.Errorf("item not found")
}

// Record the item was read after having been saved to read later.
//
// It is useful to be able to refer back to such items as it is likely they were
// looked at more closely than others.
func dbRecordReadAfterReadLater(db *sql.DB, userID int, item DBItem) error {
	query := `
		INSERT INTO rss_item_read_after_archive
		(user_id, rss_feed_id, rss_item_id)
		VALUES ($1, $2, $3)
`
	if _, err := db.Exec(query, userID, item.RSSFeedID, item.ID); err != nil {
		return fmt.Errorf("unable to insert: %s", err)
	}

	return nil
}
<commit_msg>Use inner join<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/horgh/gorse"
)

// DBItem holds the information about an input/entry that is in the database.
// TODO(will@summercat.com): Refactor to combine with gorse.DBItem. I think we
// should have one that is the generic item. This one includes a field related
// to a single user.
type DBItem struct {
	gorse.DBItem

	// Name from the rss_feed table.
	FeedName string

	// Read state from rss_item_state table
	ReadState string
}

// connectToDB opens a new connection to the database.
func connectToDB(settings *Config) (*sql.DB, error) {
	dsn := fmt.Sprintf("user=%s password=%s dbname=%s host=%s connect_timeout=10",
		settings.DBUser, settings.DBPass, settings.DBName, settings.DBHost)

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	log.Print("Opened new connection to the database.")
	return db, nil
}

// getDB connects us to the database if necessary, and returns an active
// database connection.
//
// We use the global DB variable to try to ensure we use a single connection.
func getDB(settings *Config) (*sql.DB, error) {
	// If we have a db connection, ensure that it is still available so that we
	// reconnect if it is not.
	if DB != nil {
		err := DB.Ping()
		if err == nil {
			return DB, nil
		}

		log.Printf("Database ping failed: %s", err)

		// Continue on, but set us so that we attempt to reconnect.

		DBLock.Lock()
		if DB != nil {
			_ = DB.Close()
			DB = nil
		}
		DBLock.Unlock()
	}

	DBLock.Lock()
	defer DBLock.Unlock()

	if DB != nil {
		return DB, nil
	}

	db, err := connectToDB(settings)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	// Set global
	DB = db

	return DB, nil
}

// dbCountItems retrieves a count of items.
//
// This is for pagination.
func dbCountItems(db *sql.DB, userID int, state gorse.ReadState) (int, error) {
	query := `
		SELECT COUNT(*)
		FROM rss_item ri
		LEFT JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE rf.active = true AND
			COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $3
`

	rows, err := db.Query(query, state.String(), userID, userID)
	if err != nil {
		return -1, err
	}

	if !rows.Next() {
		return -1, errors.New("count not found")
	}

	var count int
	if err := rows.Scan(&count); err != nil {
		_ = rows.Close()
		return -1, err
	}

	if err := rows.Close(); err != nil {
		return -1, fmt.Errorf("problem closing rows: %s", err)
	}

	return count, nil
}

// dbRetrieveFeedItems retrieves feed items from the database which are marked
// a given state.
func dbRetrieveFeedItems(db *sql.DB, settings *Config, order sortOrder,
	page, userID int, state gorse.ReadState) ([]DBItem, error) {

	if page < 1 {
		return nil, errors.New("invalid page number")
	}

	query := `
		SELECT
			rf.name,
			ri.id,
			ri.title,
			ri.link,
			ri.description,
			ri.publication_date,
			ri.guid
		FROM rss_item ri
		JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE rf.active = true AND
			COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $3
`

	if order == sortAscending {
		query += "ORDER BY ri.publication_date ASC, rf.name, ri.title"
	} else {
		query += "ORDER BY ri.publication_date DESC, rf.name, ri.title"
	}

	query += " LIMIT $4 OFFSET $5"

	offset := (page - 1) * pageSize

	rows, err := db.Query(query, state.String(), userID, userID, pageSize, offset)
	if err != nil {
		return nil, err
	}

	var items []DBItem
	for rows.Next() {
		item := DBItem{}

		err := rows.Scan(&item.FeedName, &item.ID, &item.Title, &item.Link,
			&item.Description, &item.PublicationDate, &item.GUID)
		if err != nil {
			log.Printf("Failed to scan row information: %s", err)
			_ = rows.Close()
			return nil, err
		}

		items = append(items, item)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("failure fetching rows: %s", err)
	}

	return items, nil
}

// Retrieve an item's information from the database. This includes the item's
// state for the given user.
func dbGetItem(db *sql.DB, itemID int64, userID int) (DBItem, error) {
	query := `
		SELECT
			ri.id,
			ri.title,
			ri.description,
			ri.link,
			ri.publication_date,
			ri.guid,
			ri.rss_feed_id,
			rf.name,
			COALESCE(ris.state, 'unread')
		FROM rss_item ri
		JOIN rss_feed rf ON ri.rss_feed_id = rf.id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE ri.id = $1 AND
			COALESCE(ris.user_id, $2) = $3
`
	rows, err := db.Query(query, itemID, userID, userID)
	if err != nil {
		return DBItem{}, err
	}

	for rows.Next() {
		item := DBItem{}

		if err := rows.Scan(&item.ID, &item.Title, &item.Description, &item.Link,
			&item.PublicationDate, &item.GUID, &item.RSSFeedID, &item.FeedName,
			&item.ReadState); err != nil {
			_ = rows.Close()
			return DBItem{}, fmt.Errorf("failed to scan row: %s", err)
		}

		if err := rows.Close(); err != nil {
			return DBItem{}, fmt.Errorf("error closing rows: %s", err)
		}

		return item, nil
	}

	if err := rows.Err(); err != nil {
		return DBItem{}, fmt.Errorf("failure fetching rows: %s", err)
	}

	return DBItem{}, fmt.Errorf("item not found")
}

// Record the item was read after having been saved to read later.
//
// It is useful to be able to refer back to such items as it is likely they were
// looked at more closely than others.
func dbRecordReadAfterReadLater(db *sql.DB, userID int, item DBItem) error {
	query := `
		INSERT INTO rss_item_read_after_archive
		(user_id, rss_feed_id, rss_item_id)
		VALUES ($1, $2, $3)
`
	if _, err := db.Exec(query, userID, item.RSSFeedID, item.ID); err != nil {
		return fmt.Errorf("unable to insert: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"runtime"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	raven "github.com/getsentry/raven-go"
	"github.com/honeycombio/beeline-go"
	libhoney "github.com/honeycombio/libhoney-go"
	librato "github.com/mihasya/go-metrics-librato"
	metrics "github.com/rcrowley/go-metrics"
	travismetrics "github.com/travis-ci/jupiter-brain/metrics"
	"github.com/travis-ci/jupiter-brain/server"
)

func main() {
	app := cli.NewApp()
	app.Usage = "Jupiter Brain API server"
	app.Author = "Travis CI"
	app.Email = "contact+jupiter-brain@travis-ci.org"
	app.Version = VersionString
	app.Compiled = GeneratedTime()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "addr",
			Usage: "host:port to listen to",
			Value: func() string {
				v := ":" + os.Getenv("PORT")
				if v == ":" {
					v = ":42161"
				}
				return v
			}(),
			EnvVar: "JUPITER_BRAIN_ADDR",
		},
		cli.StringFlag{
			Name:   "auth-token",
			Usage:  "authentication token for the api server",
			EnvVar: "JUPITER_BRAIN_AUTH_TOKEN",
		},
		cli.StringFlag{
			Name:   "vsphere-api-url",
			Usage:  "URL to vSphere API",
			EnvVar: "JUPITER_BRAIN_VSPHERE_API_URL,VSPHERE_API_URL",
		},
		cli.StringFlag{
			Name:   "vsphere-base-path",
			Usage:  "path to folder of base VMs in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_BASE_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-vm-path",
			Usage:  "path to folder where VMs will be put in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_VM_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-cluster-path",
			Usage:  "path to compute cluster that VMs will be booted in",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CLUSTER_PATH",
		},
		cli.IntFlag{
			Name:   "vsphere-concurrent-read-operations",
			Usage:  "number of concurrent fetch and list operations",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CONCURRENT_READ_OPERATIONS",
			Value:  4,
		},
		cli.IntFlag{
			Name:   "vsphere-concurrent-create-operations",
			Usage:  "number of concurrent start operations",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CONCURRENT_CREATE_OPERATIONS",
			Value:  48,
		},
		cli.IntFlag{
			Name:   "vsphere-concurrent-delete-operations",
			Usage:  "number of concurrent terminate operations",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CONCURRENT_DELETE_OPERATIONS",
			Value:  48,
		},
		cli.StringFlag{
			Name:   "database-url",
			Usage:  "URL to the PostgreSQL database",
			EnvVar: "JUPITER_BRAIN_DATABASE_URL,DATABASE_URL",
		},
		cli.IntFlag{
			Name:   "database-pool-size",
			Usage:  "pool size",
			EnvVar: "JUPITER_BRAIN_DATABASE_POOL_SIZE,DATABASE_POOL_SIZE",
			Value:  2,
		},
		cli.BoolFlag{
			Name:   "debug",
			Usage:  "enable debug logging",
			EnvVar: "JUPITER_BRAIN_DEBUG,DEBUG",
		},
		cli.StringFlag{
			Name:   "sentry-dsn",
			Usage:  "Sentry DSN to send errors to",
			EnvVar: "JUPITER_BRAIN_SENTRY_DSN,SENTRY_DSN",
		},
		cli.StringFlag{
			Name:   "sentry-environment",
			Usage:  "Environment name to pass to Sentry",
			EnvVar: "JUPITER_BRAIN_SENTRY_ENVIRONMENT,SENTRY_ENVIRONMENT",
		},
		cli.StringFlag{
			Name:   "librato-email",
			Usage:  "Email for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_EMAIL,LIBRATO_EMAIL",
		},
		cli.StringFlag{
			Name:   "librato-token",
			Usage:  "Token for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_TOKEN,LIBRATO_TOKEN",
		},
		cli.StringFlag{
			Name:   "librato-source",
			Usage:  "The source to use when sending metrics to Librato",
			EnvVar: "JUPITER_BRAIN_LIBRATO_SOURCE,LIBRATO_SOURCE",
		},
		cli.StringFlag{
			Name:   "pprof-addr",
			Usage:  "Whether to enable pprof endpoints over HTTP",
			EnvVar: "JUPITER_BRAIN_PPROF_ADDR",
		},
		cli.DurationFlag{
			Name:   "request-timeout",
			Usage:  "The max time an incoming HTTP request can take before timing out",
			EnvVar: "JUPITER_BRAIN_REQUEST_TIMEOUT",
			Value:  3 * time.Minute,
		},
		cli.StringFlag{
			Name:   "honeycomb-write-key",
			Usage:  "The write key for Honeycomb",
			EnvVar: "JUPITER_BRAIN_HONEYCOMB_WRITE_KEY",
		},
		cli.StringFlag{
			Name:   "honeycomb-dataset",
			Usage:  "The dataset name for Honeycomb",
			EnvVar: "JUPITER_BRAIN_HONEYCOMB_DATASET",
		},
	}
	app.Action = runServer

	app.RunAndExitOnError()
}

func runServer(c *cli.Context) {
	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})

	if c.String("librato-email") != "" && c.String("librato-token") != "" && c.String("librato-source") != "" {
		logrus.Info("starting librato metrics reporter")

		go librato.Librato(
			metrics.DefaultRegistry,
			time.Minute,
			c.String("librato-email"),
			c.String("librato-token"),
			c.String("librato-source"),
			[]float64{0.50, 0.75, 0.90, 0.95, 0.99, 0.999, 1.0},
			time.Millisecond,
		)
	}
	go travismetrics.ReportMemstatsMetrics()

	if c.String("honeycomb-write-key") != "" && c.String("honeycomb-dataset") != "" {
		if c.String("honeycomb-dataset") != "" {
			libhoney.Init(libhoney.Config{
				WriteKey: c.String("honeycomb-write-key"),
				Dataset:  c.String("honeycomb-dataset"),
			})
			defer libhoney.Close()

			libhoney.AddDynamicField("num_goroutines", func() interface{} { return runtime.NumGoroutine() })
			libhoney.AddField("jupiter_brain_version", c.App.Version)
			libhoney.AddField("jupiter_brain_source", c.String("librato-source"))
		}

		if c.String("honeycomb-request-dataset") != "" {
			beeline.Init(beeline.Config{
				WriteKey: c.String("honeycomb-write-key"),
				Dataset:  c.String("honeycomb-request-dataset"),
			})
		}
	}

	raven.SetDSN(c.String("sentry-dsn"))
	raven.SetRelease(VersionString)
	if c.String("sentry-environment") != "" {
		raven.SetEnvironment(c.String("sentry-environment"))
	}

	server.Main(&server.Config{
		Addr:              c.String("addr"),
		AuthToken:         c.String("auth-token"),
		Debug:             c.Bool("debug"),
		SentryDSN:         c.String("sentry-dsn"),
		SentryEnvironment: c.String("sentry-environment"),

		VSphereURL:                        c.String("vsphere-api-url"),
		VSphereBasePath:                   c.String("vsphere-base-path"),
		VSphereVMPath:                     c.String("vsphere-vm-path"),
		VSphereClusterPath:                c.String("vsphere-cluster-path"),
		VSphereConcurrentReadOperations:   c.Int("vsphere-concurrent-read-operations"),
		VSphereConcurrentCreateOperations: c.Int("vsphere-concurrent-create-operations"),
		VSphereConcurrentDeleteOperations: c.Int("vsphere-concurrent-delete-operations"),

		DatabaseURL:      c.String("database-url"),
		DatabasePoolSize: c.Int("database-pool-size"),

		PprofAddr:      c.String("pprof-addr"),
		RequestTimeout: c.Duration("request-timeout"),
	})
}
<commit_msg>Add CLI flag for request dataset<commit_after>package main

import (
	"os"
	"runtime"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	raven "github.com/getsentry/raven-go"
	"github.com/honeycombio/beeline-go"
	libhoney "github.com/honeycombio/libhoney-go"
	librato "github.com/mihasya/go-metrics-librato"
	metrics "github.com/rcrowley/go-metrics"
	travismetrics "github.com/travis-ci/jupiter-brain/metrics"
	"github.com/travis-ci/jupiter-brain/server"
)

func main() {
	app := cli.NewApp()
	app.Usage = "Jupiter Brain API server"
	app.Author = "Travis CI"
	app.Email = "contact+jupiter-brain@travis-ci.org"
	app.Version = VersionString
	app.Compiled = GeneratedTime()
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "addr",
			Usage: "host:port to listen to",
			Value: func() string {
				v := ":" + os.Getenv("PORT")
				if v == ":" {
					v = ":42161"
				}
				return v
			}(),
			EnvVar: "JUPITER_BRAIN_ADDR",
		},
		cli.StringFlag{
			Name:   "auth-token",
			Usage:  "authentication token for the api server",
			EnvVar: "JUPITER_BRAIN_AUTH_TOKEN",
		},
		cli.StringFlag{
			Name:   "vsphere-api-url",
			Usage:  "URL to vSphere API",
			EnvVar: "JUPITER_BRAIN_VSPHERE_API_URL,VSPHERE_API_URL",
		},
		cli.StringFlag{
			Name:   "vsphere-base-path",
			Usage:  "path to folder of base VMs in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_BASE_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-vm-path",
			Usage:  "path to folder where VMs will be put in vSphere inventory",
			EnvVar: "JUPITER_BRAIN_VSPHERE_VM_PATH",
		},
		cli.StringFlag{
			Name:   "vsphere-cluster-path",
			Usage:  "path to compute cluster that VMs will be booted in",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CLUSTER_PATH",
		},
		cli.IntFlag{
			Name:   "vsphere-concurrent-read-operations",
			Usage:  "number of concurrent fetch and list operations",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CONCURRENT_READ_OPERATIONS",
			Value:  4,
		},
		cli.IntFlag{
			Name:   "vsphere-concurrent-create-operations",
			Usage:  "number of concurrent start operations",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CONCURRENT_CREATE_OPERATIONS",
			Value:  48,
		},
		cli.IntFlag{
			Name:   "vsphere-concurrent-delete-operations",
			Usage:  "number of concurrent terminate operations",
			EnvVar: "JUPITER_BRAIN_VSPHERE_CONCURRENT_DELETE_OPERATIONS",
			Value:  48,
		},
		cli.StringFlag{
			Name:   "database-url",
			Usage:  "URL to the PostgreSQL database",
			EnvVar: "JUPITER_BRAIN_DATABASE_URL,DATABASE_URL",
		},
		cli.IntFlag{
			Name:   "database-pool-size",
			Usage:  "pool size",
			EnvVar: "JUPITER_BRAIN_DATABASE_POOL_SIZE,DATABASE_POOL_SIZE",
			Value:  2,
		},
		cli.BoolFlag{
			Name:   "debug",
			Usage:  "enable debug logging",
			EnvVar: "JUPITER_BRAIN_DEBUG,DEBUG",
		},
		cli.StringFlag{
			Name:   "sentry-dsn",
			Usage:  "Sentry DSN to send errors to",
			EnvVar: "JUPITER_BRAIN_SENTRY_DSN,SENTRY_DSN",
		},
		cli.StringFlag{
			Name:   "sentry-environment",
			Usage:  "Environment name to pass to Sentry",
			EnvVar: "JUPITER_BRAIN_SENTRY_ENVIRONMENT,SENTRY_ENVIRONMENT",
		},
		cli.StringFlag{
			Name:   "librato-email",
			Usage:  "Email for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_EMAIL,LIBRATO_EMAIL",
		},
		cli.StringFlag{
			Name:   "librato-token",
			Usage:  "Token for Librato account to send metrics to",
			EnvVar: "JUPITER_BRAIN_LIBRATO_TOKEN,LIBRATO_TOKEN",
		},
		cli.StringFlag{
			Name:   "librato-source",
			Usage:  "The source to use when sending metrics to Librato",
			EnvVar: "JUPITER_BRAIN_LIBRATO_SOURCE,LIBRATO_SOURCE",
		},
		cli.StringFlag{
			Name:   "pprof-addr",
			Usage:  "Whether to enable pprof endpoints over HTTP",
			EnvVar: "JUPITER_BRAIN_PPROF_ADDR",
		},
		cli.DurationFlag{
			Name:   "request-timeout",
			Usage:  "The max time an incoming HTTP request can take before timing out",
			EnvVar: "JUPITER_BRAIN_REQUEST_TIMEOUT",
			Value:  3 * time.Minute,
		},
		cli.StringFlag{
			Name:   "honeycomb-write-key",
			Usage:  "The write key for Honeycomb",
			EnvVar: "JUPITER_BRAIN_HONEYCOMB_WRITE_KEY",
		},
		cli.StringFlag{
			Name:   "honeycomb-dataset",
			Usage:  "The dataset name for Honeycomb",
			EnvVar: "JUPITER_BRAIN_HONEYCOMB_DATASET",
		},
		cli.StringFlag{
			Name:   "honeycomb-request-dataset",
			Usage:  "The dataset name for Honeycomb to track HTTP requests",
			EnvVar: "JUPITER_BRAIN_HONEYCOMB_REQUEST_DATASET",
		},
	}
	app.Action = runServer

	app.RunAndExitOnError()
}

func runServer(c *cli.Context) {
	logrus.SetFormatter(&logrus.TextFormatter{DisableColors: true})

	if c.String("librato-email") != "" && c.String("librato-token") != "" && c.String("librato-source") != "" {
		logrus.Info("starting librato metrics reporter")

		go librato.Librato(
			metrics.DefaultRegistry,
			time.Minute,
			c.String("librato-email"),
			c.String("librato-token"),
			c.String("librato-source"),
			[]float64{0.50, 0.75, 0.90, 0.95, 0.99, 0.999, 1.0},
			time.Millisecond,
		)
	}
	go travismetrics.ReportMemstatsMetrics()

	if c.String("honeycomb-write-key") != "" && c.String("honeycomb-dataset") != "" {
		if c.String("honeycomb-dataset") != "" {
			libhoney.Init(libhoney.Config{
				WriteKey: c.String("honeycomb-write-key"),
				Dataset:  c.String("honeycomb-dataset"),
			})
			defer libhoney.Close()

			libhoney.AddDynamicField("num_goroutines", func() interface{} { return runtime.NumGoroutine() })
			libhoney.AddField("jupiter_brain_version", c.App.Version)
			libhoney.AddField("jupiter_brain_source", c.String("librato-source"))
		}

		if c.String("honeycomb-request-dataset") != "" {
			beeline.Init(beeline.Config{
				WriteKey: c.String("honeycomb-write-key"),
				Dataset:  c.String("honeycomb-request-dataset"),
			})
		}
	}

	raven.SetDSN(c.String("sentry-dsn"))
	raven.SetRelease(VersionString)
	if c.String("sentry-environment") != "" {
		raven.SetEnvironment(c.String("sentry-environment"))
	}

	server.Main(&server.Config{
		Addr:              c.String("addr"),
		AuthToken:         c.String("auth-token"),
		Debug:             c.Bool("debug"),
		SentryDSN:         c.String("sentry-dsn"),
		SentryEnvironment: c.String("sentry-environment"),

		VSphereURL:                        c.String("vsphere-api-url"),
		VSphereBasePath:                   c.String("vsphere-base-path"),
		VSphereVMPath:                     c.String("vsphere-vm-path"),
		VSphereClusterPath:                c.String("vsphere-cluster-path"),
		VSphereConcurrentReadOperations:   c.Int("vsphere-concurrent-read-operations"),
		VSphereConcurrentCreateOperations: c.Int("vsphere-concurrent-create-operations"),
		VSphereConcurrentDeleteOperations: c.Int("vsphere-concurrent-delete-operations"),

		DatabaseURL:      c.String("database-url"),
		DatabasePoolSize: c.Int("database-pool-size"),

		PprofAddr:      c.String("pprof-addr"),
		RequestTimeout: c.Duration("request-timeout"),
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/datawire/teleproxy/pkg/kubeapply"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

func envBool(name string) bool {
	val := os.Getenv(name)
	switch strings.TrimSpace(strings.ToLower(val)) {
	case "true":
		return true
	case "yes":
		return true
	case "1":
		return true
	case "false":
		return false
	case "no":
		return false
	case "0":
		return false
	case "":
		return false
	}

	return true
}

// Version holds the version of the code. This is intended to be overridden at build time.
var Version = "(unknown version)"

func adapt(f func(*cobra.Command, []string) error) func(*cobra.Command, []string) {
	return func(cmd *cobra.Command, args []string) {
		err := f(cmd, args)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
}

func main() {
	var ka = &cobra.Command{
		Use:   "kubeapply",
		Short: "kubeapply",
		Long:  "kubeapply - the way kubectl aught to work",
	}

	debug := ka.Flags().Bool("debug", envBool("KUBEAPPLY_DEBUG"), "enable debug mode")
	dryRun := ka.Flags().Bool("dry-run", envBool("KUBEAPPLY_DRYRUN"), "enable dry-run mode")
	timeout := ka.Flags().DurationP("timeout", "t", 60*time.Second, "timeout to wait for applied yaml to be ready")
	showVersion := ka.Flags().Bool("version", false, "output version information and exit")
	files := ka.Flags().StringArrayP("", "f", nil, "files to apply")

	ka.Run = adapt(func(cmd *cobra.Command, args []string) error {
		if *showVersion {
			fmt.Println(Version)
			return nil
		}
		if len(args) > 0 {
			return errors.Errorf("extra args: %v", args)
		}
		if len(*files) == 0 {
			return errors.Errorf("at least one file argument is required")
		}
		return kubeapply.Kubeapply(nil, *timeout, *debug, *dryRun, *files...)
	})

	err := ka.Execute()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>use strconv.ParseBool<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"github.com/datawire/teleproxy/pkg/kubeapply"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

func envBool(name string) bool {
	val, _ := strconv.ParseBool(name)
	return val
}

// Version holds the version of the code. This is intended to be overridden at build time.
var Version = "(unknown version)"

func adapt(f func(*cobra.Command, []string) error) func(*cobra.Command, []string) {
	return func(cmd *cobra.Command, args []string) {
		err := f(cmd, args)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			os.Exit(1)
		}
	}
}

func main() {
	var ka = &cobra.Command{
		Use:   "kubeapply",
		Short: "kubeapply",
		Long:  "kubeapply - the way kubectl aught to work",
	}

	debug := ka.Flags().Bool("debug", envBool("KUBEAPPLY_DEBUG"), "enable debug mode")
	dryRun := ka.Flags().Bool("dry-run", envBool("KUBEAPPLY_DRYRUN"), "enable dry-run mode")
	timeout := ka.Flags().DurationP("timeout", "t", 60*time.Second, "timeout to wait for applied yaml to be ready")
	showVersion := ka.Flags().Bool("version", false, "output version information and exit")
	files := ka.Flags().StringArrayP("", "f", nil, "files to apply")

	ka.Run = adapt(func(cmd *cobra.Command, args []string) error {
		if *showVersion {
			fmt.Println(Version)
			return nil
		}
		if len(args) > 0 {
			return errors.Errorf("extra args: %v", args)
		}
		if len(*files) == 0 {
			return errors.Errorf("at least one file argument is required")
		}
		return kubeapply.Kubeapply(nil, *timeout, *debug, *dryRun, *files...)
	})

	err := ka.Execute()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"

	"github.com/codegangsta/cli"
	"golang.org/x/net/context"
	"li.lan/labs/plasma/rpcprotos"
)

var NewAddressCommand = cli.Command{
	Name:   "newaddress",
	Usage:  "gets the next address in the HD chain",
	Action: newAddress,
}

func newAddress(ctx *cli.Context) {
	client := getClient(ctx)

	ctxb := context.Background()
	addr, err := client.NewAddress(ctxb, &lnrpc.NewAddressRequest{})
	if err != nil {
		fatal(err)
	}

	fmt.Println(json.Marshal(addr))
}

var SendManyCommand = cli.Command{
	Name: "sendmany",
	Usage: "create and broadcast a transaction paying the specified " +
		"amount(s) to the passed address(es)",
	Action: sendMany,
}

func sendMany(ctx *cli.Context) {
	var amountToAddr map[string]int64

	fmt.Println(ctx.Args())

	jsonMap := ctx.Args().Get(0)
	if err := json.Unmarshal([]byte(jsonMap), &amountToAddr); err != nil {
		fatal(err)
	}

	fmt.Println("map: %v", amountToAddr)

	ctxb := context.Background()
	client := getClient(ctx)

	txid, err := client.SendMany(ctxb, &lnrpc.SendManyRequest{amountToAddr})
	if err != nil {
		fatal(err)
	}

	fmt.Println(json.Marshal(txid))
}
<commit_msg>cmd/lncli: properly pretty print json response<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"os"

	"github.com/codegangsta/cli"
	"golang.org/x/net/context"
	"li.lan/labs/plasma/rpcprotos"
)

func printRespJson(resp interface{}) {
	b, err := json.Marshal(resp)
	if err != nil {
		fatal(err)
	}

	var out bytes.Buffer
	json.Indent(&out, b, "", "\t")
	out.WriteTo(os.Stdout)
}

var NewAddressCommand = cli.Command{
	Name:   "newaddress",
	Usage:  "gets the next address in the HD chain",
	Action: newAddress,
}

func newAddress(ctx *cli.Context) {
	client := getClient(ctx)

	ctxb := context.Background()
	addr, err := client.NewAddress(ctxb, &lnrpc.NewAddressRequest{})
	if err != nil {
		fatal(err)
	}

	printRespJson(addr)
}

var SendManyCommand = cli.Command{
	Name: "sendmany",
	Usage: "create and broadcast a transaction paying the specified " +
		"amount(s) to the passed address(es)",
	Action: sendMany,
}

func sendMany(ctx *cli.Context) {
	var amountToAddr map[string]int64

	jsonMap := ctx.Args().Get(0)
	if err := json.Unmarshal([]byte(jsonMap), &amountToAddr); err != nil {
		fatal(err)
	}

	ctxb := context.Background()
	client := getClient(ctx)

	txid, err := client.SendMany(ctxb, &lnrpc.SendManyRequest{amountToAddr})
	if err != nil {
		fatal(err)
	}

	printRespJson(txid)
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 IBM Corporation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import "fmt"

func main() {
    fmt.Printf("IBM header.\n")
}
<commit_msg>Delete ibm-bad.go<commit_after><|endoftext|>


<commit_before>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2018
	All Rights Reserved
	Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

// An RPC Server tool, import the services as modules
package main

import (
	"fmt"
	"os"

	// Frameworks
	gopi "github.com/djthorpe/gopi"

	// Modules
	_ "github.com/djthorpe/gopi/sys/logger"
	_ "github.com/djthorpe/gopi/sys/rpc/grpc"
	_ "github.com/djthorpe/gopi/sys/rpc/mdns"

	// RPC Clients
	hw "github.com/djthorpe/gopi/cmd/rpc/helloworld"
)

////////////////////////////////////////////////////////////////////////////////

func RecordMatches(app *gopi.AppInstance, record *gopi.RPCServiceRecord) bool {
	if service_type, err := gopi.RPCServiceType(app.Service(), 0); err != nil {
		app.Logger.Error("RecordMatches: %v", err)
		return false
	} else {
		return service_type == record.Type
	}
}

func RunClient(app *gopi.AppInstance, client *hw.MyGreeterClient) {
	name, _ := app.AppFlags.GetString("name")
	if message, err := client.SayHello(name); err != nil {
		app.Logger.Error("RunClient: %v", client.Conn().Name(), err)
	} else {
		fmt.Printf("%v says '%v'\n\n", client.Conn().Name(), message)
	}
}

func HandleEvent(app *gopi.AppInstance, evt gopi.RPCEvent) {
	pool := app.ModuleInstance("rpc/clientpool").(gopi.RPCClientPool)

	switch evt.Type() {
	case gopi.RPC_EVENT_SERVICE_RECORD:
		// Create a connection if the service record is correct type
		if RecordMatches(app, evt.ServiceRecord()) {
			if _, err := pool.Connect(evt.ServiceRecord(), 0); err != nil {
				app.Logger.Error("Connect: %v", err)
			}
		}
	case gopi.RPC_EVENT_CLIENT_CONNECTED:
		conn := evt.Source().(gopi.RPCClientConn)
		if client := pool.NewClient("mutablelogic.Helloworld", conn); client == nil {
			app.Logger.Error("Connect: Unable to create client")
		} else {
			RunClient(app, client.(*hw.MyGreeterClient))
			if err := pool.Disconnect(conn); err != nil {
				app.Logger.Error("Disconnect: %v", err)
			}
		}
	case gopi.RPC_EVENT_CLIENT_DISCONNECT:
		// Send a terminate signal to end
		app.SendSignal()
	default:
		app.Logger.Warn("Unhandled event type: %v", evt.Type())
	}
}

func EventLoop(app *gopi.AppInstance, done <-chan struct{}) error {
	// Obtain client connection
	pool := app.ModuleInstance("rpc/clientpool").(gopi.RPCClientPool)
	if pool == nil {
		return gopi.ErrAppError
	}

	// Subscribe to events
	poolevents := pool.Subscribe()

FOR_LOOP:
	for {
		select {
		case <-done:
			break FOR_LOOP
		case evt := <-poolevents:
			if evt != nil {
				HandleEvent(app, evt.(gopi.RPCEvent))
			}
		}
	}

	// Unsubscribe from events
	pool.Unsubscribe(poolevents)

	// Return success
	return nil
}

func Main(app *gopi.AppInstance, done chan<- struct{}) error {

	// Wait until CTRL+C is pressed or SIGTERM signal
	app.Logger.Info("Waiting for CTRL+C")
	app.WaitForSignal()

	// Success
	done <- gopi.DONE
	return nil
}

////////////////////////////////////////////////////////////////////////////////

func main() {
	// Create the configuration
	config := gopi.NewAppConfig("rpc/client/helloworld:grpc")

	// Name argument
	config.AppFlags.FlagString("name", "", "Your name")

	// Set the RPCServiceRecord for server discovery
	config.Service = "helloworld"

	// Run the command line tool
	os.Exit(gopi.CommandLineTool(config, Main, EventLoop))
}
<commit_msg>Added load averages for darwin<commit_after><|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"net"
	"net/url"
	"os"
	"strings"
	"time"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/cfhttp"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	loggingclient "code.cloudfoundry.org/diego-logging-client"
	"code.cloudfoundry.org/diego-ssh/authenticators"
	"code.cloudfoundry.org/diego-ssh/cmd/ssh-proxy/config"
	"code.cloudfoundry.org/diego-ssh/healthcheck"
	"code.cloudfoundry.org/diego-ssh/helpers"
	"code.cloudfoundry.org/diego-ssh/proxy"
	"code.cloudfoundry.org/diego-ssh/server"
	"code.cloudfoundry.org/go-loggregator/runtimeemitter"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/locket"
	"github.com/hashicorp/consul/api"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/crypto/ssh"
)

var configPath = flag.String(
	"config",
	"",
	"Path to SSH Proxy config.",
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	flag.Parse()

	sshProxyConfig, err := config.NewSSHProxyConfig(*configPath)
	if err != nil {
		logger, _ := lagerflags.New("ssh-proxy")
		logger.Fatal("failed-to-parse-config", err)
	}

	logger, reconfigurableSink := lagerflags.NewFromConfig("ssh-proxy", sshProxyConfig.LagerConfig)

	cfhttp.Initialize(time.Duration(sshProxyConfig.CommunicationTimeout))

	metronClient, err := initializeMetron(logger, sshProxyConfig)
	if err != nil {
		logger.Error("failed-to-initialize-metron-client", err)
		os.Exit(1)
	}

	proxySSHServerConfig, err := configureProxy(logger, sshProxyConfig)
	if err != nil {
		logger.Error("configure-failed", err)
		os.Exit(1)
	}

	tlsConfig, err := sshProxyConfig.BackendsTLSConfig()
	if err != nil {
		logger.Error("failed-to-get-tls-config", err)
		os.Exit(1)
	}
	sshProxy := proxy.New(logger, proxySSHServerConfig, metronClient, tlsConfig)
	server := server.NewServer(logger, sshProxyConfig.Address, sshProxy, time.Duration(sshProxyConfig.IdleConnectionTimeout))

	healthCheckHandler := healthcheck.NewHandler(logger)

	members := grouper.Members{
		{"ssh-proxy", server},
	}

	if !sshProxyConfig.DisableHealthCheckServer {
		httpServer := http_server.New(sshProxyConfig.HealthCheckAddress, healthCheckHandler)
		members = append(members, grouper.Member{"healthcheck", httpServer})
	}

	if sshProxyConfig.EnableConsulServiceRegistration {
		consulClient, err := consuladapter.NewClientFromUrl(sshProxyConfig.ConsulCluster)
		if err != nil {
			logger.Fatal("new-client-failed", err)
		}

		registrationRunner := initializeRegistrationRunner(logger, consulClient, sshProxyConfig.Address, clock.NewClock())
		members = append(members, grouper.Member{"registration-runner", registrationRunner})
	}

	if sshProxyConfig.DebugAddress != "" {
		members = append(grouper.Members{{
			"debug-server", debugserver.Runner(sshProxyConfig.DebugAddress, reconfigurableSink),
		}}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)
	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
	os.Exit(0)
}

func configureProxy(logger lager.Logger, sshProxyConfig config.SSHProxyConfig) (*ssh.ServerConfig, error) {
	if sshProxyConfig.BBSAddress == "" {
		err := errors.New("bbsAddress is required")
		logger.Fatal("bbs-address-required", err)
	}

	url, err := url.Parse(sshProxyConfig.BBSAddress)
	if err != nil {
		logger.Fatal("failed-to-parse-bbs-address", err)
	}

	bbsClient := initializeBBSClient(
		logger,
		sshProxyConfig.BBSAddress,
		sshProxyConfig.BBSCACert,
		sshProxyConfig.BBSClientCert,
		sshProxyConfig.BBSClientKey,
		sshProxyConfig.BBSClientSessionCacheSize,
		sshProxyConfig.BBSMaxIdleConnsPerHost,
	)
	permissionsBuilder := authenticators.NewPermissionsBuilder(bbsClient, sshProxyConfig.ConnectToInstanceAddress)

	authens := []authenticators.PasswordAuthenticator{}

	if sshProxyConfig.EnableDiegoAuth {
		diegoAuthenticator := authenticators.NewDiegoProxyAuthenticator(logger, []byte(sshProxyConfig.DiegoCredentials), permissionsBuilder)
		authens = append(authens, diegoAuthenticator)
	}

	if sshProxyConfig.EnableCFAuth {
		if sshProxyConfig.CCAPIURL == "" {
			return nil, errors.New("ccAPIURL is required for Cloud Foundry authentication")
		}

		_, err = url.Parse(sshProxyConfig.CCAPIURL)
		if err != nil {
			return nil, err
		}

		if sshProxyConfig.UAAPassword == "" {
			return nil, errors.New("UAA password is required for Cloud Foundry authentication")
		}

		if sshProxyConfig.UAAUsername == "" {
			return nil, errors.New("UAA username is required for Cloud Foundry authentication")
		}

		if sshProxyConfig.UAATokenURL == "" {
			return nil, errors.New("uaaTokenURL is required for Cloud Foundry authentication")
		}

		_, err = url.Parse(sshProxyConfig.UAATokenURL)
		if err != nil {
			return nil, err
		}

		client, err := helpers.NewHTTPSClient(sshProxyConfig.SkipCertVerify, []string{sshProxyConfig.UAACACert, sshProxyConfig.CCAPICACert}, time.Duration(sshProxyConfig.CommunicationTimeout))
		if err != nil {
			return nil, err
		}

		cfAuthenticator := authenticators.NewCFAuthenticator(
			logger,
			client,
			sshProxyConfig.CCAPIURL,
			sshProxyConfig.UAATokenURL,
			sshProxyConfig.UAAUsername,
			sshProxyConfig.UAAPassword,
			permissionsBuilder,
		)
		authens = append(authens, cfAuthenticator)
	}

	authenticator := authenticators.NewCompositeAuthenticator(authens...)

	sshConfig := &ssh.ServerConfig{
		ServerVersion:    "SSH-2.0-diego-ssh-proxy",
		PasswordCallback: authenticator.Authenticate,
		AuthLogCallback: func(cmd ssh.ConnMetadata, method string, err error) {
			if err != nil {
				logger.Error("authentication-failed", err, lager.Data{"user": cmd.User()})
			} else {
				logger.Info("authentication-attempted", lager.Data{"user": cmd.User()})
			}
		},
	}

	sshConfig.SetDefaults()

	if sshProxyConfig.HostKey == "" {
		err := errors.New("hostKey is required")
		logger.Fatal("host-key-required", err)
	}

	key, err := parsePrivateKey(logger, sshProxyConfig.HostKey)
	if err != nil {
		logger.Fatal("failed-to-parse-host-key", err)
	}

	sshConfig.AddHostKey(key)

	if sshProxyConfig.AllowedCiphers != "" {
		sshConfig.Config.Ciphers = strings.Split(sshProxyConfig.AllowedCiphers, ",")
	} else {
		sshConfig.Config.Ciphers = []string{"chacha20-poly1305@openssh.com", "aes128-gcm@openssh.com", "aes256-ctr", "aes192-ctr", "aes128-ctr"}
	}

	if sshProxyConfig.AllowedMACs != "" {
		sshConfig.Config.MACs = strings.Split(sshProxyConfig.AllowedMACs, ",")
	} else {
		sshConfig.Config.MACs = []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256"}
	}

	if sshProxyConfig.AllowedKeyExchanges != "" {
		sshConfig.Config.KeyExchanges = strings.Split(sshProxyConfig.AllowedKeyExchanges, ",")
	} else {
		sshConfig.Config.KeyExchanges = []string{"curve25519-sha256@libssh.org"}
	}

	return sshConfig, err
}

func parsePrivateKey(logger lager.Logger, encodedKey string) (ssh.Signer, error) {
	key, err := ssh.ParsePrivateKey([]byte(encodedKey))
	if err != nil {
		logger.Error("failed-to-parse-private-key", err)
		return nil, err
	}
	return key, nil
}

func initializeBBSClient(
	logger lager.Logger,
	bbsAddress,
	bbsCACert,
	bbsClientCert,
	bbsClientKey string,
	bbsClientSessionCacheSize,
	bbsMaxIdleConnsPerHost int,
) bbs.InternalClient {
	bbsClient, err := bbs.NewClient(bbsAddress, bbsCACert, bbsClientCert, bbsClientKey, bbsClientSessionCacheSize, bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, listenAddress string, clock clock.Clock) ifrit.Runner {
	_, portString, err := net.SplitHostPort(listenAddress)
	if err != nil {
		logger.Fatal("failed-invalid-listen-address", err)
	}
	portNum, err := net.LookupPort("tcp", portString)
	if err != nil {
		logger.Fatal("failed-invalid-listen-port", err)
	}

	registration := &api.AgentServiceRegistration{
		Name: "ssh-proxy",
		Port: portNum,
		Check: &api.AgentServiceCheck{
			TTL: "20s",
		},
	}

	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}

func initializeMetron(logger lager.Logger, locketConfig config.SSHProxyConfig) (loggingclient.IngressClient, error) {
	client, err := loggingclient.NewIngressClient(locketConfig.LoggregatorConfig)
	if err != nil {
		return nil, err
	}

	if locketConfig.LoggregatorConfig.UseV2API {
		emitter := runtimeemitter.NewV1(client)
		go emitter.Run()
	}

	return client, nil
}
<commit_msg>Revert "Revert "Switches ssh-proxy to use new initializer for BBS client""<commit_after>package main

import (
	"errors"
	"flag"
	"net"
	"net/url"
	"os"
	"strings"
	"time"

	"code.cloudfoundry.org/bbs"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/consuladapter"
	"code.cloudfoundry.org/debugserver"
	loggingclient "code.cloudfoundry.org/diego-logging-client"
	"code.cloudfoundry.org/diego-ssh/authenticators"
	"code.cloudfoundry.org/diego-ssh/cmd/ssh-proxy/config"
	"code.cloudfoundry.org/diego-ssh/healthcheck"
	"code.cloudfoundry.org/diego-ssh/helpers"
	"code.cloudfoundry.org/diego-ssh/proxy"
	"code.cloudfoundry.org/diego-ssh/server"
	"code.cloudfoundry.org/go-loggregator/runtimeemitter"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagerflags"
	"code.cloudfoundry.org/locket"
	"github.com/hashicorp/consul/api"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/crypto/ssh"
)

var configPath = flag.String(
	"config",
	"",
	"Path to SSH Proxy config.",
)

func main() {
	debugserver.AddFlags(flag.CommandLine)
	flag.Parse()

	sshProxyConfig, err := config.NewSSHProxyConfig(*configPath)
	if err != nil {
		logger, _ := lagerflags.New("ssh-proxy")
		logger.Fatal("failed-to-parse-config", err)
	}

	logger, reconfigurableSink := lagerflags.NewFromConfig("ssh-proxy", sshProxyConfig.LagerConfig)

	metronClient, err := initializeMetron(logger, sshProxyConfig)
	if err != nil {
		logger.Error("failed-to-initialize-metron-client", err)
		os.Exit(1)
	}

	proxySSHServerConfig, err := configureProxy(logger, sshProxyConfig)
	if err != nil {
		logger.Error("configure-failed", err)
		os.Exit(1)
	}

	tlsConfig, err := sshProxyConfig.BackendsTLSConfig()
	if err != nil {
		logger.Error("failed-to-get-tls-config", err)
		os.Exit(1)
	}
	sshProxy := proxy.New(logger, proxySSHServerConfig, metronClient, tlsConfig)
	server := server.NewServer(logger, sshProxyConfig.Address, sshProxy, time.Duration(sshProxyConfig.IdleConnectionTimeout))

	healthCheckHandler := healthcheck.NewHandler(logger)

	members := grouper.Members{
		{"ssh-proxy", server},
	}

	if !sshProxyConfig.DisableHealthCheckServer {
		httpServer := http_server.New(sshProxyConfig.HealthCheckAddress, healthCheckHandler)
		members = append(members, grouper.Member{"healthcheck", httpServer})
	}

	if sshProxyConfig.EnableConsulServiceRegistration {
		consulClient, err := consuladapter.NewClientFromUrl(sshProxyConfig.ConsulCluster)
		if err != nil {
			logger.Fatal("new-client-failed", err)
		}

		registrationRunner := initializeRegistrationRunner(logger, consulClient, sshProxyConfig.Address, clock.NewClock())
		members = append(members, grouper.Member{"registration-runner", registrationRunner})
	}

	if sshProxyConfig.DebugAddress != "" {
		members = append(grouper.Members{{
			"debug-server", debugserver.Runner(sshProxyConfig.DebugAddress, reconfigurableSink),
		}}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)
	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
	os.Exit(0)
}

func configureProxy(logger lager.Logger, sshProxyConfig config.SSHProxyConfig) (*ssh.ServerConfig, error) {
	if sshProxyConfig.BBSAddress == "" {
		err := errors.New("bbsAddress is required")
		logger.Fatal("bbs-address-required", err)
	}

	url, err := url.Parse(sshProxyConfig.BBSAddress)
	if err != nil {
		logger.Fatal("failed-to-parse-bbs-address", err)
	}

	bbsClient := initializeBBSClient(logger, sshProxyConfig)
	permissionsBuilder := authenticators.NewPermissionsBuilder(bbsClient, sshProxyConfig.ConnectToInstanceAddress)

	authens := []authenticators.PasswordAuthenticator{}

	if sshProxyConfig.EnableDiegoAuth {
		diegoAuthenticator := authenticators.NewDiegoProxyAuthenticator(logger, []byte(sshProxyConfig.DiegoCredentials), permissionsBuilder)
		authens = append(authens, diegoAuthenticator)
	}

	if sshProxyConfig.EnableCFAuth {
		if sshProxyConfig.CCAPIURL == "" {
			return nil, errors.New("ccAPIURL is required for Cloud Foundry authentication")
		}

		_, err = url.Parse(sshProxyConfig.CCAPIURL)
		if err != nil {
			return nil, err
		}

		if sshProxyConfig.UAAPassword == "" {
			return nil, errors.New("UAA password is required for Cloud Foundry authentication")
		}

		if sshProxyConfig.UAAUsername == "" {
			return nil, errors.New("UAA username is required for Cloud Foundry authentication")
		}

		if sshProxyConfig.UAATokenURL == "" {
			return nil, errors.New("uaaTokenURL is required for Cloud Foundry authentication")
		}

		_, err = url.Parse(sshProxyConfig.UAATokenURL)
		if err != nil {
			return nil, err
		}

		client, err := helpers.NewHTTPSClient(sshProxyConfig.SkipCertVerify, []string{sshProxyConfig.UAACACert, sshProxyConfig.CCAPICACert}, time.Duration(sshProxyConfig.CommunicationTimeout))
		if err != nil {
			return nil, err
		}

		cfAuthenticator := authenticators.NewCFAuthenticator(
			logger,
			client,
			sshProxyConfig.CCAPIURL,
			sshProxyConfig.UAATokenURL,
			sshProxyConfig.UAAUsername,
			sshProxyConfig.UAAPassword,
			permissionsBuilder,
		)
		authens = append(authens, cfAuthenticator)
	}

	authenticator := authenticators.NewCompositeAuthenticator(authens...)

	sshConfig := &ssh.ServerConfig{
		ServerVersion:    "SSH-2.0-diego-ssh-proxy",
		PasswordCallback: authenticator.Authenticate,
		AuthLogCallback: func(cmd ssh.ConnMetadata, method string, err error) {
			if err != nil {
				logger.Error("authentication-failed", err, lager.Data{"user": cmd.User()})
			} else {
				logger.Info("authentication-attempted", lager.Data{"user": cmd.User()})
			}
		},
	}

	sshConfig.SetDefaults()

	if sshProxyConfig.HostKey == "" {
		err := errors.New("hostKey is required")
		logger.Fatal("host-key-required", err)
	}

	key, err := parsePrivateKey(logger, sshProxyConfig.HostKey)
	if err != nil {
		logger.Fatal("failed-to-parse-host-key", err)
	}

	sshConfig.AddHostKey(key)

	if sshProxyConfig.AllowedCiphers != "" {
		sshConfig.Config.Ciphers = strings.Split(sshProxyConfig.AllowedCiphers, ",")
	} else {
		sshConfig.Config.Ciphers = []string{"chacha20-poly1305@openssh.com", "aes128-gcm@openssh.com", "aes256-ctr", "aes192-ctr", "aes128-ctr"}
	}

	if sshProxyConfig.AllowedMACs != "" {
		sshConfig.Config.MACs = strings.Split(sshProxyConfig.AllowedMACs, ",")
	} else {
		sshConfig.Config.MACs = []string{"hmac-sha2-256-etm@openssh.com", "hmac-sha2-256"}
	}

	if sshProxyConfig.AllowedKeyExchanges != "" {
		sshConfig.Config.KeyExchanges = strings.Split(sshProxyConfig.AllowedKeyExchanges, ",")
	} else {
		sshConfig.Config.KeyExchanges = []string{"curve25519-sha256@libssh.org"}
	}

	return sshConfig, err
}

func parsePrivateKey(logger lager.Logger, encodedKey string) (ssh.Signer, error) {
	key, err := ssh.ParsePrivateKey([]byte(encodedKey))
	if err != nil {
		logger.Error("failed-to-parse-private-key", err)
		return nil, err
	}
	return key, nil
}

func initializeBBSClient(logger lager.Logger, sshProxyConfig config.SSHProxyConfig) bbs.InternalClient {
	bbsClient, err := bbs.NewClientWithConfig(bbs.ClientConfig{
		URL:                    sshProxyConfig.BBSAddress,
		IsTLS:                  true,
		CAFile:                 sshProxyConfig.BBSCACert,
		CertFile:               sshProxyConfig.BBSClientCert,
		KeyFile:                sshProxyConfig.BBSClientKey,
		ClientSessionCacheSize: sshProxyConfig.BBSClientSessionCacheSize,
		MaxIdleConnsPerHost:    sshProxyConfig.BBSMaxIdleConnsPerHost,
		RequestTimeout:         time.Duration(sshProxyConfig.CommunicationTimeout),
	})
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}

func initializeRegistrationRunner(logger lager.Logger, consulClient consuladapter.Client, listenAddress string, clock clock.Clock) ifrit.Runner {
	_, portString, err := net.SplitHostPort(listenAddress)
	if err != nil {
		logger.Fatal("failed-invalid-listen-address", err)
	}
	portNum, err := net.LookupPort("tcp", portString)
	if err != nil {
		logger.Fatal("failed-invalid-listen-port", err)
	}

	registration := &api.AgentServiceRegistration{
		Name: "ssh-proxy",
		Port: portNum,
		Check: &api.AgentServiceCheck{
			TTL: "20s",
		},
	}

	return locket.NewRegistrationRunner(logger, registration, consulClient, locket.RetryInterval, clock)
}

func initializeMetron(logger lager.Logger, locketConfig config.SSHProxyConfig) (loggingclient.IngressClient, error) {
	client, err := loggingclient.NewIngressClient(locketConfig.LoggregatorConfig)
	if err != nil {
		return nil, err
	}

	if locketConfig.LoggregatorConfig.UseV2API {
		emitter := runtimeemitter.NewV1(client)
		go emitter.Run()
	}

	return client, nil
}
<|endoftext|>


<commit_before>// The idx package provides a metadata index for metrics

package idx

import (
	"encoding/hex"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	schema "gopkg.in/raintank/schema.v1"
)

var (
	BothBranchAndLeaf  = errors.New("node can't be both branch and leaf")
	BranchUnderLeaf    = errors.New("can't add branch under leaf")
	errInvalidQuery    = errors.New("invalid query")
	errInvalidIdString = errors.New("invalid ID string")
)

//go:generate msgp
type Node struct {
	Path        string
	Leaf        bool
	Defs        []Archive
	HasChildren bool
}

type Archive struct {
	schema.MetricDefinition
	SchemaId uint16 // index in mdata.schemas (not persisted)
	AggId    uint16 // index in mdata.aggregations (not persisted)
	LastSave uint32 // last time the metricDefinition was saved to a backend store (cassandra)
}

type MetricID struct {
	org int
	key [16]byte
}

func NewMetricIDFromString(s string) (MetricID, error) {
	id := MetricID{}
	err := id.FromString(s)
	return id, err
}

func (id *MetricID) FromString(s string) error {
	splits := strings.Split(s, ".")
	if len(splits) != 2 || len(splits[1]) != 32 {
		return errInvalidIdString
	}

	var err error
	id.org, err = strconv.Atoi(splits[0])
	if err != nil {
		return err
	}

	dst := make([]byte, 16)
	n, err := hex.Decode(dst, []byte(splits[1]))
	if n != 16 {
		return errInvalidIdString
	}
	copy(id.key[:], dst)
	return nil
}

func (id *MetricID) String() string {
	return fmt.Sprintf("%d.%x", id.org, id.key)
}

// used primarily by tests, for convenience
func NewArchiveBare(name string) Archive {
	return Archive{
		MetricDefinition: schema.MetricDefinition{
			Name: name,
		},
	}
}

/*
Currently the index is solely used for supporting Graphite style queries.
So, the index only needs to be able to search by a pattern that matches the
MetricDefinition.Name field. In future we plan to extend the searching
capabilities to include the other fields in the definition.

Note:

* metrictank is a multi-tenant system where different orgs cannot see each
  other's data

* any given metric may appear multiple times, under different organisations

* Each metric path can be mapped to multiple metricDefinitions in the case that
  fields other then the Name vary.  The most common occurrence of this is when
  the Interval at which the metric is being collected has changed.

Interface

* Init()
  This is the initialization step performed at startup. This method should
  block until the index is ready to handle searches.

* Stop():
 This will be called when metrictank is shutting down.

* AddOrUpdate(*schema.MetricData, int32) Archive:
  Every metric received will result in a call to this method to ensure the
  metric has been added to the index. The method is passed the metricData
  payload and the partition id of the metric

* Get(string) (Archive, bool):
  This method should return the MetricDefintion with the passed Id.

* GetPath(string) (Archive, bool) []Archive:
  This method should return the archives under the given path

* List(int) []Archive:
  This method should return all MetricDefinitions for the passed OrgId.  If the
  passed OrgId is "-1", then all metricDefinitions across all organisations
  should be returned.

* Find(int, string, int64) ([]Node, error):
  This method provides searches.  The method is passed an OrgId, a query
  pattern and a unix timestamp. Searches should return all nodes that match for
  the given OrgId and OrgId -1.  The pattern should be handled in the same way
  Graphite would. see https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards
  And the unix stimestamp is used to ignore series that have been stale since
  the timestamp.

* Delete(int, string) ([]Archive, error):
  This method is used for deleting items from the index. The method is passed
  an OrgId and a query pattern.  If the pattern matches a branch node, then
  all leaf nodes on that branch should also be deleted. So if the pattern is
  "*", all items in the index should be deleted.  A copy of all of the
  metricDefinitions deleted are returned.

* Prune(int, time.Time) ([]Archive, error):
  This method should delete all metrics from the index for the passed org where
  the last time the metric was seen is older then the passed timestamp. If the org
  passed is -1, then the all orgs should be examined for stale metrics to be deleted.
  The method returns a list of the metricDefinitions deleted from the index and any
  error encountered.

* Tags(int, string, int64) ([]string, error):
  This method returns a list of all tag keys associated with the metrics of a given
  organization. The return values are filtered by the regex in the second parameter.
  If the third parameter is >0 then only metrics will be accounted of which the
  LastUpdate time is >= the given value.

* TagDetails(int, string, string, int64) map[string]uint64:
  This method returns a list of all values associated with a given tag key in the
  given org. The occurences of each value is counted and the count is referred to by
  the metric names in the returned map.
  If the third parameter is not "" it will be used as a regular expression to filter
  the values before accouting for them.
  If the fourth parameter is > 0 then the metrics will be filtered and only those
  of which the LastUpdate time is >= the from timestamp will be considered while
  the others are being ignored.

* FindByTag(int, []string, int64) ([]string, error):
  This method takes a list of expressions in the format key<operator>value.
  The allowed operators are: =, !=, =~, !=~.
  It returns a slice of metric names that match the given conditions, the
  conditions are logically AND-ed.
  If the third argument is > 0 then the results will be filtered and only those
  where the LastUpdate time is >= from will be returned as results.
*/

type MetricIndex interface {
	Init() error
	Stop()
	AddOrUpdate(*schema.MetricData, int32) Archive
	Get(string) (Archive, bool)
	GetPath(int, string) []Archive
	Delete(int, string) ([]Archive, error)
	Find(int, string, int64) ([]Node, error)
	List(int) []Archive
	Prune(int, time.Time) ([]Archive, error)
	FindByTag(int, []string, int64) ([]string, error)
	Tags(int, string, int64) ([]string, error)
	TagDetails(int, string, string, int64) (map[string]uint64, error)
}
<commit_msg>rework MetricIndex documentation<commit_after>// The idx package provides a metadata index for metrics

package idx

import (
	"encoding/hex"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	schema "gopkg.in/raintank/schema.v1"
)

var (
	BothBranchAndLeaf  = errors.New("node can't be both branch and leaf")
	BranchUnderLeaf    = errors.New("can't add branch under leaf")
	errInvalidQuery    = errors.New("invalid query")
	errInvalidIdString = errors.New("invalid ID string")
)

//go:generate msgp
type Node struct {
	Path        string
	Leaf        bool
	Defs        []Archive
	HasChildren bool
}

type Archive struct {
	schema.MetricDefinition
	SchemaId uint16 // index in mdata.schemas (not persisted)
	AggId    uint16 // index in mdata.aggregations (not persisted)
	LastSave uint32 // last time the metricDefinition was saved to a backend store (cassandra)
}

type MetricID struct {
	org int
	key [16]byte
}

func NewMetricIDFromString(s string) (MetricID, error) {
	id := MetricID{}
	err := id.FromString(s)
	return id, err
}

func (id *MetricID) FromString(s string) error {
	splits := strings.Split(s, ".")
	if len(splits) != 2 || len(splits[1]) != 32 {
		return errInvalidIdString
	}

	var err error
	id.org, err = strconv.Atoi(splits[0])
	if err != nil {
		return err
	}

	dst := make([]byte, 16)
	n, err := hex.Decode(dst, []byte(splits[1]))
	if n != 16 {
		return errInvalidIdString
	}
	copy(id.key[:], dst)
	return nil
}

func (id *MetricID) String() string {
	return fmt.Sprintf("%d.%x", id.org, id.key)
}

// used primarily by tests, for convenience
func NewArchiveBare(name string) Archive {
	return Archive{
		MetricDefinition: schema.MetricDefinition{
			Name: name,
		},
	}
}

// The MetricIndex interface supports Graphite style queries.
// Note:
// * metrictank is a multi-tenant system where different orgs cannot see each
//   other's data, and any given metric name may appear multiple times,
//   under different organisations
//
// * Each metric path can be mapped to multiple metricDefinitions in the case that
//   fields other then the Name vary.  The most common occurrence of this is when
//   the Interval at which the metric is being collected has changed.
type MetricIndex interface {
	// Init initializes the index at startup and
	// blocks until the index is ready for use.
	Init() error

	// Stop shuts down the index.
	Stop()

	// AddOrUpdate makes sure a metric is known in the index,
	// and should be called for every received metric.
	AddOrUpdate(*schema.MetricData, int32) Archive

	// Get returns the archive for the requested id.
	Get(string) (Archive, bool)

	// GetPath returns the archives under the given path.
	GetPath(int, string) []Archive

	// Delete deletes items from the index for the given org and query.
	// If the pattern matches a branch node, then
	// all leaf nodes on that branch are deleted. So if the pattern is
	// "*", all items in the index are deleted.
	// It returns a copy of all of the Archives deleted.
	Delete(int, string) ([]Archive, error)

	// Find searches the index.  The method is passed an OrgId, a query
	// pattern and a unix timestamp. Searches should return all nodes that match for
	// the given OrgId and OrgId -1.  The pattern should be handled in the same way
	// Graphite would. see https://graphite.readthedocs.io/en/latest/render_api.html#paths-and-wildcards
	// And the unix stimestamp is used to ignore series that have been stale since
	// the timestamp.
	Find(int, string, int64) ([]Node, error)

	// List returns all Archives for the passed OrgId, or for all organisations if -1 is provided.
	List(int) []Archive

	// Prune deletes all metrics from the index for the passed org where
	// the last time the metric was seen is older then the passed timestamp. If the org
	// passed is -1, then the all orgs should be examined for stale metrics to be deleted.
	// It returns all Archives deleted and any error encountered.
	Prune(int, time.Time) ([]Archive, error)

	// FindByTag takes a list of expressions in the format key<operator>value.
	// The allowed operators are: =, !=, =~, !=~.
	// It returns a slice of metric names that match the given conditions, the
	// conditions are logically AND-ed.
	// If the third argument is > 0 then the results will be filtered and only those
	// where the LastUpdate time is >= from will be returned as results.
	FindByTag(int, []string, int64) ([]string, error)

	// Tags returns a list of all tag keys associated with the metrics of a given
	// organization. The return values are filtered by the regex in the second parameter.
	// If the third parameter is >0 then only metrics will be accounted of which the
	// LastUpdate time is >= the given value.
	Tags(int, string, int64) ([]string, error)

	// TagDetails returns a list of all values associated with a given tag key in the
	// given org. The occurences of each value is counted and the count is referred to by
	// the metric names in the returned map.
	// If the third parameter is not "" it will be used as a regular expression to filter
	// the values before accounting for them.
	// If the fourth parameter is > 0 then only those metrics of which the LastUpdate
	// time is >= the from timestamp will be included.
	TagDetails(int, string, string, int64) (map[string]uint64, error)
}
<|endoftext|>


<commit_before>package app

import (
	"errors"
	"fmt"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/unit"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/fs"
	"github.com/timeredbull/tsuru/log"
	"github.com/timeredbull/tsuru/repository"
	"labix.org/v2/mgo/bson"
	"launchpad.net/goyaml"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"time"
)

const confSep = "========"

type EnvVar struct {
	Name   string
	Value  string
	Public bool
}

func (e *EnvVar) String() string {
	var value, suffix string
	if e.Public {
		value = e.Value
	} else {
		value = "***"
		suffix = " (private variable)"
	}
	return fmt.Sprintf("%s=%s%s", e.Name, value, suffix)
}

type App struct {
	Env       map[string]EnvVar
	Framework string
	Name      string
	State     string
	Units     []unit.Unit
	Teams     []string
	Logs      []Log
	fsystem   fs.Fs
}

type Log struct {
	Date    time.Time
	Message string
}

type conf struct {
	PreRestart string `yaml:"pre-restart"`
	PosRestart string `yaml:"pos-restart"`
}

func AllApps() ([]App, error) {
	var apps []App
	err := db.Session.Apps().Find(nil).All(&apps)
	return apps, err
}

func (a *App) Get() error {
	return db.Session.Apps().Find(bson.M{"name": a.Name}).One(&a)
}

func (a *App) Create() error {
	a.State = "PENDING"
	err := db.Session.Apps().Insert(a)
	if err != nil {
		return err
	}
	a.Log(fmt.Sprintf("creating app %s", a.Name))
	cmd := exec.Command("juju", "deploy", "--repository=/home/charms", "local:"+a.Framework, a.Name)
	log.Printf("deploying %s with name %s", a.Framework, a.Name)
	out, err := cmd.CombinedOutput()
	a.Log(string(out))
	if err != nil {
		return err
	}
	a.Log(fmt.Sprintf("app %s successfully created", a.Name))
	return nil
}

func (a *App) Destroy() error {
	err := db.Session.Apps().Remove(bson.M{"name": a.Name})
	if err != nil {
		return err
	}
	go func(a *App) {
		p, _ := repository.GetBarePath(a.Name)
		a.fs().RemoveAll(p)
	}(a)
	u := a.unit()
	cmd := exec.Command("juju", "destroy-service", a.Name)
	log.Printf("destroying %s with name %s", a.Framework, a.Name)
	out, err := cmd.CombinedOutput()
	log.Printf(string(out))
	if err != nil {
		return err
	}
	cmd = exec.Command("juju", "terminate-machine", strconv.Itoa(u.Machine))
	log.Printf("terminating machine %d", u.Machine)
	out, err = cmd.CombinedOutput()
	log.Printf(string(out))
	if err != nil {
		return err
	}
	return nil
}

func (a *App) AddOrUpdateUnit(u *unit.Unit) {
	for i, unt := range a.Units {
		if unt.InstanceId == u.InstanceId {
			a.Units[i].Ip = u.Ip
			a.Units[i].AgentState = u.AgentState
			a.Units[i].InstanceState = u.InstanceState
			return
		}
	}
	a.Units = append(a.Units, *u)
}

func (a *App) fs() fs.Fs {
	if a.fsystem == nil {
		a.fsystem = fs.OsFs{}
	}
	return a.fsystem
}

func (a *App) findTeam(team *auth.Team) int {
	for i, t := range a.Teams {
		if t == team.Name {
			return i
		}
	}
	return -1
}

func (a *App) hasTeam(team *auth.Team) bool {
	return a.findTeam(team) > -1
}

func (a *App) GrantAccess(team *auth.Team) error {
	if a.hasTeam(team) {
		return errors.New("This team has already access to this app")
	}
	a.Teams = append(a.Teams, team.Name)
	return nil
}

func (a *App) RevokeAccess(team *auth.Team) error {
	index := a.findTeam(team)
	if index < 0 {
		return errors.New("This team does not have access to this app")
	}
	last := len(a.Teams) - 1
	a.Teams[index] = a.Teams[last]
	a.Teams = a.Teams[:last]
	return nil
}

func (a *App) GetTeams() (teams []auth.Team) {
	db.Session.Teams().Find(bson.M{"name": bson.M{"$in": a.Teams}}).All(&teams)
	return
}

func (a *App) setTeams(teams []auth.Team) {
	a.Teams = make([]string, len(teams))
	for i, team := range teams {
		a.Teams[i] = team.Name
	}
}

func (a *App) CheckUserAccess(user *auth.User) bool {
	for _, team := range a.GetTeams() {
		if team.ContainsUser(user) {
			return true
		}
	}
	return false
}

func (a *App) SetEnv(name, value string) {
	if a.Env == nil {
		a.Env = make(map[string]EnvVar)
	}
	env := EnvVar{
		Name:   name,
		Value:  value,
		Public: true,
	}
	a.Env[name] = env
	a.Log(fmt.Sprintf("setting env %s with value %s", name, value))
}

func (a *App) GetEnv(name string) (env EnvVar, err error) {
	var ok bool
	if env, ok = a.Env[name]; !ok {
		err = errors.New("Environment variable not declared for this app.")
	}
	return
}

func deployHookAbsPath(p string) (string, error) {
	repoPath, err := config.GetString("git:unit-repo")
	if err != nil {
		return "", nil
	}
	return path.Join(repoPath, p), nil
}

/*
* Returns app.conf located at app's git repository
 */
func (a *App) conf() (conf, error) {
	var c conf
	u := a.unit()
	uRepo, err := repository.GetPath()
	if err != nil {
		a.Log(fmt.Sprintf("Got error while getting repository path: %s", err.Error()))
		return c, err
	}
	cPath := path.Join(uRepo, "app.conf")
	cmd := fmt.Sprintf(`echo "%s";cat %s`, confSep, cPath)
	o, err := u.Command(cmd)
	if err != nil {
		a.Log(fmt.Sprintf("Got error while executing command: %s... Skipping hooks execution", err.Error()))
		return c, nil
	}
	data := strings.Split(string(o), confSep)[1]
	err = goyaml.Unmarshal([]byte(data), &c)
	if err != nil {
		a.Log(fmt.Sprintf("Got error while parsing yaml: %s", err.Error()))
		return c, err
	}
	return c, nil
}

/*
* preRestart is responsible for running user's pre-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) preRestart(c conf) error {
	if !a.hasRestartHooks(c) {
		a.Log("app.conf file does not exists or is in the right place. Skipping...")
		return nil
	}
	if c.PreRestart == "" {
		a.Log("pre-restart hook section in app conf does not exists... Skipping...")
		return nil
	}
	u := a.unit()
	p, err := deployHookAbsPath(c.PreRestart)
	if err != nil {
		a.Log(fmt.Sprintf("Error obtaining absolute path to hook: %s. Skipping...", err))
		return nil
	}
	out, err := u.Command("/bin/bash", p)
	a.Log("Executing pre-restart hook...")
	a.Log(fmt.Sprintf("Output of pre-restart hook: %s", string(out)))
	return err
}

/*
* posRestart is responsible for running user's pos-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) posRestart(c conf) error {
	if !a.hasRestartHooks(c) {
		a.Log("app.conf file does not exists or is in the right place. Skipping...")
		return nil
	}
	if c.PosRestart == "" {
		a.Log("pos-restart hook section in app conf does not exists... Skipping...")
		return nil
	}
	u := a.unit()
	p, err := deployHookAbsPath(c.PosRestart)
	if err != nil {
		a.Log(fmt.Sprintf("Error obtaining absolute path to hook: %s. Skipping...", err))
		return nil
	}
	out, err := u.Command("/bin/bash", p)
	a.Log("Executing pos-restart hook...")
	a.Log(fmt.Sprintf("Output of pos-restart hook: %s", string(out)))
	return err
}

func (a *App) hasRestartHooks(c conf) bool {
	return !(c.PreRestart == "" && c.PosRestart == "")
}

func (a *App) updateHooks() error {
	u := a.unit()
	a.Log("executting hook dependencies")
	out, err := u.ExecuteHook("dependencies")
	a.Log(string(out))
	if err != nil {
		return err
	}
	a.Log("executting hook reload-gunicorn")
	out, err = u.ExecuteHook("reload-gunicorn")
	a.Log(string(out))
	if err != nil {
		return err
	}
	return nil
}

func (a *App) unit() unit.Unit {
	if len(a.Units) > 0 {
		return a.Units[0]
	}
	return unit.Unit{}
}

func (a *App) Log(message string) error {
	log.Printf(message)
	l := Log{Date: time.Now(), Message: message}
	a.Logs = append(a.Logs, l)
	return db.Session.Apps().Update(bson.M{"name": a.Name}, a)
}

// GetApps returns all apps that the given team has access to
func GetApps(team *auth.Team) (apps []App, err error) {
	if team == nil {
		err = errors.New("You must provide the team.")
		return
	}
	err = db.Session.Apps().Find(bson.M{"teams": team.Name}).All(&apps)
	return
}
<commit_msg>api/app: added ServiceName to the EnvVar struct<commit_after>package app

import (
	"errors"
	"fmt"
	"github.com/timeredbull/tsuru/api/auth"
	"github.com/timeredbull/tsuru/api/unit"
	"github.com/timeredbull/tsuru/config"
	"github.com/timeredbull/tsuru/db"
	"github.com/timeredbull/tsuru/fs"
	"github.com/timeredbull/tsuru/log"
	"github.com/timeredbull/tsuru/repository"
	"labix.org/v2/mgo/bson"
	"launchpad.net/goyaml"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"time"
)

const confSep = "========"

type EnvVar struct {
	Name        string
	Value       string
	Public      bool
	ServiceName string
}

func (e *EnvVar) String() string {
	var value, suffix string
	if e.Public {
		value = e.Value
	} else {
		value = "***"
		suffix = " (private variable)"
	}
	return fmt.Sprintf("%s=%s%s", e.Name, value, suffix)
}

type App struct {
	Env       map[string]EnvVar
	Framework string
	Name      string
	State     string
	Units     []unit.Unit
	Teams     []string
	Logs      []Log
	fsystem   fs.Fs
}

type Log struct {
	Date    time.Time
	Message string
}

type conf struct {
	PreRestart string `yaml:"pre-restart"`
	PosRestart string `yaml:"pos-restart"`
}

func AllApps() ([]App, error) {
	var apps []App
	err := db.Session.Apps().Find(nil).All(&apps)
	return apps, err
}

func (a *App) Get() error {
	return db.Session.Apps().Find(bson.M{"name": a.Name}).One(&a)
}

func (a *App) Create() error {
	a.State = "PENDING"
	err := db.Session.Apps().Insert(a)
	if err != nil {
		return err
	}
	a.Log(fmt.Sprintf("creating app %s", a.Name))
	cmd := exec.Command("juju", "deploy", "--repository=/home/charms", "local:"+a.Framework, a.Name)
	log.Printf("deploying %s with name %s", a.Framework, a.Name)
	out, err := cmd.CombinedOutput()
	a.Log(string(out))
	if err != nil {
		return err
	}
	a.Log(fmt.Sprintf("app %s successfully created", a.Name))
	return nil
}

func (a *App) Destroy() error {
	err := db.Session.Apps().Remove(bson.M{"name": a.Name})
	if err != nil {
		return err
	}
	go func(a *App) {
		p, _ := repository.GetBarePath(a.Name)
		a.fs().RemoveAll(p)
	}(a)
	u := a.unit()
	cmd := exec.Command("juju", "destroy-service", a.Name)
	log.Printf("destroying %s with name %s", a.Framework, a.Name)
	out, err := cmd.CombinedOutput()
	log.Printf(string(out))
	if err != nil {
		return err
	}
	cmd = exec.Command("juju", "terminate-machine", strconv.Itoa(u.Machine))
	log.Printf("terminating machine %d", u.Machine)
	out, err = cmd.CombinedOutput()
	log.Printf(string(out))
	if err != nil {
		return err
	}
	return nil
}

func (a *App) AddOrUpdateUnit(u *unit.Unit) {
	for i, unt := range a.Units {
		if unt.InstanceId == u.InstanceId {
			a.Units[i].Ip = u.Ip
			a.Units[i].AgentState = u.AgentState
			a.Units[i].InstanceState = u.InstanceState
			return
		}
	}
	a.Units = append(a.Units, *u)
}

func (a *App) fs() fs.Fs {
	if a.fsystem == nil {
		a.fsystem = fs.OsFs{}
	}
	return a.fsystem
}

func (a *App) findTeam(team *auth.Team) int {
	for i, t := range a.Teams {
		if t == team.Name {
			return i
		}
	}
	return -1
}

func (a *App) hasTeam(team *auth.Team) bool {
	return a.findTeam(team) > -1
}

func (a *App) GrantAccess(team *auth.Team) error {
	if a.hasTeam(team) {
		return errors.New("This team has already access to this app")
	}
	a.Teams = append(a.Teams, team.Name)
	return nil
}

func (a *App) RevokeAccess(team *auth.Team) error {
	index := a.findTeam(team)
	if index < 0 {
		return errors.New("This team does not have access to this app")
	}
	last := len(a.Teams) - 1
	a.Teams[index] = a.Teams[last]
	a.Teams = a.Teams[:last]
	return nil
}

func (a *App) GetTeams() (teams []auth.Team) {
	db.Session.Teams().Find(bson.M{"name": bson.M{"$in": a.Teams}}).All(&teams)
	return
}

func (a *App) setTeams(teams []auth.Team) {
	a.Teams = make([]string, len(teams))
	for i, team := range teams {
		a.Teams[i] = team.Name
	}
}

func (a *App) CheckUserAccess(user *auth.User) bool {
	for _, team := range a.GetTeams() {
		if team.ContainsUser(user) {
			return true
		}
	}
	return false
}

func (a *App) SetEnv(name, value string) {
	if a.Env == nil {
		a.Env = make(map[string]EnvVar)
	}
	env := EnvVar{
		Name:   name,
		Value:  value,
		Public: true,
	}
	a.Env[name] = env
	a.Log(fmt.Sprintf("setting env %s with value %s", name, value))
}

func (a *App) GetEnv(name string) (env EnvVar, err error) {
	var ok bool
	if env, ok = a.Env[name]; !ok {
		err = errors.New("Environment variable not declared for this app.")
	}
	return
}

func deployHookAbsPath(p string) (string, error) {
	repoPath, err := config.GetString("git:unit-repo")
	if err != nil {
		return "", nil
	}
	return path.Join(repoPath, p), nil
}

/*
* Returns app.conf located at app's git repository
 */
func (a *App) conf() (conf, error) {
	var c conf
	u := a.unit()
	uRepo, err := repository.GetPath()
	if err != nil {
		a.Log(fmt.Sprintf("Got error while getting repository path: %s", err.Error()))
		return c, err
	}
	cPath := path.Join(uRepo, "app.conf")
	cmd := fmt.Sprintf(`echo "%s";cat %s`, confSep, cPath)
	o, err := u.Command(cmd)
	if err != nil {
		a.Log(fmt.Sprintf("Got error while executing command: %s... Skipping hooks execution", err.Error()))
		return c, nil
	}
	data := strings.Split(string(o), confSep)[1]
	err = goyaml.Unmarshal([]byte(data), &c)
	if err != nil {
		a.Log(fmt.Sprintf("Got error while parsing yaml: %s", err.Error()))
		return c, err
	}
	return c, nil
}

/*
* preRestart is responsible for running user's pre-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) preRestart(c conf) error {
	if !a.hasRestartHooks(c) {
		a.Log("app.conf file does not exists or is in the right place. Skipping...")
		return nil
	}
	if c.PreRestart == "" {
		a.Log("pre-restart hook section in app conf does not exists... Skipping...")
		return nil
	}
	u := a.unit()
	p, err := deployHookAbsPath(c.PreRestart)
	if err != nil {
		a.Log(fmt.Sprintf("Error obtaining absolute path to hook: %s. Skipping...", err))
		return nil
	}
	out, err := u.Command("/bin/bash", p)
	a.Log("Executing pre-restart hook...")
	a.Log(fmt.Sprintf("Output of pre-restart hook: %s", string(out)))
	return err
}

/*
* posRestart is responsible for running user's pos-restart script.
* The path to this script can be found at the app.conf file, at the root of user's app repository.
 */
func (a *App) posRestart(c conf) error {
	if !a.hasRestartHooks(c) {
		a.Log("app.conf file does not exists or is in the right place. Skipping...")
		return nil
	}
	if c.PosRestart == "" {
		a.Log("pos-restart hook section in app conf does not exists... Skipping...")
		return nil
	}
	u := a.unit()
	p, err := deployHookAbsPath(c.PosRestart)
	if err != nil {
		a.Log(fmt.Sprintf("Error obtaining absolute path to hook: %s. Skipping...", err))
		return nil
	}
	out, err := u.Command("/bin/bash", p)
	a.Log("Executing pos-restart hook...")
	a.Log(fmt.Sprintf("Output of pos-restart hook: %s", string(out)))
	return err
}

func (a *App) hasRestartHooks(c conf) bool {
	return !(c.PreRestart == "" && c.PosRestart == "")
}

func (a *App) updateHooks() error {
	u := a.unit()
	a.Log("executting hook dependencies")
	out, err := u.ExecuteHook("dependencies")
	a.Log(string(out))
	if err != nil {
		return err
	}
	a.Log("executting hook reload-gunicorn")
	out, err = u.ExecuteHook("reload-gunicorn")
	a.Log(string(out))
	if err != nil {
		return err
	}
	return nil
}

func (a *App) unit() unit.Unit {
	if len(a.Units) > 0 {
		return a.Units[0]
	}
	return unit.Unit{}
}

func (a *App) Log(message string) error {
	log.Printf(message)
	l := Log{Date: time.Now(), Message: message}
	a.Logs = append(a.Logs, l)
	return db.Session.Apps().Update(bson.M{"name": a.Name}, a)
}

// GetApps returns all apps that the given team has access to
func GetApps(team *auth.Team) (apps []App, err error) {
	if team == nil {
		err = errors.New("You must provide the team.")
		return
	}
	err = db.Session.Apps().Find(bson.M{"teams": team.Name}).All(&apps)
	return
}
<|endoftext|>


<commit_before>package api

import (
	"context"

	"github.com/manifoldco/torus-cli/apitypes"
)

type UpdatesClient struct {
	client *Client
}

func (c *UpdatesClient) Check(ctx context.Context) (*apitypes.Updates, error) {
	req, _, err := c.client.NewRequest("GET", "/updates", nil, nil, false)
	if err != nil {
		return nil, err
	}

	var needsUpdate apitypes.Updates
	_, err = c.client.Do(ctx, req, &needsUpdate, nil, nil)
	return &needsUpdate, nil
}
<commit_msg>return error if actual call fails<commit_after>package api

import (
	"context"

	"github.com/manifoldco/torus-cli/apitypes"
)

type UpdatesClient struct {
	client *Client
}

func (c *UpdatesClient) Check(ctx context.Context) (*apitypes.Updates, error) {
	req, _, err := c.client.NewRequest("GET", "/updates", nil, nil, false)
	if err != nil {
		return nil, err
	}

	var needsUpdate apitypes.Updates
	if _, err := c.client.Do(ctx, req, &needsUpdate, nil, nil); err != nil {
		return nil, err
	}

	return &needsUpdate, nil
}
<|endoftext|>