<commit_before>package main

import (
	"bytes"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"github.com/lightningnetwork/lnd/lncfg"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/walletunlocker"
	"github.com/urfave/cli"
	"gopkg.in/macaroon.v2"
)

var (
	errNoProfileFile = errors.New("no profile file found")
)

// profileEntry is a struct that represents all settings for one specific
// profile.
type profileEntry struct {
	Name        string       `json:"name"`
	RPCServer   string       `json:"rpcserver"`
	LndDir      string       `json:"lnddir"`
	Chain       string       `json:"chain"`
	Network     string       `json:"network"`
	NoMacaroons bool         `json:"no-macaroons,omitempty"`
	TLSCert     string       `json:"tlscert"`
	Macaroons   *macaroonJar `json:"macaroons"`
}

// cert returns the profile's TLS certificate as a x509 certificate pool.
func (e *profileEntry) cert() (*x509.CertPool, error) {
	if e.TLSCert == "" {
		return nil, nil
	}

	cp := x509.NewCertPool()
	if !cp.AppendCertsFromPEM([]byte(e.TLSCert)) {
		return nil, fmt.Errorf("credentials: failed to append " +
			"certificate")
	}
	return cp, nil
}

// getGlobalOptions returns the global connection options. If a profile file
// exists, these global options might be read from a predefined profile. If no
// profile exists, the global options from the command line are returned as an
// ephemeral profile entry.
func getGlobalOptions(ctx *cli.Context, skipMacaroons bool) (*profileEntry,
	error) {

	var profileName string

	// Try to load the default profile file and depending on its existence
	// what profile to use.
	f, err := loadProfileFile(defaultProfileFile)
	switch {
	// The legacy case where no profile file exists and the user also didn't
	// request to use one. We only consider the global options here.
	case err == errNoProfileFile && !ctx.GlobalIsSet("profile"):
		return profileFromContext(ctx, false, skipMacaroons)

	// The file doesn't exist but the user specified an explicit profile.
	case err == errNoProfileFile && ctx.GlobalIsSet("profile"):
		return nil, fmt.Errorf("profile file %s does not exist",
			defaultProfileFile)

	// There is a file but we couldn't read/parse it.
	case err != nil:
		return nil, fmt.Errorf("could not read profile file %s: "+
			"%v", defaultProfileFile, err)

	// The user explicitly disabled the use of profiles for this command by
	// setting the flag to an empty string. We fall back to the default/old
	// behavior.
	case ctx.GlobalIsSet("profile") && ctx.GlobalString("profile") == "":
		return profileFromContext(ctx, false, skipMacaroons)

	// There is a file, but no default profile is specified. The user also
	// didn't specify a profile to use so we fall back to the default/old
	// behavior.
	case !ctx.GlobalIsSet("profile") && len(f.Default) == 0:
		return profileFromContext(ctx, false, skipMacaroons)

	// The user didn't specify a profile but there is a default one defined.
	case !ctx.GlobalIsSet("profile") && len(f.Default) > 0:
		profileName = f.Default

	// The user specified a specific profile to use.
	case ctx.GlobalIsSet("profile"):
		profileName = ctx.GlobalString("profile")
	}

	// If we got to here, we do have a profile file and know the name of the
	// profile to use. Now we just need to make sure it does exist.
	for _, prof := range f.Profiles {
		if prof.Name == profileName {
			return prof, nil
		}
	}

	return nil, fmt.Errorf("profile '%s' not found in file %s", profileName,
		defaultProfileFile)
}

// profileFromContext creates an ephemeral profile entry from the global options
// set in the CLI context.
func profileFromContext(ctx *cli.Context, store, skipMacaroons bool) (
	*profileEntry, error) {

	// Parse the paths of the cert and macaroon. This will validate the
	// chain and network value as well.
	tlsCertPath, macPath, err := extractPathArgs(ctx)
	if err != nil {
		return nil, err
	}

	// Load the certificate file now, if specified. We store it as plain PEM
	// directly.
	var tlsCert []byte
	if lnrpc.FileExists(tlsCertPath) {
		var err error
		tlsCert, err = ioutil.ReadFile(tlsCertPath)
		if err != nil {
			return nil, fmt.Errorf("could not load TLS cert file "+
				"%s: %v", tlsCertPath, err)
		}
	}

	entry := &profileEntry{
		RPCServer:   ctx.GlobalString("rpcserver"),
		LndDir:      lncfg.CleanAndExpandPath(ctx.GlobalString("lnddir")),
		Chain:       ctx.GlobalString("chain"),
		Network:     ctx.GlobalString("network"),
		NoMacaroons: ctx.GlobalBool("no-macaroons"),
		TLSCert:     string(tlsCert),
	}

	// If we aren't using macaroons in general (flag --no-macaroons) or
	// don't need macaroons for this command (wallet unlocker), we can now
	// return already.
	if skipMacaroons || ctx.GlobalBool("no-macaroons") {
		return entry, nil
	}

	// Now load and possibly encrypt the macaroon file.
	macBytes, err := ioutil.ReadFile(macPath)
	if err != nil {
		return nil, fmt.Errorf("unable to read macaroon path (check "+
			"the network setting!): %v", err)
	}
	mac := &macaroon.Macaroon{}
	if err = mac.UnmarshalBinary(macBytes); err != nil {
		return nil, fmt.Errorf("unable to decode macaroon: %v", err)
	}

	var pw []byte
	if store {
		// Read a password from the terminal. If it's empty, we won't
		// encrypt the macaroon and store it plaintext.
		pw, err = capturePassword(
			"Enter password to encrypt macaroon with or leave "+
				"blank to store in plaintext: ", true,
			walletunlocker.ValidatePassword,
		)
		if err != nil {
			return nil, fmt.Errorf("unable to get encryption "+
				"password: %v", err)
		}
	}
	macEntry := &macaroonEntry{}
	if err = macEntry.storeMacaroon(mac, pw); err != nil {
		return nil, fmt.Errorf("unable to store macaroon: %v", err)
	}

	// We determine the name of the macaroon from the file itself but cut
	// off the ".macaroon" at the end.
	macEntry.Name = path.Base(macPath)
	if path.Ext(macEntry.Name) == "macaroon" {
		macEntry.Name = strings.TrimSuffix(macEntry.Name, ".macaroon")
	}

	// Now that we have the macaroon jar as well, let's return the entry
	// with all the values populated.
	entry.Macaroons = &macaroonJar{
		Default: macEntry.Name,
		Timeout: ctx.GlobalInt64("macaroontimeout"),
		IP:      ctx.GlobalString("macaroonip"),
		Jar:     []*macaroonEntry{macEntry},
	}

	return entry, nil
}

// loadProfileFile tries to load the file specified and JSON deserialize it into
// the profile file struct.
func loadProfileFile(file string) (*profileFile, error) {
	if !lnrpc.FileExists(file) {
		return nil, errNoProfileFile
	}

	content, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("could not load profile file %s: %v",
			file, err)
	}
	f := &profileFile{}
	err = f.unmarshalJSON(content)
	if err != nil {
		return nil, fmt.Errorf("could not unmarshal profile file %s: "+
			"%v", file, err)
	}
	return f, nil
}

// saveProfileFile stores the given profile file struct in the specified file,
// overwriting it if it already existed.
func saveProfileFile(file string, f *profileFile) error {
	content, err := f.marshalJSON()
	if err != nil {
		return fmt.Errorf("could not marshal profile: %v", err)
	}
	return ioutil.WriteFile(file, content, 0644)
}

// profileFile is a struct that represents the whole content of a profile file.
type profileFile struct {
	Default  string          `json:"default,omitempty"`
	Profiles []*profileEntry `json:"profiles"`
}

// unmarshalJSON tries to parse the given JSON and unmarshal it into the
// receiving instance.
func (f *profileFile) unmarshalJSON(content []byte) error {
	return json.Unmarshal(content, f)
}

// marshalJSON serializes the receiving instance to formatted/indented JSON.
func (f *profileFile) marshalJSON() ([]byte, error) {
	b, err := json.Marshal(f)
	if err != nil {
		return nil, fmt.Errorf("error JSON marshalling profile: %v",
			err)
	}

	var out bytes.Buffer
	err = json.Indent(&out, b, "", "  ")
	if err != nil {
		return nil, fmt.Errorf("error indenting profile JSON: %v", err)
	}
	out.WriteString("\n")
	return out.Bytes(), nil
}
<commit_msg>lncli: check for existence of tls file (#5416)<commit_after>package main

import (
	"bytes"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"github.com/lightningnetwork/lnd/lncfg"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/walletunlocker"
	"github.com/urfave/cli"
	"gopkg.in/macaroon.v2"
)

var (
	errNoProfileFile = errors.New("no profile file found")
)

// profileEntry is a struct that represents all settings for one specific
// profile.
type profileEntry struct {
	Name        string       `json:"name"`
	RPCServer   string       `json:"rpcserver"`
	LndDir      string       `json:"lnddir"`
	Chain       string       `json:"chain"`
	Network     string       `json:"network"`
	NoMacaroons bool         `json:"no-macaroons,omitempty"`
	TLSCert     string       `json:"tlscert"`
	Macaroons   *macaroonJar `json:"macaroons"`
}

// cert returns the profile's TLS certificate as a x509 certificate pool.
func (e *profileEntry) cert() (*x509.CertPool, error) {
	if e.TLSCert == "" {
		return nil, nil
	}

	cp := x509.NewCertPool()
	if !cp.AppendCertsFromPEM([]byte(e.TLSCert)) {
		return nil, fmt.Errorf("credentials: failed to append " +
			"certificate")
	}
	return cp, nil
}

// getGlobalOptions returns the global connection options. If a profile file
// exists, these global options might be read from a predefined profile. If no
// profile exists, the global options from the command line are returned as an
// ephemeral profile entry.
func getGlobalOptions(ctx *cli.Context, skipMacaroons bool) (*profileEntry,
	error) {

	var profileName string

	// Try to load the default profile file and depending on its existence
	// what profile to use.
	f, err := loadProfileFile(defaultProfileFile)
	switch {
	// The legacy case where no profile file exists and the user also didn't
	// request to use one. We only consider the global options here.
	case err == errNoProfileFile && !ctx.GlobalIsSet("profile"):
		return profileFromContext(ctx, false, skipMacaroons)

	// The file doesn't exist but the user specified an explicit profile.
	case err == errNoProfileFile && ctx.GlobalIsSet("profile"):
		return nil, fmt.Errorf("profile file %s does not exist",
			defaultProfileFile)

	// There is a file but we couldn't read/parse it.
	case err != nil:
		return nil, fmt.Errorf("could not read profile file %s: "+
			"%v", defaultProfileFile, err)

	// The user explicitly disabled the use of profiles for this command by
	// setting the flag to an empty string. We fall back to the default/old
	// behavior.
	case ctx.GlobalIsSet("profile") && ctx.GlobalString("profile") == "":
		return profileFromContext(ctx, false, skipMacaroons)

	// There is a file, but no default profile is specified. The user also
	// didn't specify a profile to use so we fall back to the default/old
	// behavior.
	case !ctx.GlobalIsSet("profile") && len(f.Default) == 0:
		return profileFromContext(ctx, false, skipMacaroons)

	// The user didn't specify a profile but there is a default one defined.
	case !ctx.GlobalIsSet("profile") && len(f.Default) > 0:
		profileName = f.Default

	// The user specified a specific profile to use.
	case ctx.GlobalIsSet("profile"):
		profileName = ctx.GlobalString("profile")
	}

	// If we got to here, we do have a profile file and know the name of the
	// profile to use. Now we just need to make sure it does exist.
	for _, prof := range f.Profiles {
		if prof.Name == profileName {
			return prof, nil
		}
	}

	return nil, fmt.Errorf("profile '%s' not found in file %s", profileName,
		defaultProfileFile)
}

// profileFromContext creates an ephemeral profile entry from the global options
// set in the CLI context.
func profileFromContext(ctx *cli.Context, store, skipMacaroons bool) (
	*profileEntry, error) {

	// Parse the paths of the cert and macaroon. This will validate the
	// chain and network value as well.
	tlsCertPath, macPath, err := extractPathArgs(ctx)
	if err != nil {
		return nil, err
	}

	// Load the certificate file now, if specified. We store it as plain PEM
	// directly.
	var tlsCert []byte
	if tlsCertPath != "" {
		var err error
		tlsCert, err = ioutil.ReadFile(tlsCertPath)
		if err != nil {
			return nil, fmt.Errorf("could not load TLS cert file: %v", err)
		}
	}

	entry := &profileEntry{
		RPCServer:   ctx.GlobalString("rpcserver"),
		LndDir:      lncfg.CleanAndExpandPath(ctx.GlobalString("lnddir")),
		Chain:       ctx.GlobalString("chain"),
		Network:     ctx.GlobalString("network"),
		NoMacaroons: ctx.GlobalBool("no-macaroons"),
		TLSCert:     string(tlsCert),
	}

	// If we aren't using macaroons in general (flag --no-macaroons) or
	// don't need macaroons for this command (wallet unlocker), we can now
	// return already.
	if skipMacaroons || ctx.GlobalBool("no-macaroons") {
		return entry, nil
	}

	// Now load and possibly encrypt the macaroon file.
	macBytes, err := ioutil.ReadFile(macPath)
	if err != nil {
		return nil, fmt.Errorf("unable to read macaroon path (check "+
			"the network setting!): %v", err)
	}
	mac := &macaroon.Macaroon{}
	if err = mac.UnmarshalBinary(macBytes); err != nil {
		return nil, fmt.Errorf("unable to decode macaroon: %v", err)
	}

	var pw []byte
	if store {
		// Read a password from the terminal. If it's empty, we won't
		// encrypt the macaroon and store it plaintext.
		pw, err = capturePassword(
			"Enter password to encrypt macaroon with or leave "+
				"blank to store in plaintext: ", true,
			walletunlocker.ValidatePassword,
		)
		if err != nil {
			return nil, fmt.Errorf("unable to get encryption "+
				"password: %v", err)
		}
	}
	macEntry := &macaroonEntry{}
	if err = macEntry.storeMacaroon(mac, pw); err != nil {
		return nil, fmt.Errorf("unable to store macaroon: %v", err)
	}

	// We determine the name of the macaroon from the file itself but cut
	// off the ".macaroon" at the end.
	macEntry.Name = path.Base(macPath)
	if path.Ext(macEntry.Name) == "macaroon" {
		macEntry.Name = strings.TrimSuffix(macEntry.Name, ".macaroon")
	}

	// Now that we have the macaroon jar as well, let's return the entry
	// with all the values populated.
	entry.Macaroons = &macaroonJar{
		Default: macEntry.Name,
		Timeout: ctx.GlobalInt64("macaroontimeout"),
		IP:      ctx.GlobalString("macaroonip"),
		Jar:     []*macaroonEntry{macEntry},
	}

	return entry, nil
}

// loadProfileFile tries to load the file specified and JSON deserialize it into
// the profile file struct.
func loadProfileFile(file string) (*profileFile, error) {
	if !lnrpc.FileExists(file) {
		return nil, errNoProfileFile
	}

	content, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("could not load profile file %s: %v",
			file, err)
	}
	f := &profileFile{}
	err = f.unmarshalJSON(content)
	if err != nil {
		return nil, fmt.Errorf("could not unmarshal profile file %s: "+
			"%v", file, err)
	}
	return f, nil
}

// saveProfileFile stores the given profile file struct in the specified file,
// overwriting it if it already existed.
func saveProfileFile(file string, f *profileFile) error {
	content, err := f.marshalJSON()
	if err != nil {
		return fmt.Errorf("could not marshal profile: %v", err)
	}
	return ioutil.WriteFile(file, content, 0644)
}

// profileFile is a struct that represents the whole content of a profile file.
type profileFile struct {
	Default  string          `json:"default,omitempty"`
	Profiles []*profileEntry `json:"profiles"`
}

// unmarshalJSON tries to parse the given JSON and unmarshal it into the
// receiving instance.
func (f *profileFile) unmarshalJSON(content []byte) error {
	return json.Unmarshal(content, f)
}

// marshalJSON serializes the receiving instance to formatted/indented JSON.
func (f *profileFile) marshalJSON() ([]byte, error) {
	b, err := json.Marshal(f)
	if err != nil {
		return nil, fmt.Errorf("error JSON marshalling profile: %v",
			err)
	}

	var out bytes.Buffer
	err = json.Indent(&out, b, "", "  ")
	if err != nil {
		return nil, fmt.Errorf("error indenting profile JSON: %v", err)
	}
	out.WriteString("\n")
	return out.Bytes(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.200"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.167", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.201 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.201"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.168", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.114"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.81", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.115 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.115"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.82", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.202"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.169", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.203 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.203"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.170", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"net/http"
	"time"

	"github.com/cloudfoundry-incubator/auction/auctiontypes"
	"github.com/cloudfoundry-incubator/auction/communication/http/auction_http_client"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/rep/cmd/rep/testrunner"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/pivotal-golang/lager/lagertest"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
)

var runner *testrunner.Runner

var _ = Describe("The Rep", func() {
	var (
		fakeExecutor    *ghttp.Server
		bbs             *Bbs.BBS
		pollingInterval time.Duration
	)

	BeforeEach(func() {
		fakeExecutor = ghttp.NewServer()
		// these tests only look for the start of a sequence of requests
		fakeExecutor.AllowUnhandledRequests = true
		fakeExecutor.RouteToHandler("GET", "/ping", ghttp.RespondWith(http.StatusOK, nil))

		bbs = Bbs.NewBBS(etcdRunner.Adapter(), timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		pollingInterval = 50 * time.Millisecond

		runner = testrunner.New(
			representativePath,
			cellID,
			"the-stack",
			"the-lrp-host",
			fakeExecutor.URL(),
			fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"info",
			auctionServerPort,
			time.Second,
			pollingInterval,
		)

		runner.Start()
	})

	AfterEach(func(done Done) {
		runner.KillWithFire()
		fakeExecutor.Close()
		close(done)
	})

	claimActual := func(a models.ActualLRP) *models.ActualLRP {
		_, err := bbs.CreateActualLRP(a)
		Ω(err).ShouldNot(HaveOccurred())
		claimed, err := bbs.ClaimActualLRP(a)
		return claimed
	}

	Describe("when an interrupt signal is sent to the representative", func() {
		BeforeEach(func() {
			runner.Stop()
		})

		It("should die", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(0))
		})
	})

	Describe("maintaining presence", func() {
		var cellPresence models.CellPresence

		BeforeEach(func() {
			Eventually(bbs.Cells).Should(HaveLen(1))
			cells, err := bbs.Cells()
			Ω(err).ShouldNot(HaveOccurred())
			cellPresence = cells[0]
		})

		It("should maintain presence", func() {
			Ω(cellPresence.Stack).Should(Equal("the-stack"))
			Ω(cellPresence.CellID).Should(Equal(cellID))
		})

		Context("when the presence fails to be maintained", func() {
			It("should not exit, but keep trying to maintain presence at the same ID", func() {
				etcdRunner.Stop()
				etcdRunner.Start()

				Eventually(bbs.Cells, 5).Should(HaveLen(1))
				cells, err := bbs.Cells()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(cells[0]).Should(Equal(cellPresence))

				Ω(runner.Session).ShouldNot(gexec.Exit())
			})
		})
	})

	Describe("when a task is written to the BBS", func() {
		BeforeEach(func() {
			fakeExecutor.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/containers/the-task-guid"),
				ghttp.RespondWith(http.StatusCreated, `{"executor_guid":"executor-guid","guid":"guid-123"}`)),
			)

			bbs.DesireTask(models.Task{
				TaskGuid: "the-task-guid",
				Stack:    "the-stack",
				Action: &models.RunAction{
					Path: "cat",
					Args: []string{"/tmp/file"},
				},
			})
		})

		It("makes a request to the executor", func() {
			Eventually(fakeExecutor.ReceivedRequests).Should(HaveLen(2))
		})
	})

	Describe("acting as an auction representative", func() {
		Describe("reporting the state of its resources", func() {
			BeforeEach(func() {
				fakeExecutor.RouteToHandler("GET", "/resources/total", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
					MemoryMB:   1024,
					DiskMB:     2048,
					Containers: 4,
				}))
				fakeExecutor.RouteToHandler("GET", "/resources/remaining", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
					MemoryMB:   512,
					DiskMB:     1024,
					Containers: 2,
				}))
				fakeExecutor.RouteToHandler("GET", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, []executor.Container{}))
			})

			It("makes a request to the executor", func() {
				Eventually(bbs.Cells).Should(HaveLen(1))
				cells, err := bbs.Cells()
				Ω(err).ShouldNot(HaveOccurred())

				client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

				state, err := client.State()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(state.TotalResources).Should(Equal(auctiontypes.Resources{
					MemoryMB:   1024,
					DiskMB:     2048,
					Containers: 4,
				}))
				Ω(state.AvailableResources).Should(Equal(auctiontypes.Resources{
					MemoryMB:   512,
					DiskMB:     1024,
					Containers: 2,
				}))
				Ω(state.Stack).Should(Equal("the-stack"))
			})
		})

		Describe("performing work", func() {
			Describe("starting LRPs", func() {
				var lrpStartAuction models.LRPStartAuction

				BeforeEach(func() {
					lrpStartAuction = models.LRPStartAuction{
						DesiredLRP: models.DesiredLRP{
							ProcessGuid: "the-process-guid",
							MemoryMB:    2,
							DiskMB:      2,
							Stack:       "the-stack",
							Domain:      "the-domain",
						},
						InstanceGuid: "the-instance-guid",
						Index:        1,
					}
					_, err := bbs.CreateActualLRP(models.NewActualLRP("the-process-guid", "the-instance-guid", "", "the-domain", 1, "UNCLAIMED"))
					Ω(err).ShouldNot(HaveOccurred())

					fakeExecutor.RouteToHandler("POST", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.Container{}))
					fakeExecutor.RouteToHandler("POST", "/containers/the-instance-guid/run", ghttp.RespondWith(http.StatusOK, ""))
				})

				var claimedActualLRPs = func(bbs *Bbs.BBS) func() ([]models.ActualLRP, error) {
					return func() ([]models.ActualLRP, error) {
						actualLRPs, err := bbs.ActualLRPs()
						if err != nil {
							return []models.ActualLRP{}, err
						}

						result := []models.ActualLRP{}
						for _, actualLRP := range actualLRPs {
							if actualLRP.State == models.ActualLRPStateClaimed {
								result = append(result, actualLRP)
							}
						}

						return result, nil
					}
				}

				It("makes a request to executor to allocate and run the container, and marks the state as claimed in the BBS", func() {
					Eventually(bbs.Cells).Should(HaveLen(1))
					cells, err := bbs.Cells()
					Ω(err).ShouldNot(HaveOccurred())

					client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

					Ω(bbs.RunningActualLRPs()).Should(BeEmpty())

					works := auctiontypes.Work{
						LRPStarts: []models.LRPStartAuction{lrpStartAuction},
					}
					failedWorks, err := client.Perform(works)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(failedWorks.LRPStarts).Should(BeEmpty())

					Eventually(claimedActualLRPs(bbs)).Should(HaveLen(1))
					actualLRPs, err := claimedActualLRPs(bbs)()
					Ω(err).ShouldNot(HaveOccurred())
					Ω(actualLRPs[0].ProcessGuid).Should(Equal("the-process-guid"))
					Ω(actualLRPs[0].State).Should(Equal(models.ActualLRPStateClaimed))
				})
			})

			Describe("running tasks", func() {
				var task models.Task

				BeforeEach(func() {
					task = models.Task{
						TaskGuid: "the-task-guid",
						MemoryMB: 2,
						DiskMB:   2,
						Stack:    "the-stack",
						Domain:   "the-domain",
						Action: &models.RunAction{
							Path: "date",
						},
					}
					err := bbs.DesireTask(task)
					Ω(err).ShouldNot(HaveOccurred())

					fakeExecutor.RouteToHandler("POST", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.Container{}))
					fakeExecutor.RouteToHandler("POST", "/containers/the-task-guid/run", ghttp.RespondWith(http.StatusOK, ""))
				})

				It("makes a request to executor to allocate and run the container, and marks the task as running in the BBS", func() {
					Eventually(bbs.Cells).Should(HaveLen(1))
					cells, err := bbs.Cells()
					Ω(err).ShouldNot(HaveOccurred())

					client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

					Ω(bbs.PendingTasks()).Should(HaveLen(1))
					Ω(bbs.RunningTasks()).Should(BeEmpty())

					works := auctiontypes.Work{
						Tasks: []models.Task{task},
					}
					failedWorks, err := client.Perform(works)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(failedWorks.Tasks).Should(BeEmpty())

					Eventually(bbs.PendingTasks).Should(BeEmpty())
					Eventually(bbs.RunningTasks).Should(HaveLen(1))
					runningTasks, err := bbs.RunningTasks()
					Ω(err).ShouldNot(HaveOccurred())
					Ω(runningTasks[0].TaskGuid).Should(Equal("the-task-guid"))
					Ω(runningTasks[0].State).Should(Equal(models.TaskStateRunning))
				})
			})
		})
	})

	Describe("polling the BBS for tasks to reap", func() {
		var task models.Task

		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			task = models.Task{
				TaskGuid: "a-new-task-guid",
				Domain:   "the-domain",
				Action: &models.RunAction{
					Path: "the-path",
					Args: []string{},
				},
				Stack: "the-stack",
			}

			err := bbs.DesireTask(task)
			Ω(err).ShouldNot(HaveOccurred())

			err = bbs.StartTask(task.TaskGuid, cellID)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("eventually marks tasks with no corresponding container as failed", func() {
			Eventually(bbs.CompletedTasks, 5*pollingInterval).Should(HaveLen(1))

			completedTasks, err := bbs.CompletedTasks()
			Ω(err).ShouldNot(HaveOccurred())

			Ω(completedTasks[0].TaskGuid).Should(Equal(task.TaskGuid))
			Ω(completedTasks[0].Failed).Should(BeTrue())
		})
	})

	Describe("polling the BBS for actual LRPs to reap", func() {
		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			claimActual(models.NewActualLRP("process-guid", "a-new-instance-guid", cellID, "the-domain", 0, ""))
		})

		It("eventually reaps actual LRPs with no corresponding container", func() {
			Eventually(bbs.ActualLRPs, 5*pollingInterval).Should(BeEmpty())
		})
	})

	Describe("when a StopLRPInstance request comes in", func() {
		var runningLRP models.ActualLRP

		BeforeEach(func() {
			fakeExecutor.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("DELETE", "/containers/some-instance-guid"),
					ghttp.RespondWith(http.StatusOK, nil),
				),
			)

			runningLRP = models.ActualLRP{
				ProcessGuid:  "some-process-guid",
				InstanceGuid: "some-instance-guid",
				Domain:       "the-domain",
				Index:        3,

				CellID: cellID,

				State: models.ActualLRPStateRunning,
			}

			_, err := bbs.StartActualLRP(runningLRP)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("should delete the container and resolve the StopLRPInstance", func() {
			err := bbs.RequestStopLRPInstance(runningLRP)
			Ω(err).ShouldNot(HaveOccurred())

			findDeleteRequest := func() string {
				for _, req := range fakeExecutor.ReceivedRequests() {
					if req.Method == "DELETE" {
						return req.URL.Path
					}
				}
				return ""
			}

			Eventually(findDeleteRequest).Should(Equal("/containers/some-instance-guid"))
			Eventually(bbs.ActualLRPs).Should(BeEmpty())
		})
	})
})
<commit_msg>Don't leak etcdAdapter in test<commit_after>package main_test

import (
	"fmt"
	"net/http"
	"time"

	"github.com/cloudfoundry-incubator/auction/auctiontypes"
	"github.com/cloudfoundry-incubator/auction/communication/http/auction_http_client"
	"github.com/cloudfoundry-incubator/executor"
	"github.com/cloudfoundry-incubator/rep/cmd/rep/testrunner"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/storeadapter"
	"github.com/pivotal-golang/lager/lagertest"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
)

var runner *testrunner.Runner
var etcdAdapter storeadapter.StoreAdapter

var _ = Describe("The Rep", func() {
	var (
		fakeExecutor    *ghttp.Server
		bbs             *Bbs.BBS
		pollingInterval time.Duration
	)

	BeforeEach(func() {
		fakeExecutor = ghttp.NewServer()
		// these tests only look for the start of a sequence of requests
		fakeExecutor.AllowUnhandledRequests = true
		fakeExecutor.RouteToHandler("GET", "/ping", ghttp.RespondWith(http.StatusOK, nil))

		etcdAdapter = etcdRunner.Adapter()
		bbs = Bbs.NewBBS(etcdAdapter, timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		pollingInterval = 50 * time.Millisecond

		runner = testrunner.New(
			representativePath,
			cellID,
			"the-stack",
			"the-lrp-host",
			fakeExecutor.URL(),
			fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"info",
			auctionServerPort,
			time.Second,
			pollingInterval,
		)

		runner.Start()
	})

	AfterEach(func(done Done) {
		etcdAdapter.Disconnect()
		runner.KillWithFire()
		fakeExecutor.Close()
		close(done)
	})

	claimActual := func(a models.ActualLRP) *models.ActualLRP {
		_, err := bbs.CreateActualLRP(a)
		Ω(err).ShouldNot(HaveOccurred())
		claimed, err := bbs.ClaimActualLRP(a)
		return claimed
	}

	Describe("when an interrupt signal is sent to the representative", func() {
		BeforeEach(func() {
			runner.Stop()
		})

		It("should die", func() {
			Eventually(runner.Session.ExitCode).Should(Equal(0))
		})
	})

	Describe("maintaining presence", func() {
		var cellPresence models.CellPresence

		BeforeEach(func() {
			Eventually(bbs.Cells).Should(HaveLen(1))
			cells, err := bbs.Cells()
			Ω(err).ShouldNot(HaveOccurred())
			cellPresence = cells[0]
		})

		It("should maintain presence", func() {
			Ω(cellPresence.Stack).Should(Equal("the-stack"))
			Ω(cellPresence.CellID).Should(Equal(cellID))
		})

		Context("when the presence fails to be maintained", func() {
			It("should not exit, but keep trying to maintain presence at the same ID", func() {
				etcdRunner.Stop()
				etcdRunner.Start()

				Eventually(bbs.Cells, 5).Should(HaveLen(1))
				cells, err := bbs.Cells()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(cells[0]).Should(Equal(cellPresence))

				Ω(runner.Session).ShouldNot(gexec.Exit())
			})
		})
	})

	Describe("when a task is written to the BBS", func() {
		BeforeEach(func() {
			fakeExecutor.AppendHandlers(ghttp.CombineHandlers(
				ghttp.VerifyRequest("POST", "/containers/the-task-guid"),
				ghttp.RespondWith(http.StatusCreated, `{"executor_guid":"executor-guid","guid":"guid-123"}`)),
			)

			bbs.DesireTask(models.Task{
				TaskGuid: "the-task-guid",
				Stack:    "the-stack",
				Action: &models.RunAction{
					Path: "cat",
					Args: []string{"/tmp/file"},
				},
			})
		})

		It("makes a request to the executor", func() {
			Eventually(fakeExecutor.ReceivedRequests).Should(HaveLen(2))
		})
	})

	Describe("acting as an auction representative", func() {
		Describe("reporting the state of its resources", func() {
			BeforeEach(func() {
				fakeExecutor.RouteToHandler("GET", "/resources/total", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
					MemoryMB:   1024,
					DiskMB:     2048,
					Containers: 4,
				}))
				fakeExecutor.RouteToHandler("GET", "/resources/remaining", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.ExecutorResources{
					MemoryMB:   512,
					DiskMB:     1024,
					Containers: 2,
				}))
				fakeExecutor.RouteToHandler("GET", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, []executor.Container{}))
			})

			It("makes a request to the executor", func() {
				Eventually(bbs.Cells).Should(HaveLen(1))
				cells, err := bbs.Cells()
				Ω(err).ShouldNot(HaveOccurred())

				client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

				state, err := client.State()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(state.TotalResources).Should(Equal(auctiontypes.Resources{
					MemoryMB:   1024,
					DiskMB:     2048,
					Containers: 4,
				}))
				Ω(state.AvailableResources).Should(Equal(auctiontypes.Resources{
					MemoryMB:   512,
					DiskMB:     1024,
					Containers: 2,
				}))
				Ω(state.Stack).Should(Equal("the-stack"))
			})
		})

		Describe("performing work", func() {
			Describe("starting LRPs", func() {
				var lrpStartAuction models.LRPStartAuction

				BeforeEach(func() {
					lrpStartAuction = models.LRPStartAuction{
						DesiredLRP: models.DesiredLRP{
							ProcessGuid: "the-process-guid",
							MemoryMB:    2,
							DiskMB:      2,
							Stack:       "the-stack",
							Domain:      "the-domain",
						},
						InstanceGuid: "the-instance-guid",
						Index:        1,
					}
					_, err := bbs.CreateActualLRP(models.NewActualLRP("the-process-guid", "the-instance-guid", "", "the-domain", 1, "UNCLAIMED"))
					Ω(err).ShouldNot(HaveOccurred())

					fakeExecutor.RouteToHandler("POST", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.Container{}))
					fakeExecutor.RouteToHandler("POST", "/containers/the-instance-guid/run", ghttp.RespondWith(http.StatusOK, ""))
				})

				var claimedActualLRPs = func(bbs *Bbs.BBS) func() ([]models.ActualLRP, error) {
					return func() ([]models.ActualLRP, error) {
						actualLRPs, err := bbs.ActualLRPs()
						if err != nil {
							return []models.ActualLRP{}, err
						}

						result := []models.ActualLRP{}
						for _, actualLRP := range actualLRPs {
							if actualLRP.State == models.ActualLRPStateClaimed {
								result = append(result, actualLRP)
							}
						}

						return result, nil
					}
				}

				It("makes a request to executor to allocate and run the container, and marks the state as claimed in the BBS", func() {
					Eventually(bbs.Cells).Should(HaveLen(1))
					cells, err := bbs.Cells()
					Ω(err).ShouldNot(HaveOccurred())

					client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

					Ω(bbs.RunningActualLRPs()).Should(BeEmpty())

					works := auctiontypes.Work{
						LRPStarts: []models.LRPStartAuction{lrpStartAuction},
					}
					failedWorks, err := client.Perform(works)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(failedWorks.LRPStarts).Should(BeEmpty())

					Eventually(claimedActualLRPs(bbs)).Should(HaveLen(1))
					actualLRPs, err := claimedActualLRPs(bbs)()
					Ω(err).ShouldNot(HaveOccurred())
					Ω(actualLRPs[0].ProcessGuid).Should(Equal("the-process-guid"))
					Ω(actualLRPs[0].State).Should(Equal(models.ActualLRPStateClaimed))
				})
			})

			Describe("running tasks", func() {
				var task models.Task

				BeforeEach(func() {
					task = models.Task{
						TaskGuid: "the-task-guid",
						MemoryMB: 2,
						DiskMB:   2,
						Stack:    "the-stack",
						Domain:   "the-domain",
						Action: &models.RunAction{
							Path: "date",
						},
					}
					err := bbs.DesireTask(task)
					Ω(err).ShouldNot(HaveOccurred())

					fakeExecutor.RouteToHandler("POST", "/containers", ghttp.RespondWithJSONEncoded(http.StatusOK, executor.Container{}))
					fakeExecutor.RouteToHandler("POST", "/containers/the-task-guid/run", ghttp.RespondWith(http.StatusOK, ""))
				})

				It("makes a request to executor to allocate and run the container, and marks the task as running in the BBS", func() {
					Eventually(bbs.Cells).Should(HaveLen(1))
					cells, err := bbs.Cells()
					Ω(err).ShouldNot(HaveOccurred())

					client := auction_http_client.New(http.DefaultClient, cells[0].CellID, cells[0].RepAddress, lagertest.NewTestLogger("auction-client"))

					Ω(bbs.PendingTasks()).Should(HaveLen(1))
					Ω(bbs.RunningTasks()).Should(BeEmpty())

					works := auctiontypes.Work{
						Tasks: []models.Task{task},
					}
					failedWorks, err := client.Perform(works)
					Ω(err).ShouldNot(HaveOccurred())
					Ω(failedWorks.Tasks).Should(BeEmpty())

					Eventually(bbs.PendingTasks).Should(BeEmpty())
					Eventually(bbs.RunningTasks).Should(HaveLen(1))
					runningTasks, err := bbs.RunningTasks()
					Ω(err).ShouldNot(HaveOccurred())
					Ω(runningTasks[0].TaskGuid).Should(Equal("the-task-guid"))
					Ω(runningTasks[0].State).Should(Equal(models.TaskStateRunning))
				})
			})
		})
	})

	Describe("polling the BBS for tasks to reap", func() {
		var task models.Task

		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			task = models.Task{
				TaskGuid: "a-new-task-guid",
				Domain:   "the-domain",
				Action: &models.RunAction{
					Path: "the-path",
					Args: []string{},
				},
				Stack: "the-stack",
			}

			err := bbs.DesireTask(task)
			Ω(err).ShouldNot(HaveOccurred())

			err = bbs.StartTask(task.TaskGuid, cellID)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("eventually marks tasks with no corresponding container as failed", func() {
			Eventually(bbs.CompletedTasks, 5*pollingInterval).Should(HaveLen(1))

			completedTasks, err := bbs.CompletedTasks()
			Ω(err).ShouldNot(HaveOccurred())

			Ω(completedTasks[0].TaskGuid).Should(Equal(task.TaskGuid))
			Ω(completedTasks[0].Failed).Should(BeTrue())
		})
	})

	Describe("polling the BBS for actual LRPs to reap", func() {
		BeforeEach(func() {
			fakeExecutor.RouteToHandler(
				"GET",
				"/containers",
				ghttp.RespondWith(http.StatusOK, "[]"),
			)

			claimActual(models.NewActualLRP("process-guid", "a-new-instance-guid", cellID, "the-domain", 0, ""))
		})

		It("eventually reaps actual LRPs with no corresponding container", func() {
			Eventually(bbs.ActualLRPs, 5*pollingInterval).Should(BeEmpty())
		})
	})

	Describe("when a StopLRPInstance request comes in", func() {
		var runningLRP models.ActualLRP

		BeforeEach(func() {
			fakeExecutor.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("DELETE", "/containers/some-instance-guid"),
					ghttp.RespondWith(http.StatusOK, nil),
				),
			)

			runningLRP = models.ActualLRP{
				ProcessGuid:  "some-process-guid",
				InstanceGuid: "some-instance-guid",
				Domain:       "the-domain",
				Index:        3,

				CellID: cellID,

				State: models.ActualLRPStateRunning,
			}

			_, err := bbs.StartActualLRP(runningLRP)
			Ω(err).ShouldNot(HaveOccurred())
		})

		It("should delete the container and resolve the StopLRPInstance", func() {
			err := bbs.RequestStopLRPInstance(runningLRP)
			Ω(err).ShouldNot(HaveOccurred())

			findDeleteRequest := func() string {
				for _, req := range fakeExecutor.ReceivedRequests() {
					if req.Method == "DELETE" {
						return req.URL.Path
					}
				}
				return ""
			}

			Eventually(findDeleteRequest).Should(Equal("/containers/some-instance-guid"))
			Eventually(bbs.ActualLRPs).Should(BeEmpty())
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Command sblookup is a tool for looking up URLs via the command-line.
//
// The tool reads one URL per line from STDIN and checks every URL against
// the Safe Browsing API. The "Safe" or "Unsafe" verdict is printed to STDOUT.
// If an error occurred, debug information may be printed to STDERR.
//
// To build the tool:
//	$ go get github.com/google/safebrowsing/cmd/sblookup
//
// Example usage:
//	$ sblookup -apikey $APIKEY
//	https://google.com
//	Safe URL: https://google.com
//	http://bad1url.org
//	Unsafe URL: [{bad1url.org {MALWARE ANY_PLATFORM URL}}]
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"

	"github.com/google/safebrowsing"
)

var (
	apiKeyFlag   = flag.String("apikey", "", "specify your Safe Browsing API key")
	databaseFlag = flag.String("db", "", "path to the Safe Browsing database. By default persistent storage is disabled (not recommended).")
)

const usage = `sblookup: command-line tool to lookup URLs with Safe Browsing.

Tool reads one URL per line from STDIN and checks every URL against the
Safe Browsing API. The Safe or Unsafe verdict is printed to STDOUT. If an error
occurred, debug information may be printed to STDERR.

Exit codes:
  0     if all URLs were looked up an are safe.
  1     if at least one URL is not safe.
  128   if at least one URL lookup failed.

Usage: %s -apikey=$APIKEY

`

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if *apiKeyFlag == "" {
		fmt.Fprintln(os.Stderr, "No -apikey specified")
		os.Exit(1)
	}
	sb, err := safebrowsing.NewSafeBrowser(safebrowsing.Config{
		APIKey: *apiKeyFlag,
		DBPath: *databaseFlag,
		Logger: os.Stderr,
	})
	if err != nil {
		fmt.Fprintln(os.Stderr, "Unable to initialize Safe Browsing client: ", err)
		os.Exit(1)
	}

	scanner := bufio.NewScanner(os.Stdin)
	code := 0
	for scanner.Scan() {
		url := scanner.Text()
		threats, err := sb.LookupURLs([]string{url})
		if err != nil {
			fmt.Fprintln(os.Stderr, "Lookup error:", err)
			if code != 0 {
				code = 128 // Invalid argument.
			}
		}
		if len(threats[0]) == 0 {
			fmt.Fprintln(os.Stdout, "Safe URL:", url)
		} else {
			fmt.Fprintln(os.Stdout, "Unsafe URL:", threats[0])
			if code != 0 {
				code = 1
			}
		}
	}
	if scanner.Err() != nil {
		fmt.Fprintln(os.Stderr, "Unable to read input:", scanner.Err())
		if code != 0 {
			code = 128 // Invalid argument.
		}
	}
	os.Exit(code)
}
<commit_msg>Fix sblookup error handling<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Command sblookup is a tool for looking up URLs via the command-line.
//
// The tool reads one URL per line from STDIN and checks every URL against
// the Safe Browsing API. The "Safe" or "Unsafe" verdict is printed to STDOUT.
// If an error occurred, debug information may be printed to STDERR.
//
// To build the tool:
//	$ go get github.com/google/safebrowsing/cmd/sblookup
//
// Example usage:
//	$ sblookup -apikey $APIKEY
//	https://google.com
//	Safe URL: https://google.com
//	http://bad1url.org
//	Unsafe URL: [{bad1url.org {MALWARE ANY_PLATFORM URL}}]
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"

	"github.com/google/safebrowsing"
)

var (
	apiKeyFlag   = flag.String("apikey", "", "specify your Safe Browsing API key")
	databaseFlag = flag.String("db", "", "path to the Safe Browsing database. By default persistent storage is disabled (not recommended).")
)

const usage = `sblookup: command-line tool to lookup URLs with Safe Browsing.

Tool reads one URL per line from STDIN and checks every URL against the
Safe Browsing API. The Safe or Unsafe verdict is printed to STDOUT. If an error
occurred, debug information may be printed to STDERR.

Exit codes (bitwise OR of following codes):
  0  if and only if all URLs were looked up and are safe.
  1  if at least one URL is not safe.
  2  if at least one URL lookup failed.
  4  if the input was invalid.

Usage: %s -apikey=$APIKEY

`

const (
	codeSafe = (1 << iota) / 2 // Sequence of 0, 1, 2, 4, 8, etc...
	codeUnsafe
	codeFailed
	codeInvalid
)

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	if *apiKeyFlag == "" {
		fmt.Fprintln(os.Stderr, "No -apikey specified")
		os.Exit(codeInvalid)
	}
	sb, err := safebrowsing.NewSafeBrowser(safebrowsing.Config{
		APIKey: *apiKeyFlag,
		DBPath: *databaseFlag,
		Logger: os.Stderr,
	})
	if err != nil {
		fmt.Fprintln(os.Stderr, "Unable to initialize Safe Browsing client: ", err)
		os.Exit(codeInvalid)
	}

	scanner := bufio.NewScanner(os.Stdin)
	code := codeSafe
	for scanner.Scan() {
		url := scanner.Text()
		threats, err := sb.LookupURLs([]string{url})
		if err != nil {
			fmt.Fprintln(os.Stdout, "Unknown URL:", url)
			fmt.Fprintln(os.Stderr, "Lookup error:", err)
			code |= codeFailed
		} else if len(threats[0]) == 0 {
			fmt.Fprintln(os.Stdout, "Safe URL:", url)
		} else {
			fmt.Fprintln(os.Stdout, "Unsafe URL:", threats[0])
			code |= codeUnsafe
		}
	}
	if scanner.Err() != nil {
		fmt.Fprintln(os.Stderr, "Unable to read input:", scanner.Err())
		code |= codeInvalid
	}
	os.Exit(code)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"unicode/utf8"
)

const ENDL = '\n'

func fold(b []byte, width int) []byte {
	runes := bytes.Runes(b)
	if len(runes) < width {
		return b
	} else {
		lineBreak := bytes.IndexRune(b, ENDL)
		if len(bytes.Runes(b[:lineBreak])) < width {
			lineLen := lineBreak + utf8.RuneLen(ENDL)
			return append(b[0:lineLen], fold(b[lineLen:], width)...)
		} else {
			curLine := []byte(string(append(runes[0:width], '\n')))
			nextStart := len(curLine) - utf8.RuneLen(runes[width])
			return append(curLine, fold(b[nextStart:], width)...)
		}
	}
}

func main() {
	flag.Parse()
	fileCount := len(flag.Args())
	if fileCount > 0 {
		for i := 0; i < fileCount; i++ {
			input, _ := ioutil.ReadFile(flag.Args()[i])
			result := string(fold(input, 80))
			fmt.Print(result)
		}
	} else {
	}
}
<commit_msg>Make fold work without arguments<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"unicode/utf8"
)

const ENDL = '\n'

func fold(b []byte, width int) []byte {
	runes := bytes.Runes(b)
	if len(runes) < width {
		return b
	} else {
		lineBreak := bytes.IndexRune(b, ENDL)
		if len(bytes.Runes(b[:lineBreak])) < width {
			lineLen := lineBreak + utf8.RuneLen(ENDL)
			return append(b[0:lineLen], fold(b[lineLen:], width)...)
		} else {
			curLine := []byte(string(append(runes[0:width], '\n')))
			nextStart := len(curLine) - utf8.RuneLen(runes[width])
			return append(curLine, fold(b[nextStart:], width)...)
		}
	}
}

func main() {
	flag.Parse()
	fileCount := len(flag.Args())
	var input []byte
	if fileCount > 0 {
		for i := 0; i < fileCount; i++ {
			input, _ = ioutil.ReadFile(flag.Args()[i])

		}
	} else {
		input, _ = ioutil.ReadAll(os.Stdin)
	}
	result := string(fold(input, 60))
	fmt.Print(result)
}
<|endoftext|>


<commit_before>// Copyright 2014 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package watch

import (
	"errors"
	"os"
	"path/filepath"
	"sync"

	"github.com/limetext/lime/backend/log"
	"gopkg.in/fsnotify.v1"
)

type (
	FileChangedCallback interface {
		FileChanged(string)
	}
	FileCreatedCallback interface {
		FileCreated(string)
	}
	FileRemovedCallback interface {
		FileRemoved(string)
	}
	FileRenamedCallback interface {
		FileRenamed(string)
	}

	// Wrapper around fsnotify watcher to suit lime needs
	// Enables:
	// 		- Watching directories, we will have less individual watchers
	// 		- Have multiple subscribers on one file or directory resolves #285
	// 		- Watching a path which doesn't exist yet
	// 		- Watching and applying action on certain events
	Watcher struct {
		wchr     *fsnotify.Watcher
		watched  map[string][]interface{}
		watchers []string // helper variable for paths we created watcher on
		dirs     []string // helper variable for dirs we are watching
		lock     sync.Mutex
	}
)

func NewWatcher() (*Watcher, error) {
	wchr, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	w := &Watcher{wchr: wchr}
	w.watched = make(map[string][]interface{})
	w.watchers = make([]string, 0)
	w.dirs = make([]string, 0)

	return w, nil
}

func (w *Watcher) Watch(name string, cb interface{}) error {
	log.Finest("Watch(%s)", name)
	fi, err := os.Stat(name)
	isDir := err == nil && fi.IsDir()
	// If the file doesn't exist currently we will add watcher for file
	// directory and look for create event inside the directory
	if os.IsNotExist(err) {
		log.Fine("%s doesn't exist, Watching parent directory", name)
		if err := w.Watch(filepath.Dir(name), nil); err != nil {
			return err
		}
	}
	w.lock.Lock()
	defer w.lock.Unlock()
	if err := w.add(name, cb); err != nil {
		if !isDir {
			return err
		}
		if exist(w.dirs, name) {
			log.Debug("%s is watched already", name)
			return nil
		}
	}
	// If exists in watchers we are already watching the path
	// Or
	// If the file is under one of watched dirs
	//
	// no need to create watcher
	if exist(w.watchers, name) || (!isDir && exist(w.dirs, filepath.Dir(name))) {
		return nil
	}
	if err := w.watch(name); err != nil {
		return err
	}
	if isDir {
		w.flushDir(name)
	}
	return nil
}

func (w *Watcher) add(name string, cb interface{}) error {
	numok := 0
	if _, ok := cb.(FileChangedCallback); ok {
		numok++
	}
	if _, ok := cb.(FileCreatedCallback); ok {
		numok++
	}
	if _, ok := cb.(FileRemovedCallback); ok {
		numok++
	}
	if _, ok := cb.(FileRenamedCallback); ok {
		numok++
	}
	if numok == 0 {
		return errors.New("The callback argument does satisfy any File*Callback interfaces")
	}
	w.watched[name] = append(w.watched[name], cb)
	return nil
}

func (w *Watcher) watch(name string) error {
	if err := w.wchr.Add(name); err != nil {
		return err
	}
	w.watchers = append(w.watchers, name)
	return nil
}

// Remove watchers created on files under this directory because
// one watcher on the parent directory is enough for all of them
func (w *Watcher) flushDir(name string) {
	if exist(w.dirs, name) {
		return
	}
	w.dirs = append(w.dirs, name)
	for _, p := range w.watchers {
		if filepath.Dir(p) == name {
			if err := w.removeWatch(p); err != nil {
				log.Error("Couldn't unwatch file %s: %s", p, err)
			}
		}
	}
}

func (w *Watcher) UnWatch(name string, cb interface{}) error {
	log.Finest("UnWatch(%s)", name)
	w.lock.Lock()
	defer w.lock.Unlock()
	if cb == nil {
		return w.unWatch(name)
	}
	for i, c := range w.watched[name] {
		if c == cb {
			w.watched[name][i] = w.watched[name][len(w.watched[name])-1]
			w.watched[name][len(w.watched[name])-1] = nil
			w.watched[name] = w.watched[name][:len(w.watched[name])-1]
			break
		}
	}
	if len(w.watched[name]) == 0 {
		w.unWatch(name)
	}
	return nil
}

func (w *Watcher) unWatch(name string) error {
	delete(w.watched, name)
	if err := w.removeWatch(name); err != nil {
		return err
	}
	return nil
}

func (w *Watcher) removeWatch(name string) error {
	if err := w.wchr.Remove(name); err != nil {
		return err
	}
	w.watchers = remove(w.watchers, name)
	if exist(w.dirs, name) {
		w.removeDir(name)
	}
	return nil
}

// Put back watchers on watching files under the directory
func (w *Watcher) removeDir(name string) {
	for p, _ := range w.watched {
		if filepath.Dir(p) == name {
			if err := w.watch(p); err != nil {
				log.Error("Could not watch: %s", err)
				continue
			}
		}
	}
	w.dirs = remove(w.dirs, name)
}

func (w *Watcher) Observe() {
	for {
		select {
		case ev := <-w.wchr.Events:
			func() {
				w.lock.Lock()
				defer w.lock.Unlock()
				w.apply(ev)
				name := ev.Name
				// If the name refers to a directory run all watched
				// callbacks for wathed files under the directory
				if exist(w.dirs, name) {
					for p, _ := range w.watched {
						if filepath.Dir(p) == name {
							ev.Name = p
							w.apply(ev)
						}
					}
				}
				dir := filepath.Dir(name)
				// The watcher will be removed if the file is deleted
				// so we need to watch the parent directory for when the
				// file is created again
				if ev.Op&fsnotify.Remove != 0 {
					w.watchers = remove(w.watchers, name)
					w.lock.Unlock()
					w.Watch(dir, nil)
					w.lock.Lock()
				}
				// We will apply parent directory FileChanged callbacks to,
				// if one of the files inside the directory has changed
				if cbs, exist := w.watched[dir]; ev.Op&fsnotify.Write != 0 && exist {
					for _, cb := range cbs {
						if c, ok := cb.(FileChangedCallback); ok {
							c.FileChanged(dir)
						}
					}
				}

			}()
		case err := <-w.wchr.Errors:
			log.Error("Watcher error: %s", err)
		}
	}
}

func (w *Watcher) apply(ev fsnotify.Event) {
	for _, cb := range w.watched[ev.Name] {
		if ev.Op&fsnotify.Create != 0 {
			if c, ok := cb.(FileCreatedCallback); ok {
				c.FileCreated(ev.Name)
			}
		}
		if ev.Op&fsnotify.Write != 0 {
			if c, ok := cb.(FileChangedCallback); ok {
				c.FileChanged(ev.Name)
			}
		}
		if ev.Op&fsnotify.Remove != 0 {
			if c, ok := cb.(FileRemovedCallback); ok {
				c.FileRemoved(ev.Name)
			}
		}
		if ev.Op&fsnotify.Rename != 0 {
			if c, ok := cb.(FileRenamedCallback); ok {
				c.FileRenamed(ev.Name)
			}
		}
	}
}
<commit_msg>Don't unwatch a directory on flushing the parent directory<commit_after>// Copyright 2014 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package watch

import (
	"errors"
	"os"
	"path/filepath"
	"sync"

	"github.com/limetext/lime/backend/log"
	"gopkg.in/fsnotify.v1"
)

type (
	FileChangedCallback interface {
		FileChanged(string)
	}
	FileCreatedCallback interface {
		FileCreated(string)
	}
	FileRemovedCallback interface {
		FileRemoved(string)
	}
	FileRenamedCallback interface {
		FileRenamed(string)
	}

	// Wrapper around fsnotify watcher to suit lime needs
	// Enables:
	// 		- Watching directories, we will have less individual watchers
	// 		- Have multiple subscribers on one file or directory resolves #285
	// 		- Watching a path which doesn't exist yet
	// 		- Watching and applying action on certain events
	Watcher struct {
		wchr     *fsnotify.Watcher
		watched  map[string][]interface{}
		watchers []string // helper variable for paths we created watcher on
		dirs     []string // helper variable for dirs we are watching
		lock     sync.Mutex
	}
)

func NewWatcher() (*Watcher, error) {
	wchr, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}
	w := &Watcher{wchr: wchr}
	w.watched = make(map[string][]interface{})
	w.watchers = make([]string, 0)
	w.dirs = make([]string, 0)

	return w, nil
}

func (w *Watcher) Watch(name string, cb interface{}) error {
	log.Finest("Watch(%s)", name)
	fi, err := os.Stat(name)
	isDir := err == nil && fi.IsDir()
	// If the file doesn't exist currently we will add watcher for file
	// directory and look for create event inside the directory
	if os.IsNotExist(err) {
		log.Fine("%s doesn't exist, Watching parent directory", name)
		if err := w.Watch(filepath.Dir(name), nil); err != nil {
			return err
		}
	}
	w.lock.Lock()
	defer w.lock.Unlock()
	if err := w.add(name, cb); err != nil {
		if !isDir {
			return err
		}
		if exist(w.dirs, name) {
			log.Debug("%s is watched already", name)
			return nil
		}
	}
	// If exists in watchers we are already watching the path
	// Or
	// If the file is under one of watched dirs
	//
	// no need to create watcher
	if exist(w.watchers, name) || (!isDir && exist(w.dirs, filepath.Dir(name))) {
		return nil
	}
	if err := w.watch(name); err != nil {
		return err
	}
	if isDir {
		w.flushDir(name)
	}
	return nil
}

func (w *Watcher) add(name string, cb interface{}) error {
	numok := 0
	if _, ok := cb.(FileChangedCallback); ok {
		numok++
	}
	if _, ok := cb.(FileCreatedCallback); ok {
		numok++
	}
	if _, ok := cb.(FileRemovedCallback); ok {
		numok++
	}
	if _, ok := cb.(FileRenamedCallback); ok {
		numok++
	}
	if numok == 0 {
		return errors.New("The callback argument does satisfy any File*Callback interfaces")
	}
	w.watched[name] = append(w.watched[name], cb)
	return nil
}

func (w *Watcher) watch(name string) error {
	if err := w.wchr.Add(name); err != nil {
		return err
	}
	w.watchers = append(w.watchers, name)
	return nil
}

// Remove watchers created on files under this directory because
// one watcher on the parent directory is enough for all of them
func (w *Watcher) flushDir(name string) {
	if exist(w.dirs, name) {
		return
	}
	w.dirs = append(w.dirs, name)
	for _, p := range w.watchers {
		if filepath.Dir(p) == name && !exist(w.dirs, p) {
			if err := w.removeWatch(p); err != nil {
				log.Error("Couldn't unwatch file %s: %s", p, err)
			}
		}
	}
}

func (w *Watcher) UnWatch(name string, cb interface{}) error {
	log.Finest("UnWatch(%s)", name)
	w.lock.Lock()
	defer w.lock.Unlock()
	if cb == nil {
		return w.unWatch(name)
	}
	for i, c := range w.watched[name] {
		if c == cb {
			w.watched[name][i] = w.watched[name][len(w.watched[name])-1]
			w.watched[name][len(w.watched[name])-1] = nil
			w.watched[name] = w.watched[name][:len(w.watched[name])-1]
			break
		}
	}
	if len(w.watched[name]) == 0 {
		w.unWatch(name)
	}
	return nil
}

func (w *Watcher) unWatch(name string) error {
	delete(w.watched, name)
	if err := w.removeWatch(name); err != nil {
		return err
	}
	return nil
}

func (w *Watcher) removeWatch(name string) error {
	if err := w.wchr.Remove(name); err != nil {
		return err
	}
	w.watchers = remove(w.watchers, name)
	if exist(w.dirs, name) {
		w.removeDir(name)
	}
	return nil
}

// Put back watchers on watching files under the directory
func (w *Watcher) removeDir(name string) {
	for p, _ := range w.watched {
		if filepath.Dir(p) == name {
			if err := w.watch(p); err != nil {
				log.Error("Could not watch: %s", err)
				continue
			}
		}
	}
	w.dirs = remove(w.dirs, name)
}

func (w *Watcher) Observe() {
	for {
		select {
		case ev := <-w.wchr.Events:
			func() {
				w.lock.Lock()
				defer w.lock.Unlock()
				w.apply(ev)
				name := ev.Name
				// If the name refers to a directory run all watched
				// callbacks for wathed files under the directory
				if exist(w.dirs, name) {
					for p, _ := range w.watched {
						if filepath.Dir(p) == name {
							ev.Name = p
							w.apply(ev)
						}
					}
				}
				dir := filepath.Dir(name)
				// The watcher will be removed if the file is deleted
				// so we need to watch the parent directory for when the
				// file is created again
				if ev.Op&fsnotify.Remove != 0 {
					w.watchers = remove(w.watchers, name)
					w.lock.Unlock()
					w.Watch(dir, nil)
					w.lock.Lock()
				}
				// We will apply parent directory FileChanged callbacks to,
				// if one of the files inside the directory has changed
				if cbs, exist := w.watched[dir]; ev.Op&fsnotify.Write != 0 && exist {
					for _, cb := range cbs {
						if c, ok := cb.(FileChangedCallback); ok {
							c.FileChanged(dir)
						}
					}
				}

			}()
		case err := <-w.wchr.Errors:
			log.Error("Watcher error: %s", err)
		}
	}
}

func (w *Watcher) apply(ev fsnotify.Event) {
	for _, cb := range w.watched[ev.Name] {
		if ev.Op&fsnotify.Create != 0 {
			if c, ok := cb.(FileCreatedCallback); ok {
				c.FileCreated(ev.Name)
			}
		}
		if ev.Op&fsnotify.Write != 0 {
			if c, ok := cb.(FileChangedCallback); ok {
				c.FileChanged(ev.Name)
			}
		}
		if ev.Op&fsnotify.Remove != 0 {
			if c, ok := cb.(FileRemovedCallback); ok {
				c.FileRemoved(ev.Name)
			}
		}
		if ev.Op&fsnotify.Rename != 0 {
			if c, ok := cb.(FileRenamedCallback); ok {
				c.FileRenamed(ev.Name)
			}
		}
	}
}
<|endoftext|>


<commit_before>package bagit

import (
	"fmt"
	"github.com/APTrust/easy-store/util"
	"strings"
)

// BagItProfile is a slightly modified version of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
// In this version, the Tag-Files-Required list is not a list of
// strings, but a list of TagFile objects, like the BagInfo object
// in bagit-profiles. This lets us validate the presence and value
// of specific tags in any tag file the same way bagit-profile lets
// us validate tags in the bagit.txt file.
type BagItProfile struct {
	// AcceptBagItVersion is a list of BagIt versions to accept.
	// For example, ["0.96", "0.97"]
	AcceptBagItVersion []string `json:"Accept-BagIt-Version"`
	// AcceptSerialization is a list of BagIt serialization formats
	// to accept. For example, ["application/zip", "application/tar"]
	AcceptSerialization []string `json:"Accept-Serialization"`
	// AllowFetchTxt indicates whether we allow a fetch.txt file in the bag.
	AllowFetchTxt bool `json:"Allow-Fetch.txt"`
	// AllowMiscTopLevelFiles indicates whether we allow files in the top-level
	// directory other than payload manifests and tag manifests.
	AllowMiscTopLevelFiles bool `json:"Allow-Misc-Top-Level-Files"`
	// AllowMiscDirectories indicates whether we allow miscellaneous
	// directories to exist outside the data directory. These non-data
	// directories often contain custom tag files whose checksums may
	// not appear in any manifest.
	AllowMiscDirectories bool `json:"Allow-Misc-Directories"`
	// BagItProfileInfo contains descriptive information about this
	// BagIt profile.
	BagItProfileInfo BagItProfileInfo `json:"BagIt-Profile-Info"`
	// ManifestsRequired is a list of payload manifests that must be
	// in the bag. Values in this list are the algoritm names. So, to
	// require manifest-md5.txt and manifest-sha256.txt, the list
	// should contain just ["md5", "sha256"].
	ManifestsRequired []string `json:"Manifests-Required"`
	// Serialization can be "required", "optional" or "forbidden."
	Serialization string `json:"Serialization"`
	// TagFiles is a list of TagFile objects, each of which describes
	// a tag file in the bag. Here, we differ from the bagit-profiles
	// specification in that ALL tag files in the list are objects
	// instead of strings, and the objects describe tags we expect to
	// find in the files. Since TagFile objects have a Required property,
	// we omit bagit-profiles' TagFilesRequired, because that would be
	// redundant.
	TagFilesRequired map[string]map[string]*TagDefinition `json:"Tag-Files-Required"`
	// TagManifestsRequired is a list of required tag manifests. Like
	// ManifestsRequired, the list contains only the names of the
	// required hashing algorithms. E.g. ["md5", "sha256"]
	TagManifestsRequired []string `json:"Tag-Manifests-Required"`
}

// BagItProfileInfo contains some basic info about the bagit profile
// and the sender, adhering to the structure of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
type BagItProfileInfo struct {
	// BagItProfileIdentifier is the URL where this bagit profile can be found.
	BagItProfileIdentifier string `json:"BagIt-Profile-Identifier"`
	// ContactEmail is the email address of the person maintaining this
	// bagit profile.
	ContactEmail string `json:"Contact-Email"`
	// ContactName is the name of the person maintaining this profile.
	ContactName string `json:"Contact-Name"`
	// ExternamDescription describes what this profile is for. For example,
	// "BagIt profile for ingesting content into APTrust."
	ExternalDescription string `json:"External-Description"`
	// SourceOrganization is the name of the organization maintaining this
	// profile.
	SourceOrganization string `json:"Source-Organization"`
	// Version is the version number of this profile. E.g "1.2".
	Version string `json:"Version"`
}

// LoadBagItProfile loads a BagItProfile from the specified file path.
func LoadBagItProfile(filePath string) (*BagItProfile, error) {
	profile := &BagItProfile{}
	err := util.LoadJson(filePath, profile)
	// The BagIt profile spec is a little out of alignment with our
	// TagDefinition struct, so we have to copy the tag names into
	// the struct from the JSON map keys.
	if err == nil && profile != nil && profile.TagFilesRequired != nil {
		for _, tagDefinitions := range profile.TagFilesRequired {
			for tagName, tagDef := range tagDefinitions {
				tagDef.Label = tagName
			}
		}
	}
	return profile, err
}

// Validate returns a list of errors describing why the profile is not
// valid. If the profile is valid, this returns an empty slice.
func (profile *BagItProfile) Validate() []error {
	errs := make([]error, 0)
	if profile.AcceptBagItVersion == nil || len(profile.AcceptBagItVersion) == 0 {
		errs = append(errs, fmt.Errorf("Accept-BagIt-Version must accept at least one BagIt version."))
	}
	if profile.ManifestsRequired == nil || len(profile.ManifestsRequired) == 0 {
		errs = append(errs, fmt.Errorf("Manifests-Required must require at least one algorithm."))
	}
	if _, hasBagit := profile.TagFilesRequired["bagit.txt"]; !hasBagit {
		errs = append(errs, fmt.Errorf("Tag-Files-Required is missing bagit.txt."))
	}
	if _, hasBaginfo := profile.TagFilesRequired["bag-info.txt"]; !hasBaginfo {
		errs = append(errs, fmt.Errorf("Tag-Files-Required is missing bag-info.txt."))
	}
	return errs
}

// RequiredTagDirs returns a list of directories that the profile says
// should contain required tag files. The validator should permit the
// presence of these directories even when AllowMiscDirectories is false.
func (profile *BagItProfile) RequiredTagDirs() []string {
	dirs := make([]string, 0)
	for filename := range profile.TagFilesRequired {
		// Use "/" instead of os.PathSeparator, because BagIt spec
		// says manifests and tag manifests should use "/".
		parts := strings.Split(filename, "/")
		if len(parts) > 1 && !util.StringListContains(dirs, parts[0]) {
			dirs = append(dirs, parts[0])
		}
	}
	return dirs
}
<commit_msg>Fixed typo in comment<commit_after>package bagit

import (
	"fmt"
	"github.com/APTrust/easy-store/util"
	"strings"
)

// BagItProfile is a slightly modified version of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
// In this version, the Tag-Files-Required list is not a list of
// strings, but a list of TagFile objects, like the BagInfo object
// in bagit-profiles. This lets us validate the presence and value
// of specific tags in any tag file the same way bagit-profile lets
// us validate tags in the bagit.txt file.
type BagItProfile struct {
	// AcceptBagItVersion is a list of BagIt versions to accept.
	// For example, ["0.96", "0.97"]
	AcceptBagItVersion []string `json:"Accept-BagIt-Version"`
	// AcceptSerialization is a list of BagIt serialization formats
	// to accept. For example, ["application/zip", "application/tar"]
	AcceptSerialization []string `json:"Accept-Serialization"`
	// AllowFetchTxt indicates whether we allow a fetch.txt file in the bag.
	AllowFetchTxt bool `json:"Allow-Fetch.txt"`
	// AllowMiscTopLevelFiles indicates whether we allow files in the top-level
	// directory other than payload manifests and tag manifests.
	AllowMiscTopLevelFiles bool `json:"Allow-Misc-Top-Level-Files"`
	// AllowMiscDirectories indicates whether we allow miscellaneous
	// directories to exist outside the data directory. These non-data
	// directories often contain custom tag files whose checksums may
	// not appear in any manifest.
	AllowMiscDirectories bool `json:"Allow-Misc-Directories"`
	// BagItProfileInfo contains descriptive information about this
	// BagIt profile.
	BagItProfileInfo BagItProfileInfo `json:"BagIt-Profile-Info"`
	// ManifestsRequired is a list of payload manifests that must be
	// in the bag. Values in this list are the algoritm names. So, to
	// require manifest-md5.txt and manifest-sha256.txt, the list
	// should contain just ["md5", "sha256"].
	ManifestsRequired []string `json:"Manifests-Required"`
	// Serialization can be "required", "optional" or "forbidden."
	Serialization string `json:"Serialization"`
	// TagFiles is a list of TagFile objects, each of which describes
	// a tag file in the bag. Here, we differ from the bagit-profiles
	// specification in that ALL tag files in the list are objects
	// instead of strings, and the objects describe tags we expect to
	// find in the files. Since TagFile objects have a Required property,
	// we omit bagit-profiles' TagFilesRequired, because that would be
	// redundant.
	TagFilesRequired map[string]map[string]*TagDefinition `json:"Tag-Files-Required"`
	// TagManifestsRequired is a list of required tag manifests. Like
	// ManifestsRequired, the list contains only the names of the
	// required hashing algorithms. E.g. ["md5", "sha256"]
	TagManifestsRequired []string `json:"Tag-Manifests-Required"`
}

// BagItProfileInfo contains some basic info about the bagit profile
// and the sender, adhering to the structure of bagit-profiles at
// https://github.com/ruebot/bagit-profiles.
type BagItProfileInfo struct {
	// BagItProfileIdentifier is the URL where this bagit profile can be found.
	BagItProfileIdentifier string `json:"BagIt-Profile-Identifier"`
	// ContactEmail is the email address of the person maintaining this
	// bagit profile.
	ContactEmail string `json:"Contact-Email"`
	// ContactName is the name of the person maintaining this profile.
	ContactName string `json:"Contact-Name"`
	// ExternalDescription describes what this profile is for. For example,
	// "BagIt profile for ingesting content into APTrust."
	ExternalDescription string `json:"External-Description"`
	// SourceOrganization is the name of the organization maintaining this
	// profile.
	SourceOrganization string `json:"Source-Organization"`
	// Version is the version number of this profile. E.g "1.2".
	Version string `json:"Version"`
}

// LoadBagItProfile loads a BagItProfile from the specified file path.
func LoadBagItProfile(filePath string) (*BagItProfile, error) {
	profile := &BagItProfile{}
	err := util.LoadJson(filePath, profile)
	// The BagIt profile spec is a little out of alignment with our
	// TagDefinition struct, so we have to copy the tag names into
	// the struct from the JSON map keys.
	if err == nil && profile != nil && profile.TagFilesRequired != nil {
		for _, tagDefinitions := range profile.TagFilesRequired {
			for tagName, tagDef := range tagDefinitions {
				tagDef.Label = tagName
			}
		}
	}
	return profile, err
}

// Validate returns a list of errors describing why the profile is not
// valid. If the profile is valid, this returns an empty slice.
func (profile *BagItProfile) Validate() []error {
	errs := make([]error, 0)
	if profile.AcceptBagItVersion == nil || len(profile.AcceptBagItVersion) == 0 {
		errs = append(errs, fmt.Errorf("Accept-BagIt-Version must accept at least one BagIt version."))
	}
	if profile.ManifestsRequired == nil || len(profile.ManifestsRequired) == 0 {
		errs = append(errs, fmt.Errorf("Manifests-Required must require at least one algorithm."))
	}
	if _, hasBagit := profile.TagFilesRequired["bagit.txt"]; !hasBagit {
		errs = append(errs, fmt.Errorf("Tag-Files-Required is missing bagit.txt."))
	}
	if _, hasBaginfo := profile.TagFilesRequired["bag-info.txt"]; !hasBaginfo {
		errs = append(errs, fmt.Errorf("Tag-Files-Required is missing bag-info.txt."))
	}
	return errs
}

// RequiredTagDirs returns a list of directories that the profile says
// should contain required tag files. The validator should permit the
// presence of these directories even when AllowMiscDirectories is false.
func (profile *BagItProfile) RequiredTagDirs() []string {
	dirs := make([]string, 0)
	for filename := range profile.TagFilesRequired {
		// Use "/" instead of os.PathSeparator, because BagIt spec
		// says manifests and tag manifests should use "/".
		parts := strings.Split(filename, "/")
		if len(parts) > 1 && !util.StringListContains(dirs, parts[0]) {
			dirs = append(dirs, parts[0])
		}
	}
	return dirs
}
<|endoftext|>