<commit_before><commit_msg>fix loader bug<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"

	_ "github.com/jacobsa/oglematchers"
	_ "github.com/jacobsa/ogletest"
)

func init() {
	flag.IntVar(&port, "port", 8080, "The port at which to serve http.")
	flag.StringVar(&host, "host", "127.0.0.1", "The host at which to serve http.")
}

func main() {
	flag.Parse()

	// initialize file system
	// initialize shell

	// initialize a watcher
	// adjust watcher

	// initialize parser
	// initialize executor
	// initialize server

	// initialize scanner
	// start scanning

	// serve http
}

var (
	port int
	host string
)
<commit_msg>First draft of main (there are still kinks to work out).<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/smartystreets/goconvey/web/server/api"
	"github.com/smartystreets/goconvey/web/server/contract"
	exec "github.com/smartystreets/goconvey/web/server/executor"
	parse "github.com/smartystreets/goconvey/web/server/parser"
	"github.com/smartystreets/goconvey/web/server/system"
	watch "github.com/smartystreets/goconvey/web/server/watcher"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"time"

	// dependencies:
	_ "github.com/jacobsa/oglematchers"
	_ "github.com/jacobsa/ogletest"
)

func init() {
	oneSecond, _ := time.ParseDuration("1s")
	flag.IntVar(&port, "port", 8081, "The port at which to serve http.")
	flag.StringVar(&host, "host", "127.0.0.1", "The host at which to serve http.")
	flag.DurationVar(&nap, "poll", oneSecond, "The interval to wait between polling the file system for changes (default: 1s)")
}

func main() {
	flag.Parse()

	monitor, server := wireup()

	go monitor.ScanForever()

	serveHTTP(server)
}

func serveHTTP(server contract.Server) {
	serveStaticResources()
	serveAjaxMethods(server)
	activateServer()
}

func serveStaticResources() {
	_, file, _, _ := runtime.Caller(0)
	here := filepath.Dir(file)
	static := filepath.Join(here, "/web/client")
	http.Handle("/", http.FileServer(http.Dir(static)))
}

func serveAjaxMethods(server contract.Server) {
	http.HandleFunc("/watch", server.Watch)
	http.HandleFunc("/latest", server.Results)
	http.HandleFunc("/execute", server.Execute)
	http.HandleFunc("/status", server.Status)
}

func activateServer() {
	err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil)
	if err != nil {
		fmt.Println(err)
	}
}

func wireup() (*contract.Monitor, contract.Server) {
	working, err := os.Getwd()
	if err != nil {
		panic(err)
	}

	fs := system.NewFileSystem()
	shell := system.NewShell()

	watcher := watch.NewWatcher(fs, shell)
	watcher.Adjust(working)

	parser := parse.NewParser(parse.ParsePackageResults)
	tester := exec.NewConcurrentTester(shell)
	executor := exec.NewExecutor(tester, parser)
	server := api.NewHTTPServer(watcher, executor)
	scanner := watch.NewScanner(fs, watcher)
	monitor := contract.NewMonitor(scanner, watcher, executor, server, func() { time.Sleep(nap) })

	return monitor, server
}

var (
	port int
	host string
	nap  time.Duration
)
<|endoftext|>


<commit_before>package concordances

import (
	"fmt"

	log "github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	var results []neoReadStruct
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		MATCH (canonical)<-[:EQUIVALENT_TO]-(leafNode:Thing)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, leafNode.authority as authority, leafNode.authorityValue as authorityValue
		UNION ALL

		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		WHERE exists(canonical.leiCode)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, 'LEI' as authority, canonical.leiCode as authorityValue
		UNION ALL
		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		WHERE exists(canonical.iso31661)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, 'ISO-3166-1' as authority, canonical.iso31661 as authorityValue
		UNION ALL
		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		MATCH (canonical)<-[:EQUIVALENT_TO]-(leafNode:Thing)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, 'UPP' as authority, leafNode.uuid as authorityValue
        `,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("error accessing Concordance datastore for identifier: %v", identifiers)
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, results)

}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	var results []neoReadStruct

	authorityProperty, found := AuthorityFromURI(authority)
	if !found {
		return Concordances{}, false, nil
	}

	var query *neoism.CypherQuery

	if authorityProperty == "UPP" {
		// We need to treat the UPP authority slightly different as it's stored elsewhere.
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Concept)
		WHERE p.uuid IN {authorityValue}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, p.uuid as UUID, 'UPP' as authority, p.uuid as authorityValue`,

			Parameters: neoism.Props{
				"authorityValue": identifierValues,
			},
			Result: &results,
		}
	} else if authorityProperty == "LEI" {
		// We've gotta treat LEI special like as well.
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Concept)
		WHERE p.leiCode IN {authorityValue}
		AND exists(p.prefUUID)
		RETURN DISTINCT p.prefUUID AS canonicalUUID, labels(p) AS types, p.uuid as UUID, 'LEI' as authority, p.leiCode as authorityValue`,

			Parameters: neoism.Props{
				"authorityValue": identifierValues,
			},
			Result: &results,
		}
	} else if authorityProperty == "ISO-3166-1" {
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Concept)
		WHERE exists(p.iso31661)
		AND p.iso31661 IN {authorityValue}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, p.uuid as UUID, 'ISO-3166-1' as authority, p.iso31661 as authorityValue
			`,
			Parameters: neoism.Props{
				"authorityValue": identifierValues,
			},
			Result: &results,
		}
	} else {
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Concept)
		WHERE p.authority = {authority} AND p.authorityValue IN {authorityValue}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, p.uuid as UUID, p.authority as authority, p.authorityValue as authorityValue`,

			Parameters: neoism.Props{
				"authorityValue": identifierValues,
				"authority":      authorityProperty,
			},
			Result: &results,
		}
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("error accessing Concordance datastore for identifier: %v", identifierValues)
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, results)
}

func processCypherQueryToConcordances(pcw CypherDriver, q *neoism.CypherQuery, results []neoReadStruct) (concordances Concordances, found bool, err error) {
	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{q})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", q.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore")
	}

	concordances = neoReadStructToConcordances(results, pcw.env)

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}
	return concordances, true, nil
}

func neoReadStructToConcordances(neo []neoReadStruct, env string) (concordances Concordances) {
	concordances = Concordances{
		Concordance: []Concordance{},
	}
	for _, neoCon := range neo {
		var con = Concordance{}
		var concept = Concept{}

		concept.ID = mapper.IDURL(neoCon.CanonicalUUID)
		concept.APIURL = mapper.APIURL(neoCon.CanonicalUUID, neoCon.Types, env)
		authorityURI, found := AuthorityToURI(neoCon.Authority)
		if !found {
			log.Debugf("Unsupported authority: %s", neoCon.Authority)
			continue
		}
		con.Identifier = Identifier{Authority: authorityURI, IdentifierValue: neoCon.AuthorityValue}

		con.Concept = concept
		concordances.Concordance = append(concordances.Concordance, con)
	}
	return concordances
}

// Map of authority to URI for the supported concordance IDs
var authorityMap = map[string]string{
	"TME":             "http://api.ft.com/system/FT-TME",
	"FACTSET":         "http://api.ft.com/system/FACTSET",
	"UPP":             "http://api.ft.com/system/UPP",
	"LEI":             "http://api.ft.com/system/LEI",
	"Smartlogic":      "http://api.ft.com/system/SMARTLOGIC",
	"ManagedLocation": "http://api.ft.com/system/MANAGEDLOCATION",
	"ISO-3166-1":      "http://api.ft.com/system/ISO-3166-1",
	"Geonames":        "http://api.ft.com/system/GEONAMES",
	"Wikidata":        "http://api.ft.com/system/WIKIDATA",
	"DBPedia":         "http://api.ft.com/system/DBPEDIA",
}

func AuthorityFromURI(uri string) (string, bool) {
	for a, u := range authorityMap {
		if u == uri {
			return a, true
		}
	}
	return "", false
}

func AuthorityToURI(authority string) (string, bool) {
	authorityURI, found := authorityMap[authority]
	return authorityURI, found
}
<commit_msg>Fix things not showing on fetch by authority (#29)<commit_after>package concordances

import (
	"fmt"

	log "github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	var results []neoReadStruct
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		MATCH (canonical)<-[:EQUIVALENT_TO]-(leafNode:Thing)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, leafNode.authority as authority, leafNode.authorityValue as authorityValue
		UNION ALL

		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		WHERE exists(canonical.leiCode)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, 'LEI' as authority, canonical.leiCode as authorityValue
		UNION ALL
		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		WHERE exists(canonical.iso31661)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, 'ISO-3166-1' as authority, canonical.iso31661 as authorityValue
		UNION ALL
		MATCH (p:Thing)
		WHERE p.uuid in {identifiers}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		MATCH (canonical)<-[:EQUIVALENT_TO]-(leafNode:Thing)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, 'UPP' as authority, leafNode.uuid as authorityValue
        `,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("error accessing Concordance datastore for identifier: %v", identifiers)
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, results)

}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	var results []neoReadStruct

	authorityProperty, found := AuthorityFromURI(authority)
	if !found {
		return Concordances{}, false, nil
	}

	var query *neoism.CypherQuery

	if authorityProperty == "UPP" {
		// We need to treat the UPP authority slightly different as it's stored elsewhere.
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Thing)
		WHERE p.uuid IN {authorityValue}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, p.uuid as UUID, 'UPP' as authority, p.uuid as authorityValue`,

			Parameters: neoism.Props{
				"authorityValue": identifierValues,
			},
			Result: &results,
		}
	} else if authorityProperty == "LEI" {
		// We've gotta treat LEI special like as well.
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Concept)
		WHERE p.leiCode IN {authorityValue}
		AND exists(p.prefUUID)
		RETURN DISTINCT p.prefUUID AS canonicalUUID, labels(p) AS types, p.uuid as UUID, 'LEI' as authority, p.leiCode as authorityValue`,

			Parameters: neoism.Props{
				"authorityValue": identifierValues,
			},
			Result: &results,
		}
	} else if authorityProperty == "ISO-3166-1" {
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Concept)
		WHERE exists(p.iso31661)
		AND p.iso31661 IN {authorityValue}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, p.uuid as UUID, 'ISO-3166-1' as authority, p.iso31661 as authorityValue
			`,
			Parameters: neoism.Props{
				"authorityValue": identifierValues,
			},
			Result: &results,
		}
	} else {
		query = &neoism.CypherQuery{
			Statement: `
		MATCH (p:Thing)
		WHERE p.authority = {authority} AND p.authorityValue IN {authorityValue}
		MATCH (p)-[:EQUIVALENT_TO]->(canonical:Concept)
		RETURN DISTINCT canonical.prefUUID AS canonicalUUID, labels(canonical) AS types, p.uuid as UUID, p.authority as authority, p.authorityValue as authorityValue`,

			Parameters: neoism.Props{
				"authorityValue": identifierValues,
				"authority":      authorityProperty,
			},
			Result: &results,
		}
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("error accessing Concordance datastore for identifier: %v", identifierValues)
	}

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	return processCypherQueryToConcordances(pcw, query, results)
}

func processCypherQueryToConcordances(pcw CypherDriver, q *neoism.CypherQuery, results []neoReadStruct) (concordances Concordances, found bool, err error) {
	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{q})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", q.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore")
	}

	concordances = neoReadStructToConcordances(results, pcw.env)

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}
	return concordances, true, nil
}

func neoReadStructToConcordances(neo []neoReadStruct, env string) (concordances Concordances) {
	concordances = Concordances{
		Concordance: []Concordance{},
	}
	for _, neoCon := range neo {
		var con = Concordance{}
		var concept = Concept{}

		concept.ID = mapper.IDURL(neoCon.CanonicalUUID)
		concept.APIURL = mapper.APIURL(neoCon.CanonicalUUID, neoCon.Types, env)
		authorityURI, found := AuthorityToURI(neoCon.Authority)
		if !found {
			log.Debugf("Unsupported authority: %s", neoCon.Authority)
			continue
		}
		con.Identifier = Identifier{Authority: authorityURI, IdentifierValue: neoCon.AuthorityValue}

		con.Concept = concept
		concordances.Concordance = append(concordances.Concordance, con)
	}
	return concordances
}

// Map of authority to URI for the supported concordance IDs
var authorityMap = map[string]string{
	"TME":             "http://api.ft.com/system/FT-TME",
	"FACTSET":         "http://api.ft.com/system/FACTSET",
	"UPP":             "http://api.ft.com/system/UPP",
	"LEI":             "http://api.ft.com/system/LEI",
	"Smartlogic":      "http://api.ft.com/system/SMARTLOGIC",
	"ManagedLocation": "http://api.ft.com/system/MANAGEDLOCATION",
	"ISO-3166-1":      "http://api.ft.com/system/ISO-3166-1",
	"Geonames":        "http://api.ft.com/system/GEONAMES",
	"Wikidata":        "http://api.ft.com/system/WIKIDATA",
	"DBPedia":         "http://api.ft.com/system/DBPEDIA",
}

func AuthorityFromURI(uri string) (string, bool) {
	for a, u := range authorityMap {
		if u == uri {
			return a, true
		}
	}
	return "", false
}

func AuthorityToURI(authority string) (string, bool) {
	authorityURI, found := authorityMap[authority]
	return authorityURI, found
}
<|endoftext|>


<commit_before>package shh

import (
	"fmt"
	"net/url"
	"regexp"
	"runtime"
	"time"
)

const (
	VERSION                          = "0.9.6"
	DEFAULT_EMPTY_STRING             = ""
	DEFAULT_INTERVAL                 = "60s"                                                              // Default tick interval for pollers
	DEFAULT_OUTPUTTER                = "stdoutl2metder"                                                   // Default outputter
	DEFAULT_POLLERS                  = "conntrack,cpu,df,disk,listen,load,mem,nif,ntpdate,processes,self" // Default pollers
	DEFAULT_PROFILE_PORT             = "0"                                                                // Default profile port, 0 disables
	DEFAULT_DF_TYPES                 = "btrfs,ext3,ext4,xfs"                                              // Default fs types to report df for
	DEFAULT_DF_LOOP                  = false                                                              // Default to not reporting df metrics for loop back filesystems
	DEFAULT_NIF_DEVICES              = "eth0,lo"                                                          // Default interfaces to report stats for
	DEFAULT_NTPDATE_SERVERS          = "0.pool.ntp.org,1.pool.ntp.org"                                    // Default to the pool.ntp.org servers
	DEFAULT_CPU_AGGR                 = true                                                               // Default whether to only report aggregate CPU
	DEFAULT_SYSLOGNG_SOCKET          = "/var/lib/syslog-ng/syslog-ng.ctl"                                 // Default location of the syslog-ng socket
	DEFAULT_SELF_POLLER_MODE         = "minimal"                                                          // Default to only minimal set of self metrics
	DEFAULT_SOCKSTAT_PROTOS          = "TCP,UDP,TCP6,UDP6"                                                // Default protocols to report sockstats on
	DEFAULT_PERCENTAGES              = ""                                                                 // Default pollers where publishing perc metrics is allowed
	DEFAULT_FULL                     = ""                                                                 // Default list of pollers who should report full metrycs
	DEFAULT_LIBRATO_URL              = "https://metrics-api.librato.com/v1/metrics"                       // Default librato url to submit metrics to
	DEFAULT_LIBRATO_BATCH_SIZE       = 500                                                                // Default submission count
	DEFAULT_LIBRATO_BATCH_TIMEOUT    = "10s"                                                              // Default submission after
	DEFAULT_LIBRATO_ROUND            = true                                                               // Round measure_time to interval
	DEFAULT_LISTEN_ADDR              = "unix,#shh"                                                        // listen on UDS #shh
	DEFAULT_DISK_FILTER              = "(xv|s)d"                                                          // xvd* and sd* by default
	DEFAULT_PROCESSES_REGEX          = `\A\z`                                                             // Regex of processes to pull additional stats about
	DEFAULT_TICKS                    = 100                                                                // Default number of clock ticks per second (see _SC_CLK_TCK)
	DEFAULT_PAGE_SIZE                = 4096                                                               // Default system page size (see getconf PAGESIZE)
	DEFAULT_NAGIOS3_METRIC_NAMES     = "NUMSERVICES,NUMHOSTS,AVGACTSVCLAT,AVGACTHSTLAT,NUMHSTACTCHK5M,NUMSVCACTCHK5M,NUMHSTACTCHK1M,NUMSVCACTCHK1M"
	DEFAULT_SPLUNK_PEERS_SKIP_VERIFY = false
	DEFAULT_NETWORK_TIMEOUT          = "5s"
	DEFAULT_REDIS_INFO               = "clients:connected_clients;memory:used_memory,used_memory_rss;stats:instantaneous_ops_per_sec;keyspace:db0.keys" // semi colon seperated section:keya,keyb list
	DEFAULT_REDIS_URL                = "tcp://localhost:6379/0?timeout=10s&maxidle=1"
)

var (
	start = time.Now()
)

type Config struct {
	Interval              time.Duration
	Outputter             string
	Pollers               []string
	Source                string
	Prefix                string
	ProfilePort           string
	Percentages           []string
	Full                  []string
	DfTypes               []string
	DfLoop                bool
	Listen                string
	ListenTimeout         time.Duration
	NifDevices            []string
	NtpdateServers        []string
	CpuOnlyAggregate      bool
	LibratoUrl            *url.URL
	LibratoUser           string
	LibratoToken          string
	LibratoBatchSize      int
	LibratoBatchTimeout   time.Duration
	LibratoRound          bool
	NetworkTimeout        time.Duration
	CarbonHost            string
	SockStatProtos        []string
	StatsdHost            string
	StatsdProto           string
	SyslogngSocket        string
	Start                 time.Time
	DiskFilter            *regexp.Regexp
	UserAgent             string
	ProcessesRegex        *regexp.Regexp
	Ticks                 int
	PageSize              int
	Nagios3MetricNames    []string
	SplunkPeersSkipVerify bool
	SplunkPeersUrl        *url.URL
	FolsomBaseUrl         *url.URL
	RedisUrl              *url.URL
	RedisInfo             string
}

func GetConfig() (config Config) {
	config.Interval = GetEnvWithDefaultDuration("SHH_INTERVAL", DEFAULT_INTERVAL)                                          // Polling Interval
	config.Outputter = GetEnvWithDefault("SHH_OUTPUTTER", DEFAULT_OUTPUTTER)                                               // Outputter
	config.Pollers = GetEnvWithDefaultStrings("SHH_POLLERS", DEFAULT_POLLERS)                                              // Pollers to poll
	config.Source = GetEnvWithDefault("SHH_SOURCE", DEFAULT_EMPTY_STRING)                                                  // Source to emit
	config.Prefix = GetEnvWithDefault("SHH_PREFIX", DEFAULT_EMPTY_STRING)                                                  // Metric prefix to use
	config.ProfilePort = GetEnvWithDefault("SHH_PROFILE_PORT", DEFAULT_PROFILE_PORT)                                       // Profile Port
	config.Percentages = GetEnvWithDefaultStrings("SHH_PERCENTAGES", DEFAULT_PERCENTAGES)                                  // Use Percentages for these pollers
	config.Full = GetEnvWithDefaultStrings("SHH_FULL", DEFAULT_FULL)                                                       // Report full measurements for these pollers
	config.DfTypes = GetEnvWithDefaultStrings("SHH_DF_TYPES", DEFAULT_DF_TYPES)                                            // Default DF types
	config.DfLoop = GetEnvWithDefaultBool("SHH_DF_LOOP", DEFAULT_DF_LOOP)                                                  // Report df metrics for loop back filesystmes or not
	config.Listen = GetEnvWithDefault("SHH_LISTEN", DEFAULT_LISTEN_ADDR)                                                   // Default network socket info for listen
	config.ListenTimeout = GetEnvWithDefaultDuration("SHH_LISTEN_TIMEOUT", config.Interval.String())                       // Listen Poller Socket Timeout
	config.NifDevices = GetEnvWithDefaultStrings("SHH_NIF_DEVICES", DEFAULT_NIF_DEVICES)                                   // Devices to poll
	config.NtpdateServers = GetEnvWithDefaultStrings("SHH_NTPDATE_SERVERS", DEFAULT_NTPDATE_SERVERS)                       // NTP Servers
	config.CpuOnlyAggregate = GetEnvWithDefaultBool("SHH_CPU_AGGR", DEFAULT_CPU_AGGR)                                      // Whether to only report aggregate CPU usage
	config.LibratoUrl = GetEnvWithDefaultURL("SHH_LIBRATO_URL", DEFAULT_LIBRATO_URL)                                       // The Librato API End-Point
	config.LibratoUser = GetEnvWithDefault("SHH_LIBRATO_USER", DEFAULT_EMPTY_STRING)                                       // The Librato API User
	config.LibratoToken = GetEnvWithDefault("SHH_LIBRATO_TOKEN", DEFAULT_EMPTY_STRING)                                     // The Librato API TOken
	config.LibratoBatchSize = GetEnvWithDefaultInt("SHH_LIBRATO_BATCH_SIZE", DEFAULT_LIBRATO_BATCH_SIZE)                   // The max number of metrics to submit in a single request
	config.LibratoBatchTimeout = GetEnvWithDefaultDuration("SHH_LIBRATO_BATCH_TIMEOUT", DEFAULT_LIBRATO_BATCH_TIMEOUT)     // The max time metrics will sit un-delivered
	config.LibratoRound = GetEnvWithDefaultBool("SHH_LIBRATO_ROUND", DEFAULT_LIBRATO_ROUND)                                // Should we round measurement times to the nearest Interval when submitting to Librato
	config.CarbonHost = GetEnvWithDefault("SHH_CARBON_HOST", DEFAULT_EMPTY_STRING)                                         // Where the Carbon Outputter sends it's data
	config.SockStatProtos = GetEnvWithDefaultStrings("SHH_SOCKSTAT_PROTOS", DEFAULT_SOCKSTAT_PROTOS)                       // Protocols to report sockstats about
	config.StatsdHost = GetEnvWithDefault("SHH_STATSD_HOST", DEFAULT_EMPTY_STRING)                                         // Where the Statsd Outputter sends it's data
	config.StatsdProto = GetEnvWithDefault("SHH_STATSD_PROTO", "udp")                                                      // Whether the Stats Outputter uses TCP or UDP
	config.SyslogngSocket = GetEnvWithDefault("SHH_SYSLOGNG_SOCKET", DEFAULT_SYSLOGNG_SOCKET)                              // The location of the syslog-ng socket
	config.ProcessesRegex = GetEnvWithDefaultRegexp("SHH_PROCESSES_REGEX", DEFAULT_PROCESSES_REGEX)                        // The regex to match process names against for collecting additional measurements
	config.Ticks = GetEnvWithDefaultInt("SHH_TICKS", DEFAULT_TICKS)                                                        // Number of ticks per CPU cycle. It's normally 100, but you can check with `getconf CLK_TCK`
	config.PageSize = GetEnvWithDefaultInt("SHH_PAGE_SIZE", DEFAULT_PAGE_SIZE)                                             // System Page Size. It's usually 4096, but you can check with `getconf PAGESIZE`
	config.Nagios3MetricNames = GetEnvWithDefaultStrings("SHH_NAGIOS3_METRIC_NAMES", DEFAULT_NAGIOS3_METRIC_NAMES)         // Which nagios3stats metrics names should we poll
	config.SplunkPeersSkipVerify = GetEnvWithDefaultBool("SHH_SPLUNK_PEERS_SKIP_VERIFY", DEFAULT_SPLUNK_PEERS_SKIP_VERIFY) // If SHH_SPLUNK_PEERS_URL is https, do we need to skip verification?
	config.SplunkPeersUrl = GetEnvWithDefaultURL("SHH_SPLUNK_PEERS_URL", DEFAULT_EMPTY_STRING)                             // URL of splunk search peers api endpoint. Ex: https://user:pass@localhost:8089/services/search/distributed/peers?count=-1
	config.FolsomBaseUrl = GetEnvWithDefaultURL("SHH_FOLSOM_BASE_URL", DEFAULT_EMPTY_STRING)                               // URL of splunk search peers api endpoint. Ex: https://user:pass@localhost:8089/services/search/distributed/peers?count=-1
	config.RedisUrl = GetEnvWithDefaultURL("SHH_REDIS_URL", DEFAULT_REDIS_URL)                                             // URL of redis endpoint. Ex: tcp://auth:pass@localhost:6379/0?timeout=10s&maxidle=1"
	config.RedisInfo = GetEnvWithDefault("SHH_REDIS_INFO", DEFAULT_REDIS_INFO)                                             // section:key1,key2;section2:key1,key2
	config.NetworkTimeout = GetEnvWithDefaultDuration("NETWORK_TIMEOUT", DEFAULT_NETWORK_TIMEOUT)                          // The maximum time to wait for network requests to respond (for both dial and first header when applicable)

	tmp := GetEnvWithDefault("SHH_DISK_FILTER", DEFAULT_DISK_FILTER)
	config.DiskFilter = regexp.MustCompile(tmp)
	config.UserAgent = fmt.Sprintf("shh/%s (%s; %s; %s; %s)", VERSION, runtime.Version(), runtime.GOOS, runtime.GOARCH, runtime.Compiler)
	config.Start = start // Start time
	return
}
<commit_msg>version bump<commit_after>package shh

import (
	"fmt"
	"net/url"
	"regexp"
	"runtime"
	"time"
)

const (
	VERSION                          = "0.9.7"
	DEFAULT_EMPTY_STRING             = ""
	DEFAULT_INTERVAL                 = "60s"                                                              // Default tick interval for pollers
	DEFAULT_OUTPUTTER                = "stdoutl2metder"                                                   // Default outputter
	DEFAULT_POLLERS                  = "conntrack,cpu,df,disk,listen,load,mem,nif,ntpdate,processes,self" // Default pollers
	DEFAULT_PROFILE_PORT             = "0"                                                                // Default profile port, 0 disables
	DEFAULT_DF_TYPES                 = "btrfs,ext3,ext4,xfs"                                              // Default fs types to report df for
	DEFAULT_DF_LOOP                  = false                                                              // Default to not reporting df metrics for loop back filesystems
	DEFAULT_NIF_DEVICES              = "eth0,lo"                                                          // Default interfaces to report stats for
	DEFAULT_NTPDATE_SERVERS          = "0.pool.ntp.org,1.pool.ntp.org"                                    // Default to the pool.ntp.org servers
	DEFAULT_CPU_AGGR                 = true                                                               // Default whether to only report aggregate CPU
	DEFAULT_SYSLOGNG_SOCKET          = "/var/lib/syslog-ng/syslog-ng.ctl"                                 // Default location of the syslog-ng socket
	DEFAULT_SELF_POLLER_MODE         = "minimal"                                                          // Default to only minimal set of self metrics
	DEFAULT_SOCKSTAT_PROTOS          = "TCP,UDP,TCP6,UDP6"                                                // Default protocols to report sockstats on
	DEFAULT_PERCENTAGES              = ""                                                                 // Default pollers where publishing perc metrics is allowed
	DEFAULT_FULL                     = ""                                                                 // Default list of pollers who should report full metrycs
	DEFAULT_LIBRATO_URL              = "https://metrics-api.librato.com/v1/metrics"                       // Default librato url to submit metrics to
	DEFAULT_LIBRATO_BATCH_SIZE       = 500                                                                // Default submission count
	DEFAULT_LIBRATO_BATCH_TIMEOUT    = "10s"                                                              // Default submission after
	DEFAULT_LIBRATO_ROUND            = true                                                               // Round measure_time to interval
	DEFAULT_LISTEN_ADDR              = "unix,#shh"                                                        // listen on UDS #shh
	DEFAULT_DISK_FILTER              = "(xv|s)d"                                                          // xvd* and sd* by default
	DEFAULT_PROCESSES_REGEX          = `\A\z`                                                             // Regex of processes to pull additional stats about
	DEFAULT_TICKS                    = 100                                                                // Default number of clock ticks per second (see _SC_CLK_TCK)
	DEFAULT_PAGE_SIZE                = 4096                                                               // Default system page size (see getconf PAGESIZE)
	DEFAULT_NAGIOS3_METRIC_NAMES     = "NUMSERVICES,NUMHOSTS,AVGACTSVCLAT,AVGACTHSTLAT,NUMHSTACTCHK5M,NUMSVCACTCHK5M,NUMHSTACTCHK1M,NUMSVCACTCHK1M"
	DEFAULT_SPLUNK_PEERS_SKIP_VERIFY = false
	DEFAULT_NETWORK_TIMEOUT          = "5s"
	DEFAULT_REDIS_INFO               = "clients:connected_clients;memory:used_memory,used_memory_rss;stats:instantaneous_ops_per_sec;keyspace:db0.keys" // semi colon seperated section:keya,keyb list
	DEFAULT_REDIS_URL                = "tcp://localhost:6379/0?timeout=10s&maxidle=1"
)

var (
	start = time.Now()
)

type Config struct {
	Interval              time.Duration
	Outputter             string
	Pollers               []string
	Source                string
	Prefix                string
	ProfilePort           string
	Percentages           []string
	Full                  []string
	DfTypes               []string
	DfLoop                bool
	Listen                string
	ListenTimeout         time.Duration
	NifDevices            []string
	NtpdateServers        []string
	CpuOnlyAggregate      bool
	LibratoUrl            *url.URL
	LibratoUser           string
	LibratoToken          string
	LibratoBatchSize      int
	LibratoBatchTimeout   time.Duration
	LibratoRound          bool
	NetworkTimeout        time.Duration
	CarbonHost            string
	SockStatProtos        []string
	StatsdHost            string
	StatsdProto           string
	SyslogngSocket        string
	Start                 time.Time
	DiskFilter            *regexp.Regexp
	UserAgent             string
	ProcessesRegex        *regexp.Regexp
	Ticks                 int
	PageSize              int
	Nagios3MetricNames    []string
	SplunkPeersSkipVerify bool
	SplunkPeersUrl        *url.URL
	FolsomBaseUrl         *url.URL
	RedisUrl              *url.URL
	RedisInfo             string
}

func GetConfig() (config Config) {
	config.Interval = GetEnvWithDefaultDuration("SHH_INTERVAL", DEFAULT_INTERVAL)                                          // Polling Interval
	config.Outputter = GetEnvWithDefault("SHH_OUTPUTTER", DEFAULT_OUTPUTTER)                                               // Outputter
	config.Pollers = GetEnvWithDefaultStrings("SHH_POLLERS", DEFAULT_POLLERS)                                              // Pollers to poll
	config.Source = GetEnvWithDefault("SHH_SOURCE", DEFAULT_EMPTY_STRING)                                                  // Source to emit
	config.Prefix = GetEnvWithDefault("SHH_PREFIX", DEFAULT_EMPTY_STRING)                                                  // Metric prefix to use
	config.ProfilePort = GetEnvWithDefault("SHH_PROFILE_PORT", DEFAULT_PROFILE_PORT)                                       // Profile Port
	config.Percentages = GetEnvWithDefaultStrings("SHH_PERCENTAGES", DEFAULT_PERCENTAGES)                                  // Use Percentages for these pollers
	config.Full = GetEnvWithDefaultStrings("SHH_FULL", DEFAULT_FULL)                                                       // Report full measurements for these pollers
	config.DfTypes = GetEnvWithDefaultStrings("SHH_DF_TYPES", DEFAULT_DF_TYPES)                                            // Default DF types
	config.DfLoop = GetEnvWithDefaultBool("SHH_DF_LOOP", DEFAULT_DF_LOOP)                                                  // Report df metrics for loop back filesystmes or not
	config.Listen = GetEnvWithDefault("SHH_LISTEN", DEFAULT_LISTEN_ADDR)                                                   // Default network socket info for listen
	config.ListenTimeout = GetEnvWithDefaultDuration("SHH_LISTEN_TIMEOUT", config.Interval.String())                       // Listen Poller Socket Timeout
	config.NifDevices = GetEnvWithDefaultStrings("SHH_NIF_DEVICES", DEFAULT_NIF_DEVICES)                                   // Devices to poll
	config.NtpdateServers = GetEnvWithDefaultStrings("SHH_NTPDATE_SERVERS", DEFAULT_NTPDATE_SERVERS)                       // NTP Servers
	config.CpuOnlyAggregate = GetEnvWithDefaultBool("SHH_CPU_AGGR", DEFAULT_CPU_AGGR)                                      // Whether to only report aggregate CPU usage
	config.LibratoUrl = GetEnvWithDefaultURL("SHH_LIBRATO_URL", DEFAULT_LIBRATO_URL)                                       // The Librato API End-Point
	config.LibratoUser = GetEnvWithDefault("SHH_LIBRATO_USER", DEFAULT_EMPTY_STRING)                                       // The Librato API User
	config.LibratoToken = GetEnvWithDefault("SHH_LIBRATO_TOKEN", DEFAULT_EMPTY_STRING)                                     // The Librato API TOken
	config.LibratoBatchSize = GetEnvWithDefaultInt("SHH_LIBRATO_BATCH_SIZE", DEFAULT_LIBRATO_BATCH_SIZE)                   // The max number of metrics to submit in a single request
	config.LibratoBatchTimeout = GetEnvWithDefaultDuration("SHH_LIBRATO_BATCH_TIMEOUT", DEFAULT_LIBRATO_BATCH_TIMEOUT)     // The max time metrics will sit un-delivered
	config.LibratoRound = GetEnvWithDefaultBool("SHH_LIBRATO_ROUND", DEFAULT_LIBRATO_ROUND)                                // Should we round measurement times to the nearest Interval when submitting to Librato
	config.CarbonHost = GetEnvWithDefault("SHH_CARBON_HOST", DEFAULT_EMPTY_STRING)                                         // Where the Carbon Outputter sends it's data
	config.SockStatProtos = GetEnvWithDefaultStrings("SHH_SOCKSTAT_PROTOS", DEFAULT_SOCKSTAT_PROTOS)                       // Protocols to report sockstats about
	config.StatsdHost = GetEnvWithDefault("SHH_STATSD_HOST", DEFAULT_EMPTY_STRING)                                         // Where the Statsd Outputter sends it's data
	config.StatsdProto = GetEnvWithDefault("SHH_STATSD_PROTO", "udp")                                                      // Whether the Stats Outputter uses TCP or UDP
	config.SyslogngSocket = GetEnvWithDefault("SHH_SYSLOGNG_SOCKET", DEFAULT_SYSLOGNG_SOCKET)                              // The location of the syslog-ng socket
	config.ProcessesRegex = GetEnvWithDefaultRegexp("SHH_PROCESSES_REGEX", DEFAULT_PROCESSES_REGEX)                        // The regex to match process names against for collecting additional measurements
	config.Ticks = GetEnvWithDefaultInt("SHH_TICKS", DEFAULT_TICKS)                                                        // Number of ticks per CPU cycle. It's normally 100, but you can check with `getconf CLK_TCK`
	config.PageSize = GetEnvWithDefaultInt("SHH_PAGE_SIZE", DEFAULT_PAGE_SIZE)                                             // System Page Size. It's usually 4096, but you can check with `getconf PAGESIZE`
	config.Nagios3MetricNames = GetEnvWithDefaultStrings("SHH_NAGIOS3_METRIC_NAMES", DEFAULT_NAGIOS3_METRIC_NAMES)         // Which nagios3stats metrics names should we poll
	config.SplunkPeersSkipVerify = GetEnvWithDefaultBool("SHH_SPLUNK_PEERS_SKIP_VERIFY", DEFAULT_SPLUNK_PEERS_SKIP_VERIFY) // If SHH_SPLUNK_PEERS_URL is https, do we need to skip verification?
	config.SplunkPeersUrl = GetEnvWithDefaultURL("SHH_SPLUNK_PEERS_URL", DEFAULT_EMPTY_STRING)                             // URL of splunk search peers api endpoint. Ex: https://user:pass@localhost:8089/services/search/distributed/peers?count=-1
	config.FolsomBaseUrl = GetEnvWithDefaultURL("SHH_FOLSOM_BASE_URL", DEFAULT_EMPTY_STRING)                               // URL of splunk search peers api endpoint. Ex: https://user:pass@localhost:8089/services/search/distributed/peers?count=-1
	config.RedisUrl = GetEnvWithDefaultURL("SHH_REDIS_URL", DEFAULT_REDIS_URL)                                             // URL of redis endpoint. Ex: tcp://auth:pass@localhost:6379/0?timeout=10s&maxidle=1"
	config.RedisInfo = GetEnvWithDefault("SHH_REDIS_INFO", DEFAULT_REDIS_INFO)                                             // section:key1,key2;section2:key1,key2
	config.NetworkTimeout = GetEnvWithDefaultDuration("NETWORK_TIMEOUT", DEFAULT_NETWORK_TIMEOUT)                          // The maximum time to wait for network requests to respond (for both dial and first header when applicable)

	tmp := GetEnvWithDefault("SHH_DISK_FILTER", DEFAULT_DISK_FILTER)
	config.DiskFilter = regexp.MustCompile(tmp)
	config.UserAgent = fmt.Sprintf("shh/%s (%s; %s; %s; %s)", VERSION, runtime.Version(), runtime.GOOS, runtime.GOARCH, runtime.Compiler)
	config.Start = start // Start time
	return
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

type configFile struct {
	Listen               string
	Threads              int
	Workers              int
	Retries              int
	APITimeout           int
	CacheDir             string
	FastStart            bool
	Secret               string `xml:",omitempty"`
	ProxyAddr            string `xml:",omitempty"`
	RealRemoteAddrHeader string `xml:",omitempty"`
	Logging              logConfig
}

type logConfig struct {
	LogFile string

	Debug        bool
	DebugLogFile string

	LogRequests bool
	CensorLog   bool
}

var conf configFile

func genSecret() string {
	buf := make([]byte, 32)
	io.ReadFull(rand.Reader, buf)
	return fmt.Sprintf("%x", buf)
}

var defaultConfig = configFile{
	Listen:     "127.0.0.1:3748",
	Threads:    0,
	Workers:    10,
	Retries:    3,
	APITimeout: 60,
	CacheDir:   "cache/",
	Logging: logConfig{
		CensorLog: true,
	},
}

func createConfig() {
	defaultConfig.Secret = genSecret()
	confXML, _ := xml.MarshalIndent(defaultConfig, "", "  ")

	err := ioutil.WriteFile("apiproxy.xml.default", confXML, 0600)
	if err != nil {
		log.Fatalf("Error creating config file apiproxy.xml.default: %s", err)
	} else {
		log.Fatalf("Created new config file apiproxy.xml.default")
	}
}

func loadConfig(filename string) (configFile, error) {
	conf, err := ioutil.ReadFile(filename)
	if err != nil {
		return defaultConfig, err
	}

	newConfig := defaultConfig
	err = xml.Unmarshal(conf, &newConfig)
	if err != nil {
		return defaultConfig, err
	}

	if newConfig.CacheDir == "" {
		return defaultConfig, fmt.Errorf("Need cache directory")
	}

	if newConfig.Logging.Debug {
		debug = true
	}

	return newConfig, nil
}
<commit_msg>removed unused secret generation<commit_after>package main

import (
	"crypto/rand"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"log"
)

type configFile struct {
	Listen               string
	Threads              int
	Workers              int
	Retries              int
	APITimeout           int
	CacheDir             string
	FastStart            bool
	Secret               string `xml:",omitempty"`
	ProxyAddr            string `xml:",omitempty"`
	RealRemoteAddrHeader string `xml:",omitempty"`
	Logging              logConfig
}

type logConfig struct {
	LogFile string

	Debug        bool
	DebugLogFile string

	LogRequests bool
	CensorLog   bool
}

var conf configFile

func genSecret() string {
	buf := make([]byte, 32)
	io.ReadFull(rand.Reader, buf)
	return fmt.Sprintf("%x", buf)
}

var defaultConfig = configFile{
	Listen:     "127.0.0.1:3748",
	Threads:    0,
	Workers:    10,
	Retries:    3,
	APITimeout: 60,
	CacheDir:   "cache/",
	Logging: logConfig{
		CensorLog: true,
	},
}

func createConfig() {
	//  Secret for eventual control code
	//	defaultConfig.Secret = genSecret()
	confXML, _ := xml.MarshalIndent(defaultConfig, "", "  ")

	err := ioutil.WriteFile("apiproxy.xml.default", confXML, 0600)
	if err != nil {
		log.Fatalf("Error creating config file apiproxy.xml.default: %s", err)
	} else {
		log.Fatalf("Created new config file apiproxy.xml.default")
	}
}

func loadConfig(filename string) (configFile, error) {
	conf, err := ioutil.ReadFile(filename)
	if err != nil {
		return defaultConfig, err
	}

	newConfig := defaultConfig
	err = xml.Unmarshal(conf, &newConfig)
	if err != nil {
		return defaultConfig, err
	}

	if newConfig.CacheDir == "" {
		return defaultConfig, fmt.Errorf("Need cache directory")
	}

	if newConfig.Logging.Debug {
		debug = true
	}

	return newConfig, nil
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"log"
	"os/user"
	"path/filepath"
	"strings"

	"os"

	"github.com/spf13/viper"
)

// TablesConf informations
type TablesConf struct {
	Name        string   `mapstructure:"name"`
	Permissions []string `mapstructure:"permissions"`
	Fields      []string `mapstructure:"fields"`
}

// AccessConf informations
type AccessConf struct {
	Restrict bool
	Tables   []TablesConf
}

// Prest basic config
type Prest struct {
	// HTTPPort Declare which http port the PREST used
	HTTPPort        int
	PGHost          string
	PGPort          int
	PGUser          string
	PGPass          string
	PGDatabase      string
	PGMaxIdleConn   int
	PGMAxOpenConn   int
	PGConnTimeout   int
	JWTKey          string
	MigrationsPath  string
	QueriesPath     string
	AccessConf      AccessConf
	CORSAllowOrigin []string
}

// PrestConf config variable
var PrestConf *Prest

func viperCfg() {
	filePath := os.Getenv("PREST_CONF")
	if filePath == "" {
		filePath = "prest.toml"
	}
	replacer := strings.NewReplacer(".", "_")
	viper.SetEnvPrefix("PREST")
	viper.AutomaticEnv()
	viper.SetEnvKeyReplacer(replacer)
	viper.SetConfigFile(filePath)
	viper.SetConfigType("toml")
	viper.SetDefault("http.port", 3000)
	viper.SetDefault("pg.host", "127.0.0.1")
	viper.SetDefault("pg.port", 5432)
	viper.SetDefault("pg.maxidleconn", 10)
	viper.SetDefault("pg.maxopenconn", 10)
	viper.SetDefault("pg.conntimeout", 10)

	user, err := user.Current()
	if err != nil {
		log.Println("{viperCfg}", err)
	}

	viper.SetDefault("queries.location", filepath.Join(user.HomeDir, "queries"))
}

// Parse pREST config
func Parse(cfg *Prest) (err error) {
	err = viper.ReadInConfig()
	if err != nil {
		return
	}
	cfg.HTTPPort = viper.GetInt("http.port")
	cfg.PGHost = viper.GetString("pg.host")
	cfg.PGPort = viper.GetInt("pg.port")
	cfg.PGUser = viper.GetString("pg.user")
	cfg.PGPass = viper.GetString("pg.pass")
	cfg.PGDatabase = viper.GetString("pg.database")
	cfg.PGMaxIdleConn = viper.GetInt("pg.maxidleconn")
	cfg.PGMAxOpenConn = viper.GetInt("pg.maxopenconn")
	cfg.PGConnTimeout = viper.GetInt("pg.conntimeout")
	cfg.JWTKey = viper.GetString("jwt.key")
	cfg.MigrationsPath = viper.GetString("migrations")
	cfg.AccessConf.Restrict = viper.GetBool("access.restrict")
	cfg.QueriesPath = viper.GetString("queries.location")
	cfg.CORSAllowOrigin = viper.GetStringSlice("cors.alloworigin")

	var t []TablesConf
	err = viper.UnmarshalKey("access.tables", &t)
	if err != nil {
		return err
	}

	cfg.AccessConf.Tables = t

	return
}

// Load configuration
func Load() {
	viperCfg()
	PrestConf = &Prest{}
	Parse(PrestConf)

	if !PrestConf.AccessConf.Restrict {
		fmt.Println("You are running pREST in public mode.")
	}

	if _, err := os.Stat(PrestConf.QueriesPath); os.IsNotExist(err) {
		os.MkdirAll(PrestConf.QueriesPath, 0777)
	}
}
<commit_msg>file no more mandatory (#144)<commit_after>package config

import (
	"fmt"
	"log"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"strings"

	"github.com/spf13/viper"
)

// TablesConf informations
type TablesConf struct {
	Name        string   `mapstructure:"name"`
	Permissions []string `mapstructure:"permissions"`
	Fields      []string `mapstructure:"fields"`
}

// AccessConf informations
type AccessConf struct {
	Restrict bool
	Tables   []TablesConf
}

// Prest basic config
type Prest struct {
	// HTTPPort Declare which http port the PREST used
	HTTPPort        int
	PGHost          string
	PGPort          int
	PGUser          string
	PGPass          string
	PGDatabase      string
	PGMaxIdleConn   int
	PGMAxOpenConn   int
	PGConnTimeout   int
	JWTKey          string
	MigrationsPath  string
	QueriesPath     string
	AccessConf      AccessConf
	CORSAllowOrigin []string
}

// PrestConf config variable
var PrestConf *Prest

func viperCfg() {
	filePath := os.Getenv("PREST_CONF")
	dir, file := path.Split(filePath)
	file = strings.TrimSuffix(file, filepath.Ext(file))
	replacer := strings.NewReplacer(".", "_")
	viper.SetEnvPrefix("PREST")
	viper.AutomaticEnv()
	viper.SetEnvKeyReplacer(replacer)
	viper.AddConfigPath(dir)
	viper.SetConfigName(file)
	viper.SetConfigType("toml")
	viper.SetDefault("http.port", 3000)
	viper.SetDefault("pg.host", "127.0.0.1")
	viper.SetDefault("pg.port", 5432)
	viper.SetDefault("pg.maxidleconn", 10)
	viper.SetDefault("pg.maxopenconn", 10)
	viper.SetDefault("pg.conntimeout", 10)

	user, err := user.Current()
	if err != nil {
		log.Println("{viperCfg}", err)
	}

	viper.SetDefault("queries.location", filepath.Join(user.HomeDir, "queries"))
}

// Parse pREST config
func Parse(cfg *Prest) (err error) {
	err = viper.ReadInConfig()
	if err != nil {
		return
	}
	cfg.HTTPPort = viper.GetInt("http.port")
	cfg.PGHost = viper.GetString("pg.host")
	cfg.PGPort = viper.GetInt("pg.port")
	cfg.PGUser = viper.GetString("pg.user")
	cfg.PGPass = viper.GetString("pg.pass")
	cfg.PGDatabase = viper.GetString("pg.database")
	cfg.PGMaxIdleConn = viper.GetInt("pg.maxidleconn")
	cfg.PGMAxOpenConn = viper.GetInt("pg.maxopenconn")
	cfg.PGConnTimeout = viper.GetInt("pg.conntimeout")
	cfg.JWTKey = viper.GetString("jwt.key")
	cfg.MigrationsPath = viper.GetString("migrations")
	cfg.AccessConf.Restrict = viper.GetBool("access.restrict")
	cfg.QueriesPath = viper.GetString("queries.location")
	cfg.CORSAllowOrigin = viper.GetStringSlice("cors.alloworigin")

	var t []TablesConf
	err = viper.UnmarshalKey("access.tables", &t)
	if err != nil {
		return err
	}

	cfg.AccessConf.Tables = t

	return
}

// Load configuration
func Load() {
	viperCfg()
	PrestConf = &Prest{}
	Parse(PrestConf)

	if !PrestConf.AccessConf.Restrict {
		fmt.Println("You are running pREST in public mode.")
	}

	if _, err := os.Stat(PrestConf.QueriesPath); os.IsNotExist(err) {
		os.MkdirAll(PrestConf.QueriesPath, 0777)
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/gopkg.in/yaml.v2"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/iam"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/github.com/tj/go-debug"
import "strings"
import "io/ioutil"
import "os"

/*

# lambda-phage config file sample

name: my-first-lambda-function
description: provides some sample stuff
archive: my-first-lambda-function.zip
entryPoint: "index.handler"
memorySize: 128
runtime: nodejs
timeout: 5
regions:
  - us-east-1
iamPolicy:
  # arn:
  name: lambda_basic_execution
location:
  s3bucket: test-bucket
  s3key: my-first-function/
  s3region: us-east-1
  #s3ObjectVersion
*/

type IamRole struct {
	Arn  *string `yaml:"arn"`
	Name *string `yaml:"name"`
}

type Location struct {
	S3Bucket        *string
	S3Key           *string
	S3ObjectVersion *string
	S3Region        *string
}

type Config struct {
	fName        string
	Name         *string
	Project      string
	Arn          *string
	Description  *string
	Archive      *string
	EntryPoint   *string `yaml:"entryPoint"`
	MemorySize   *int64  `yaml:"memorySize"`
	Runtime      *string
	Timeout      *int64
	Regions      []*string `yaml:"regions"`
	IamRole      *IamRole  `yaml:"iamRole"`
	Location     *Location
	EventSources []*EventSource `yaml:"eventSources"`
}

type EventSource struct {
	Type               string
	ApiName            *string `yaml:"apiName"`
	ApiDeploymentStage *string `yaml:"apiDeploymentStage"`
	ApiSecurity        *string `yaml:"apiSecurity"`
}

// writes this config to a YAML file
func (c *Config) writeToFile(fName string) error {
	return writeToYamlFile(c, fName)
}

// copies properties from all the config objects into
// the receiver config object
func (c *Config) merge(others ...*Config) *Config {
	for _, other := range others {
		c.Name = other.Name
		c.Project = other.Project
		c.Arn = other.Arn
		c.Description = other.Description
		c.Archive = other.Archive
		c.EntryPoint = other.EntryPoint
		c.MemorySize = other.MemorySize
		c.Runtime = other.Runtime
		c.Timeout = other.Timeout
		c.Regions = other.Regions
		c.IamRole = other.IamRole
		c.Location = other.Location
		c.EventSources = other.EventSources
	}
	return c
}

// returns the arn for the role specified
func (c *Config) getRoleArn() (*string, error) {
	if c.IamRole.Arn == nil &&
		c.IamRole.Name == nil {
		return nil, fmt.Errorf("Missing ARN config!")
	}

	// if the config file has an ARN listed,
	// that takes precedence
	if c.IamRole.Arn != nil {
		return c.IamRole.Arn, nil
	} else if c.IamRole.Name != nil {
		// look up the iam role name
		iamRole, err := getIamPolicy(*c.IamRole.Name)
		if err != nil {
			return nil, err
		}
		return iamRole, err
	} else {
		// TODO: create a default standard role
		// and update config file
	}

	return nil, fmt.Errorf("how did you get here")
}

// returns a normalized S3 path to a file
// based on config information
//
// requires the file name of the archive you'll upload
//
// returns the bucket and the key
func (c *Config) getS3Info(fName string) (bucket, key *string) {
	debug := debug.Debug("config.getS3Info")
	loc := c.Location
	if loc == nil {
		debug("no upload location info found")
		return nil, nil
	}

	if loc.S3Bucket == nil {
		// TODO: make these return an error instead??
		debug("upload location info found, but s3 bucket missing")
		return nil, nil
	}

	b := *loc.S3Bucket
	var k string
	if loc.S3Key == nil {
		debug("no s3 key in location config, using file name")
		// no key in config?
		// then the key is the name of the file
		// being passed in
		k = fName
	} else if loc.S3Key != nil &&
		len(*loc.S3Key) > 0 {
		// key in config? let's see
		// if it looks like a zip file
		if strings.Index(*loc.S3Key, ".zip") > -1 {
			// great, we can use this one for the key
			k = *loc.S3Key
		} else {
			// if there's no .zip in the s3Key config
			// setting, then assume this is to
			// be the first part in a directory
			dir := *loc.S3Key
			sl := []byte("/")
			if dir[len(dir)-1] != sl[0] {
				dir += "/"
			}

			k = dir + fName
		}
	} else {
		debug("empty s3key found in config file")
		k = fName
	}

	return &b, &k
}

// gets an IAM policy by name
func getIamPolicy(name string) (*string, error) {
	debug := debug.Debug("getIamPolicy")
	i := iam.New(nil)

	debug("getting iam role")

	r, err := i.GetRole(&iam.GetRoleInput{
		RoleName: &name,
	})

	if err != nil {
		return aws.String(""), err
	}

	return r.Role.Arn, nil
}

func writeToYamlFile(data interface{}, fName string) error {
	d, err := yaml.Marshal(data)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(fName, d, os.FileMode(0644))
	if err != nil {
		return err
	}
}
<commit_msg>fix missing return<commit_after>package main

import "fmt"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/gopkg.in/yaml.v2"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/github.com/aws/aws-sdk-go/service/iam"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws"
import "github.com/hopkinsth/lambda-phage/Godeps/_workspace/src/github.com/tj/go-debug"
import "strings"
import "io/ioutil"
import "os"

/*

# lambda-phage config file sample

name: my-first-lambda-function
description: provides some sample stuff
archive: my-first-lambda-function.zip
entryPoint: "index.handler"
memorySize: 128
runtime: nodejs
timeout: 5
regions:
  - us-east-1
iamPolicy:
  # arn:
  name: lambda_basic_execution
location:
  s3bucket: test-bucket
  s3key: my-first-function/
  s3region: us-east-1
  #s3ObjectVersion
*/

type IamRole struct {
	Arn  *string `yaml:"arn"`
	Name *string `yaml:"name"`
}

type Location struct {
	S3Bucket        *string
	S3Key           *string
	S3ObjectVersion *string
	S3Region        *string
}

type Config struct {
	fName        string
	Name         *string
	Project      string
	Arn          *string
	Description  *string
	Archive      *string
	EntryPoint   *string `yaml:"entryPoint"`
	MemorySize   *int64  `yaml:"memorySize"`
	Runtime      *string
	Timeout      *int64
	Regions      []*string `yaml:"regions"`
	IamRole      *IamRole  `yaml:"iamRole"`
	Location     *Location
	EventSources []*EventSource `yaml:"eventSources"`
}

type EventSource struct {
	Type               string
	ApiName            *string `yaml:"apiName"`
	ApiDeploymentStage *string `yaml:"apiDeploymentStage"`
	ApiSecurity        *string `yaml:"apiSecurity"`
}

// writes this config to a YAML file
func (c *Config) writeToFile(fName string) error {
	return writeToYamlFile(c, fName)
}

// copies properties from all the config objects into
// the receiver config object
func (c *Config) merge(others ...*Config) *Config {
	for _, other := range others {
		c.Name = other.Name
		c.Project = other.Project
		c.Arn = other.Arn
		c.Description = other.Description
		c.Archive = other.Archive
		c.EntryPoint = other.EntryPoint
		c.MemorySize = other.MemorySize
		c.Runtime = other.Runtime
		c.Timeout = other.Timeout
		c.Regions = other.Regions
		c.IamRole = other.IamRole
		c.Location = other.Location
		c.EventSources = other.EventSources
	}
	return c
}

// returns the arn for the role specified
func (c *Config) getRoleArn() (*string, error) {
	if c.IamRole.Arn == nil &&
		c.IamRole.Name == nil {
		return nil, fmt.Errorf("Missing ARN config!")
	}

	// if the config file has an ARN listed,
	// that takes precedence
	if c.IamRole.Arn != nil {
		return c.IamRole.Arn, nil
	} else if c.IamRole.Name != nil {
		// look up the iam role name
		iamRole, err := getIamPolicy(*c.IamRole.Name)
		if err != nil {
			return nil, err
		}
		return iamRole, err
	} else {
		// TODO: create a default standard role
		// and update config file
	}

	return nil, fmt.Errorf("how did you get here")
}

// returns a normalized S3 path to a file
// based on config information
//
// requires the file name of the archive you'll upload
//
// returns the bucket and the key
func (c *Config) getS3Info(fName string) (bucket, key *string) {
	debug := debug.Debug("config.getS3Info")
	loc := c.Location
	if loc == nil {
		debug("no upload location info found")
		return nil, nil
	}

	if loc.S3Bucket == nil {
		// TODO: make these return an error instead??
		debug("upload location info found, but s3 bucket missing")
		return nil, nil
	}

	b := *loc.S3Bucket
	var k string
	if loc.S3Key == nil {
		debug("no s3 key in location config, using file name")
		// no key in config?
		// then the key is the name of the file
		// being passed in
		k = fName
	} else if loc.S3Key != nil &&
		len(*loc.S3Key) > 0 {
		// key in config? let's see
		// if it looks like a zip file
		if strings.Index(*loc.S3Key, ".zip") > -1 {
			// great, we can use this one for the key
			k = *loc.S3Key
		} else {
			// if there's no .zip in the s3Key config
			// setting, then assume this is to
			// be the first part in a directory
			dir := *loc.S3Key
			sl := []byte("/")
			if dir[len(dir)-1] != sl[0] {
				dir += "/"
			}

			k = dir + fName
		}
	} else {
		debug("empty s3key found in config file")
		k = fName
	}

	return &b, &k
}

// gets an IAM policy by name
func getIamPolicy(name string) (*string, error) {
	debug := debug.Debug("getIamPolicy")
	i := iam.New(nil)

	debug("getting iam role")

	r, err := i.GetRole(&iam.GetRoleInput{
		RoleName: &name,
	})

	if err != nil {
		return aws.String(""), err
	}

	return r.Role.Arn, nil
}

func writeToYamlFile(data interface{}, fName string) error {
	d, err := yaml.Marshal(data)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(fName, d, os.FileMode(0644))
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"time"
)

type cacheEntry struct {
	ModTime  time.Time
	Contents interface{}
}

type Config struct {
	Cache         map[string]cacheEntry
	BuildFileList func(map[string]string) []string
}

var C Config

func init() {
	log.Println("Initializing configuration cache")
	C.Cache = make(map[string]cacheEntry)
}

func lookupvar(key, path string) interface{} {
	var f interface{}
	i, err := os.Stat(path)
	_, ok := C.Cache[path]
	if os.IsNotExist(err) {
		log.Println("Config does not exist", path)
		if ok {
			log.Println("Purging", path, "from cache")
			delete(C.Cache, path)
		}
	}
	if err != nil {
		return nil
	}

	if C.Cache[path].ModTime.Before(i.ModTime()) || !ok {
		log.Println("Stale cache for", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}

		err = json.Unmarshal(data, &f)
		if err != nil {
			log.Println("Cannot parse", path)
			return nil
		}

		log.Println("Updating cache for", path)
		C.Cache[path] = cacheEntry{
			ModTime:  i.ModTime(),
			Contents: f,
		}

		return f.(map[string]interface{})[key]
	} else {
		log.Println("Found cache for", path)
		return C.Cache[path].Contents.(map[string]interface{})[key]
	}
}

func (c *Config) Lookup(context map[string]string, key string) interface{} {
	var value interface{}

	for _, fpath := range c.BuildFileList(context) {
		log.Println("Context", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value
		}
	}

	log.Println("Context", context, "Key", key, "not found")
	return nil
}
<commit_msg>Code cleanup - do away with exports<commit_after>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"time"
)

type cacheEntry struct {
	modTime  time.Time
	contents interface{}
}

type config struct {
	cache         map[string]cacheEntry
	buildFileList func(map[string]string) []string
}

var c config

func init() {
	log.Println("Initializing configuration cache")
	c.cache = make(map[string]cacheEntry)
}

func lookupvar(key, path string) interface{} {
	var f interface{}
	i, err := os.Stat(path)
	_, ok := c.cache[path]
	if os.IsNotExist(err) {
		log.Println("Config does not exist", path)
		if ok {
			log.Println("Purging", path, "from cache")
			delete(c.cache, path)
		}
	}
	if err != nil {
		return nil
	}

	if c.cache[path].modTime.Before(i.ModTime()) || !ok {
		log.Println("Stale cache for", path)
		data, err := ioutil.ReadFile(path)
		if err != nil {
			return nil
		}

		err = json.Unmarshal(data, &f)
		if err != nil {
			log.Println("Cannot parse", path)
			return nil
		}

		log.Println("Updating cache for", path)
		c.cache[path] = cacheEntry{
			modTime:  i.ModTime(),
			contents: f,
		}

		return f.(map[string]interface{})[key]
	}

	log.Println("Found cache for", path)
	return c.cache[path].contents.(map[string]interface{})[key]
}

func SetFileListBuilder(f func(map[string]string) []string) {
	c.buildFileList = f
}

func Lookup(context map[string]string, key string) interface{} {
	var value interface{}

	for _, fpath := range c.buildFileList(context) {
		log.Println("Context", context, "Looking up", key, "in", fpath)
		value = lookupvar(key, fpath)
		if value != nil {
			log.Println("Context:", context, "Found", key, value)
			return value
		}
	}

	log.Println("Context", context, "Key", key, "not found")
	return nil
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"io/ioutil"
)

type Config struct {
	Nick      string
	Host      string
	RealName  string
	User      string
	Networks  []string
	Servers   map[string][]string
	Channels  map[string][]string
	Passwords map[string]string
	Plugins   []string
	Ignore    []string
	Logpath   string
}

func ReadConfig(path string) (Config, error) {
	var config Config

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return config, err
	}

	err = json.Unmarshal(data, &config)
	if err != nil {
		return config, err
	}

	return config, nil
}
<commit_msg>It's hierryfying and does not build… yet<commit_after>package config

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"path"
)

type Config struct {
	Nick      string
	Host      string
	RealName  string
	User      string
	Networks  []string
	Servers   map[string][]string
	Channels  map[string][]string
	Passwords map[string]string
	Plugins   []string
	Ignore    []string
	Logpath   string
	ConfigDir string
}

type Context struct {
	Network string
	Target  string
	Source  string
}

var C Config

func lookupvar(key, path string) interface{} {
	var f interface{}
	_, err := os.Stat(path)
	if err != nil {
		return ""
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return ""
	}

	err = json.Unmarshal(data, &f)

	return f.(map[string]interface{})[key]
}

func (c *Config) Lookup(context Context, key string) interface{} {
	var fpath string
	var value interface{}
	if context.Network != "" {
		if context.Source != "" {
			if context.Target != "" {
				fpath = path.Join(c.ConfigDir, context.Network, context.Source, context.Target+".json")

				log.Println("Context:", context, "Looking up", key, "in", fpath)
				value = lookupvar(key, fpath)
				if value != nil {
					return value
				}
			}
			return lookupvar(key, path.Join(c.ConfigDir, context.Network, context.Source+".json"))
		}
		return lookupvar(key, path.Join(c.ConfigDir, context.Network+".json"))
	}
	return lookupvar(key, path.Join(c.ConfigDir, "common.json"))
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/gcfg"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
)

type Configuration struct {
	ImageName           string   `json:"image_name"`
	MountHomeTo         string   `json:"mount_home_to"`
	ContainerUsername   string   `json:"container_username"`
	Shell               string   `json:"shell"`
	BlacklistUserConfig []string `json:"blacklist_user_config"`
	BlacklistSetup      bool
	DisableUserConfig   bool   `json:"disable_user_config"`
	MountHome           bool   `json:"mount_home"`
	MountTmp            bool   `json:"mount_tmp"`
	MountDockerSocket   bool   `json:"mount_docker_socket"`
	DockerSocket        string `json:"docker_socket"`
}

func (c Configuration) Dump() string {
	return fmt.Sprintf("ImageName %s MountHomeTo %s ContainerUsername %s Shell %s DockerSocket %s", c.ImageName, c.MountHomeTo, c.ContainerUsername, c.Shell, c.DockerSocket)
}

type configInterpolation struct {
	Home string
	User string
}

var defaultConfig = Configuration{
	ImageName:           "busybox",
	MountHomeTo:         "%h",
	ContainerUsername:   "%u",
	Shell:               "/bin/ash",
	MountHome:           true,
	MountTmp:            true,
	BlacklistUserConfig: []string{"image_name", "shell", "container_username", "mount_home_to", "mount_tmp", "mount_docker_socket"},
	BlacklistSetup:      false,
	DisableUserConfig:   false,
	MountDockerSocket:   false,
	DockerSocket:        "/var/run/docker.sock",
}

func loadAllConfig(user string, homedir string) (config Configuration, err error) {
	globalconfig, err := loadConfig(loadableFile("/etc/dockersh"), user)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return config, errors.New("could not load config")
	}
	localconfig, err := loadConfig(loadableFile(fmt.Sprintf("%s/.dockersh", homedir)), user)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return config, errors.New("could not load config")
	}
	return mergeConfigs(mergeConfigs(defaultConfig, globalconfig), localconfig), nil
}

type loadableFile string

func (fn loadableFile) Getcontents() []byte {
	localConfigFile, err := os.Open(string(fn))
	if err != nil {
	}
	b, err := ioutil.ReadAll(localConfigFile)
	localConfigFile.Close()
	return b
}

func loadConfig(filename loadableFile, user string) (config Configuration, err error) {
	bytes := filename.Getcontents()
	if err != nil {
		return
	}
	return (loadConfigFromString(bytes, user))
}

func mergeConfigs(old Configuration, new Configuration) (ret Configuration) {
	return old
}

func loadConfigFromString(bytes []byte, user string) (config Configuration, err error) {
	inicfg := struct {
		Dockersh Configuration
		Profile  map[string]*Configuration
	}{}
	err = gcfg.ReadStringInto(&inicfg, string(bytes))
	if err != nil {
		return
	}
	return mergeConfigs(inicfg.Dockersh, *inicfg.Profile[user]), nil
}
<commit_msg>Fix segfault if no user config found<commit_after>package main

import (
	"code.google.com/p/gcfg"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
)

type Configuration struct {
	ImageName           string   `json:"image_name"`
	MountHomeTo         string   `json:"mount_home_to"`
	ContainerUsername   string   `json:"container_username"`
	Shell               string   `json:"shell"`
	BlacklistUserConfig []string `json:"blacklist_user_config"`
	BlacklistSetup      bool
	DisableUserConfig   bool   `json:"disable_user_config"`
	MountHome           bool   `json:"mount_home"`
	MountTmp            bool   `json:"mount_tmp"`
	MountDockerSocket   bool   `json:"mount_docker_socket"`
	DockerSocket        string `json:"docker_socket"`
}

func (c Configuration) Dump() string {
	return fmt.Sprintf("ImageName %s MountHomeTo %s ContainerUsername %s Shell %s DockerSocket %s", c.ImageName, c.MountHomeTo, c.ContainerUsername, c.Shell, c.DockerSocket)
}

type configInterpolation struct {
	Home string
	User string
}

var defaultConfig = Configuration{
	ImageName:           "busybox",
	MountHomeTo:         "%h",
	ContainerUsername:   "%u",
	Shell:               "/bin/ash",
	MountHome:           true,
	MountTmp:            true,
	BlacklistUserConfig: []string{"image_name", "shell", "container_username", "mount_home_to", "mount_tmp", "mount_docker_socket"},
	BlacklistSetup:      false,
	DisableUserConfig:   false,
	MountDockerSocket:   false,
	DockerSocket:        "/var/run/docker.sock",
}

func loadAllConfig(user string, homedir string) (config Configuration, err error) {
	globalconfig, err := loadConfig(loadableFile("/etc/dockersh"), user)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return config, errors.New("could not load config")
	}
	localconfig, err := loadConfig(loadableFile(fmt.Sprintf("%s/.dockersh", homedir)), user)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not load config: %v", err)
		return config, errors.New("could not load config")
	}
	return mergeConfigs(mergeConfigs(defaultConfig, globalconfig), localconfig), nil
}

type loadableFile string

func (fn loadableFile) Getcontents() []byte {
	localConfigFile, err := os.Open(string(fn))
	if err != nil {
	}
	b, err := ioutil.ReadAll(localConfigFile)
	localConfigFile.Close()
	return b
}

func loadConfig(filename loadableFile, user string) (config Configuration, err error) {
	bytes := filename.Getcontents()
	if err != nil {
		return
	}
	return (loadConfigFromString(bytes, user))
}

func mergeConfigs(old Configuration, new Configuration) (ret Configuration) {
	return old
}

func loadConfigFromString(bytes []byte, user string) (config Configuration, err error) {
	inicfg := struct {
		Dockersh Configuration
		Profile  map[string]*Configuration
	}{}
	err = gcfg.ReadStringInto(&inicfg, string(bytes))
	if err != nil {
		return
	}
	if inicfg.Profile[user] == nil {
		return inicfg.Dockersh, nil
	}
	return mergeConfigs(inicfg.Dockersh, *inicfg.Profile[user]), nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/hashicorp/consul/api"
	"log"
	"os"
)

// build consul client
func (c *config) buildClient() error {
	var cc *api.Config // client configuration
	var err error      // general error holder

	// build default client config
	cc = api.DefaultConfig()

	// overwrite address if needed
	if c.consulAddr != "" {
		cc.Address = c.consulAddr
	}

	// overwrite scheme if needed
	if c.consulScheme != "" {
		cc.Scheme = c.consulScheme
	}

	// overwrite dc if needed
	if c.consulDc != "" {
		cc.Datacenter = c.consulDc
	}

	// overwrite token if needed
	if c.consulToken != "" {
		cc.Token = c.consulToken
	}

	// populate client wrapper
	c.consulClient, err = api.NewClient(cc)

	// return last error
	return err
}

// fetch keys from the store and write to a backup file
func (c *config) backupKeys() (int, error) {
	var kvps api.KVPairs       // list of requested kv pairs
	var opts *api.QueryOptions // client query options
	var count int              // key count
	var data []byte            // read keys
	var err error              // general error holder

	// build query options
	opts = &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}

	// get all keys
	if kvps, _, err = c.consulClient.KV().List(c.consulPrefix, opts); err != nil {
		return count, err
	}

	// transform paths
	c.transformPaths(kvps)

	// set count
	count = len(kvps)

	// encode and return
	if data, err = json.MarshalIndent(kvps, "", "  "); err != nil {
		return count, err
	}

	// write data
	if err = c.writeBackupFile(data); err != nil {
		return count, err
	}

	// return key count - no error
	return count, nil
}

// read keys from a backup file and restore to consul
func (c *config) restoreKeys() (int, error) {
	var kvps api.KVPairs // decoded kv pairs
	var count int        // key count
	var data []byte      // read json data
	var err error        // general error holder

	// read json data from file
	if data, err = c.readBackupFile(); err != nil {
		return count, err
	}

	// doing a dump?
	if c.dataDump {
		// dump data
		if err = dumpData(data, c.plainDump); err != nil {
			return err
		}
		// exit clean
		os.Exit(0)
	}

	// decode data
	if err = json.Unmarshal(data, &kvps); err != nil {
		return count, err
	}

	// transform paths
	c.transformPaths(kvps)

	// set count
	count = len(kvps)

	// delete tree before restore if requested
	if c.delTree {
		if _, err := c.consulClient.KV().DeleteTree(c.consulPrefix, nil); err != nil {
			return count, err
		}
	}

	// loop through keys
	for _, kv := range kvps {
		// write key
		if _, err = c.consulClient.KV().Put(kv, nil); err != nil {
			log.Printf("[Warning] Failed to restore %s: %s",
				kv.Key, err.Error())
		}
	}

	// return key count - no error
	return count, nil
}
<commit_msg>oops, fixup problem caused by #1<commit_after>package main

import (
	"encoding/json"
	"github.com/hashicorp/consul/api"
	"log"
	"os"
)

// build consul client
func (c *config) buildClient() error {
	var cc *api.Config // client configuration
	var err error      // general error holder

	// build default client config
	cc = api.DefaultConfig()

	// overwrite address if needed
	if c.consulAddr != "" {
		cc.Address = c.consulAddr
	}

	// overwrite scheme if needed
	if c.consulScheme != "" {
		cc.Scheme = c.consulScheme
	}

	// overwrite dc if needed
	if c.consulDc != "" {
		cc.Datacenter = c.consulDc
	}

	// overwrite token if needed
	if c.consulToken != "" {
		cc.Token = c.consulToken
	}

	// populate client wrapper
	c.consulClient, err = api.NewClient(cc)

	// return last error
	return err
}

// fetch keys from the store and write to a backup file
func (c *config) backupKeys() (int, error) {
	var kvps api.KVPairs       // list of requested kv pairs
	var opts *api.QueryOptions // client query options
	var count int              // key count
	var data []byte            // read keys
	var err error              // general error holder

	// build query options
	opts = &api.QueryOptions{
		AllowStale:        false,
		RequireConsistent: true,
	}

	// get all keys
	if kvps, _, err = c.consulClient.KV().List(c.consulPrefix, opts); err != nil {
		return count, err
	}

	// transform paths
	c.transformPaths(kvps)

	// set count
	count = len(kvps)

	// encode and return
	if data, err = json.MarshalIndent(kvps, "", "  "); err != nil {
		return count, err
	}

	// write data
	if err = c.writeBackupFile(data); err != nil {
		return count, err
	}

	// return key count - no error
	return count, nil
}

// read keys from a backup file and restore to consul
func (c *config) restoreKeys() (int, error) {
	var kvps api.KVPairs // decoded kv pairs
	var count int        // key count
	var data []byte      // read json data
	var err error        // general error holder

	// read json data from file
	if data, err = c.readBackupFile(); err != nil {
		return count, err
	}

	// doing a dump?
	if c.dataDump {
		// dump data
		if err = dumpData(data, c.plainDump); err != nil {
			return count, err
		}
		// exit clean
		os.Exit(0)
	}

	// decode data
	if err = json.Unmarshal(data, &kvps); err != nil {
		return count, err
	}

	// transform paths
	c.transformPaths(kvps)

	// set count
	count = len(kvps)

	// delete tree before restore if requested
	if c.delTree {
		if _, err := c.consulClient.KV().DeleteTree(c.consulPrefix, nil); err != nil {
			return count, err
		}
	}

	// loop through keys
	for _, kv := range kvps {
		// write key
		if _, err = c.consulClient.KV().Put(kv, nil); err != nil {
			log.Printf("[Warning] Failed to restore %s: %s",
				kv.Key, err.Error())
		}
	}

	// return key count - no error
	return count, nil
}
<|endoftext|>


<commit_before>package conio

import (
	"bytes"
	"fmt"
	"strings"
	"unicode"
)

func KeyFuncIncSearch(this *Buffer) Result {
	var searchBuf bytes.Buffer
	foundStr := ""
	searchStr := ""
	lastFoundPos := len(this.Session.Histories) - 1
	Backspace(this.Cursor - this.ViewStart)
	for {
		drawStr := fmt.Sprintf("(i-search)[%s]:%s", searchStr, foundStr)
		drawWidth := GetStringWidth(drawStr)
		fmt.Print(drawStr)
		fmt.Print("  ")
		Backspace(2)
		charcode, _, _ := GetKey()
		Backspace(drawWidth)
		switch charcode {
		case '\b':
			searchBuf.Reset()
			// chop last char
			var lastchar rune
			for i, c := range searchStr {
				if i > 0 {
					searchBuf.WriteRune(lastchar)
				}
				lastchar = c
			}
			searchStr = searchBuf.String()
		case '\r': // ENTER
			this.ViewStart = 0
			this.Length = 0
			this.Cursor = 0
			this.ReplaceAndRepaint(0, foundStr)
			return CONTINUE
		case rune('g' & 0x1F):
			w := 0
			var i int
			for i = this.ViewStart; i < this.Cursor; i++ {
				w += GetCharWidth(this.Buffer[i])
				PutRep(this.Buffer[i], 1)
			}
			bs := 0
			for {
				if i >= this.Length {
					if drawWidth > w {
						PutRep(' ', drawWidth-w)
						bs += (drawWidth - w)
					}
					break
				}
				w1 := GetCharWidth(this.Buffer[i])
				if w+w1 >= this.ViewWidth {
					break
				}
				PutRep(this.Buffer[i], 1)
				w += w1
				bs += w1
				i++
			}
			Backspace(bs)
			return CONTINUE
		case rune('r' & 0x1F):
			for i := lastFoundPos; i >= 0; i-- {
				p := this.Session.Histories[i]
				if strings.Contains(p.Line, searchStr) {
					foundStr = p.Line
					lastFoundPos = i
					break
				}
			}
		default:
			if unicode.IsControl(charcode) {
				break
			}
			searchBuf.WriteRune(charcode)
			searchStr = searchBuf.String()
			for i := len(this.Session.Histories) - 1; ; i-- {
				if i < 0 {
					foundStr = ""
					break
				}
				p := this.Session.Histories[i]
				if strings.Contains(p.Line, searchStr) {
					foundStr = p.Line
					lastFoundPos = i
					break
				}
			}
		}
	}
}
<commit_msg>Not print noise on C-r in incremental-search<commit_after>package conio

import (
	"bytes"
	"fmt"
	"strings"
	"unicode"
)

func KeyFuncIncSearch(this *Buffer) Result {
	var searchBuf bytes.Buffer
	foundStr := ""
	searchStr := ""
	lastDrawWidth := 0
	lastFoundPos := len(this.Session.Histories) - 1
	Backspace(this.Cursor - this.ViewStart)
	for {
		drawStr := fmt.Sprintf("(i-search)[%s]:%s", searchStr, foundStr)
		drawWidth := GetStringWidth(drawStr)
		fmt.Print(drawStr)
		if lastDrawWidth > drawWidth {
			n := lastDrawWidth - drawWidth
			PutRep(' ', n)
			Backspace(n)
		}
		lastDrawWidth = drawWidth
		charcode, _, _ := GetKey()
		Backspace(drawWidth)
		switch charcode {
		case '\b':
			searchBuf.Reset()
			// chop last char
			var lastchar rune
			for i, c := range searchStr {
				if i > 0 {
					searchBuf.WriteRune(lastchar)
				}
				lastchar = c
			}
			searchStr = searchBuf.String()
		case '\r': // ENTER
			this.ViewStart = 0
			this.Length = 0
			this.Cursor = 0
			this.ReplaceAndRepaint(0, foundStr)
			return CONTINUE
		case rune('g' & 0x1F):
			w := 0
			var i int
			for i = this.ViewStart; i < this.Cursor; i++ {
				w += GetCharWidth(this.Buffer[i])
				PutRep(this.Buffer[i], 1)
			}
			bs := 0
			for {
				if i >= this.Length {
					if drawWidth > w {
						PutRep(' ', drawWidth-w)
						bs += (drawWidth - w)
					}
					break
				}
				w1 := GetCharWidth(this.Buffer[i])
				if w+w1 >= this.ViewWidth {
					break
				}
				PutRep(this.Buffer[i], 1)
				w += w1
				bs += w1
				i++
			}
			Backspace(bs)
			return CONTINUE
		case rune('r' & 0x1F):
			for i := lastFoundPos - 1; ; i-- {
				if i < 0 {
					i = len(this.Session.Histories) - 1
				}
				if i == lastFoundPos {
					break
				}
				p := this.Session.Histories[i]
				if strings.Contains(p.Line, searchStr) && foundStr != p.Line {
					foundStr = p.Line
					lastFoundPos = i
					break
				}
			}
		default:
			if unicode.IsControl(charcode) {
				break
			}
			searchBuf.WriteRune(charcode)
			searchStr = searchBuf.String()
			for i := len(this.Session.Histories) - 1; ; i-- {
				if i < 0 {
					foundStr = ""
					break
				}
				p := this.Session.Histories[i]
				if strings.Contains(p.Line, searchStr) {
					foundStr = p.Line
					lastFoundPos = i
					break
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestLoadProcessConfigReal(t *testing.T) {
	_, err := LoadProcessConfig("./test/config1.json")
	assert.NoError(t, err)
}

func TestLoadProcessConfig1(t *testing.T) {
	template := []string{"./cmd1", "%{uploads_dir}", "%{download_files}"}
	job_sub := "projects/dummy-gcp-proj/subscriptions/test-job-subscription"
	job_pull_interval := 60
	job_sus_delay := float64(600)
	job_sus_interval := float64(540)
	prog_topic := "projects/dummy-gcp-proj/topics/test-progress-topic"

	d := map[string]interface{}{
		"command": map[string]interface{}{
			"template": template,
		},
		"job": map[string]interface{}{
			"subscription":  job_sub,
			"pull_interval": job_pull_interval,
			"sustainer": map[string]interface{}{
				"delay":    job_sus_delay,
				"interval": job_sus_interval,
			},
		},
		"progress": map[string]interface{}{
			"topic": prog_topic,
		},
	}

	dir, err := ioutil.TempDir("", "process_config")
	assert.NoError(t, err)
	defer os.RemoveAll(dir)

	fpath := filepath.Join(dir, "config.json")
	s, err := json.Marshal(d)
	assert.NoError(t, err)
	err = ioutil.WriteFile(fpath, s, 0666)
	assert.NoError(t, err)

	config, err := LoadProcessConfig(fpath)
	assert.NoError(t, err)

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, template, config.Command.Template)
		assert.Nil(t, config.Command.Options)
		assert.Equal(t, false, config.Command.Dryrun)
	}

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, job_sub, config.Job.Subscription)
		assert.Equal(t, job_pull_interval, config.Job.PullInterval)
		if assert.NotNil(t, config.Job.Sustainer) {
			assert.Equal(t, job_sus_delay, config.Job.Sustainer.Delay)
			assert.Equal(t, job_sus_interval, config.Job.Sustainer.Interval)
		}
	}
}

func TestLoadProcessConfig2(t *testing.T) {
	template := []string{"%{attrs.key}"}
	commands := map[string][]string{
		"key1": []string{"./cmd1", "%{uploads_dir}", "%{download_files.foo}", "%{download_files.bar}"},
		"key2": []string{"./cmd2", "%{uploads_dir}", "%{download_files}"},
	}
	job_sub := "projects/dummy-gcp-proj/subscriptions/test-job-subscription"
	job_pull_interval := 60
	job_sus_delay := float64(600)
	job_sus_interval := float64(540)
	prog_topic := "projects/dummy-gcp-proj/topics/test-progress-topic"

	d := map[string]interface{}{
		"command": map[string]interface{}{
			"template": template,
			"options": commands,
		},
		"job": map[string]interface{}{
			"subscription":  job_sub,
			"pull_interval": job_pull_interval,
			"sustainer": map[string]interface{}{
				"delay":    job_sus_delay,
				"interval": job_sus_interval,
			},
		},
		"progress": map[string]interface{}{
			"topic": prog_topic,
		},
	}

	dir, err := ioutil.TempDir("", "process_config")
	assert.NoError(t, err)
	defer os.RemoveAll(dir)

	fpath := filepath.Join(dir, "config2.json")
	s, err := json.Marshal(d)
	assert.NoError(t, err)
	err = ioutil.WriteFile(fpath, s, 0666)
	assert.NoError(t, err)

	config, err := LoadProcessConfig(fpath)
	assert.NoError(t, err)

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, template, config.Command.Template)
		assert.Equal(t, commands, config.Command.Options)
		assert.Equal(t, false, config.Command.Dryrun)
	}

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, job_sub, config.Job.Subscription)
		assert.Equal(t, job_pull_interval, config.Job.PullInterval)
		if assert.NotNil(t, config.Job.Sustainer) {
			assert.Equal(t, job_sus_delay, config.Job.Sustainer.Delay)
			assert.Equal(t, job_sus_interval, config.Job.Sustainer.Interval)
		}
	}
}
<commit_msg>:green_heart: Add assertions for progress.topic<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestLoadProcessConfigReal(t *testing.T) {
	_, err := LoadProcessConfig("./test/config1.json")
	assert.NoError(t, err)
}

func TestLoadProcessConfig1(t *testing.T) {
	template := []string{"./cmd1", "%{uploads_dir}", "%{download_files}"}
	job_sub := "projects/dummy-gcp-proj/subscriptions/test-job-subscription"
	job_pull_interval := 60
	job_sus_delay := float64(600)
	job_sus_interval := float64(540)
	prog_topic := "projects/dummy-gcp-proj/topics/test-progress-topic"

	d := map[string]interface{}{
		"command": map[string]interface{}{
			"template": template,
		},
		"job": map[string]interface{}{
			"subscription":  job_sub,
			"pull_interval": job_pull_interval,
			"sustainer": map[string]interface{}{
				"delay":    job_sus_delay,
				"interval": job_sus_interval,
			},
		},
		"progress": map[string]interface{}{
			"topic": prog_topic,
		},
	}

	dir, err := ioutil.TempDir("", "process_config")
	assert.NoError(t, err)
	defer os.RemoveAll(dir)

	fpath := filepath.Join(dir, "config.json")
	s, err := json.Marshal(d)
	assert.NoError(t, err)
	err = ioutil.WriteFile(fpath, s, 0666)
	assert.NoError(t, err)

	config, err := LoadProcessConfig(fpath)
	assert.NoError(t, err)

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, template, config.Command.Template)
		assert.Nil(t, config.Command.Options)
		assert.Equal(t, false, config.Command.Dryrun)
	}

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, job_sub, config.Job.Subscription)
		assert.Equal(t, job_pull_interval, config.Job.PullInterval)
		if assert.NotNil(t, config.Job.Sustainer) {
			assert.Equal(t, job_sus_delay, config.Job.Sustainer.Delay)
			assert.Equal(t, job_sus_interval, config.Job.Sustainer.Interval)
		}
	}

	if assert.NotNil(t, config.Progress) {
		assert.Equal(t, prog_topic, config.Progress.Topic)
	}
}

func TestLoadProcessConfig2(t *testing.T) {
	template := []string{"%{attrs.key}"}
	commands := map[string][]string{
		"key1": []string{"./cmd1", "%{uploads_dir}", "%{download_files.foo}", "%{download_files.bar}"},
		"key2": []string{"./cmd2", "%{uploads_dir}", "%{download_files}"},
	}
	job_sub := "projects/dummy-gcp-proj/subscriptions/test-job-subscription"
	job_pull_interval := 60
	job_sus_delay := float64(600)
	job_sus_interval := float64(540)
	prog_topic := "projects/dummy-gcp-proj/topics/test-progress-topic"

	d := map[string]interface{}{
		"command": map[string]interface{}{
			"template": template,
			"options": commands,
		},
		"job": map[string]interface{}{
			"subscription":  job_sub,
			"pull_interval": job_pull_interval,
			"sustainer": map[string]interface{}{
				"delay":    job_sus_delay,
				"interval": job_sus_interval,
			},
		},
		"progress": map[string]interface{}{
			"topic": prog_topic,
		},
	}

	dir, err := ioutil.TempDir("", "process_config")
	assert.NoError(t, err)
	defer os.RemoveAll(dir)

	fpath := filepath.Join(dir, "config2.json")
	s, err := json.Marshal(d)
	assert.NoError(t, err)
	err = ioutil.WriteFile(fpath, s, 0666)
	assert.NoError(t, err)

	config, err := LoadProcessConfig(fpath)
	assert.NoError(t, err)

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, template, config.Command.Template)
		assert.Equal(t, commands, config.Command.Options)
		assert.Equal(t, false, config.Command.Dryrun)
	}

	if assert.NotNil(t, config.Job) {
		assert.Equal(t, job_sub, config.Job.Subscription)
		assert.Equal(t, job_pull_interval, config.Job.PullInterval)
		if assert.NotNil(t, config.Job.Sustainer) {
			assert.Equal(t, job_sus_delay, config.Job.Sustainer.Delay)
			assert.Equal(t, job_sus_interval, config.Job.Sustainer.Interval)
		}
	}

	if assert.NotNil(t, config.Progress) {
		assert.Equal(t, prog_topic, config.Progress.Topic)
	}
}
<|endoftext|>