<commit_before>package config

import (
	"bytes"
	"fmt"
	"net"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/gomail"
	"github.com/go-redis/redis"
	"github.com/spf13/viper"
)

var (
	// Version of the release (see scripts/build.sh script)
	Version string
	// BuildTime is ISO-8601 UTC string representation of the time of
	// the build
	BuildTime string
	// BuildMode is the build mode of the release. Should be either
	// production or development.
	BuildMode = "development"
)

var isDevRelease = (BuildMode == "development")

// Filename is the default configuration filename that cozy
// search for
const Filename = "cozy"

// Paths is the list of directories used to search for a
// configuration file
var Paths = []string{
	".cozy",
	"$HOME/.cozy",
	"/etc/cozy",
}

// hardcodedRegistry is the default registry used if no configuration is set
// for registries.
var hardcodedRegistry, _ = url.Parse("https://apps-registry.cozy.io/")

// SubdomainType specify how subdomains are structured.
type SubdomainType int

const (
	// FlatSubdomains is the value for apps subdomains like
	// https://<user>-<app>.<domain>/
	FlatSubdomains SubdomainType = iota + 1
	// NestedSubdomains is the value for apps subdomains like
	// https://<app>.<user>.<domain>/ (used by default)
	NestedSubdomains
)

const (
	// SchemeFile is the URL scheme used to configure a file filesystem.
	SchemeFile = "file"
	// SchemeMem is the URL scheme used to configure an in-memory filesystem.
	SchemeMem = "mem"
	// SchemeSwift is the URL scheme used to configure a swift filesystem.
	SchemeSwift = "swift"
)

// AdminSecretFileName is the name of the file containing the administration
// hashed passphrase.
const AdminSecretFileName = "cozy-admin-passphrase" // #nosec

var config *Config
var log = logger.WithNamespace("config")

// Config contains the configuration values of the application
type Config struct {
	Host       string
	Port       int
	Assets     string
	Doctypes   string
	Subdomains SubdomainType
	AdminHost  string
	AdminPort  int
	NoReply    string
	Hooks      string
	GeoDB      string

	AutoUpdates AutoUpdates
	Fs          Fs
	CouchDB     CouchDB
	Jobs        Jobs
	Konnectors  Konnectors
	Mail        *gomail.DialerOptions

	Cache                       RedisConfig
	Lock                        RedisConfig
	SessionStorage              RedisConfig
	DownloadStorage             RedisConfig
	KonnectorsOauthStateStorage RedisConfig
	Realtime                    RedisConfig

	Contexts   map[string]interface{}
	Registries map[string][]*url.URL

	DisableCSP bool
}

// Fs contains the configuration values of the file-system
type Fs struct {
	Auth *url.Userinfo
	URL  *url.URL
}

// CouchDB contains the configuration values of the database
type CouchDB struct {
	Auth *url.Userinfo
	URL  *url.URL
}

// Jobs contains the configuration values for the jobs and triggers synchronization
type Jobs struct {
	Workers int
	Redis   RedisConfig
}

// Konnectors contains the configuration values for the konnectors
type Konnectors struct {
	Cmd string
}

// AutoUpdates contains the configuration values for auto updates
type AutoUpdates struct {
	Activated bool
	Schedule  string
}

// RedisConfig contains the configuration values for a redis system
type RedisConfig struct {
	Auth *url.Userinfo
	URL  *url.URL

	opt *redis.Options
	cli *redis.Client
}

// NewRedisConfig creates a redis configuration and its associated client.
func NewRedisConfig(u string) RedisConfig {
	var conf RedisConfig
	if u == "" {
		return conf
	}
	opt, err := redis.ParseURL(u)
	if err != nil {
		log.Errorf("can't parse redis URL(%s), ignoring", u)
		return conf
	}
	parsedURL, user, err := parseURL(u)
	if err != nil {
		log.Errorf("can't parse redis URL(%s), ignoring", u)
		return conf
	}
	conf.Auth = user
	conf.URL = parsedURL
	conf.cli = redis.NewClient(opt)
	conf.opt = opt
	return conf
}

// FsURL returns a copy of the filesystem URL
func FsURL() *url.URL {
	return config.Fs.URL
}

// ServerAddr returns the address on which the stack is run
func ServerAddr() string {
	return net.JoinHostPort(config.Host, strconv.Itoa(config.Port))
}

// AdminServerAddr returns the address on which the administration is listening
func AdminServerAddr() string {
	return net.JoinHostPort(config.AdminHost, strconv.Itoa(config.AdminPort))
}

// CouchURL returns the CouchDB string url
func CouchURL() *url.URL {
	return config.CouchDB.URL
}

// Client returns the redis.Client for a RedisConfig
func (rc *RedisConfig) Client() *redis.Client {
	return rc.cli
}

// IsDevRelease returns whether or not the binary is a development
// release
func IsDevRelease() bool {
	return isDevRelease
}

// GetConfig returns the configured instance of Config
func GetConfig() *Config {
	return config
}

// Setup Viper to read the environment and the optional config file
func Setup(cfgFile string) (err error) {
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.SetEnvPrefix("cozy")
	viper.AutomaticEnv()

	if cfgFile == "" {
		for _, ext := range viper.SupportedExts {
			var file string
			file, err = FindConfigFile(Filename + "." + ext)
			if file != "" && err == nil {
				cfgFile = file
				break
			}
		}
	}

	if cfgFile == "" {
		return UseViper(viper.GetViper())
	}

	log.Debugf("Using config file: %s", cfgFile)

	tmpl := template.New(filepath.Base(cfgFile))
	tmpl = tmpl.Option("missingkey=zero")
	tmpl, err = tmpl.ParseFiles(cfgFile)
	if err != nil {
		return fmt.Errorf("Unable to open and parse configuration file template %s: %s", cfgFile, err)
	}

	dest := new(bytes.Buffer)
	ctxt := &struct{ Env map[string]string }{Env: envMap()}
	err = tmpl.ExecuteTemplate(dest, filepath.Base(cfgFile), ctxt)
	if err != nil {
		return fmt.Errorf("Template error for config file %s: %s", cfgFile, err)
	}

	if ext := filepath.Ext(cfgFile); len(ext) > 0 {
		viper.SetConfigType(ext[1:])
	}
	if err := viper.ReadConfig(dest); err != nil {
		if _, isParseErr := err.(viper.ConfigParseError); isParseErr {
			log.Errorf("Failed to read cozy-stack configurations from %s", cfgFile)
			log.Errorf(dest.String())
			return err
		}
	}

	return UseViper(viper.GetViper())
}

func envMap() map[string]string {
	env := make(map[string]string)
	for _, i := range os.Environ() {
		sep := strings.Index(i, "=")
		env[i[0:sep]] = i[sep+1:]
	}
	return env
}

// UseViper sets the configured instance of Config
func UseViper(v *viper.Viper) error {
	fsURL, err := url.Parse(v.GetString("fs.url"))
	if err != nil {
		return err
	}
	if fsURL.Scheme == "file" {
		if fsPath := fsURL.Path; fsPath != "" && !path.IsAbs(fsPath) {
			return fmt.Errorf("Filesystem path should be absolute, was: %q", fsPath)
		}
	}

	couchURL, couchAuth, err := parseURL(v.GetString("couchdb.url"))
	if err != nil {
		return err
	}
	if couchURL.Path == "" {
		couchURL.Path = "/"
	}

	regs, err := makeRegistries(v)
	if err != nil {
		return err
	}

	var subdomains SubdomainType
	if subs := v.GetString("subdomains"); subs != "" {
		switch subs {
		case "flat":
			subdomains = FlatSubdomains
		case "nested":
			subdomains = NestedSubdomains
		default:
			return fmt.Errorf(`Subdomains mode should either be "flat" or "nested", was: %q`, subs)
		}
	} else {
		subdomains = NestedSubdomains
	}

	config = &Config{
		Host:       v.GetString("host"),
		Port:       v.GetInt("port"),
		Subdomains: subdomains,
		AdminHost:  v.GetString("admin.host"),
		AdminPort:  v.GetInt("admin.port"),
		Assets:     v.GetString("assets"),
		Doctypes:   v.GetString("doctypes"),
		NoReply:    v.GetString("mail.noreply_address"),
		Hooks:      v.GetString("hooks"),
		GeoDB:      v.GetString("geodb"),
		Fs: Fs{
			URL: fsURL,
		},
		CouchDB: CouchDB{
			Auth: couchAuth,
			URL:  couchURL,
		},
		Jobs: Jobs{
			Workers: v.GetInt("jobs.workers"),
			Redis:   NewRedisConfig(v.GetString("jobs.url")),
		},
		Konnectors: Konnectors{
			Cmd: v.GetString("konnectors.cmd"),
		},
		AutoUpdates: AutoUpdates{
			Activated: v.GetString("auto_updates.schedule") != "",
			Schedule:  v.GetString("auto_updates.schedule"),
		},
		Cache:                       NewRedisConfig(v.GetString("cache.url")),
		Lock:                        NewRedisConfig(v.GetString("lock.url")),
		SessionStorage:              NewRedisConfig(v.GetString("sessions.url")),
		DownloadStorage:             NewRedisConfig(v.GetString("downloads.url")),
		KonnectorsOauthStateStorage: NewRedisConfig(v.GetString("konnectors.oauthstate")),
		Realtime:                    NewRedisConfig(v.GetString("realtime.url")),
		Mail: &gomail.DialerOptions{
			Host:                      v.GetString("mail.host"),
			Port:                      v.GetInt("mail.port"),
			Username:                  v.GetString("mail.username"),
			Password:                  v.GetString("mail.password"),
			DisableTLS:                v.GetBool("mail.disable_tls"),
			SkipCertificateValidation: v.GetBool("mail.skip_certificate_validation"),
		},
		Contexts:   v.GetStringMap("contexts"),
		Registries: regs,
	}

	loggerRedis := NewRedisConfig(v.GetString("log.redis"))
	return logger.Init(logger.Options{
		Level:  v.GetString("log.level"),
		Syslog: v.GetBool("log.syslog"),
		Redis:  loggerRedis.Client(),
	})
}

func makeRegistries(v *viper.Viper) (map[string][]*url.URL, error) {
	regs := make(map[string][]*url.URL)

	regsSlice := v.GetStringSlice("registries")
	if len(regsSlice) > 0 {
		urlList := make([]*url.URL, len(regsSlice))
		for i, s := range regsSlice {
			u, err := url.Parse(s)
			if err != nil {
				return nil, err
			}
			urlList[i] = u
		}
		regs["default"] = urlList
	} else {
		for k, v := range v.GetStringMap("registries") {
			list, ok := v.([]interface{})
			if !ok {
				return nil, fmt.Errorf(
					"Bad format in the registries section of the configuration file: "+
						"should be a list of strings, got %#v", v)
			}
			urlList := make([]*url.URL, len(list))
			for i, s := range list {
				u, err := url.Parse(s.(string))
				if err != nil {
					return nil, err
				}
				urlList[i] = u
			}
			regs[k] = urlList
		}
	}

	defaults, ok := regs["default"]
	if !ok {
		defaults = []*url.URL{hardcodedRegistry}
		regs["default"] = defaults
	}
	for ctx, urls := range regs {
		if ctx == "default" {
			continue
		}
		regs[ctx] = append(urls, defaults...)
	}

	return regs, nil
}

const defaultTestConfig = `
host: localhost
port: 8080
assets: ./assets
subdomains: nested

fs:
  url: mem://test

couchdb:
    url: http://localhost:5984/

cache:
    url: redis://localhost:6379/0

log:
    level: info

jobs:
    workers: 2
`

// UseTestFile can be used in a test file to inject a configuration
// from a cozy.test.* file. If it can not find this file in your
// $HOME/.cozy directory it will use the default one.
func UseTestFile() {
	v := viper.New()
	v.SetConfigName("cozy.test")
	v.AddConfigPath("$HOME/.cozy")

	if err := v.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
			panic(fmt.Errorf("fatal error test config file: %s", err))
		}
		UseTestYAML(defaultTestConfig)
		return
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}
}

// UseTestYAML can be used in a test file to inject a configuration
// from a YAML string.
func UseTestYAML(yaml string) {
	v := viper.New()
	v.SetConfigType("yaml")

	if err := v.ReadConfig(strings.NewReader(yaml)); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}
}

// FindConfigFile search in the Paths directories for the file with the given
// name. It returns an error if it cannot find it or if an error occurs while
// searching.
func FindConfigFile(name string) (string, error) {
	for _, cp := range Paths {
		filename := filepath.Join(utils.AbsPath(cp), name)
		ok, err := utils.FileExists(filename)
		if err != nil {
			return "", err
		}
		if ok {
			return filename, nil
		}
	}
	return "", fmt.Errorf("Could not find config file %s", name)
}

func parseURL(u string) (*url.URL, *url.Userinfo, error) {
	parsedURL, err := url.Parse(u)
	if err != nil {
		return nil, nil, err
	}
	user := parsedURL.User
	parsedURL.User = nil
	return parsedURL, user, nil
}
<commit_msg>Avoid using / as fs path<commit_after>package config

import (
	"bytes"
	"fmt"
	"net"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/gomail"
	"github.com/go-redis/redis"
	"github.com/spf13/viper"
)

var (
	// Version of the release (see scripts/build.sh script)
	Version string
	// BuildTime is ISO-8601 UTC string representation of the time of
	// the build
	BuildTime string
	// BuildMode is the build mode of the release. Should be either
	// production or development.
	BuildMode = "development"
)

var isDevRelease = (BuildMode == "development")

// Filename is the default configuration filename that cozy
// search for
const Filename = "cozy"

// Paths is the list of directories used to search for a
// configuration file
var Paths = []string{
	".cozy",
	"$HOME/.cozy",
	"/etc/cozy",
}

// hardcodedRegistry is the default registry used if no configuration is set
// for registries.
var hardcodedRegistry, _ = url.Parse("https://apps-registry.cozy.io/")

// SubdomainType specify how subdomains are structured.
type SubdomainType int

const (
	// FlatSubdomains is the value for apps subdomains like
	// https://<user>-<app>.<domain>/
	FlatSubdomains SubdomainType = iota + 1
	// NestedSubdomains is the value for apps subdomains like
	// https://<app>.<user>.<domain>/ (used by default)
	NestedSubdomains
)

const (
	// SchemeFile is the URL scheme used to configure a file filesystem.
	SchemeFile = "file"
	// SchemeMem is the URL scheme used to configure an in-memory filesystem.
	SchemeMem = "mem"
	// SchemeSwift is the URL scheme used to configure a swift filesystem.
	SchemeSwift = "swift"
)

// AdminSecretFileName is the name of the file containing the administration
// hashed passphrase.
const AdminSecretFileName = "cozy-admin-passphrase" // #nosec

var config *Config
var log = logger.WithNamespace("config")

// Config contains the configuration values of the application
type Config struct {
	Host       string
	Port       int
	Assets     string
	Doctypes   string
	Subdomains SubdomainType
	AdminHost  string
	AdminPort  int
	NoReply    string
	Hooks      string
	GeoDB      string

	AutoUpdates AutoUpdates
	Fs          Fs
	CouchDB     CouchDB
	Jobs        Jobs
	Konnectors  Konnectors
	Mail        *gomail.DialerOptions

	Cache                       RedisConfig
	Lock                        RedisConfig
	SessionStorage              RedisConfig
	DownloadStorage             RedisConfig
	KonnectorsOauthStateStorage RedisConfig
	Realtime                    RedisConfig

	Contexts   map[string]interface{}
	Registries map[string][]*url.URL

	DisableCSP bool
}

// Fs contains the configuration values of the file-system
type Fs struct {
	Auth *url.Userinfo
	URL  *url.URL
}

// CouchDB contains the configuration values of the database
type CouchDB struct {
	Auth *url.Userinfo
	URL  *url.URL
}

// Jobs contains the configuration values for the jobs and triggers synchronization
type Jobs struct {
	Workers int
	Redis   RedisConfig
}

// Konnectors contains the configuration values for the konnectors
type Konnectors struct {
	Cmd string
}

// AutoUpdates contains the configuration values for auto updates
type AutoUpdates struct {
	Activated bool
	Schedule  string
}

// RedisConfig contains the configuration values for a redis system
type RedisConfig struct {
	Auth *url.Userinfo
	URL  *url.URL

	opt *redis.Options
	cli *redis.Client
}

// NewRedisConfig creates a redis configuration and its associated client.
func NewRedisConfig(u string) RedisConfig {
	var conf RedisConfig
	if u == "" {
		return conf
	}
	opt, err := redis.ParseURL(u)
	if err != nil {
		log.Errorf("can't parse redis URL(%s), ignoring", u)
		return conf
	}
	parsedURL, user, err := parseURL(u)
	if err != nil {
		log.Errorf("can't parse redis URL(%s), ignoring", u)
		return conf
	}
	conf.Auth = user
	conf.URL = parsedURL
	conf.cli = redis.NewClient(opt)
	conf.opt = opt
	return conf
}

// FsURL returns a copy of the filesystem URL
func FsURL() *url.URL {
	return config.Fs.URL
}

// ServerAddr returns the address on which the stack is run
func ServerAddr() string {
	return net.JoinHostPort(config.Host, strconv.Itoa(config.Port))
}

// AdminServerAddr returns the address on which the administration is listening
func AdminServerAddr() string {
	return net.JoinHostPort(config.AdminHost, strconv.Itoa(config.AdminPort))
}

// CouchURL returns the CouchDB string url
func CouchURL() *url.URL {
	return config.CouchDB.URL
}

// Client returns the redis.Client for a RedisConfig
func (rc *RedisConfig) Client() *redis.Client {
	return rc.cli
}

// IsDevRelease returns whether or not the binary is a development
// release
func IsDevRelease() bool {
	return isDevRelease
}

// GetConfig returns the configured instance of Config
func GetConfig() *Config {
	return config
}

// Setup Viper to read the environment and the optional config file
func Setup(cfgFile string) (err error) {
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.SetEnvPrefix("cozy")
	viper.AutomaticEnv()

	if cfgFile == "" {
		for _, ext := range viper.SupportedExts {
			var file string
			file, err = FindConfigFile(Filename + "." + ext)
			if file != "" && err == nil {
				cfgFile = file
				break
			}
		}
	}

	if cfgFile == "" {
		return UseViper(viper.GetViper())
	}

	log.Debugf("Using config file: %s", cfgFile)

	tmpl := template.New(filepath.Base(cfgFile))
	tmpl = tmpl.Option("missingkey=zero")
	tmpl, err = tmpl.ParseFiles(cfgFile)
	if err != nil {
		return fmt.Errorf("Unable to open and parse configuration file template %s: %s", cfgFile, err)
	}

	dest := new(bytes.Buffer)
	ctxt := &struct{ Env map[string]string }{Env: envMap()}
	err = tmpl.ExecuteTemplate(dest, filepath.Base(cfgFile), ctxt)
	if err != nil {
		return fmt.Errorf("Template error for config file %s: %s", cfgFile, err)
	}

	if ext := filepath.Ext(cfgFile); len(ext) > 0 {
		viper.SetConfigType(ext[1:])
	}
	if err := viper.ReadConfig(dest); err != nil {
		if _, isParseErr := err.(viper.ConfigParseError); isParseErr {
			log.Errorf("Failed to read cozy-stack configurations from %s", cfgFile)
			log.Errorf(dest.String())
			return err
		}
	}

	return UseViper(viper.GetViper())
}

func envMap() map[string]string {
	env := make(map[string]string)
	for _, i := range os.Environ() {
		sep := strings.Index(i, "=")
		env[i[0:sep]] = i[sep+1:]
	}
	return env
}

// UseViper sets the configured instance of Config
func UseViper(v *viper.Viper) error {
	fsURL, err := url.Parse(v.GetString("fs.url"))
	if err != nil {
		return err
	}
	if fsURL.Scheme == "file" {
		fsPath := fsURL.Path
		if fsPath != "" && !path.IsAbs(fsPath) {
			return fmt.Errorf("Filesystem path should be absolute, was: %q", fsPath)
		}
		if fsPath == "/" {
			return fmt.Errorf("Filesystem path should not be root, was: %q", fsPath)
		}
	}

	couchURL, couchAuth, err := parseURL(v.GetString("couchdb.url"))
	if err != nil {
		return err
	}
	if couchURL.Path == "" {
		couchURL.Path = "/"
	}

	regs, err := makeRegistries(v)
	if err != nil {
		return err
	}

	var subdomains SubdomainType
	if subs := v.GetString("subdomains"); subs != "" {
		switch subs {
		case "flat":
			subdomains = FlatSubdomains
		case "nested":
			subdomains = NestedSubdomains
		default:
			return fmt.Errorf(`Subdomains mode should either be "flat" or "nested", was: %q`, subs)
		}
	} else {
		subdomains = NestedSubdomains
	}

	config = &Config{
		Host:       v.GetString("host"),
		Port:       v.GetInt("port"),
		Subdomains: subdomains,
		AdminHost:  v.GetString("admin.host"),
		AdminPort:  v.GetInt("admin.port"),
		Assets:     v.GetString("assets"),
		Doctypes:   v.GetString("doctypes"),
		NoReply:    v.GetString("mail.noreply_address"),
		Hooks:      v.GetString("hooks"),
		GeoDB:      v.GetString("geodb"),
		Fs: Fs{
			URL: fsURL,
		},
		CouchDB: CouchDB{
			Auth: couchAuth,
			URL:  couchURL,
		},
		Jobs: Jobs{
			Workers: v.GetInt("jobs.workers"),
			Redis:   NewRedisConfig(v.GetString("jobs.url")),
		},
		Konnectors: Konnectors{
			Cmd: v.GetString("konnectors.cmd"),
		},
		AutoUpdates: AutoUpdates{
			Activated: v.GetString("auto_updates.schedule") != "",
			Schedule:  v.GetString("auto_updates.schedule"),
		},
		Cache:                       NewRedisConfig(v.GetString("cache.url")),
		Lock:                        NewRedisConfig(v.GetString("lock.url")),
		SessionStorage:              NewRedisConfig(v.GetString("sessions.url")),
		DownloadStorage:             NewRedisConfig(v.GetString("downloads.url")),
		KonnectorsOauthStateStorage: NewRedisConfig(v.GetString("konnectors.oauthstate")),
		Realtime:                    NewRedisConfig(v.GetString("realtime.url")),
		Mail: &gomail.DialerOptions{
			Host:                      v.GetString("mail.host"),
			Port:                      v.GetInt("mail.port"),
			Username:                  v.GetString("mail.username"),
			Password:                  v.GetString("mail.password"),
			DisableTLS:                v.GetBool("mail.disable_tls"),
			SkipCertificateValidation: v.GetBool("mail.skip_certificate_validation"),
		},
		Contexts:   v.GetStringMap("contexts"),
		Registries: regs,
	}

	loggerRedis := NewRedisConfig(v.GetString("log.redis"))
	return logger.Init(logger.Options{
		Level:  v.GetString("log.level"),
		Syslog: v.GetBool("log.syslog"),
		Redis:  loggerRedis.Client(),
	})
}

func makeRegistries(v *viper.Viper) (map[string][]*url.URL, error) {
	regs := make(map[string][]*url.URL)

	regsSlice := v.GetStringSlice("registries")
	if len(regsSlice) > 0 {
		urlList := make([]*url.URL, len(regsSlice))
		for i, s := range regsSlice {
			u, err := url.Parse(s)
			if err != nil {
				return nil, err
			}
			urlList[i] = u
		}
		regs["default"] = urlList
	} else {
		for k, v := range v.GetStringMap("registries") {
			list, ok := v.([]interface{})
			if !ok {
				return nil, fmt.Errorf(
					"Bad format in the registries section of the configuration file: "+
						"should be a list of strings, got %#v", v)
			}
			urlList := make([]*url.URL, len(list))
			for i, s := range list {
				u, err := url.Parse(s.(string))
				if err != nil {
					return nil, err
				}
				urlList[i] = u
			}
			regs[k] = urlList
		}
	}

	defaults, ok := regs["default"]
	if !ok {
		defaults = []*url.URL{hardcodedRegistry}
		regs["default"] = defaults
	}
	for ctx, urls := range regs {
		if ctx == "default" {
			continue
		}
		regs[ctx] = append(urls, defaults...)
	}

	return regs, nil
}

const defaultTestConfig = `
host: localhost
port: 8080
assets: ./assets
subdomains: nested

fs:
  url: mem://test

couchdb:
    url: http://localhost:5984/

cache:
    url: redis://localhost:6379/0

log:
    level: info

jobs:
    workers: 2
`

// UseTestFile can be used in a test file to inject a configuration
// from a cozy.test.* file. If it can not find this file in your
// $HOME/.cozy directory it will use the default one.
func UseTestFile() {
	v := viper.New()
	v.SetConfigName("cozy.test")
	v.AddConfigPath("$HOME/.cozy")

	if err := v.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
			panic(fmt.Errorf("fatal error test config file: %s", err))
		}
		UseTestYAML(defaultTestConfig)
		return
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}
}

// UseTestYAML can be used in a test file to inject a configuration
// from a YAML string.
func UseTestYAML(yaml string) {
	v := viper.New()
	v.SetConfigType("yaml")

	if err := v.ReadConfig(strings.NewReader(yaml)); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}
}

// FindConfigFile search in the Paths directories for the file with the given
// name. It returns an error if it cannot find it or if an error occurs while
// searching.
func FindConfigFile(name string) (string, error) {
	for _, cp := range Paths {
		filename := filepath.Join(utils.AbsPath(cp), name)
		ok, err := utils.FileExists(filename)
		if err != nil {
			return "", err
		}
		if ok {
			return filename, nil
		}
	}
	return "", fmt.Errorf("Could not find config file %s", name)
}

func parseURL(u string) (*url.URL, *url.Userinfo, error) {
	parsedURL, err := url.Parse(u)
	if err != nil {
		return nil, nil, err
	}
	user := parsedURL.User
	parsedURL.User = nil
	return parsedURL, user, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"sort"

	"github.com/yuuki/diamondb/lib/series"
)

type seriesMap map[string]*seriesPoint

func (sm seriesMap) SortedNames() []string {
	names := make([]string, 0, len(sm))
	for name, _ := range sm {
		names = append(names, name)
	}
	slices := sort.StringSlice(names)
	slices.Sort()
	return slices
}

func (sm1 seriesMap) Merge(sm2 seriesMap) seriesMap {
	for name, s := range sm2 {
		sm1[name] = s
	}
	return sm1
}

func (sm1 seriesMap) MergePointsToMap(sm2 seriesMap) seriesMap {
	for name, s1 := range sm1 {
		if s2, ok := sm2[name]; ok {
			points := append(s1.Points(), s2.Points()...)
			sm1[name] = newSeriesPoint(name, points, s1.Step())
		}
	}
	for name, s2 := range sm2 {
		if _, ok := sm1[name]; !ok {
			sm1[name] = s2
		}
	}
	return sm1
}

func (sm1 seriesMap) MergePointsToSlice(sm2 seriesMap) series.SeriesSlice {
	sm := sm1.MergePointsToMap(sm2)
	ss := make(series.SeriesSlice, 0, len(sm1))
	for _, name := range sm.SortedNames() {
		ss = append(ss, sm[name].ToSeries())
	}
	return ss
}
<commit_msg>Tweak it<commit_after>package storage

import (
	"sort"

	"github.com/yuuki/diamondb/lib/series"
)

type seriesMap map[string]*seriesPoint

func (sm seriesMap) SortedNames() []string {
	names := make([]string, 0, len(sm))
	for name, _ := range sm {
		names = append(names, name)
	}
	slices := sort.StringSlice(names)
	slices.Sort()
	return slices
}

func (sm1 seriesMap) Merge(sm2 seriesMap) seriesMap {
	for name, s := range sm2 {
		sm1[name] = s
	}
	return sm1
}

func (sm1 seriesMap) MergePointsToMap(sm2 seriesMap) seriesMap {
	for name, s1 := range sm1 {
		if s2, ok := sm2[name]; ok {
			points := append(s1.Points(), s2.Points()...)
			sm1[name] = newSeriesPoint(name, points, s1.Step())
		}
	}
	for name, s2 := range sm2 {
		if _, ok := sm1[name]; !ok {
			sm1[name] = s2
		}
	}
	return sm1
}

func (sm1 seriesMap) MergePointsToSlice(sm2 seriesMap) series.SeriesSlice {
	sm := sm1.MergePointsToMap(sm2)
	ss := make(series.SeriesSlice, 0, len(sm))
	for _, name := range sm.SortedNames() {
		ss = append(ss, sm[name].ToSeries())
	}
	return ss
}
<|endoftext|>


<commit_before>package config

import (
	"bytes"
	"fmt"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/gomail"
	"github.com/go-redis/redis"
	"github.com/spf13/viper"
)

const (
	// Production mode
	Production string = "production"
	// Development mode
	Development string = "development"
)

var (
	// Version of the release (see scripts/build.sh script)
	Version string
	// BuildTime is ISO-8601 UTC string representation of the time of
	// the build
	BuildTime string
	// BuildMode is the build mode of the release. Should be either
	// production or development.
	BuildMode = Development
)

// Filename is the default configuration filename that cozy
// search for
const Filename = "cozy"

// Paths is the list of directories used to search for a
// configuration file
var Paths = []string{
	".cozy",
	"$HOME/.cozy",
	"/etc/cozy",
}

const (
	// FlatSubdomains is the value for apps subdomains like https://<user>-<app>.<domain>/
	FlatSubdomains = "flat"
	// NestedSubdomains is the value for apps subdomains like https://<app>.<user>.<domain>/
	NestedSubdomains = "nested"
)

// AdminSecretFileName is the name of the file containing the administration
// hashed passphrase.
const AdminSecretFileName = "cozy-admin-passphrase" // #nosec

var config *Config

// Config contains the configuration values of the application
type Config struct {
	Host       string
	Port       int
	Assets     string
	Subdomains string
	AdminHost  string
	AdminPort  int

	Fs         Fs
	CouchDB    CouchDB
	Jobs       Jobs
	Konnectors Konnectors
	Mail       *gomail.DialerOptions
	Logger     Logger

	Cache           RedisConfig
	Lock            RedisConfig
	SessionStorage  RedisConfig
	DownloadStorage RedisConfig
}

// Fs contains the configuration values of the file-system
type Fs struct {
	URL string
}

// CouchDB contains the configuration values of the database
type CouchDB struct {
	URL string
}

// Jobs contains the configuration values for the jobs and triggers synchronization
type Jobs struct {
	Workers int
	URL     string
}

// Konnectors contains the configuration values for the konnectors
type Konnectors struct {
	Cmd string
}

// RedisConfig contains the configuration values for a redis system
type RedisConfig struct {
	URL string
}

// Lock contains the configuration values of the locking layer
type Lock struct {
	URL string
}

// Logger contains the configuration values of the logger system
type Logger struct {
	Level string
}

// FsURL returns a copy of the filesystem URL
func FsURL() *url.URL {
	u, err := url.Parse(config.Fs.URL)
	if err != nil {
		panic(fmt.Errorf("malformed configuration fs url %s", config.Fs.URL))
	}
	return u
}

// ServerAddr returns the address on which the stack is run
func ServerAddr() string {
	return net.JoinHostPort(config.Host, strconv.Itoa(config.Port))
}

// AdminServerAddr returns the address on which the administration is listening
func AdminServerAddr() string {
	return net.JoinHostPort(config.AdminHost, strconv.Itoa(config.AdminPort))
}

// CouchURL returns the CouchDB string url
func CouchURL() string {
	return config.CouchDB.URL
}

// Options returns the redis.Options for a RedisConfig
func (rc *RedisConfig) Options() *redis.Options {
	if rc.URL == "" {
		return nil
	}
	opts, err := redis.ParseURL(rc.URL)
	if err != nil {
		log.Errorf("can't parse cache.URL(%s), ignoring", rc.URL)
		return nil
	}
	return opts
}

// IsDevRelease returns whether or not the binary is a development
// release
func IsDevRelease() bool {
	return BuildMode == Development
}

// GetConfig returns the configured instance of Config
func GetConfig() *Config {
	return config
}

// Setup Viper to read the environment and the optional config file
func Setup(cfgFile string) (err error) {
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.SetEnvPrefix("cozy")
	viper.AutomaticEnv()

	if cfgFile == "" {
		for _, ext := range viper.SupportedExts {
			var file string
			file, err = FindConfigFile(Filename + "." + ext)
			if file != "" && err == nil {
				cfgFile = file
				break
			}
		}
	}

	if cfgFile == "" {
		return UseViper(viper.GetViper())
	}

	log.Debugf("Using config file: %s", cfgFile)

	tmpl := template.New(filepath.Base(cfgFile))
	tmpl = tmpl.Option("missingkey=zero")
	tmpl, err = tmpl.ParseFiles(cfgFile)
	if err != nil {
		return fmt.Errorf("Unable to open and parse configuration file template %s: %s", cfgFile, err)
	}

	dest := new(bytes.Buffer)
	ctxt := &struct{ Env map[string]string }{Env: envMap()}
	err = tmpl.ExecuteTemplate(dest, filepath.Base(cfgFile), ctxt)
	if err != nil {
		return fmt.Errorf("Template error for config file %s: %s", cfgFile, err)
	}

	if ext := filepath.Ext(cfgFile); len(ext) > 0 {
		viper.SetConfigType(ext[1:])
	}
	if err := viper.ReadConfig(dest); err != nil {
		if _, isParseErr := err.(viper.ConfigParseError); isParseErr {
			log.Errorf("Failed to read cozy-stack configurations from %s", cfgFile)
			log.Errorf(dest.String())
			return err
		}
	}

	return UseViper(viper.GetViper())
}

func envMap() map[string]string {
	env := make(map[string]string)
	for _, i := range os.Environ() {
		sep := strings.Index(i, "=")
		env[i[0:sep]] = i[sep+1:]
	}
	return env
}

// UseViper sets the configured instance of Config
func UseViper(v *viper.Viper) error {
	fsURL, err := url.Parse(v.GetString("fs.url"))
	if err != nil {
		return err
	}

	couchURL, err := url.Parse(v.GetString("couchdb.url"))
	if err != nil {
		return err
	}
	if couchURL.Path == "" {
		couchURL.Path = "/"
	}

	config = &Config{
		Host:       v.GetString("host"),
		Port:       v.GetInt("port"),
		Subdomains: v.GetString("subdomains"),
		AdminHost:  v.GetString("admin.host"),
		AdminPort:  v.GetInt("admin.port"),
		Assets:     v.GetString("assets"),
		Fs: Fs{
			URL: fsURL.String(),
		},
		CouchDB: CouchDB{
			URL: couchURL.String(),
		},
		Konnectors: Konnectors{
			Cmd: v.GetString("konnectors.cmd"),
		},
		Cache: RedisConfig{
			URL: v.GetString("cache.url"),
		},
		Lock: RedisConfig{
			URL: v.GetString("lock.url"),
		},
		SessionStorage: RedisConfig{
			URL: v.GetString("sessions.url"),
		},
		DownloadStorage: RedisConfig{
			URL: v.GetString("downloads.url"),
		},
		Mail: &gomail.DialerOptions{
			Host:                      v.GetString("mail.host"),
			Port:                      v.GetInt("mail.port"),
			Username:                  v.GetString("mail.username"),
			Password:                  v.GetString("mail.password"),
			DisableTLS:                v.GetBool("mail.disable_tls"),
			SkipCertificateValidation: v.GetBool("mail.skip_certificate_validation"),
		},
		Logger: Logger{
			Level: v.GetString("log.level"),
		},
	}

	return configureLogger()
}

const defaultTestConfig = `
host: localhost
port: 8080
assets: ./assets
subdomains: nested

fs:
  url: mem://test

couchdb:
    url: http://localhost:5984/

cache:
    url: redis://localhost:6379/0

log:
    level: info
`

// UseTestFile can be used in a test file to inject a configuration
// from a cozy.test.* file. If it can not find this file in your
// $HOME/.cozy directory it will use the default one.
func UseTestFile() {
	v := viper.New()
	v.SetConfigName("cozy.test")
	v.AddConfigPath("$HOME/.cozy")

	if err := v.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
			panic(fmt.Errorf("fatal error test config file: %s", err))
		}
		UseTestYAML(defaultTestConfig)
		return
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	return
}

// UseTestYAML can be used in a test file to inject a configuration
// from a YAML string.
func UseTestYAML(yaml string) {
	v := viper.New()
	v.SetConfigType("yaml")

	if err := v.ReadConfig(strings.NewReader(yaml)); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	return
}

// FindConfigFile search in the Paths directories for the file with the given
// name. It returns an error if it cannot find it or if an error occurs while
// searching.
func FindConfigFile(name string) (string, error) {
	for _, cp := range Paths {
		filename := filepath.Join(utils.AbsPath(cp), name)
		ok, err := utils.FileExists(filename)
		if err != nil {
			return "", err
		}
		if ok {
			return filename, nil
		}
	}
	return "", fmt.Errorf("Could not find config file %s", name)
}

func configureLogger() error {
	loggerCfg := config.Logger

	level := loggerCfg.Level
	if level == "" {
		level = "info"
	}

	logLevel, err := log.ParseLevel(level)
	if err != nil {
		return err
	}

	log.SetLevel(logLevel)
	return nil
}
<commit_msg>Set jobs config<commit_after>package config

import (
	"bytes"
	"fmt"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	log "github.com/Sirupsen/logrus"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/cozy/gomail"
	"github.com/go-redis/redis"
	"github.com/spf13/viper"
)

const (
	// Production mode
	Production string = "production"
	// Development mode
	Development string = "development"
)

var (
	// Version of the release (see scripts/build.sh script)
	Version string
	// BuildTime is ISO-8601 UTC string representation of the time of
	// the build
	BuildTime string
	// BuildMode is the build mode of the release. Should be either
	// production or development.
	BuildMode = Development
)

// Filename is the default configuration filename that cozy
// search for
const Filename = "cozy"

// Paths is the list of directories used to search for a
// configuration file
var Paths = []string{
	".cozy",
	"$HOME/.cozy",
	"/etc/cozy",
}

const (
	// FlatSubdomains is the value for apps subdomains like https://<user>-<app>.<domain>/
	FlatSubdomains = "flat"
	// NestedSubdomains is the value for apps subdomains like https://<app>.<user>.<domain>/
	NestedSubdomains = "nested"
)

// AdminSecretFileName is the name of the file containing the administration
// hashed passphrase.
const AdminSecretFileName = "cozy-admin-passphrase" // #nosec

var config *Config

// Config contains the configuration values of the application
type Config struct {
	Host       string
	Port       int
	Assets     string
	Subdomains string
	AdminHost  string
	AdminPort  int

	Fs         Fs
	CouchDB    CouchDB
	Jobs       Jobs
	Konnectors Konnectors
	Mail       *gomail.DialerOptions
	Logger     Logger

	Cache           RedisConfig
	Lock            RedisConfig
	SessionStorage  RedisConfig
	DownloadStorage RedisConfig
}

// Fs contains the configuration values of the file-system
type Fs struct {
	URL string
}

// CouchDB contains the configuration values of the database
type CouchDB struct {
	URL string
}

// Jobs contains the configuration values for the jobs and triggers synchronization
type Jobs struct {
	Workers int
	URL     string
}

// Konnectors contains the configuration values for the konnectors
type Konnectors struct {
	Cmd string
}

// RedisConfig contains the configuration values for a redis system
type RedisConfig struct {
	URL string
}

// Lock contains the configuration values of the locking layer
type Lock struct {
	URL string
}

// Logger contains the configuration values of the logger system
type Logger struct {
	Level string
}

// FsURL returns a copy of the filesystem URL
func FsURL() *url.URL {
	u, err := url.Parse(config.Fs.URL)
	if err != nil {
		panic(fmt.Errorf("malformed configuration fs url %s", config.Fs.URL))
	}
	return u
}

// ServerAddr returns the address on which the stack is run
func ServerAddr() string {
	return net.JoinHostPort(config.Host, strconv.Itoa(config.Port))
}

// AdminServerAddr returns the address on which the administration is listening
func AdminServerAddr() string {
	return net.JoinHostPort(config.AdminHost, strconv.Itoa(config.AdminPort))
}

// CouchURL returns the CouchDB string url
func CouchURL() string {
	return config.CouchDB.URL
}

// Options returns the redis.Options for a RedisConfig
func (rc *RedisConfig) Options() *redis.Options {
	if rc.URL == "" {
		return nil
	}
	opts, err := redis.ParseURL(rc.URL)
	if err != nil {
		log.Errorf("can't parse cache.URL(%s), ignoring", rc.URL)
		return nil
	}
	return opts
}

// IsDevRelease returns whether or not the binary is a development
// release
func IsDevRelease() bool {
	return BuildMode == Development
}

// GetConfig returns the configured instance of Config
func GetConfig() *Config {
	return config
}

// Setup Viper to read the environment and the optional config file
func Setup(cfgFile string) (err error) {
	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
	viper.SetEnvPrefix("cozy")
	viper.AutomaticEnv()

	if cfgFile == "" {
		for _, ext := range viper.SupportedExts {
			var file string
			file, err = FindConfigFile(Filename + "." + ext)
			if file != "" && err == nil {
				cfgFile = file
				break
			}
		}
	}

	if cfgFile == "" {
		return UseViper(viper.GetViper())
	}

	log.Debugf("Using config file: %s", cfgFile)

	tmpl := template.New(filepath.Base(cfgFile))
	tmpl = tmpl.Option("missingkey=zero")
	tmpl, err = tmpl.ParseFiles(cfgFile)
	if err != nil {
		return fmt.Errorf("Unable to open and parse configuration file template %s: %s", cfgFile, err)
	}

	dest := new(bytes.Buffer)
	ctxt := &struct{ Env map[string]string }{Env: envMap()}
	err = tmpl.ExecuteTemplate(dest, filepath.Base(cfgFile), ctxt)
	if err != nil {
		return fmt.Errorf("Template error for config file %s: %s", cfgFile, err)
	}

	if ext := filepath.Ext(cfgFile); len(ext) > 0 {
		viper.SetConfigType(ext[1:])
	}
	if err := viper.ReadConfig(dest); err != nil {
		if _, isParseErr := err.(viper.ConfigParseError); isParseErr {
			log.Errorf("Failed to read cozy-stack configurations from %s", cfgFile)
			log.Errorf(dest.String())
			return err
		}
	}

	return UseViper(viper.GetViper())
}

func envMap() map[string]string {
	env := make(map[string]string)
	for _, i := range os.Environ() {
		sep := strings.Index(i, "=")
		env[i[0:sep]] = i[sep+1:]
	}
	return env
}

// UseViper sets the configured instance of Config
func UseViper(v *viper.Viper) error {
	fsURL, err := url.Parse(v.GetString("fs.url"))
	if err != nil {
		return err
	}

	couchURL, err := url.Parse(v.GetString("couchdb.url"))
	if err != nil {
		return err
	}
	if couchURL.Path == "" {
		couchURL.Path = "/"
	}

	config = &Config{
		Host:       v.GetString("host"),
		Port:       v.GetInt("port"),
		Subdomains: v.GetString("subdomains"),
		AdminHost:  v.GetString("admin.host"),
		AdminPort:  v.GetInt("admin.port"),
		Assets:     v.GetString("assets"),
		Fs: Fs{
			URL: fsURL.String(),
		},
		CouchDB: CouchDB{
			URL: couchURL.String(),
		},
		Jobs: Jobs{
			Workers: v.GetInt("jobs.workers"),
			URL:     v.GetString("jobs.url"),
		},
		Konnectors: Konnectors{
			Cmd: v.GetString("konnectors.cmd"),
		},
		Cache: RedisConfig{
			URL: v.GetString("cache.url"),
		},
		Lock: RedisConfig{
			URL: v.GetString("lock.url"),
		},
		SessionStorage: RedisConfig{
			URL: v.GetString("sessions.url"),
		},
		DownloadStorage: RedisConfig{
			URL: v.GetString("downloads.url"),
		},
		Mail: &gomail.DialerOptions{
			Host:                      v.GetString("mail.host"),
			Port:                      v.GetInt("mail.port"),
			Username:                  v.GetString("mail.username"),
			Password:                  v.GetString("mail.password"),
			DisableTLS:                v.GetBool("mail.disable_tls"),
			SkipCertificateValidation: v.GetBool("mail.skip_certificate_validation"),
		},
		Logger: Logger{
			Level: v.GetString("log.level"),
		},
	}

	return configureLogger()
}

const defaultTestConfig = `
host: localhost
port: 8080
assets: ./assets
subdomains: nested

fs:
  url: mem://test

couchdb:
    url: http://localhost:5984/

cache:
    url: redis://localhost:6379/0

log:
    level: info
`

// UseTestFile can be used in a test file to inject a configuration
// from a cozy.test.* file. If it can not find this file in your
// $HOME/.cozy directory it will use the default one.
func UseTestFile() {
	v := viper.New()
	v.SetConfigName("cozy.test")
	v.AddConfigPath("$HOME/.cozy")

	if err := v.ReadInConfig(); err != nil {
		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
			panic(fmt.Errorf("fatal error test config file: %s", err))
		}
		UseTestYAML(defaultTestConfig)
		return
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	return
}

// UseTestYAML can be used in a test file to inject a configuration
// from a YAML string.
func UseTestYAML(yaml string) {
	v := viper.New()
	v.SetConfigType("yaml")

	if err := v.ReadConfig(strings.NewReader(yaml)); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	if err := UseViper(v); err != nil {
		panic(fmt.Errorf("fatal error test config file: %s", err))
	}

	return
}

// FindConfigFile search in the Paths directories for the file with the given
// name. It returns an error if it cannot find it or if an error occurs while
// searching.
func FindConfigFile(name string) (string, error) {
	for _, cp := range Paths {
		filename := filepath.Join(utils.AbsPath(cp), name)
		ok, err := utils.FileExists(filename)
		if err != nil {
			return "", err
		}
		if ok {
			return filename, nil
		}
	}
	return "", fmt.Errorf("Could not find config file %s", name)
}

func configureLogger() error {
	loggerCfg := config.Logger

	level := loggerCfg.Level
	if level == "" {
		level = "info"
	}

	logLevel, err := log.ParseLevel(level)
	if err != nil {
		return err
	}

	log.SetLevel(logLevel)
	return nil
}
<|endoftext|>


<commit_before>package crank

import (
	"log"
	"os"
)

// Manager manages multiple process groups
type Manager struct {
	configPath      string
	config          *ProcessConfig
	socket          *os.File
	processCount    int
	events          chan Event
	startAction     chan *ProcessConfig
	shutdownAction  chan bool
	childs          processSet
	shuttingDown    bool
	startingTracker *TimeoutTracker
	stoppingTracker *TimeoutTracker
}

func NewManager(configPath string, socket *os.File) *Manager {
	config, err := loadProcessConfig(configPath)
	if err != nil {
		log.Println("Could not load config file: ", err)
	}

	manager := &Manager{
		configPath:      configPath,
		config:          config,
		socket:          socket,
		events:          make(chan Event),
		startAction:     make(chan *ProcessConfig),
		childs:          make(processSet),
		startingTracker: NewTimeoutTracker(),
		stoppingTracker: NewTimeoutTracker(),
	}
	return manager
}

// Run starts the event loop for the manager process
func (self *Manager) Run() {
	if self.config != nil && self.config.Command != "" {
		self.startProcess(self.config)
	}

	go self.startingTracker.Run()
	go self.stoppingTracker.Run()

	for {
		select {
		// actions
		case config := <-self.startAction:
			if self.shuttingDown {
				self.log("Ignore start, manager is shutting down")
				continue
			}
			if self.childs.starting() != nil {
				self.log("Ignore start, new process is already being started")
				continue
			}
			self.startProcess(config)
		case <-self.shutdownAction:
			if self.shuttingDown {
				self.log("Already shutting down")
				continue
			}
			self.shuttingDown = true
			self.childs.each(func(p *Process) {
				self.stopProcess(p)
			})
		// timeouts
		case process := <-self.startingTracker.timeoutNotification:
			self.plog(process, "Killing, did not start in time.")
			process.Kill()
		case process := <-self.stoppingTracker.timeoutNotification:
			self.plog(process, "Killing, did not stop in time.")
			process.Kill()
		// process state transitions
		case e := <-self.events:
			switch event := e.(type) {
			case *ProcessReadyEvent:
				process := event.process
				self.startingTracker.Remove(process)
				if process != self.childs.starting() {
					self.plog(process, "Oops, some other process is ready")
					continue
				}
				self.plog(process, "Process is ready")
				current := self.childs.ready()
				if current != nil {
					self.plog(current, "Shutting down old current")
					self.stopProcess(current)
				}
				self.config = process.config
				err := self.config.save(self.configPath)
				if err != nil {
					self.log("Failed saving the config: %s", err)
				}
				self.childs.updateState(process, PROCESS_READY)
			case *ProcessExitEvent:
				process := event.process

				self.startingTracker.Remove(process)
				self.stoppingTracker.Remove(process)
				self.childs.rem(process)

				self.plog(process, "Process exited. code=%d err=%v", event.code, event.err)

				if self.childs.len() == 0 {
					goto exit
				}
			default:
				fail("Unknown event: ", e)
			}
		}
	}

exit:

	// Cleanup
	self.childs.each(func(p *Process) {
		p.Kill()
	})
}

// Restart queues and starts excecuting a restart job to replace the old process group with a new one.
func (self *Manager) Reload() {
	self.Start(self.config)
}

func (self *Manager) Start(c *ProcessConfig) {
	self.startAction <- c
}

func (self *Manager) Shutdown() {
	self.shutdownAction <- true
}

// Private methods

func (_ *Manager) log(format string, v ...interface{}) {
	log.Printf("[manager] "+format, v...)
}

func (m *Manager) plog(p *Process, format string, v ...interface{}) {
	args := make([]interface{}, 1, 1+len(v))
	args[0] = p
	args = append(args, v...)
	log.Printf("%s "+format, args...)
}

func (self *Manager) startProcess(config *ProcessConfig) {
	self.log("Starting a new process: %s", config)
	self.processCount += 1
	process, err := startProcess(self.processCount, config, self.socket, self.events)
	if err != nil {
		self.log("Failed to start the process", err)
		return
	}
	self.childs.add(process, PROCESS_STARTING)
	self.startingTracker.Add(process, process.config.StartTimeout)
}

func (self *Manager) stopProcess(process *Process) {
	if self.childs[process] == PROCESS_STOPPING {
		return
	}
	process.Shutdown()
	self.stoppingTracker.Add(process, process.config.StopTimeout)
	self.childs.updateState(process, PROCESS_STOPPING)
}
<commit_msg>Fixes manager shutdown on SIGINT/TERM<commit_after>package crank

import (
	"log"
	"os"
)

// Manager manages multiple process groups
type Manager struct {
	configPath      string
	config          *ProcessConfig
	socket          *os.File
	processCount    int
	events          chan Event
	startAction     chan *ProcessConfig
	shutdownAction  chan bool
	childs          processSet
	shuttingDown    bool
	startingTracker *TimeoutTracker
	stoppingTracker *TimeoutTracker
}

func NewManager(configPath string, socket *os.File) *Manager {
	config, err := loadProcessConfig(configPath)
	if err != nil {
		log.Println("Could not load config file: ", err)
	}

	manager := &Manager{
		configPath:      configPath,
		config:          config,
		socket:          socket,
		events:          make(chan Event),
		startAction:     make(chan *ProcessConfig),
		shutdownAction:  make(chan bool),
		childs:          make(processSet),
		startingTracker: NewTimeoutTracker(),
		stoppingTracker: NewTimeoutTracker(),
	}
	return manager
}

// Run starts the event loop for the manager process
func (self *Manager) Run() {
	if self.config != nil && self.config.Command != "" {
		self.startProcess(self.config)
	}

	go self.startingTracker.Run()
	go self.stoppingTracker.Run()

	for {
		select {
		// actions
		case config := <-self.startAction:
			if self.shuttingDown {
				self.log("Ignore start, manager is shutting down")
				continue
			}
			if self.childs.starting() != nil {
				self.log("Ignore start, new process is already being started")
				continue
			}
			self.startProcess(config)
		case <-self.shutdownAction:
			if self.shuttingDown {
				self.log("Already shutting down")
				continue
			}
			self.log("Shutting down")
			self.shuttingDown = true
			self.childs.each(func(p *Process) {
				self.stopProcess(p)
			})
		// timeouts
		case process := <-self.startingTracker.timeoutNotification:
			self.plog(process, "Killing, did not start in time.")
			process.Kill()
		case process := <-self.stoppingTracker.timeoutNotification:
			self.plog(process, "Killing, did not stop in time.")
			process.Kill()
		// process state transitions
		case e := <-self.events:
			switch event := e.(type) {
			case *ProcessReadyEvent:
				process := event.process
				self.startingTracker.Remove(process)
				if process != self.childs.starting() {
					self.plog(process, "Oops, some other process is ready")
					continue
				}
				self.plog(process, "Process is ready")
				current := self.childs.ready()
				if current != nil {
					self.plog(current, "Shutting down old current")
					self.stopProcess(current)
				}
				self.config = process.config
				err := self.config.save(self.configPath)
				if err != nil {
					self.log("Failed saving the config: %s", err)
				}
				self.childs.updateState(process, PROCESS_READY)
			case *ProcessExitEvent:
				process := event.process

				self.startingTracker.Remove(process)
				self.stoppingTracker.Remove(process)
				self.childs.rem(process)

				self.plog(process, "Process exited. code=%d err=%v", event.code, event.err)

				if self.childs.len() == 0 {
					goto exit
				}
			default:
				fail("Unknown event: ", e)
			}
		}
	}

exit:

	// Cleanup
	self.childs.each(func(p *Process) {
		p.Kill()
	})
}

// Restart queues and starts excecuting a restart job to replace the old process group with a new one.
func (self *Manager) Reload() {
	self.Start(self.config)
}

func (self *Manager) Start(c *ProcessConfig) {
	self.startAction <- c
}

func (self *Manager) Shutdown() {
	self.shutdownAction <- true
}

// Private methods

func (_ *Manager) log(format string, v ...interface{}) {
	log.Printf("[manager] "+format, v...)
}

func (m *Manager) plog(p *Process, format string, v ...interface{}) {
	args := make([]interface{}, 1, 1+len(v))
	args[0] = p
	args = append(args, v...)
	log.Printf("%s "+format, args...)
}

func (self *Manager) startProcess(config *ProcessConfig) {
	self.log("Starting a new process: %s", config)
	self.processCount += 1
	process, err := startProcess(self.processCount, config, self.socket, self.events)
	if err != nil {
		self.log("Failed to start the process", err)
		return
	}
	self.childs.add(process, PROCESS_STARTING)
	self.startingTracker.Add(process, process.config.StartTimeout)
}

func (self *Manager) stopProcess(process *Process) {
	if self.childs[process] == PROCESS_STOPPING {
		return
	}
	process.Shutdown()
	self.stoppingTracker.Add(process, process.config.StopTimeout)
	self.childs.updateState(process, PROCESS_STOPPING)
}
<|endoftext|>


<commit_before>package domain

import (
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// TODO: Group caching
// TODO: cache reloading

// Access validates access to data. It checks if a user
// has been given permission to access a particular item.
type Access struct {
	groups dai.Groups
	files  dai.Files
	users  dai.Users
}

// NewAccess creates a new Access.
func NewAccess(groups dai.Groups, files dai.Files, users dai.Users) *Access {
	return &Access{
		groups: groups,
		files:  files,
		users:  users,
	}
}

// AllowedByOwner checks to see if the user making the request has access to the
// particular item. Access is determined as follows:
// 1. If the user and the owner of the item are the same
//    or the user is in the admin group return true (has access).
// 2. Get a list of all the users groups for the item's owner.
//    For each user in the user group see if the requesting user
//    is included. If so then return true (has access).
// 3. None of the above matched - return false (no access).
func (a *Access) AllowedByOwner(owner, user string) bool {
	// Check if user and file owner are the same, or the user is
	// in the admin group.
	if user == owner || a.isAdmin(user) {
		return true
	}

	// Get the owners groups
	groups, err := a.groups.ForOwner(owner)
	if err != nil {
		// Some sort of error occurred, assume no access
		return false
	}

	// For each group go through its list of users and see if
	// they match the requesting user. If there is a match
	// then the owner has given access to the user.
	for _, group := range groups {
		users := group.Users
		for _, u := range users {
			if u == user {
				return true
			}
		}
	}

	return false
}

// isAdmin checks if a user is in the admin group.
func (a *Access) isAdmin(user string) bool {
	group, err := a.groups.ByID("admin")
	if err != nil {
		return false
	}

	for _, admin := range group.Users {
		if admin == user {
			return true
		}
	}

	return false
}

// GetFile will validate access to a file. Rather than taking a user,
// it takes an apikey and looks up the user. It returns the file if
// access has been granted, otherwise it returns the erro ErrNoAccess.
func (a *Access) GetFile(apikey, fileID string) (*schema.File, error) {
	user, err := a.users.ByAPIKey(apikey)
	if err != nil {
		// log error here
		app.Log.Error("User lookup failed", "error", err, "apikey", apikey)
		return nil, app.ErrNoAccess
	}

	file, err := a.files.ByID(fileID)
	if err != nil {
		app.Log.Error("File lookup failed", "error", err, "fileid", fileID)
		return nil, app.ErrNoAccess
	}

	if !a.AllowedByOwner(file.Owner, user.ID) {
		app.Log.Info("Access denied", "fileid", file.ID, "user", user.ID)
		return nil, app.ErrNoAccess
	}

	return file, nil
}
<commit_msg>Change to interface so we can create mocks.<commit_after>package domain

import (
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

// TODO: Group caching
// TODO: cache reloading

type Access interface {
	AllowedByOwner(owner, user string) bool
	GetFile(apikey, fileID string) (*schema.File, error)
}

// access validates access to data. It checks if a user
// has been given permission to access a particular item.
type access struct {
	groups dai.Groups
	files  dai.Files
	users  dai.Users
}

// NewAccess creates a new Access.
func NewAccess(groups dai.Groups, files dai.Files, users dai.Users) *access {
	return &access{
		groups: groups,
		files:  files,
		users:  users,
	}
}

// AllowedByOwner checks to see if the user making the request has access to the
// particular item. Access is determined as follows:
// 1. If the user and the owner of the item are the same
//    or the user is in the admin group return true (has access).
// 2. Get a list of all the users groups for the item's owner.
//    For each user in the user group see if the requesting user
//    is included. If so then return true (has access).
// 3. None of the above matched - return false (no access).
func (a *access) AllowedByOwner(owner, user string) bool {
	// Check if user and file owner are the same, or the user is
	// in the admin group.
	if user == owner || a.isAdmin(user) {
		return true
	}

	// Get the owners groups
	groups, err := a.groups.ForOwner(owner)
	if err != nil {
		// Some sort of error occurred, assume no access
		return false
	}

	// For each group go through its list of users and see if
	// they match the requesting user. If there is a match
	// then the owner has given access to the user.
	for _, group := range groups {
		users := group.Users
		for _, u := range users {
			if u == user {
				return true
			}
		}
	}

	return false
}

// isAdmin checks if a user is in the admin group.
func (a *access) isAdmin(user string) bool {
	group, err := a.groups.ByID("admin")
	if err != nil {
		return false
	}

	for _, admin := range group.Users {
		if admin == user {
			return true
		}
	}

	return false
}

// GetFile will validate access to a file. Rather than taking a user,
// it takes an apikey and looks up the user. It returns the file if
// access has been granted, otherwise it returns the erro ErrNoAccess.
func (a *access) GetFile(apikey, fileID string) (*schema.File, error) {
	user, err := a.users.ByAPIKey(apikey)
	if err != nil {
		// log error here
		app.Log.Error("User lookup failed", "error", err, "apikey", apikey)
		return nil, app.ErrNoAccess
	}

	file, err := a.files.ByID(fileID)
	if err != nil {
		app.Log.Error("File lookup failed", "error", err, "fileid", fileID)
		return nil, app.ErrNoAccess
	}

	if !a.AllowedByOwner(file.Owner, user.ID) {
		app.Log.Info("Access denied", "fileid", file.ID, "user", user.ID)
		return nil, app.ErrNoAccess
	}

	return file, nil
}
<|endoftext|>


<commit_before>package driver

//
// import (
// 	"sync"
//
// 	"github.com/docker/go-plugins-helpers/volume"
// )

//
// const ()
//
// type DODriver struct {
// 	Client string
// 	sync.Mutex
// }
//
// func newDODriver(c godo.Client) *DODriver {
// 	return DODriver{
// 		Client: c,
// 	}
// }
//
// func (d *DODriver) Create(http.Request) volume.Response       {}
// func (d *DODriver) List(http.Request) volume.Response         {}
// func (d *DODriver) Get(http.Request) volume.Response          {}
// func (d *DODriver) Remove(http.Request) volume.Response       {}
// func (d *DODriver) Path(http.Request) volume.Response         {}
// func (d *DODriver) Mount(http.Request) volume.Response        {}
// func (d *DODriver) Unmount(http.Request) volume.Response      {}
// func (d *DODriver) Capabilities(http.Request) volume.Response {}
<commit_msg>add basic driver functionality<commit_after>package driver

import (
	"sync"

	"github.com/docker/go-plugins-helpers/volume"

	"github.com/cloudflavor/miniovol/pkg/client"
)

// MinioDriver is the driver used by docker.
type MinioDriver struct {
	*sync.Mutex
	*client.MinioClient
}

func NewMinioDriver(c *client.MinioClient) MinioDriver {
	return MinioDriver{
		&sync.Mutex{},
		c,
	}
}

func (d MinioDriver) Create(volume.Request) volume.Response {

	return volume.Response{}
}
func (d MinioDriver) List(volume.Request) volume.Response {
	return volume.Response{}
}
func (d MinioDriver) Get(volume.Request) volume.Response {
	return volume.Response{}
}
func (d MinioDriver) Remove(volume.Request) volume.Response {
	return volume.Response{}
}
func (d MinioDriver) Path(volume.Request) volume.Response {
	return volume.Response{}
}
func (d MinioDriver) Mount(volume.MountRequest) volume.Response {
	return volume.Response{}
}
func (d MinioDriver) Unmount(volume.UnmountRequest) volume.Response {
	return volume.Response{}
}
func (d MinioDriver) Capabilities(volume.Request) volume.Response {
	return volume.Response{}
}
<|endoftext|>


<commit_before>package plugin

import (
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
	log "github.com/sirupsen/logrus"
)

type oauth2Config struct {
	ServerName string `json:"server_name"`
}

// OAuth2 checks the integrity of the provided OAuth headers
type OAuth2 struct {
	authRepo oauth.Repository
	storage  store.Store
}

// NewOAuth2 creates a new instance of KeyExistsMiddleware
func NewOAuth2(authRepo oauth.Repository, storage store.Store) *OAuth2 {
	return &OAuth2{authRepo, storage}
}

// GetName retrieves the plugin's name
func (h *OAuth2) GetName() string {
	return "oauth2"
}

// GetMiddlewares retrieves the plugin's middlewares
func (h *OAuth2) GetMiddlewares(rawConfig map[string]interface{}, referenceSpec *api.Spec) ([]router.Constructor, error) {
	var oauth2Config oauth2Config
	err := decode(rawConfig, &oauth2Config)
	if err != nil {
		return nil, err
	}

	manager, err := h.getManager(oauth2Config.ServerName)
	if nil != err {
		log.WithError(err).Error("OAuth Configuration for this API is incorrect, skipping...")
		return nil, err
	}

	mw := middleware.NewKeyExistsMiddleware(manager)
	return []router.Constructor{
		mw.Handler,
	}, nil
}

func (h *OAuth2) getManager(oAuthServerName string) (oauth.Manager, error) {
	oauthServer, err := h.authRepo.FindByName(oAuthServerName)
	if nil != err {
		return nil, err
	}

	managerType, err := oauth.ParseType(oauthServer.TokenStrategy.Name)
	if nil != err {
		return nil, err
	}

	return oauth.NewManagerFactory(h.storage, oauthServer.TokenStrategy.Settings).Build(managerType)
}
<commit_msg>Using the correct one<commit_after>package plugin

import (
	log "github.com/Sirupsen/logrus"
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/middleware"
	"github.com/hellofresh/janus/pkg/oauth"
	"github.com/hellofresh/janus/pkg/router"
	"github.com/hellofresh/janus/pkg/store"
)

type oauth2Config struct {
	ServerName string `json:"server_name"`
}

// OAuth2 checks the integrity of the provided OAuth headers
type OAuth2 struct {
	authRepo oauth.Repository
	storage  store.Store
}

// NewOAuth2 creates a new instance of KeyExistsMiddleware
func NewOAuth2(authRepo oauth.Repository, storage store.Store) *OAuth2 {
	return &OAuth2{authRepo, storage}
}

// GetName retrieves the plugin's name
func (h *OAuth2) GetName() string {
	return "oauth2"
}

// GetMiddlewares retrieves the plugin's middlewares
func (h *OAuth2) GetMiddlewares(rawConfig map[string]interface{}, referenceSpec *api.Spec) ([]router.Constructor, error) {
	var oauth2Config oauth2Config
	err := decode(rawConfig, &oauth2Config)
	if err != nil {
		return nil, err
	}

	manager, err := h.getManager(oauth2Config.ServerName)
	if nil != err {
		log.WithError(err).Error("OAuth Configuration for this API is incorrect, skipping...")
		return nil, err
	}

	mw := middleware.NewKeyExistsMiddleware(manager)
	return []router.Constructor{
		mw.Handler,
	}, nil
}

func (h *OAuth2) getManager(oAuthServerName string) (oauth.Manager, error) {
	oauthServer, err := h.authRepo.FindByName(oAuthServerName)
	if nil != err {
		return nil, err
	}

	managerType, err := oauth.ParseType(oauthServer.TokenStrategy.Name)
	if nil != err {
		return nil, err
	}

	return oauth.NewManagerFactory(h.storage, oauthServer.TokenStrategy.Settings).Build(managerType)
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The TestGrid Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package pubsub exports messages for interacting with pubsub.
package pubsub

import (
	"context"
	"fmt"
	"strconv"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
	"github.com/sirupsen/logrus"
)

// Subscriber creates Senders that attach to subscriptions with the specified settings.
type Subscriber interface {
	Subscribe(projID, subID string, setttings *pubsub.ReceiveSettings) Sender
}

// Client wraps a pubsub client into a Subscriber that creates Senders for pubsub subscriptions.
type Client pubsub.Client

// NewClient converts a raw pubsub client into a Subscriber.
func NewClient(client *pubsub.Client) *Client {
	return (*Client)(client)
}

// Subscribe to the specified id in the project using optional receive settings.
func (c *Client) Subscribe(projID, subID string, settings *pubsub.ReceiveSettings) Sender {
	sub := (*pubsub.Client)(c).SubscriptionInProject(subID, projID)
	if settings != nil {
		sub.ReceiveSettings = *settings
	}
	return sub.Receive
}

// Sender forwards pubsub messages to the receive function until the send context expires.
type Sender func(sendCtx context.Context, receive func(context.Context, *pubsub.Message)) error

const (
	keyBucket     = "bucketId"
	keyObject     = "objectId"
	keyEvent      = "eventType"
	keyTime       = "eventTime"
	keyGeneration = "objectGeneration"
)

// Event specifies what happened to the GCS object.
//
// See https://cloud.google.com/storage/docs/pubsub-notifications#events
type Event string

// Well-known event types.
const (
	Finalize Event = "OBJECT_FINALIZE"
	Delete   Event = "OBJECT_DELETE"
)

// Notification captures information about a change to a GCS object.
type Notification struct {
	Path       gcs.Path
	Event      Event
	Time       time.Time
	Generation int64
}

func (n Notification) String() string {
	return fmt.Sprintf("%s#%d %s at %s", n.Path, n.Generation, n.Event, n.Time)
}

// SendGCS converts GCS pubsub messages into Notification structs and sends them to receivers.
//
// Connects to the specified subscription with optionally specified settings.
// Receives pubsub messages from this subscription and converts it into a Notification struct.
//   - Nacks any message it cannot parse.
// Sends the notification to the receivers channel.
//   - Nacks messages associated with any unsent Notifications.
//   - Acks as soon as the Notification is sent.
//
// More info: https://cloud.google.com/storage/docs/pubsub-notifications#overview
func SendGCS(ctx context.Context, log logrus.FieldLogger, client Subscriber, projectID, subID string, settings *pubsub.ReceiveSettings, receivers chan<- *Notification) error {
	send := client.Subscribe(projectID, subID, settings)
	return sendToReceivers(ctx, log, send, receivers, realAcker{})
}

type acker interface {
	Ack(*pubsub.Message)
	Nack(*pubsub.Message)
}

type realAcker struct{}

func (ra realAcker) Ack(m *pubsub.Message) {
	m.Ack()
}

func (ra realAcker) Nack(m *pubsub.Message) {
	m.Nack()
}

func sendToReceivers(ctx context.Context, log logrus.FieldLogger, send Sender, receivers chan<- *Notification, result acker) error {
	return send(ctx, func(ctx context.Context, msg *pubsub.Message) {
		bucket, obj := msg.Attributes[keyBucket], msg.Attributes[keyObject]
		path, err := gcs.NewPath("gs://" + bucket + "/" + obj)
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"bucket": bucket,
				"object": obj,
				"id":     msg.ID,
			}).Error("Failed to parse path")
			result.Nack(msg)
			return
		}
		when, err := time.Parse(time.RFC3339, msg.Attributes[keyTime])
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"time": msg.Attributes[keyTime],
				"id":   msg.ID,
			}).Error("Failed to parse time")
			result.Nack(msg)
			return
		}
		gen, err := strconv.ParseInt(msg.Attributes[keyGeneration], 10, 64)
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"generation": msg.Attributes[keyGeneration],
				"id":         msg.ID,
			}).Error("Failed to parse generation")
			result.Nack(msg)
			return
		}
		notice := Notification{
			Path:       *path,
			Event:      Event(msg.Attributes[keyEvent]),
			Time:       when,
			Generation: gen,
		}
		select {
		case <-ctx.Done():
			result.Nack(msg)
		case receivers <- &notice:
			result.Ack(msg)
		}
	})

}
<commit_msg>Add subscription field to logs<commit_after>/*
Copyright 2021 The TestGrid Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package pubsub exports messages for interacting with pubsub.
package pubsub

import (
	"context"
	"fmt"
	"strconv"
	"time"

	"cloud.google.com/go/pubsub"
	"github.com/GoogleCloudPlatform/testgrid/util/gcs"
	"github.com/sirupsen/logrus"
)

// Subscriber creates Senders that attach to subscriptions with the specified settings.
type Subscriber interface {
	Subscribe(projID, subID string, setttings *pubsub.ReceiveSettings) Sender
}

// Client wraps a pubsub client into a Subscriber that creates Senders for pubsub subscriptions.
type Client pubsub.Client

// NewClient converts a raw pubsub client into a Subscriber.
func NewClient(client *pubsub.Client) *Client {
	return (*Client)(client)
}

// Subscribe to the specified id in the project using optional receive settings.
func (c *Client) Subscribe(projID, subID string, settings *pubsub.ReceiveSettings) Sender {
	sub := (*pubsub.Client)(c).SubscriptionInProject(subID, projID)
	if settings != nil {
		sub.ReceiveSettings = *settings
	}
	return sub.Receive
}

// Sender forwards pubsub messages to the receive function until the send context expires.
type Sender func(sendCtx context.Context, receive func(context.Context, *pubsub.Message)) error

const (
	keyBucket     = "bucketId"
	keyObject     = "objectId"
	keyEvent      = "eventType"
	keyTime       = "eventTime"
	keyGeneration = "objectGeneration"
)

// Event specifies what happened to the GCS object.
//
// See https://cloud.google.com/storage/docs/pubsub-notifications#events
type Event string

// Well-known event types.
const (
	Finalize Event = "OBJECT_FINALIZE"
	Delete   Event = "OBJECT_DELETE"
)

// Notification captures information about a change to a GCS object.
type Notification struct {
	Path       gcs.Path
	Event      Event
	Time       time.Time
	Generation int64
}

func (n Notification) String() string {
	return fmt.Sprintf("%s#%d %s at %s", n.Path, n.Generation, n.Event, n.Time)
}

// SendGCS converts GCS pubsub messages into Notification structs and sends them to receivers.
//
// Connects to the specified subscription with optionally specified settings.
// Receives pubsub messages from this subscription and converts it into a Notification struct.
//   - Nacks any message it cannot parse.
// Sends the notification to the receivers channel.
//   - Nacks messages associated with any unsent Notifications.
//   - Acks as soon as the Notification is sent.
//
// More info: https://cloud.google.com/storage/docs/pubsub-notifications#overview
func SendGCS(ctx context.Context, log logrus.FieldLogger, client Subscriber, projectID, subID string, settings *pubsub.ReceiveSettings, receivers chan<- *Notification) error {
	send := client.Subscribe(projectID, subID, settings)
	log = log.WithField("subscription", "pubsub://"+projectID+"/"+subID)
	return sendToReceivers(ctx, log, send, receivers, realAcker{})
}

type acker interface {
	Ack(*pubsub.Message)
	Nack(*pubsub.Message)
}

type realAcker struct{}

func (ra realAcker) Ack(m *pubsub.Message) {
	m.Ack()
}

func (ra realAcker) Nack(m *pubsub.Message) {
	m.Nack()
}

func sendToReceivers(ctx context.Context, log logrus.FieldLogger, send Sender, receivers chan<- *Notification, result acker) error {
	return send(ctx, func(ctx context.Context, msg *pubsub.Message) {
		bucket, obj := msg.Attributes[keyBucket], msg.Attributes[keyObject]
		path, err := gcs.NewPath("gs://" + bucket + "/" + obj)
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"bucket": bucket,
				"object": obj,
				"id":     msg.ID,
			}).Error("Failed to parse path")
			result.Nack(msg)
			return
		}
		when, err := time.Parse(time.RFC3339, msg.Attributes[keyTime])
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"time": msg.Attributes[keyTime],
				"id":   msg.ID,
			}).Error("Failed to parse time")
			result.Nack(msg)
			return
		}
		gen, err := strconv.ParseInt(msg.Attributes[keyGeneration], 10, 64)
		if err != nil {
			log.WithError(err).WithFields(logrus.Fields{
				"generation": msg.Attributes[keyGeneration],
				"id":         msg.ID,
			}).Error("Failed to parse generation")
			result.Nack(msg)
			return
		}
		notice := Notification{
			Path:       *path,
			Event:      Event(msg.Attributes[keyEvent]),
			Time:       when,
			Generation: gen,
		}
		select {
		case <-ctx.Done():
			result.Nack(msg)
		case receivers <- &notice:
			result.Ack(msg)
		}
	})

}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

// Package socket implements an WebSocket-based playground backend.
// Clients connect to a websocket handler and send run/kill commands, and
// the server sends the output and exit status of the running processes.
// Multiple clients running multiple processes may be served concurrently.
// The wire format is JSON and is described by the Message type.
//
// This will not run on App Engine as WebSockets are not supported there.
package socket

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"

	"code.google.com/p/go.net/websocket"
)

// Handler implements a WebSocket handler for a client connection.
var Handler = websocket.Handler(socketHandler)

const msgLimit = 1000 // max number of messages to send per session

// Message is the wire format for the websocket connection to the browser.
// It is used for both sending output messages and receiving commands, as
// distinguished by the Kind field.
type Message struct {
	Id   string // client-provided unique id for the process
	Kind string // in: "run", "kill" out: "stdout", "stderr", "end"
	Body string
}

// socketHandler handles the websocket connection for a given present session.
// It handles transcoding Messages to and from JSON format, and starting
// and killing processes.
func socketHandler(c *websocket.Conn) {
	in, out := make(chan *Message), make(chan *Message)
	errc := make(chan error, 1)

	// Decode messages from client and send to the in channel.
	go func() {
		dec := json.NewDecoder(c)
		for {
			var m Message
			if err := dec.Decode(&m); err != nil {
				errc <- err
				return
			}
			in <- &m
		}
	}()

	// Receive messages from the out channel and encode to the client.
	go func() {
		enc := json.NewEncoder(c)
		for m := range out {
			if err := enc.Encode(m); err != nil {
				errc <- err
				return
			}
		}
	}()

	// Start and kill processes and handle errors.
	proc := make(map[string]*process)
	for {
		select {
		case m := <-in:
			switch m.Kind {
			case "run":
				proc[m.Id].Kill()
				lOut := limiter(in, out)
				proc[m.Id] = startProcess(m.Id, m.Body, lOut)
			case "kill":
				proc[m.Id].Kill()
			}
		case err := <-errc:
			if err != io.EOF {
				// A encode or decode has failed; bail.
				log.Println(err)
			}
			// Shut down any running processes.
			for _, p := range proc {
				p.Kill()
			}
			return
		}
	}
}

// process represents a running process.
type process struct {
	id   string
	out  chan<- *Message
	done chan struct{} // closed when wait completes
	run  *exec.Cmd
}

// startProcess builds and runs the given program, sending its output
// and end event as Messages on the provided channel.
func startProcess(id, body string, out chan<- *Message) *process {
	p := &process{
		id:   id,
		out:  out,
		done: make(chan struct{}),
	}
	if err := p.start(body); err != nil {
		p.end(err)
		return nil
	}
	go p.wait()
	return p
}

// Kill stops the process if it is running and waits for it to exit.
func (p *process) Kill() {
	if p == nil {
		return
	}
	p.run.Process.Kill()
	<-p.done // block until process exits
}

// start builds and starts the given program, sending its output to p.out,
// and stores the running *exec.Cmd in the run field.
func (p *process) start(body string) error {
	// We "go build" and then exec the binary so that the
	// resultant *exec.Cmd is a handle to the user's program
	// (rather than the go tool process).
	// This makes Kill work.

	bin := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := bin + ".go"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// write body to x.go
	defer os.Remove(src)
	err := ioutil.WriteFile(src, []byte(body), 0666)
	if err != nil {
		return err
	}

	// build x.go, creating x
	defer os.Remove(bin)
	dir, file := filepath.Split(src)
	cmd := p.cmd(dir, "go", "build", "-o", bin, file)
	cmd.Stdout = cmd.Stderr // send compiler output to stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	// run x
	cmd = p.cmd("", bin)
	if err := cmd.Start(); err != nil {
		return err
	}
	p.run = cmd
	return nil
}

// wait waits for the running process to complete
// and sends its error state to the client.
func (p *process) wait() {
	p.end(p.run.Wait())
	close(p.done) // unblock waiting Kill calls
}

// end sends an "end" message to the client, containing the process id and the
// given error value.
func (p *process) end(err error) {
	m := &Message{Id: p.id, Kind: "end"}
	if err != nil {
		m.Body = err.Error()
	}
	p.out <- m
}

// cmd builds an *exec.Cmd that writes its standard output and error to the
// process' output channel.
func (p *process) cmd(dir string, args ...string) *exec.Cmd {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Stdout = &messageWriter{p.id, "stdout", p.out}
	cmd.Stderr = &messageWriter{p.id, "stderr", p.out}
	return cmd
}

// messageWriter is an io.Writer that converts all writes to Message sends on
// the out channel with the specified id and kind.
type messageWriter struct {
	id, kind string
	out      chan<- *Message
}

func (w *messageWriter) Write(b []byte) (n int, err error) {
	w.out <- &Message{Id: w.id, Kind: w.kind, Body: string(b)}
	return len(b), nil
}

// limiter returns a channel that wraps dest. Messages sent to the channel are
// sent to dest. After msgLimit Messages have been passed on, a "kill" Message
// is sent to the kill channel, and only "end" messages are passed.
func limiter(kill chan<- *Message, dest chan<- *Message) chan<- *Message {
	ch := make(chan *Message)
	go func() {
		n := 0
		for m := range ch {
			switch {
			case n < msgLimit || m.Kind == "end":
				dest <- m
				if m.Kind == "end" {
					return
				}
			case n == msgLimit:
				// process produced too much output. Kill it.
				kill <- &Message{Id: m.Id, Kind: "kill"}
			}
			n++
		}
	}()
	return ch
}

var tmpdir string

func init() {
	// find real path to temporary directory
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}

var uniq = make(chan int) // a source of numbers for naming temporary files

func init() {
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()
}
<commit_msg>go.talks/pkg/socket: add Environ hook<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

// Package socket implements an WebSocket-based playground backend.
// Clients connect to a websocket handler and send run/kill commands, and
// the server sends the output and exit status of the running processes.
// Multiple clients running multiple processes may be served concurrently.
// The wire format is JSON and is described by the Message type.
//
// This will not run on App Engine as WebSockets are not supported there.
package socket

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"

	"code.google.com/p/go.net/websocket"
)

// Handler implements a WebSocket handler for a client connection.
var Handler = websocket.Handler(socketHandler)

// Environ, if non-nil, is used to provide an environment to go command and
// user binary invocations.
var Environ func() []string

const msgLimit = 1000 // max number of messages to send per session

// Message is the wire format for the websocket connection to the browser.
// It is used for both sending output messages and receiving commands, as
// distinguished by the Kind field.
type Message struct {
	Id   string // client-provided unique id for the process
	Kind string // in: "run", "kill" out: "stdout", "stderr", "end"
	Body string
}

// socketHandler handles the websocket connection for a given present session.
// It handles transcoding Messages to and from JSON format, and starting
// and killing processes.
func socketHandler(c *websocket.Conn) {
	in, out := make(chan *Message), make(chan *Message)
	errc := make(chan error, 1)

	// Decode messages from client and send to the in channel.
	go func() {
		dec := json.NewDecoder(c)
		for {
			var m Message
			if err := dec.Decode(&m); err != nil {
				errc <- err
				return
			}
			in <- &m
		}
	}()

	// Receive messages from the out channel and encode to the client.
	go func() {
		enc := json.NewEncoder(c)
		for m := range out {
			if err := enc.Encode(m); err != nil {
				errc <- err
				return
			}
		}
	}()

	// Start and kill processes and handle errors.
	proc := make(map[string]*process)
	for {
		select {
		case m := <-in:
			switch m.Kind {
			case "run":
				proc[m.Id].Kill()
				lOut := limiter(in, out)
				proc[m.Id] = startProcess(m.Id, m.Body, lOut)
			case "kill":
				proc[m.Id].Kill()
			}
		case err := <-errc:
			if err != io.EOF {
				// A encode or decode has failed; bail.
				log.Println(err)
			}
			// Shut down any running processes.
			for _, p := range proc {
				p.Kill()
			}
			return
		}
	}
}

// process represents a running process.
type process struct {
	id   string
	out  chan<- *Message
	done chan struct{} // closed when wait completes
	run  *exec.Cmd
}

// startProcess builds and runs the given program, sending its output
// and end event as Messages on the provided channel.
func startProcess(id, body string, out chan<- *Message) *process {
	p := &process{
		id:   id,
		out:  out,
		done: make(chan struct{}),
	}
	if err := p.start(body); err != nil {
		p.end(err)
		return nil
	}
	go p.wait()
	return p
}

// Kill stops the process if it is running and waits for it to exit.
func (p *process) Kill() {
	if p == nil {
		return
	}
	p.run.Process.Kill()
	<-p.done // block until process exits
}

// start builds and starts the given program, sending its output to p.out,
// and stores the running *exec.Cmd in the run field.
func (p *process) start(body string) error {
	// We "go build" and then exec the binary so that the
	// resultant *exec.Cmd is a handle to the user's program
	// (rather than the go tool process).
	// This makes Kill work.

	bin := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := bin + ".go"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// write body to x.go
	defer os.Remove(src)
	err := ioutil.WriteFile(src, []byte(body), 0666)
	if err != nil {
		return err
	}

	// build x.go, creating x
	defer os.Remove(bin)
	dir, file := filepath.Split(src)
	cmd := p.cmd(dir, "go", "build", "-o", bin, file)
	cmd.Stdout = cmd.Stderr // send compiler output to stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	// run x
	cmd = p.cmd("", bin)
	if err := cmd.Start(); err != nil {
		return err
	}
	p.run = cmd
	return nil
}

// wait waits for the running process to complete
// and sends its error state to the client.
func (p *process) wait() {
	p.end(p.run.Wait())
	close(p.done) // unblock waiting Kill calls
}

// end sends an "end" message to the client, containing the process id and the
// given error value.
func (p *process) end(err error) {
	m := &Message{Id: p.id, Kind: "end"}
	if err != nil {
		m.Body = err.Error()
	}
	p.out <- m
}

// cmd builds an *exec.Cmd that writes its standard output and error to the
// process' output channel.
func (p *process) cmd(dir string, args ...string) *exec.Cmd {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	if Environ != nil {
		cmd.Env = Environ()
	}
	cmd.Stdout = &messageWriter{p.id, "stdout", p.out}
	cmd.Stderr = &messageWriter{p.id, "stderr", p.out}
	return cmd
}

// messageWriter is an io.Writer that converts all writes to Message sends on
// the out channel with the specified id and kind.
type messageWriter struct {
	id, kind string
	out      chan<- *Message
}

func (w *messageWriter) Write(b []byte) (n int, err error) {
	w.out <- &Message{Id: w.id, Kind: w.kind, Body: string(b)}
	return len(b), nil
}

// limiter returns a channel that wraps dest. Messages sent to the channel are
// sent to dest. After msgLimit Messages have been passed on, a "kill" Message
// is sent to the kill channel, and only "end" messages are passed.
func limiter(kill chan<- *Message, dest chan<- *Message) chan<- *Message {
	ch := make(chan *Message)
	go func() {
		n := 0
		for m := range ch {
			switch {
			case n < msgLimit || m.Kind == "end":
				dest <- m
				if m.Kind == "end" {
					return
				}
			case n == msgLimit:
				// process produced too much output. Kill it.
				kill <- &Message{Id: m.Id, Kind: "kill"}
			}
			n++
		}
	}()
	return ch
}

var tmpdir string

func init() {
	// find real path to temporary directory
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}

var uniq = make(chan int) // a source of numbers for naming temporary files

func init() {
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()
}
<|endoftext|>