<commit_before>package config

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"plugin"
	"reflect"
	"strconv"
	"strings"

	"golang.org/x/crypto/acme/autocert"
)

// SecretStore represents a contract for a store capable of resolving secrets.
type SecretStore interface {
	Configure(config Config) error
	GetSecret(secretName string) (string, bool)
}

// CertificateStore represents a secret store which can be used for certificates.
type CertificateStore interface {
	autocert.Cache
}

// Provider represents a configurable provider.
type Provider interface {
	Name() string
	Configure(config map[string]interface{}) error
}

// Config represents a configuration interface.
type Config interface {
	Vault() *VaultConfig
}

// TLSConfig represents TLS listener configuration.
type TLSConfig struct {
	ListenAddr  string `json:"listen"`                // The address to listen on.
	Host        string `json:"host"`                  // The hostname to whitelist.
	Email       string `json:"email,omitempty"`       // The email address for autocert.
	Certificate string `json:"certificate,omitempty"` // The certificate request.
	PrivateKey  string `json:"private,omitempty"`     // The private key for the certificate.
}

// Load loads the certificates from the cache or the configuration.
func (c *TLSConfig) Load(certCache autocert.Cache) (*tls.Config, http.Handler, error) {
	if c.Certificate != "" {
		return c.loadFromLocal(certCache)
	}

	return c.loadFromAutocert(certCache)
}

// loadFromLocal loads TLS configuration from pre-existing certificate
func (c *TLSConfig) loadFromLocal(certCache autocert.Cache) (*tls.Config, http.Handler, error) {
	if c.PrivateKey == "" {
		return &tls.Config{}, nil, errors.New("No certificate or private key configured")
	}

	// If the certificate provided is in plain text, write to file so we can read it.
	if strings.HasPrefix(c.Certificate, "---") {
		if err := ioutil.WriteFile("broker.crt", []byte(c.Certificate), os.ModePerm); err == nil {
			c.Certificate = "broker.crt"
		}
	}

	// If the private key provided is in plain text, write to file so we can read it.
	if strings.HasPrefix(c.PrivateKey, "---") {
		if err := ioutil.WriteFile("broker.key", []byte(c.PrivateKey), os.ModePerm); err == nil {
			c.PrivateKey = "broker.key"
		}
	}

	// Make sure the paths are absolute, otherwise we won't be able to read the files.
	c.Certificate = resolvePath(c.Certificate)
	c.PrivateKey = resolvePath(c.PrivateKey)

	// Load the certificate from the cert/key files.

	cer, err := tls.LoadX509KeyPair(c.Certificate, c.PrivateKey)
	return &tls.Config{
		Certificates: []tls.Certificate{cer},
	}, nil, err
}

// loadFromAutocert loads TLS configuration from Letsencrypt
func (c *TLSConfig) loadFromAutocert(certCache autocert.Cache) (*tls.Config, http.Handler, error) {
	if c.Host == "" {
		return nil, nil, errors.New("unable to request a certificate, no host name configured")
	}

	// Default to disk cache
	if certCache == nil {
		certCache = autocert.DirCache("certs")
	}

	// Create an auto-cert manager
	certManager := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		HostPolicy: autocert.HostWhitelist(c.Host),
		Email:      c.Email,
		Cache:      certCache,
	}

	return &tls.Config{
		GetCertificate: certManager.GetCertificate,
	}, certManager.HTTPHandler(nil), nil
}

// VaultConfig represents Vault configuration.
type VaultConfig struct {
	Address     string `json:"address"` // The vault address to use.
	Application string `json:"app"`     // The vault application ID to use.
}

// NewClient creates a new vault client for the configuration.
func (c *VaultConfig) NewClient(user string) (client *VaultClient, err error) {
	if c.Address == "" || c.Application == "" {
		return nil, errors.New("unable to configure Vault provider")
	}

	client = NewVaultClient(c.Address)
	err = client.Authenticate(c.Application, user)
	return
}

// ProviderConfig represents provider configuration.
type ProviderConfig struct {

	// The storage provider, this can either be specific builtin or the plugin path (file or
	// url) if the plugin is specified, it must contain a constructor function named 'New'
	// which returns an interface{}.
	Provider string `json:"provider"`

	// The configuration for a provider. This specifies various parameters to provide to the
	// specific provider during the Configure() call.
	Config map[string]interface{} `json:"config,omitempty"`
}

// LoadOrPanic loads a provider from the configuration and uses one or several builtins
// provided. If the provider is not found, it panics.
func (c *ProviderConfig) LoadOrPanic(builtins ...Provider) Provider {
	provider, err := c.Load(builtins...)
	if err != nil {
		panic(err)
	}

	return provider
}

// Load loads a provider from the configuration and uses one or several builtins provided.
func (c *ProviderConfig) Load(builtins ...Provider) (Provider, error) {
	for _, builtin := range builtins {
		if strings.ToLower(builtin.Name()) == strings.ToLower(c.Provider) {
			if err := builtin.Configure(c.Config); err != nil {
				return nil, errors.New("The provider '" + c.Provider + "' could not be loaded. " + err.Error())
			}

			return builtin, nil
		}
	}

	// Attempt to load a plugin provider
	p, err := plugin.Open(resolvePath(c.Provider))
	if err != nil {
		return nil, errors.New("The provider plugin '" + c.Provider + "' could not be opened. " + err.Error())
	}

	// Get the symbol
	sym, err := p.Lookup("New")
	if err != nil {
		return nil, errors.New("The provider '" + c.Provider + "' does not contain 'func New() interface{}' symbol")
	}

	// Resolve the
	pFactory, validFunc := sym.(*func() interface{})
	if !validFunc {
		return nil, errors.New("The provider '" + c.Provider + "' does not contain 'func New() interface{}' symbol")
	}

	// Construct the provider
	provider, validProv := ((*pFactory)()).(Provider)
	if !validProv {
		return nil, errors.New("The provider '" + c.Provider + "' does not implement 'Provider'")
	}

	// Configure the provider
	err = provider.Configure(c.Config)
	if err != nil {
		return nil, errors.New("The provider '" + c.Provider + "' could not be configured")
	}

	// Succesfully opened and configured a provider
	return provider, nil
}

// LoadProvider loads a provider from the configuration or panics if the configuration is
// specified, but the provider was not found or not able to configure. This uses the first
// provider as a default value.
func LoadProvider(config *ProviderConfig, providers ...Provider) Provider {
	if config == nil || config.Provider == "" {
		config.Provider = providers[0].Name() // Use the default one
	}

	// Load the provider according to the configuration
	return config.LoadOrPanic(providers...)
}

// Write writes the configuration to a specific writer, in JSON format.
func write(config interface{}, output io.Writer) (int, error) {
	var formatted bytes.Buffer
	body, err := json.Marshal(config)
	if err != nil {
		return 0, err
	}

	if err := json.Indent(&formatted, body, "", "\t"); err != nil {
		return 0, err
	}

	return output.Write(formatted.Bytes())
}

// createDefault writes the default configuration to disk.
func createDefault(path string, newDefault func() Config) (Config, error) {
	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.ModePerm)
	if err != nil {
		return nil, err
	}

	defer f.Close()
	c := newDefault()
	if _, err := write(c, f); err != nil {
		return nil, err
	}
	if err := f.Sync(); err != nil {
		return nil, err
	}
	return c, nil
}

// ReadOrCreate reads or creates the configuration object.
func ReadOrCreate(prefix string, path string, newDefault func() Config, stores ...SecretStore) (cfg Config, err error) {
	cfg = newDefault()
	if _, err := os.Stat(path); os.IsNotExist(err) {
		// Create a configuration and write it to a file
		if cfg, err = createDefault(path, newDefault); err != nil {
			return nil, err
		}
	} else {
		// Read the config from file
		b, err := ioutil.ReadFile(path)
		if err != nil {
			return nil, err
		}

		// Unmarshal the configuration
		if err := json.Unmarshal(b, cfg); err != nil {
			return nil, err
		}
	}

	// Apply all the store overrides, in order
	for _, store := range stores {
		if err := store.Configure(cfg); err == nil {
			declassify(cfg, prefix, store)
		}
	}

	return cfg, nil
}

// Declassify traverses the configuration and resolves secrets.
func declassify(config interface{}, prefix string, provider SecretStore) {
	original := reflect.ValueOf(config)
	declassifyRecursive(prefix, provider, original)
}

// DeclassifyRecursive traverses the configuration and resolves secrets.
func declassifyRecursive(prefix string, provider SecretStore, value reflect.Value) {
	switch value.Kind() {
	case reflect.Ptr:
		pValue := value.Elem()
		if !pValue.IsValid() {
			// Create a new struct and set the value
			pValue = reflect.New(value.Type().Elem())
			value.Set(pValue)
		}

		declassifyRecursive(prefix, provider, pValue)

	// If it is a struct we translate each field
	case reflect.Struct:
		for i := 0; i < value.NumField(); i++ {
			name := getFieldName(value.Type().Field(i))
			declassifyRecursive(prefix+"/"+name, provider, value.Field(i))
		}

	// This is a integer, we need to fetch the secret
	case reflect.Int:
		if v, ok := provider.GetSecret(prefix); ok {
			if iv, err := strconv.ParseInt(v, 10, 64); err == nil {
				value.SetInt(iv)
			}
		}

	// This is a string, we need to fetch the secret
	case reflect.String:
		if v, ok := provider.GetSecret(prefix); ok {
			value.SetString(v)
		}

	// This is a map, unmarshal and set
	case reflect.Map:
		if v, ok := provider.GetSecret(prefix); ok {
			var out map[string]interface{}
			if err := json.Unmarshal([]byte(v), &out); err == nil {
				value.Set(reflect.ValueOf(out))
			}
		}
	}
}

func getFieldName(f reflect.StructField) string {
	return strings.Replace(string(f.Tag.Get("json")), ",omitempty", "", -1)
}

func resolvePath(path string) string {

	// If it's an url, download the file
	if strings.HasPrefix(path, "http") {
		f, err := httpFile(path)
		if err != nil {
			panic(err)
		}

		// Get the downloaded file path
		path = f.Name()
	}

	// Make sure the path is absolute
	path, _ = filepath.Abs(path)
	return path
}
<commit_msg>fix config load<commit_after>package config

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"plugin"
	"reflect"
	"strconv"
	"strings"

	"golang.org/x/crypto/acme/autocert"
)

// SecretStore represents a contract for a store capable of resolving secrets.
type SecretStore interface {
	Configure(config Config) error
	GetSecret(secretName string) (string, bool)
}

// CertificateStore represents a secret store which can be used for certificates.
type CertificateStore interface {
	autocert.Cache
}

// Provider represents a configurable provider.
type Provider interface {
	Name() string
	Configure(config map[string]interface{}) error
}

// Config represents a configuration interface.
type Config interface {
	Vault() *VaultConfig
}

// TLSConfig represents TLS listener configuration.
type TLSConfig struct {
	ListenAddr  string `json:"listen"`                // The address to listen on.
	Host        string `json:"host"`                  // The hostname to whitelist.
	Email       string `json:"email,omitempty"`       // The email address for autocert.
	Certificate string `json:"certificate,omitempty"` // The certificate request.
	PrivateKey  string `json:"private,omitempty"`     // The private key for the certificate.
}

// Load loads the certificates from the cache or the configuration.
func (c *TLSConfig) Load(certCache autocert.Cache) (*tls.Config, http.Handler, error) {
	if c.Certificate != "" {
		return c.loadFromLocal(certCache)
	}

	return c.loadFromAutocert(certCache)
}

// loadFromLocal loads TLS configuration from pre-existing certificate
func (c *TLSConfig) loadFromLocal(certCache autocert.Cache) (*tls.Config, http.Handler, error) {
	if c.PrivateKey == "" {
		return &tls.Config{}, nil, errors.New("No certificate or private key configured")
	}

	// If the certificate provided is in plain text, write to file so we can read it.
	if strings.HasPrefix(c.Certificate, "---") {
		if err := ioutil.WriteFile("broker.crt", []byte(c.Certificate), os.ModePerm); err == nil {
			c.Certificate = "broker.crt"
		}
	}

	// If the private key provided is in plain text, write to file so we can read it.
	if strings.HasPrefix(c.PrivateKey, "---") {
		if err := ioutil.WriteFile("broker.key", []byte(c.PrivateKey), os.ModePerm); err == nil {
			c.PrivateKey = "broker.key"
		}
	}

	// Make sure the paths are absolute, otherwise we won't be able to read the files.
	c.Certificate = resolvePath(c.Certificate)
	c.PrivateKey = resolvePath(c.PrivateKey)

	// Load the certificate from the cert/key files.

	cer, err := tls.LoadX509KeyPair(c.Certificate, c.PrivateKey)
	return &tls.Config{
		Certificates: []tls.Certificate{cer},
	}, nil, err
}

// loadFromAutocert loads TLS configuration from Letsencrypt
func (c *TLSConfig) loadFromAutocert(certCache autocert.Cache) (*tls.Config, http.Handler, error) {
	if c.Host == "" {
		return nil, nil, errors.New("unable to request a certificate, no host name configured")
	}

	// Default to disk cache
	if certCache == nil {
		certCache = autocert.DirCache("certs")
	}

	// Create an auto-cert manager
	certManager := autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		HostPolicy: autocert.HostWhitelist(c.Host),
		Email:      c.Email,
		Cache:      certCache,
	}

	return &tls.Config{
		GetCertificate: certManager.GetCertificate,
	}, certManager.HTTPHandler(nil), nil
}

// VaultConfig represents Vault configuration.
type VaultConfig struct {
	Address     string `json:"address"` // The vault address to use.
	Application string `json:"app"`     // The vault application ID to use.
}

// NewClient creates a new vault client for the configuration.
func (c *VaultConfig) NewClient(user string) (client *VaultClient, err error) {
	if c.Address == "" || c.Application == "" {
		return nil, errors.New("unable to configure Vault provider")
	}

	client = NewVaultClient(c.Address)
	err = client.Authenticate(c.Application, user)
	return
}

// ProviderConfig represents provider configuration.
type ProviderConfig struct {

	// The storage provider, this can either be specific builtin or the plugin path (file or
	// url) if the plugin is specified, it must contain a constructor function named 'New'
	// which returns an interface{}.
	Provider string `json:"provider"`

	// The configuration for a provider. This specifies various parameters to provide to the
	// specific provider during the Configure() call.
	Config map[string]interface{} `json:"config,omitempty"`
}

// LoadOrPanic loads a provider from the configuration and uses one or several builtins
// provided. If the provider is not found, it panics.
func (c *ProviderConfig) LoadOrPanic(builtins ...Provider) Provider {
	provider, err := c.Load(builtins...)
	if err != nil {
		panic(err)
	}

	return provider
}

// Load loads a provider from the configuration and uses one or several builtins provided.
func (c *ProviderConfig) Load(builtins ...Provider) (Provider, error) {
	for _, builtin := range builtins {
		if strings.ToLower(builtin.Name()) == strings.ToLower(c.Provider) {
			if err := builtin.Configure(c.Config); err != nil {
				return nil, errors.New("The provider '" + c.Provider + "' could not be loaded. " + err.Error())
			}

			return builtin, nil
		}
	}

	// Attempt to load a plugin provider
	p, err := plugin.Open(resolvePath(c.Provider))
	if err != nil {
		return nil, errors.New("The provider plugin '" + c.Provider + "' could not be opened. " + err.Error())
	}

	// Get the symbol
	sym, err := p.Lookup("New")
	if err != nil {
		return nil, errors.New("The provider '" + c.Provider + "' does not contain 'func New() interface{}' symbol")
	}

	// Resolve the
	pFactory, validFunc := sym.(*func() interface{})
	if !validFunc {
		return nil, errors.New("The provider '" + c.Provider + "' does not contain 'func New() interface{}' symbol")
	}

	// Construct the provider
	provider, validProv := ((*pFactory)()).(Provider)
	if !validProv {
		return nil, errors.New("The provider '" + c.Provider + "' does not implement 'Provider'")
	}

	// Configure the provider
	err = provider.Configure(c.Config)
	if err != nil {
		return nil, errors.New("The provider '" + c.Provider + "' could not be configured")
	}

	// Succesfully opened and configured a provider
	return provider, nil
}

// LoadProvider loads a provider from the configuration or panics if the configuration is
// specified, but the provider was not found or not able to configure. This uses the first
// provider as a default value.
func LoadProvider(config *ProviderConfig, providers ...Provider) Provider {
	if config == nil || config.Provider == "" {
		config = &ProviderConfig{
			Provider: providers[0].Name(),
		}
	}

	// Load the provider according to the configuration
	return config.LoadOrPanic(providers...)
}

// Write writes the configuration to a specific writer, in JSON format.
func write(config interface{}, output io.Writer) (int, error) {
	var formatted bytes.Buffer
	body, err := json.Marshal(config)
	if err != nil {
		return 0, err
	}

	if err := json.Indent(&formatted, body, "", "\t"); err != nil {
		return 0, err
	}

	return output.Write(formatted.Bytes())
}

// createDefault writes the default configuration to disk.
func createDefault(path string, newDefault func() Config) (Config, error) {
	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, os.ModePerm)
	if err != nil {
		return nil, err
	}

	defer f.Close()
	c := newDefault()
	if _, err := write(c, f); err != nil {
		return nil, err
	}
	if err := f.Sync(); err != nil {
		return nil, err
	}
	return c, nil
}

// ReadOrCreate reads or creates the configuration object.
func ReadOrCreate(prefix string, path string, newDefault func() Config, stores ...SecretStore) (cfg Config, err error) {
	cfg = newDefault()
	if _, err := os.Stat(path); os.IsNotExist(err) {
		// Create a configuration and write it to a file
		if cfg, err = createDefault(path, newDefault); err != nil {
			return nil, err
		}
	} else {
		// Read the config from file
		b, err := ioutil.ReadFile(path)
		if err != nil {
			return nil, err
		}

		// Unmarshal the configuration
		if err := json.Unmarshal(b, cfg); err != nil {
			return nil, err
		}
	}

	// Apply all the store overrides, in order
	for _, store := range stores {
		if err := store.Configure(cfg); err == nil {
			declassify(cfg, prefix, store)
		}
	}

	return cfg, nil
}

// Declassify traverses the configuration and resolves secrets.
func declassify(config interface{}, prefix string, provider SecretStore) {
	original := reflect.ValueOf(config)
	declassifyRecursive(prefix, provider, original)
}

// DeclassifyRecursive traverses the configuration and resolves secrets.
func declassifyRecursive(prefix string, provider SecretStore, value reflect.Value) {
	switch value.Kind() {
	case reflect.Ptr:
		pValue := value.Elem()
		if !pValue.IsValid() {
			// Create a new struct and set the value
			pValue = reflect.New(value.Type().Elem())
			value.Set(pValue)
		}

		declassifyRecursive(prefix, provider, pValue)

	// If it is a struct we translate each field
	case reflect.Struct:
		for i := 0; i < value.NumField(); i++ {
			name := getFieldName(value.Type().Field(i))
			declassifyRecursive(prefix+"/"+name, provider, value.Field(i))
		}

	// This is a integer, we need to fetch the secret
	case reflect.Int:
		if v, ok := provider.GetSecret(prefix); ok {
			if iv, err := strconv.ParseInt(v, 10, 64); err == nil {
				value.SetInt(iv)
			}
		}

	// This is a string, we need to fetch the secret
	case reflect.String:
		if v, ok := provider.GetSecret(prefix); ok {
			value.SetString(v)
		}

	// This is a map, unmarshal and set
	case reflect.Map:
		if v, ok := provider.GetSecret(prefix); ok {
			var out map[string]interface{}
			if err := json.Unmarshal([]byte(v), &out); err == nil {
				value.Set(reflect.ValueOf(out))
			}
		}
	}
}

func getFieldName(f reflect.StructField) string {
	return strings.Replace(string(f.Tag.Get("json")), ",omitempty", "", -1)
}

func resolvePath(path string) string {

	// If it's an url, download the file
	if strings.HasPrefix(path, "http") {
		f, err := httpFile(path)
		if err != nil {
			panic(err)
		}

		// Get the downloaded file path
		path = f.Name()
	}

	// Make sure the path is absolute
	path, _ = filepath.Abs(path)
	return path
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"os"
	"regexp"
	"strings"
//   "fmt"
)

var ptn *regexp.Regexp
var parse_err error
var conf = make(map[string]string)

func init() {
	ptn, parse_err = regexp.Compile(`[#]*[A-Za-z0-9\_\-\.\t ]+[=].*`)
	if parse_err != nil {
		panic(parse_err)
	}
}

func Parse(filename string) (err error) {
	fi, err := os.Stat(filename)
	if err != nil {
		return
	}

	f, err := os.Open(filename)
	if err != nil {
		return
	}

	buff := make([]byte, fi.Size())
	f.Read(buff)
	f.Close()

	str := ptn.FindAllString(string(buff), -1)

	for i := 0; i < len(str); i++ {
		switch {
		case strings.Index(str[i], "#") == 0:
			// 说明是注释行，什么也不做
		case strings.Index(str[i], "=") > 0:
			kvs := strings.Split(str[i], "=")
			key, val := strings.TrimSpace(kvs[0]), strings.Trim(kvs[1], `"`)
			conf[key] = val
		default:
			err = errors.New("parse config files failed! please check this line:" + str[i])
			return
		}
	}
	return
}

//调试用，返回conf对象的指针，一般情况下最好不要使用
func GetAll() *map[string]string {
	return &conf
}

// 获取单个键值
func Get(key string) (value string) {
	value = conf[key]
	return
}

// 临时设置键值。本方法不会覆盖配置文件，因此重启后无法保存。
func Set(key string, value string) {
	conf[key] = value
	return
}

// 以可变参数的形式获取多个键值，并按顺序返回值的切片。
func GetMulti(keys ...string)(values []string){
	for _, key := range keys {
		values = append(values, conf[key])
	}
  return
}
<commit_msg>将GetMulti分成两个方法，分别返回字符串切片和空接口切片类型的值<commit_after>package config

import (
	"errors"
	"os"
	"regexp"
	"strings"
//   "fmt"
)

var ptn *regexp.Regexp
var parse_err error
var conf = make(map[string]string)

func init() {
	ptn, parse_err = regexp.Compile(`[#]*[A-Za-z0-9\_\-\.\t ]+[=].*`)
	if parse_err != nil {
		panic(parse_err)
	}
}

func Parse(filename string) (err error) {
	fi, err := os.Stat(filename)
	if err != nil {
		return
	}

	f, err := os.Open(filename)
	if err != nil {
		return
	}

	buff := make([]byte, fi.Size())
	f.Read(buff)
	f.Close()

	str := ptn.FindAllString(string(buff), -1)

	for i := 0; i < len(str); i++ {
		switch {
		case strings.Index(str[i], "#") == 0:
			// 说明是注释行，什么也不做
		case strings.Index(str[i], "=") > 0:
			kvs := strings.Split(str[i], "=")
			key, val := strings.TrimSpace(kvs[0]), strings.Trim(kvs[1], `"`)
			conf[key] = val
		default:
			err = errors.New("parse config files failed! please check this line:" + str[i])
			return
		}
	}
	return
}

//调试用，返回conf对象的指针，一般情况下最好不要使用
func GetAll() *map[string]string {
	return &conf
}

// 获取单个键值
func Get(key string) (value string) {
	value = conf[key]
	return
}

// 临时设置键值。本方法不会覆盖配置文件，因此重启后无法保存。
func Set(key string, value string) {
	conf[key] = value
	return
}

// 以可变参数的形式获取多个键值，并按顺序返回[]interface{}类型的切片。
func GetMulti(keys ...string)(values []interface{}){
	for _, key := range keys {
		values = append(values, conf[key])
	}
  return
}

// 以可变参数的形式获取多个键值，并按顺序返回[]string类型的切片。
func GetMultiValues(keys ...string)(values []string){
	for _, key := range keys {
		values = append(values, conf[key])
	}
  return
}
<|endoftext|>


<commit_before>package main

import (
	"path/filepath"
	"strings"
)

var defaultExcludedFiles = []string{
	// Mac OS Finder metadata
	".DS_Store",
	// Mac OS folder icon: "Icon" with ^M at the end
	string([]byte{0x49, 0x63, 0x6f, 0x6e, 0x0d}),
	// VCS folders
	".git",
	".svn",
	// ignore the manifest dir itself
	manifestDirName,
}

// Config for bitrot checks such as file/folder names to exclude.
type Config struct {
	ExcludedFiles []string
}

func DefaultConfig() *Config {
	return &Config{
		ExcludedFiles: defaultExcludedFiles,
	}
}

func (c *Config) isIgnoredPath(path string) bool {
	parts := strings.Split(path, string(filepath.Separator))
	for _, part := range parts {
		for _, ignoredName := range c.ExcludedFiles {
			if part == ignoredName {
				return true
			}
		}
	}
	return false
}
<commit_msg>Ignore Synology metadata<commit_after>package main

import (
	"path/filepath"
	"strings"
)

var defaultExcludedFiles = []string{
	// Mac OS Finder metadata
	".DS_Store",
	// Mac OS folder icon: "Icon" with ^M at the end
	string([]byte{0x49, 0x63, 0x6f, 0x6e, 0x0d}),
	// VCS folders
	".git",
	".svn",
	// Synology filesystem metadata
	"@eaDir",
	// ignore the manifest dir itself
	manifestDirName,
}

// Config for bitrot checks such as file/folder names to exclude.
type Config struct {
	ExcludedFiles []string
}

func DefaultConfig() *Config {
	return &Config{
		ExcludedFiles: defaultExcludedFiles,
	}
}

func (c *Config) isIgnoredPath(path string) bool {
	parts := strings.Split(path, string(filepath.Separator))
	for _, part := range parts {
		for _, ignoredName := range c.ExcludedFiles {
			if part == ignoredName {
				return true
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>package raft

import (
	"fmt"
	"io"
	"log"
	"time"
)

// These are the versions of the protocol (which includes RPC messages as
// well as Raft-specific log entries) that this server can _understand_. Use
// the ProtocolVersion member of the Config object to control the version of
// the protocol to use when _speaking_ to other servers. This is not currently
// written into snapshots so they are unversioned. Note that depending on the
// protocol version being spoken, some otherwise understood RPC messages may be
// refused. See isVersionCompatible for details of this logic.
//
// There are notes about the upgrade path in the description of the versions
// below. If you are starting a fresh cluster then there's no reason not to
// jump right to the latest protocol version. If you need to interoperate with
// older, unversioned Raft servers you'll need to drive the cluster through the
// different versions in order. This may best be done by moving forward across a
// series of releases of your application.
//
// Version History
//
// 0: Unversioned, original protocol, used to interoperate with unversioned
//    Raft servers. Under this version all configuration changes are propagated
//    using the deprecated RemovePeerDeprecated Raft log entry. This means that
//    server IDs are always set to be the same as the server addresses (since
//    the old log entry type cannot transmit an ID), and only AddPeer/RemovePeer
//    APIs are supported. This version can understand the new LogConfiguration
//    Raft log entry but it will never generate one.
// 1: Transitional protocol used when migrating an existing cluster to the new
//    server ID system. Server IDs are still set to be the same as server
//    addresses, but all configuration changes are propagated using the new
//    LogConfiguration Raft log entry type, which can carry full ID information.
//    This version supports the old AddPeer/RemovePeer APIs as well as the new
//    ID-based AddVoter/RemoveServer APIs which should be used when adding
//    version 2 servers to the cluster later. Note that non-voting roles are not
//    yet supported. This version sheds all interoperability with unversioned
//    Raft servers, but can interoperate with newer Raft servers running with
//    protocol version 0 since they can understand the new LogConfiguration Raft
//    log entry, and this version can still understand their RemovePeerDeprecated
//    Raft log entries. We need this protocol version as an intermediate step
//    between 0 and 2 so that servers will propagate the ID information that
//    will come from newly-added (or -cycled) servers using protocol version 2,
//    but since they are still using their address-based IDs from the previous
//    step they will still be able to track commitments and their own voting
//    status properly. If we skipped this step, servers would be started with
//    their new IDs, but they wouldn't see themselves in the old address-based
//    configuration, so none of the servers would think they had a vote.
// 2: Protocol adding full support for server IDs and new ID-based server APIs
//    (AddVoter, AddNonvoter, etc.), old AddPeer/RemovePeer APIs are no longer
//    supported. Version 1 servers should be swapped out by removing them from
//    the cluster one-by-one and re-adding them with updated configuration for
//    this protocol version, along with their server ID. The remove/add cycle
//    is required to populate their server ID. Note that removing must be done
//    by ID, which will be the old server's address.
const (
	ProtocolVersionMin = 0
	ProtocolVersionMax = 2
)

// Config provides any necessary configuration to
// the Raft server
type Config struct {
	// ProtocolVersion allows a Raft server to inter-operate with older
	// Raft servers running an older version of the code. This is used to
	// version the wire protocol as well as Raft-specific log entries that
	// the server uses when _speaking_ to other servers. There is currently
	// no auto-negotiation of versions so all servers must be manually
	// configured with compatible versions. See ProtocolVersionMin and
	// ProtocolVersionMax for the versions of the protocol that this server
	// can _understand_.
	ProtocolVersion int

	// HeartbeatTimeout specifies the time in follower state without
	// a leader before we attempt an election.
	HeartbeatTimeout time.Duration

	// ElectionTimeout specifies the time in candidate state without
	// a leader before we attempt an election.
	ElectionTimeout time.Duration

	// CommitTimeout controls the time without an Apply() operation
	// before we heartbeat to ensure a timely commit. Due to random
	// staggering, may be delayed as much as 2x this value.
	CommitTimeout time.Duration

	// MaxAppendEntries controls the maximum number of append entries
	// to send at once. We want to strike a balance between efficiency
	// and avoiding waste if the follower is going to reject because of
	// an inconsistent log.
	MaxAppendEntries int

	// If we are a member of a cluster, and RemovePeer is invoked for the
	// local node, then we forget all peers and transition into the follower state.
	// If ShutdownOnRemove is is set, we additional shutdown Raft. Otherwise,
	// we can become a leader of a cluster containing only this node.
	ShutdownOnRemove bool

	// TrailingLogs controls how many logs we leave after a snapshot. This is
	// used so that we can quickly replay logs on a follower instead of being
	// forced to send an entire snapshot.
	TrailingLogs uint64

	// SnapshotInterval controls how often we check if we should perform a snapshot.
	// We randomly stagger between this value and 2x this value to avoid the entire
	// cluster from performing a snapshot at once.
	SnapshotInterval time.Duration

	// SnapshotThreshold controls how many outstanding logs there must be before
	// we perform a snapshot. This is to prevent excessive snapshots when we can
	// just replay a small set of logs.
	SnapshotThreshold uint64

	// LeaderLeaseTimeout is used to control how long the "lease" lasts
	// for being the leader without being able to contact a quorum
	// of nodes. If we reach this interval without contact, we will
	// step down as leader.
	LeaderLeaseTimeout time.Duration

	// StartAsLeader forces Raft to start in the leader state. This should
	// never be used except for testing purposes, as it can cause a split-brain.
	StartAsLeader bool

	// The unique ID for this server across all time. For protocol version
	// > 1 this is required, and for older protocols it will be populated with
	// the server's network address.
	LocalID ServerID

	// NotifyCh is used to provide a channel that will be notified of leadership
	// changes. Raft will block writing to this channel, so it should either be
	// buffered or aggressively consumed.
	NotifyCh chan<- bool

	// LogOutput is used as a sink for logs, unless Logger is specified.
	// Defaults to os.Stderr.
	LogOutput io.Writer

	// Logger is a user-provided logger. If nil, a logger writing to LogOutput
	// is used.
	Logger *log.Logger
}

// DefaultConfig returns a Config with usable defaults.
func DefaultConfig() *Config {
	return &Config{
		ProtocolVersion:    ProtocolVersionMax,
		HeartbeatTimeout:   1000 * time.Millisecond,
		ElectionTimeout:    1000 * time.Millisecond,
		CommitTimeout:      50 * time.Millisecond,
		MaxAppendEntries:   64,
		ShutdownOnRemove:   true,
		TrailingLogs:       10240,
		SnapshotInterval:   120 * time.Second,
		SnapshotThreshold:  8192,
		LeaderLeaseTimeout: 500 * time.Millisecond,
	}
}

// ValidateConfig is used to validate a sane configuration
func ValidateConfig(config *Config) error {
	if config.ProtocolVersion < ProtocolVersionMin ||
		config.ProtocolVersion > ProtocolVersionMax {
		return fmt.Errorf("Protocol version %d must be >= %d and <= %d",
			config.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax)
	}
	if config.ProtocolVersion > 1 && len(config.LocalID) == 0 {
		return fmt.Errorf("LocalID cannot be empty")
	}
	if config.HeartbeatTimeout < 5*time.Millisecond {
		return fmt.Errorf("Heartbeat timeout is too low")
	}
	if config.ElectionTimeout < 5*time.Millisecond {
		return fmt.Errorf("Election timeout is too low")
	}
	if config.CommitTimeout < time.Millisecond {
		return fmt.Errorf("Commit timeout is too low")
	}
	if config.MaxAppendEntries <= 0 {
		return fmt.Errorf("MaxAppendEntries must be positive")
	}
	if config.MaxAppendEntries > 1024 {
		return fmt.Errorf("MaxAppendEntries is too large")
	}
	if config.SnapshotInterval < 5*time.Millisecond {
		return fmt.Errorf("Snapshot interval is too low")
	}
	if config.LeaderLeaseTimeout < 5*time.Millisecond {
		return fmt.Errorf("Leader lease timeout is too low")
	}
	if config.LeaderLeaseTimeout > config.HeartbeatTimeout {
		return fmt.Errorf("Leader lease timeout cannot be larger than heartbeat timeout")
	}
	if config.ElectionTimeout < config.HeartbeatTimeout {
		return fmt.Errorf("Election timeout must be equal or greater than Heartbeat Timeout")
	}
	return nil
}
<commit_msg>Clarifies protocol version 0 comment.<commit_after>package raft

import (
	"fmt"
	"io"
	"log"
	"time"
)

// These are the versions of the protocol (which includes RPC messages as
// well as Raft-specific log entries) that this server can _understand_. Use
// the ProtocolVersion member of the Config object to control the version of
// the protocol to use when _speaking_ to other servers. This is not currently
// written into snapshots so they are unversioned. Note that depending on the
// protocol version being spoken, some otherwise understood RPC messages may be
// refused. See isVersionCompatible for details of this logic.
//
// There are notes about the upgrade path in the description of the versions
// below. If you are starting a fresh cluster then there's no reason not to
// jump right to the latest protocol version. If you need to interoperate with
// older, unversioned Raft servers you'll need to drive the cluster through the
// different versions in order. This may best be done by moving forward across a
// series of releases of your application.
//
// Version History
//
// 0: Unversioned, original protocol, used to interoperate with unversioned
//    Raft servers. Under this version all configuration changes are propagated
//    using the deprecated RemovePeerDeprecated Raft log entry. This means that
//    server IDs are always set to be the same as the server addresses (since
//    the old log entry type cannot transmit an ID), and only AddPeer/RemovePeer
//    APIs are supported. Newer Raft servers running at protocol version 0 can
//    understand the new LogConfiguration Raft log entry but will never generate
//    one so they can remain compatible with unversioned Raft servers in the
//    cluster.
// 1: Transitional protocol used when migrating an existing cluster to the new
//    server ID system. Server IDs are still set to be the same as server
//    addresses, but all configuration changes are propagated using the new
//    LogConfiguration Raft log entry type, which can carry full ID information.
//    This version supports the old AddPeer/RemovePeer APIs as well as the new
//    ID-based AddVoter/RemoveServer APIs which should be used when adding
//    version 2 servers to the cluster later. Note that non-voting roles are not
//    yet supported. This version sheds all interoperability with unversioned
//    Raft servers, but can interoperate with newer Raft servers running with
//    protocol version 0 since they can understand the new LogConfiguration Raft
//    log entry, and this version can still understand their RemovePeerDeprecated
//    Raft log entries. We need this protocol version as an intermediate step
//    between 0 and 2 so that servers will propagate the ID information that
//    will come from newly-added (or -cycled) servers using protocol version 2,
//    but since they are still using their address-based IDs from the previous
//    step they will still be able to track commitments and their own voting
//    status properly. If we skipped this step, servers would be started with
//    their new IDs, but they wouldn't see themselves in the old address-based
//    configuration, so none of the servers would think they had a vote.
// 2: Protocol adding full support for server IDs and new ID-based server APIs
//    (AddVoter, AddNonvoter, etc.), old AddPeer/RemovePeer APIs are no longer
//    supported. Version 1 servers should be swapped out by removing them from
//    the cluster one-by-one and re-adding them with updated configuration for
//    this protocol version, along with their server ID. The remove/add cycle
//    is required to populate their server ID. Note that removing must be done
//    by ID, which will be the old server's address.
const (
	ProtocolVersionMin = 0
	ProtocolVersionMax = 2
)

// Config provides any necessary configuration to
// the Raft server
type Config struct {
	// ProtocolVersion allows a Raft server to inter-operate with older
	// Raft servers running an older version of the code. This is used to
	// version the wire protocol as well as Raft-specific log entries that
	// the server uses when _speaking_ to other servers. There is currently
	// no auto-negotiation of versions so all servers must be manually
	// configured with compatible versions. See ProtocolVersionMin and
	// ProtocolVersionMax for the versions of the protocol that this server
	// can _understand_.
	ProtocolVersion int

	// HeartbeatTimeout specifies the time in follower state without
	// a leader before we attempt an election.
	HeartbeatTimeout time.Duration

	// ElectionTimeout specifies the time in candidate state without
	// a leader before we attempt an election.
	ElectionTimeout time.Duration

	// CommitTimeout controls the time without an Apply() operation
	// before we heartbeat to ensure a timely commit. Due to random
	// staggering, may be delayed as much as 2x this value.
	CommitTimeout time.Duration

	// MaxAppendEntries controls the maximum number of append entries
	// to send at once. We want to strike a balance between efficiency
	// and avoiding waste if the follower is going to reject because of
	// an inconsistent log.
	MaxAppendEntries int

	// If we are a member of a cluster, and RemovePeer is invoked for the
	// local node, then we forget all peers and transition into the follower state.
	// If ShutdownOnRemove is is set, we additional shutdown Raft. Otherwise,
	// we can become a leader of a cluster containing only this node.
	ShutdownOnRemove bool

	// TrailingLogs controls how many logs we leave after a snapshot. This is
	// used so that we can quickly replay logs on a follower instead of being
	// forced to send an entire snapshot.
	TrailingLogs uint64

	// SnapshotInterval controls how often we check if we should perform a snapshot.
	// We randomly stagger between this value and 2x this value to avoid the entire
	// cluster from performing a snapshot at once.
	SnapshotInterval time.Duration

	// SnapshotThreshold controls how many outstanding logs there must be before
	// we perform a snapshot. This is to prevent excessive snapshots when we can
	// just replay a small set of logs.
	SnapshotThreshold uint64

	// LeaderLeaseTimeout is used to control how long the "lease" lasts
	// for being the leader without being able to contact a quorum
	// of nodes. If we reach this interval without contact, we will
	// step down as leader.
	LeaderLeaseTimeout time.Duration

	// StartAsLeader forces Raft to start in the leader state. This should
	// never be used except for testing purposes, as it can cause a split-brain.
	StartAsLeader bool

	// The unique ID for this server across all time. For protocol version
	// > 1 this is required, and for older protocols it will be populated with
	// the server's network address.
	LocalID ServerID

	// NotifyCh is used to provide a channel that will be notified of leadership
	// changes. Raft will block writing to this channel, so it should either be
	// buffered or aggressively consumed.
	NotifyCh chan<- bool

	// LogOutput is used as a sink for logs, unless Logger is specified.
	// Defaults to os.Stderr.
	LogOutput io.Writer

	// Logger is a user-provided logger. If nil, a logger writing to LogOutput
	// is used.
	Logger *log.Logger
}

// DefaultConfig returns a Config with usable defaults.
func DefaultConfig() *Config {
	return &Config{
		ProtocolVersion:    ProtocolVersionMax,
		HeartbeatTimeout:   1000 * time.Millisecond,
		ElectionTimeout:    1000 * time.Millisecond,
		CommitTimeout:      50 * time.Millisecond,
		MaxAppendEntries:   64,
		ShutdownOnRemove:   true,
		TrailingLogs:       10240,
		SnapshotInterval:   120 * time.Second,
		SnapshotThreshold:  8192,
		LeaderLeaseTimeout: 500 * time.Millisecond,
	}
}

// ValidateConfig is used to validate a sane configuration
func ValidateConfig(config *Config) error {
	if config.ProtocolVersion < ProtocolVersionMin ||
		config.ProtocolVersion > ProtocolVersionMax {
		return fmt.Errorf("Protocol version %d must be >= %d and <= %d",
			config.ProtocolVersion, ProtocolVersionMin, ProtocolVersionMax)
	}
	if config.ProtocolVersion > 1 && len(config.LocalID) == 0 {
		return fmt.Errorf("LocalID cannot be empty")
	}
	if config.HeartbeatTimeout < 5*time.Millisecond {
		return fmt.Errorf("Heartbeat timeout is too low")
	}
	if config.ElectionTimeout < 5*time.Millisecond {
		return fmt.Errorf("Election timeout is too low")
	}
	if config.CommitTimeout < time.Millisecond {
		return fmt.Errorf("Commit timeout is too low")
	}
	if config.MaxAppendEntries <= 0 {
		return fmt.Errorf("MaxAppendEntries must be positive")
	}
	if config.MaxAppendEntries > 1024 {
		return fmt.Errorf("MaxAppendEntries is too large")
	}
	if config.SnapshotInterval < 5*time.Millisecond {
		return fmt.Errorf("Snapshot interval is too low")
	}
	if config.LeaderLeaseTimeout < 5*time.Millisecond {
		return fmt.Errorf("Leader lease timeout is too low")
	}
	if config.LeaderLeaseTimeout > config.HeartbeatTimeout {
		return fmt.Errorf("Leader lease timeout cannot be larger than heartbeat timeout")
	}
	if config.ElectionTimeout < config.HeartbeatTimeout {
		return fmt.Errorf("Election timeout must be equal or greater than Heartbeat Timeout")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"errors"
	"fmt"
	"path"
	"strconv"
	"strings"

	"gopkg.in/yaml.v2"
)

// Moby is the type of a Moby config file
type Moby struct {
	Kernel struct {
		Image   string
		Cmdline string
	}
	Init   string
	System []MobyImage
	Daemon []MobyImage
	Files  []struct {
		Path     string
		Contents string
	}
	Outputs []struct {
		Format string
	}
}

// MobyImage is the type of an image config, based on Compose
type MobyImage struct {
	Name         string
	Image        string
	Capabilities []string
	Binds        []string
	OomScoreAdj  int64 `yaml:"oom_score_adj"`
	Command      []string
	NetworkMode  string `yaml:"network_mode"`
	Pid          string
	Ipc          string
}

const riddler = "mobylinux/riddler:7d4545d8b8ac2700971a83f12a3446a76db28c14@sha256:11b7310df6482fc38aa52b419c2ef1065d7b9207c633d47554e13aa99f6c0b72"

// NewConfig parses a config file
func NewConfig(config []byte) (*Moby, error) {
	m := Moby{}

	err := yaml.Unmarshal(config, &m)
	if err != nil {
		return &m, err
	}

	return &m, nil
}

// ConfigToRun converts a config to a series of arguments for docker run
func ConfigToRun(order int, path string, image *MobyImage) []string {
	// riddler arguments
	so := fmt.Sprintf("%03d", order)
	args := []string{"-v", "/var/run/docker.sock:/var/run/docker.sock", riddler, image.Image, "/containers/" + path + "/" + so + "-" + image.Name}
	// docker arguments
	args = append(args, "--cap-drop", "all")
	for _, cap := range image.Capabilities {
		if strings.ToUpper(cap)[0:4] == "CAP_" {
			cap = cap[4:]
		}
		args = append(args, "--cap-add", cap)
	}
	if image.OomScoreAdj != 0 {
		args = append(args, "--oom-score-adj", strconv.FormatInt(image.OomScoreAdj, 10))
	}
	if image.NetworkMode != "" {
		// TODO only "host" supported
		args = append(args, "--net="+image.NetworkMode)
	}
	if image.Pid != "" {
		// TODO only "host" supported
		args = append(args, "--pid="+image.Pid)
	}
	if image.Ipc != "" {
		// TODO only "host" supported
		args = append(args, "--ipc="+image.Pid)
	}
	for _, bind := range image.Binds {
		args = append(args, "-v", bind)
	}
	// image
	args = append(args, image.Image)
	// command
	args = append(args, image.Command...)

	return args
}

func filesystem(m *Moby) (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	defer tw.Close()

	for _, f := range m.Files {
		if f.Path == "" {
			return buf, errors.New("Did not specify path for file")
		}
		if f.Contents == "" {
			return buf, errors.New("Contents of file not specified")
		}
		// we need all the leading directories
		parts := strings.Split(path.Dir(f.Path), "/")
		root := ""
		for _, p := range parts {
			if p == "." || p == "/" {
				continue
			}
			if root == "" {
				root = p
			} else {
				root = root + "/" + p
			}
			hdr := &tar.Header{
				Name:     root,
				Typeflag: tar.TypeDir,
				Mode:     0700,
			}
			err := tw.WriteHeader(hdr)
			if err != nil {
				return buf, err
			}
		}
		hdr := &tar.Header{
			Name: f.Path,
			Mode: 0600,
			Size: int64(len(f.Contents)),
		}
		err := tw.WriteHeader(hdr)
		if err != nil {
			return buf, err
		}
		_, err = tw.Write([]byte(f.Contents))
		if err != nil {
			return buf, err
		}
	}
	return buf, nil
}
<commit_msg>Fix typo/copy paste for IPC namespaces<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"errors"
	"fmt"
	"path"
	"strconv"
	"strings"

	"gopkg.in/yaml.v2"
)

// Moby is the type of a Moby config file
type Moby struct {
	Kernel struct {
		Image   string
		Cmdline string
	}
	Init   string
	System []MobyImage
	Daemon []MobyImage
	Files  []struct {
		Path     string
		Contents string
	}
	Outputs []struct {
		Format string
	}
}

// MobyImage is the type of an image config, based on Compose
type MobyImage struct {
	Name         string
	Image        string
	Capabilities []string
	Binds        []string
	OomScoreAdj  int64 `yaml:"oom_score_adj"`
	Command      []string
	NetworkMode  string `yaml:"network_mode"`
	Pid          string
	Ipc          string
}

const riddler = "mobylinux/riddler:7d4545d8b8ac2700971a83f12a3446a76db28c14@sha256:11b7310df6482fc38aa52b419c2ef1065d7b9207c633d47554e13aa99f6c0b72"

// NewConfig parses a config file
func NewConfig(config []byte) (*Moby, error) {
	m := Moby{}

	err := yaml.Unmarshal(config, &m)
	if err != nil {
		return &m, err
	}

	return &m, nil
}

// ConfigToRun converts a config to a series of arguments for docker run
func ConfigToRun(order int, path string, image *MobyImage) []string {
	// riddler arguments
	so := fmt.Sprintf("%03d", order)
	args := []string{"-v", "/var/run/docker.sock:/var/run/docker.sock", riddler, image.Image, "/containers/" + path + "/" + so + "-" + image.Name}
	// docker arguments
	args = append(args, "--cap-drop", "all")
	for _, cap := range image.Capabilities {
		if strings.ToUpper(cap)[0:4] == "CAP_" {
			cap = cap[4:]
		}
		args = append(args, "--cap-add", cap)
	}
	if image.OomScoreAdj != 0 {
		args = append(args, "--oom-score-adj", strconv.FormatInt(image.OomScoreAdj, 10))
	}
	if image.NetworkMode != "" {
		// TODO only "host" supported
		args = append(args, "--net="+image.NetworkMode)
	}
	if image.Pid != "" {
		// TODO only "host" supported
		args = append(args, "--pid="+image.Pid)
	}
	if image.Ipc != "" {
		// TODO only "host" supported
		args = append(args, "--ipc="+image.Ipc)
	}
	for _, bind := range image.Binds {
		args = append(args, "-v", bind)
	}
	// image
	args = append(args, image.Image)
	// command
	args = append(args, image.Command...)

	return args
}

func filesystem(m *Moby) (*bytes.Buffer, error) {
	buf := new(bytes.Buffer)
	tw := tar.NewWriter(buf)
	defer tw.Close()

	for _, f := range m.Files {
		if f.Path == "" {
			return buf, errors.New("Did not specify path for file")
		}
		if f.Contents == "" {
			return buf, errors.New("Contents of file not specified")
		}
		// we need all the leading directories
		parts := strings.Split(path.Dir(f.Path), "/")
		root := ""
		for _, p := range parts {
			if p == "." || p == "/" {
				continue
			}
			if root == "" {
				root = p
			} else {
				root = root + "/" + p
			}
			hdr := &tar.Header{
				Name:     root,
				Typeflag: tar.TypeDir,
				Mode:     0700,
			}
			err := tw.WriteHeader(hdr)
			if err != nil {
				return buf, err
			}
		}
		hdr := &tar.Header{
			Name: f.Path,
			Mode: 0600,
			Size: int64(len(f.Contents)),
		}
		err := tw.WriteHeader(hdr)
		if err != nil {
			return buf, err
		}
		_, err = tw.Write([]byte(f.Contents))
		if err != nil {
			return buf, err
		}
	}
	return buf, nil
}
<|endoftext|>


<commit_before>package parser

import (
	"github.com/eliothedeman/common-practice/core"
	"github.com/eliothedeman/go-mxl"
)

var (
	PITCH_TO_INT = map[string]int{
		"a": 0,
		"b": 2,
		"c": 3,
		"d": 5,
		"e": 7,
		"f": 8,
		"g": 10,
		"A": 0,
		"B": 2,
		"C": 3,
		"D": 5,
		"E": 7,
		"F": 8,
		"G": 10,
	}
)

// toCommon converts a parsed mxl doc to a common practice score
type toCommon struct {
	Doc *mxl.MXLDoc
}

// Do converts an mxl piece to a core.Piece
func (t *toCommon) Do(m *Manager) {
	p := &core.Piece{}
	p.Title = t.Doc.Title
	p.Parts = make([]core.Part, len(t.Doc.Parts))
	for i := 0; i < len(p.Parts); i++ {

	}

}

// parseParts mxl.Part -> core.Part
func parseParts(p []mxl.Part) []core.Part {
	n := make([]core.Part, len(p))
	for i := range p {
		n[i].Name = p[i].Id
		n[i].Movements[i] = parseMovements(p[i].Measures)
	}
}

// parseMeasurs mxl.Measure -> core.Measure
func parseMeasures(m []mxl.Measure) []core.Measure {
	n := make([]core.Measure, len(m))
	for i := range m {
		n[i].Key.Fifths = m[i].Atters.Key.Fifths
		n[i].Notes = parseNotes(m[i].Notes)
	}
}

// parsenotes mxl.Note -> core.Note
func parseNotes(notes []mxl.Note) []core.Note {
	n := make([]core.Note, len(notes))
	for i := range m {
		// if the note is actually a rest, don't bother figuring out the pitch
		if notes[i].Rest.Local == "rest" {
			n[i].Rest = true
		} else {
			// a notes pitch is calculated by xml pitch(0-11) * octave + acidental (-2...2)
			n[i].Pitch = core.Pitch(parsePitch(notes[i].Pitch) + notes[i].Accidental)
		}

	}
}

// parsePitch mxl.Pitch -> core.Pitch
func parsePitch(m mxl.Pitch) core.Pitch {
	return core.Pitch(m.Octave*11 + PITCH_TO_INT[m.Step])
}
<commit_msg>added final parsing of parts<commit_after>package parser

import (
	"github.com/eliothedeman/common-practice/core"
	"github.com/eliothedeman/go-mxl"
)

var (
	PITCH_TO_INT = map[string]int{
		"a": 0,
		"b": 2,
		"c": 3,
		"d": 5,
		"e": 7,
		"f": 8,
		"g": 10,
		"A": 0,
		"B": 2,
		"C": 3,
		"D": 5,
		"E": 7,
		"F": 8,
		"G": 10,
	}
)

// toCommon converts a parsed mxl doc to a common practice score
type toCommon struct {
	Doc *mxl.MXLDoc
}

// Do converts an mxl piece to a core.Piece
func (t *toCommon) Do(m *Manager) {
	p := &core.Piece{}
	p.Title = t.Doc.Title
	p.Parts = make([]core.Part, len(t.Doc.Parts))
	for i := 0; i < len(p.Parts); i++ {
		p.Parts[i] = parseParts(t.Doc.Parts[i])
	}

}

// parseParts mxl.Part -> core.Part
func parseParts(p []mxl.Part) []core.Part {
	n := make([]core.Part, len(p))
	for i := range p {
		n[i].Name = p[i].Id
		n[i].Movements[i] = parseMovements(p[i].Measures)
	}
}

// parseMeasurs mxl.Measure -> core.Measure
func parseMeasures(m []mxl.Measure) []core.Measure {
	n := make([]core.Measure, len(m))
	for i := range m {
		n[i].Key.Fifths = m[i].Atters.Key.Fifths
		n[i].Notes = parseNotes(m[i].Notes)
	}
}

// parsenotes mxl.Note -> core.Note
func parseNotes(notes []mxl.Note) []core.Note {
	n := make([]core.Note, len(notes))
	for i := range m {
		// if the note is actually a rest, don't bother figuring out the pitch
		if notes[i].Rest.Local == "rest" {
			n[i].Rest = true
		} else {
			// a notes pitch is calculated by mxl.Pitch(0-11) * octave + acidental(-2...2)
			n[i].Pitch = core.Pitch(parsePitch(notes[i].Pitch) + notes[i].Accidental)
		}

	}
}

// parsePitch mxl.Pitch -> core.Pitch
func parsePitch(m mxl.Pitch) core.Pitch {
	return core.Pitch(m.Octave*11 + PITCH_TO_INT[m.Step])
}
<|endoftext|>


<commit_before>package queue

import (
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	peer "github.com/jbenet/go-ipfs/peer"
)

// ChanQueue makes any PeerQueue synchronizable through channels.
type ChanQueue struct {
	Queue   PeerQueue
	EnqChan chan<- peer.Peer
	DeqChan <-chan peer.Peer
}

// NewChanQueue creates a ChanQueue by wrapping pq.
func NewChanQueue(ctx context.Context, pq PeerQueue) *ChanQueue {
	cq := &ChanQueue{Queue: pq}
	cq.process(ctx)
	return cq
}

func (cq *ChanQueue) process(ctx context.Context) {

	// construct the channels here to be able to use them bidirectionally
	enqChan := make(chan peer.Peer, 10)
	deqChan := make(chan peer.Peer, 10)

	cq.EnqChan = enqChan
	cq.DeqChan = deqChan

	go func() {
		defer close(deqChan)

		var next peer.Peer
		var item peer.Peer
		var more bool

		for {
			if cq.Queue.Len() == 0 {
				select {
				case next, more = <-enqChan:
					if !more {
						return
					}

				case <-ctx.Done():
					return
				}

			} else {
				next = cq.Queue.Dequeue()
			}

			select {
			case item, more = <-enqChan:
				if !more {
					return
				}

				cq.Queue.Enqueue(item)
				cq.Queue.Enqueue(next)
				next = nil

			case deqChan <- next:
				next = nil

			case <-ctx.Done():
				return
			}
		}

	}()
}
<commit_msg>peer/queue/sync: remove buffering<commit_after>package queue

import (
	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	peer "github.com/jbenet/go-ipfs/peer"
)

// ChanQueue makes any PeerQueue synchronizable through channels.
type ChanQueue struct {
	Queue   PeerQueue
	EnqChan chan<- peer.Peer
	DeqChan <-chan peer.Peer
}

// NewChanQueue creates a ChanQueue by wrapping pq.
func NewChanQueue(ctx context.Context, pq PeerQueue) *ChanQueue {
	cq := &ChanQueue{Queue: pq}
	cq.process(ctx)
	return cq
}

func (cq *ChanQueue) process(ctx context.Context) {

	// construct the channels here to be able to use them bidirectionally
	enqChan := make(chan peer.Peer)
	deqChan := make(chan peer.Peer)

	cq.EnqChan = enqChan
	cq.DeqChan = deqChan

	go func() {
		defer close(deqChan)

		var next peer.Peer
		var item peer.Peer
		var more bool

		for {
			if cq.Queue.Len() == 0 {
				select {
				case next, more = <-enqChan:
					if !more {
						return
					}

				case <-ctx.Done():
					return
				}

			} else {
				next = cq.Queue.Dequeue()
			}

			select {
			case item, more = <-enqChan:
				if !more {
					return
				}

				cq.Queue.Enqueue(item)
				cq.Queue.Enqueue(next)
				next = nil

			case deqChan <- next:
				next = nil

			case <-ctx.Done():
				return
			}
		}

	}()
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tiff

// A tiff image file contains one or more images. The metadata
// of each image is contained in an Image File Directory (IFD),
// which contains entries of 12 bytes each and is described
// on page 14-16 of the specification. An IFD entry consists of
//
//  - a tag, which describes the signification of the entry,
//  - the data type and length of the entry,
//  - the data itself or a pointer to it if it is more than 4 bytes.
//
// The presence of a length means that each IFD is effectively an array.

const (
	ifdLen = 12 // Length of an IFD entry in bytes.
)

// imageMode represents the mode of the image.
type imageMode int

const (
	mBilevel imageMode = iota
	mPaletted
	mGray
	mGrayInvert
	mRGB
	mRGBA
	mNRGBA
)
<commit_msg>add ifd8Len const<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tiff

// A tiff image file contains one or more images. The metadata
// of each image is contained in an Image File Directory (IFD),
// which contains entries of 12 bytes each and is described
// on page 14-16 of the specification. An IFD entry consists of
//
//  - a tag, which describes the signification of the entry,
//  - the data type and length of the entry,
//  - the data itself or a pointer to it if it is more than 4 bytes.
//
// The presence of a length means that each IFD is effectively an array.

const (
	ifdLen  = 12 // Length of an IFD entry in bytes.
	ifd8Len = 20
)

// imageMode represents the mode of the image.
type imageMode int

const (
	mBilevel imageMode = iota
	mPaletted
	mGray
	mGrayInvert
	mRGB
	mRGBA
	mNRGBA
)
<|endoftext|>


<commit_before>package dotweb

//Global define
const(
	// Version current version
	Version = "1.5.7.5"
)

//Log define
const (
	LogTarget_Default     = "dotweb_default"
	LogTarget_HttpRequest = "dotweb_request"
	LogTarget_HttpServer  = "dotweb_server"
	LogTarget_RequestTimeout = "dotweb_req_timeout"

	LogLevel_Debug = "debug"
	LogLevel_Info  = "info"
	LogLevel_Warn  = "warn"
	LogLevel_Error = "error"
)

//Http define
const (
	CharsetUTF8       = "charset=utf-8"
	DefaultServerName = "dotweb"
)

// MIME types
const (
	MIMEApplicationJSON                  = "application/json"
	MIMEApplicationJSONCharsetUTF8       = MIMEApplicationJSON + "; " + CharsetUTF8
	MIMEApplicationJavaScript            = "application/javascript"
	MIMEApplicationJavaScriptCharsetUTF8 = MIMEApplicationJavaScript + "; " + CharsetUTF8
	MIMEApplicationXML                   = "application/xml"
	MIMEApplicationXMLCharsetUTF8        = MIMEApplicationXML + "; " + CharsetUTF8
	MIMEApplicationForm                  = "application/x-www-form-urlencoded"
	MIMEApplicationProtobuf              = "application/protobuf"
	MIMEApplicationMsgpack               = "application/msgpack"
	MIMETextHTML                         = "text/html"
	MIMETextHTMLCharsetUTF8              = MIMETextHTML + "; " + CharsetUTF8
	MIMETextPlain                        = "text/plain"
	MIMETextPlainCharsetUTF8             = MIMETextPlain + "; " + CharsetUTF8
	MIMEMultipartForm                    = "multipart/form-data"
	MIMEOctetStream                      = "application/octet-stream"
)

// Headers
const (
	HeaderAcceptEncoding                = "Accept-Encoding"
	HeaderAllow                         = "Allow"
	HeaderAuthorization                 = "Authorization"
	HeaderContentDisposition            = "Content-Disposition"
	HeaderContentEncoding               = "Content-Encoding"
	HeaderContentLength                 = "Content-Length"
	HeaderContentType                   = "Content-Type"
	HeaderCookie                        = "Cookie"
	HeaderSetCookie                     = "Set-Cookie"
	HeaderIfModifiedSince               = "If-Modified-Since"
	HeaderLastModified                  = "Last-Modified"
	HeaderLocation                      = "Location"
	HeaderUpgrade                       = "Upgrade"
	HeaderVary                          = "Vary"
	HeaderWWWAuthenticate               = "WWW-Authenticate"
	HeaderXRequestedWith                = "X-Requested-With"
	HeaderXForwardedProto               = "X-Forwarded-Proto"
	HeaderXHTTPMethodOverride           = "X-HTTP-Method-Override"
	HeaderXForwardedFor                 = "X-Forwarded-For"
	HeaderXRealIP                       = "X-Real-IP"
	HeaderServer                        = "Server"
	HeaderOrigin                        = "Origin"
	HeaderAccessControlRequestMethod    = "Access-Control-Request-Method"
	HeaderAccessControlRequestHeaders   = "Access-Control-Request-Headers"
	HeaderAccessControlAllowOrigin      = "Access-Control-Allow-Origin"
	HeaderAccessControlAllowMethods     = "Access-Control-Allow-Methods"
	HeaderAccessControlAllowHeaders     = "Access-Control-Allow-Headers"
	HeaderAccessControlAllowCredentials = "Access-Control-Allow-Credentials"
	HeaderAccessControlExposeHeaders    = "Access-Control-Expose-Headers"
	HeaderAccessControlMaxAge           = "Access-Control-Max-Age"
	HeaderP3P                           = "P3P"
	HeaderCacheControl                  = "Cache-control"

	// Security
	HeaderStrictTransportSecurity = "Strict-Transport-Security"
	HeaderXContentTypeOptions     = "X-Content-Type-Options"
	HeaderXXSSProtection          = "X-XSS-Protection"
	HeaderXFrameOptions           = "X-Frame-Options"
	HeaderContentSecurityPolicy   = "Content-Security-Policy"
	HeaderXCSRFToken              = "X-CSRF-Token"
)
<commit_msg>#### Version 1.5.7.6 * New Feature: Add Renderer.RegisterTemplateFunc, used to register template func in renderer * Detail:   - now inner support inner func like unescaped   - you can view example on example/render * Example:   ``` golang   app.HttpServer.Renderer().RegisterTemplateFunc("echo", func(x string) interface{}{   		return "echo:" + x   	})   ``` * 2018-09-07<commit_after>package dotweb

//Global define
const(
	// Version current version
	Version = "1.5.7.6"
)

//Log define
const (
	LogTarget_Default     = "dotweb_default"
	LogTarget_HttpRequest = "dotweb_request"
	LogTarget_HttpServer  = "dotweb_server"
	LogTarget_RequestTimeout = "dotweb_req_timeout"

	LogLevel_Debug = "debug"
	LogLevel_Info  = "info"
	LogLevel_Warn  = "warn"
	LogLevel_Error = "error"
)

//Http define
const (
	CharsetUTF8       = "charset=utf-8"
	DefaultServerName = "dotweb"
)

// MIME types
const (
	MIMEApplicationJSON                  = "application/json"
	MIMEApplicationJSONCharsetUTF8       = MIMEApplicationJSON + "; " + CharsetUTF8
	MIMEApplicationJavaScript            = "application/javascript"
	MIMEApplicationJavaScriptCharsetUTF8 = MIMEApplicationJavaScript + "; " + CharsetUTF8
	MIMEApplicationXML                   = "application/xml"
	MIMEApplicationXMLCharsetUTF8        = MIMEApplicationXML + "; " + CharsetUTF8
	MIMEApplicationForm                  = "application/x-www-form-urlencoded"
	MIMEApplicationProtobuf              = "application/protobuf"
	MIMEApplicationMsgpack               = "application/msgpack"
	MIMETextHTML                         = "text/html"
	MIMETextHTMLCharsetUTF8              = MIMETextHTML + "; " + CharsetUTF8
	MIMETextPlain                        = "text/plain"
	MIMETextPlainCharsetUTF8             = MIMETextPlain + "; " + CharsetUTF8
	MIMEMultipartForm                    = "multipart/form-data"
	MIMEOctetStream                      = "application/octet-stream"
)

// Headers
const (
	HeaderAcceptEncoding                = "Accept-Encoding"
	HeaderAllow                         = "Allow"
	HeaderAuthorization                 = "Authorization"
	HeaderContentDisposition            = "Content-Disposition"
	HeaderContentEncoding               = "Content-Encoding"
	HeaderContentLength                 = "Content-Length"
	HeaderContentType                   = "Content-Type"
	HeaderCookie                        = "Cookie"
	HeaderSetCookie                     = "Set-Cookie"
	HeaderIfModifiedSince               = "If-Modified-Since"
	HeaderLastModified                  = "Last-Modified"
	HeaderLocation                      = "Location"
	HeaderUpgrade                       = "Upgrade"
	HeaderVary                          = "Vary"
	HeaderWWWAuthenticate               = "WWW-Authenticate"
	HeaderXRequestedWith                = "X-Requested-With"
	HeaderXForwardedProto               = "X-Forwarded-Proto"
	HeaderXHTTPMethodOverride           = "X-HTTP-Method-Override"
	HeaderXForwardedFor                 = "X-Forwarded-For"
	HeaderXRealIP                       = "X-Real-IP"
	HeaderServer                        = "Server"
	HeaderOrigin                        = "Origin"
	HeaderAccessControlRequestMethod    = "Access-Control-Request-Method"
	HeaderAccessControlRequestHeaders   = "Access-Control-Request-Headers"
	HeaderAccessControlAllowOrigin      = "Access-Control-Allow-Origin"
	HeaderAccessControlAllowMethods     = "Access-Control-Allow-Methods"
	HeaderAccessControlAllowHeaders     = "Access-Control-Allow-Headers"
	HeaderAccessControlAllowCredentials = "Access-Control-Allow-Credentials"
	HeaderAccessControlExposeHeaders    = "Access-Control-Expose-Headers"
	HeaderAccessControlMaxAge           = "Access-Control-Max-Age"
	HeaderP3P                           = "P3P"
	HeaderCacheControl                  = "Cache-control"

	// Security
	HeaderStrictTransportSecurity = "Strict-Transport-Security"
	HeaderXContentTypeOptions     = "X-Content-Type-Options"
	HeaderXXSSProtection          = "X-XSS-Protection"
	HeaderXFrameOptions           = "X-Frame-Options"
	HeaderContentSecurityPolicy   = "Content-Security-Policy"
	HeaderXCSRFToken              = "X-CSRF-Token"
)
<|endoftext|>


<commit_before>package beaverguide

import (
    "net/url"
    "strings"
    "sync"
    "github.com/PuerkitoBio/goquery"
)

// Course collection
type Courses []Course

// Course structure
type Course struct {
    Code string
    Title string
    URL string
    Department string
    Students int
    Class int
    Value float32
    Teachers []string // TODO: "Additional Teachers"
    Availability string // TODO: Store in array (compulsory, optional, others)
    Prerequisites string
    Content string // TODO: Handle lists?
    Teaching string // TODO: Store in array (MT, LT, ST)
    Formative string
    Readings string
    Assessments string
    Program int
}

// Build course collection
func GetCourses(program_type string) (program_courses Courses) {
    var wg sync.WaitGroup
    program := GetDocument(PROGRAMMES_URL[program_type])
    courses := program.Find("table tr td p a")

    wg.Add(courses.Length())

    courses.Each(func(i int, s *goquery.Selection) {
        go func(s *goquery.Selection) {
            defer wg.Done()
            course_item := strings.Split(s.Text(), " ")
            course_item_url, _ := s.Attr("href")
            parsed_url, _ := url.Parse(PROGRAMMES_URL[program_type])
            parsed_relative, _ := url.Parse(course_item_url)

            course_object := new(Course)
            course_object.Code = course_item[0]
            course_object.Title = course_item[1]
            course_object.URL = parsed_url.ResolveReference(parsed_relative).String()

            if program_type == "Graduate" {
                course_object.Program = 1
            } else if program_type == "Research" {
                course_object.Program = 2
            } else {
                course_object.Program = 0
            }

            program_courses = append(program_courses, *course_object)
        }(s)
    })
    wg.Wait()
    return
}

// Sort interface
func (slice Courses) Len() int {
    return len(slice)
}

func (slice Courses) Less(i, j int) bool {
    return slice[i].Code < slice[j].Code;
}

func (slice Courses) Swap(i, j int) {
    slice[i], slice[j] = slice[j], slice[i]
}<commit_msg>Added GetAllCourses() (self-explanatory); changing pointers; and fixed course title<commit_after>package beaverguide

import (
    "net/url"
    "strings"
    "sort"
    "sync"
    "github.com/PuerkitoBio/goquery"
)

// Course collection
type Courses []*Course

// Course structure
type Course struct {
    Code string
    Title string
    URL string
    Department string
    Students int
    Class int
    Value float32
    Teachers []string // TODO: "Additional Teachers"
    Availability string // TODO: Store in array (compulsory, optional, others)
    Prerequisites string
    Content string // TODO: Handle lists?
    Teaching string // TODO: Store in array (MT, LT, ST)
    Formative string
    Readings string
    Assessments string
    Program int
}

// Build course collection for a specified program
func GetCourses(program_type string) (program_courses Courses) {
    var wg sync.WaitGroup
    program := GetDocument(PROGRAMMES_URL[program_type])
    courses := program.Find("table tr td p a")

    wg.Add(courses.Length())

    courses.Each(func(i int, s *goquery.Selection) {
        go func(s *goquery.Selection) {
            defer wg.Done()
            course_item := strings.SplitN(s.Text(), " ", 2)
            course_item_url, _ := s.Attr("href")
            parsed_url, _ := url.Parse(PROGRAMMES_URL[program_type])
            parsed_relative, _ := url.Parse(course_item_url)

            course_object := new(Course)
            course_object.Code = course_item[0]
            course_object.Title = course_item[1]
            course_object.URL = parsed_url.ResolveReference(parsed_relative).String()

            if program_type == "Graduate" {
                course_object.Program = 1
            } else if program_type == "Research" {
                course_object.Program = 2
            } else {
                course_object.Program = 0
            }

            program_courses = append(program_courses, course_object)
        }(s)
    })
    wg.Wait()
    return
}

// Build course collection for all programmes
func GetAllCourses() (all_courses Courses) {
    var wg sync.WaitGroup
    wg.Add(len(PROGRAMMES_URL))

    for program, _ := range PROGRAMMES_URL {
        go func(program string) {
            defer wg.Done()
            all_courses = append(all_courses, GetCourses(program)...)
        }(program)
    }

    wg.Wait()
    sort.Sort(all_courses)
    return
}

// Sort interface
func (slice Courses) Len() int {
    return len(slice)
}

func (slice Courses) Less(i, j int) bool {
    return slice[i].Code < slice[j].Code;
}

func (slice Courses) Swap(i, j int) {
    slice[i], slice[j] = slice[j], slice[i]
}<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
	"fmt"
	"net/http"
	"strings"

	gorp "gopkg.in/gorp.v2"

	"github.com/sapcc/limes/pkg/collector"
	"github.com/sapcc/limes/pkg/db"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/reports"
	"github.com/sapcc/limes/pkg/util"
)

//ListDomains handles GET /v1/domains.
func (p *v1Provider) ListDomains(w http.ResponseWriter, r *http.Request) {
	if !p.CheckToken(r).Require(w, "domain:list") {
		return
	}

	domains, err := reports.GetDomains(p.Cluster, nil, db.DB, reports.ReadFilter(r))
	if ReturnError(w, err) {
		return
	}

	ReturnJSON(w, 200, map[string]interface{}{"domains": domains})
}

//GetDomain handles GET /v1/domains/:domain_id.
func (p *v1Provider) GetDomain(w http.ResponseWriter, r *http.Request) {
	if !p.CheckToken(r).Require(w, "domain:show") {
		return
	}
	dbDomain := p.FindDomainFromRequest(w, r)
	if dbDomain == nil {
		return
	}

	domains, err := reports.GetDomains(p.Cluster, &dbDomain.ID, db.DB, reports.ReadFilter(r))
	if ReturnError(w, err) {
		return
	}
	if len(domains) == 0 {
		http.Error(w, "no resource data found for domain", 500)
		return
	}

	ReturnJSON(w, 200, map[string]interface{}{"domain": domains[0]})
}

//DiscoverDomains handles POST /v1/domains/discover.
func (p *v1Provider) DiscoverDomains(w http.ResponseWriter, r *http.Request) {
	if !p.CheckToken(r).Require(w, "domain:discover") {
		return
	}

	newDomainUUIDs, err := collector.ScanDomains(p.Cluster, collector.ScanDomainsOpts{})
	if ReturnError(w, err) {
		return
	}

	if len(newDomainUUIDs) == 0 {
		w.WriteHeader(204)
		return
	}
	ReturnJSON(w, 202, map[string]interface{}{"new_domains": util.IDsToJSON(newDomainUUIDs)})
}

//PutDomain handles PUT /v1/domains/:domain_id.
func (p *v1Provider) PutDomain(w http.ResponseWriter, r *http.Request) {
	token := p.CheckToken(r)
	canRaise := token.Check("domain:raise")
	canLower := token.Check("domain:lower")
	if !canRaise && !canLower {
		token.Require(w, "domain:raise") //produce standard Unauthorized response
		return
	}

	dbDomain := p.FindDomainFromRequest(w, r)
	if dbDomain == nil {
		return
	}

	//parse request body
	var parseTarget struct {
		Domain struct {
			Services ServiceQuotas `json:"services"`
		} `json:"domain"`
	}
	parseTarget.Domain.Services = make(ServiceQuotas)
	if !RequireJSON(w, r, &parseTarget) {
		return
	}
	serviceQuotas := parseTarget.Domain.Services

	//start a transaction for the quota updates
	tx, err := db.DB.Begin()
	if ReturnError(w, err) {
		return
	}
	defer db.RollbackUnlessCommitted(tx)

	//gather a report on the domain's quotas to decide whether a quota update is legal
	domainReports, err := reports.GetDomains(p.Cluster, &dbDomain.ID, db.DB, reports.Filter{})
	if ReturnError(w, err) {
		return
	}
	if len(domainReports) == 0 {
		http.Error(w, "no resource data found for domain", 500)
		return
	}
	domainReport := domainReports[0]

	//check all services for resources to update
	var services []db.DomainService
	_, err = tx.Select(&services,
		`SELECT * FROM domain_services WHERE domain_id = $1 ORDER BY type`, dbDomain.ID)
	if ReturnError(w, err) {
		return
	}
	var resourcesToUpdate []db.DomainResource
	var resourcesToUpdateAsUntyped []interface{}
	var errors []string

	var auditTrail util.AuditTrail
	for _, srv := range services {
		resourceQuotas, exists := serviceQuotas[srv.Type]
		if !exists {
			continue
		}

		//check all resources
		var resources []db.DomainResource
		_, err = tx.Select(&resources,
			`SELECT * FROM domain_resources WHERE service_id = $1 ORDER BY name`, srv.ID)
		if ReturnError(w, err) {
			return
		}
		for _, res := range resources {
			newQuota, exists := resourceQuotas[res.Name]
			if !exists {
				continue
			}
			if res.Quota == newQuota {
				continue //nothing to do
			}

			err := checkDomainQuotaUpdate(srv, res, domainReport, newQuota, canRaise, canLower)
			if err != nil {
				errors = append(errors, err.Error())
				continue
			}

			//take a copy of the loop variable (it will be updated by the loop, so if
			//we didn't take a copy manually, the resourcesToUpdateAsUntyped list
			//would contain only identical pointers)
			res := res
			auditTrail.Add("set quota %s.%s = %d -> %d for domain %s by user %s (%s)",
				srv.Type, res.Name, res.Quota, newQuota,
				dbDomain.UUID, token.UserUUID, token.UserName,
			)
			res.Quota = newQuota
			resourcesToUpdate = append(resourcesToUpdate, res)
			resourcesToUpdateAsUntyped = append(resourcesToUpdateAsUntyped, &res)
		}
	}

	//if not legal, report errors to the user
	if len(errors) > 0 {
		http.Error(w, strings.Join(errors, "\n"), 422)
		return
	}

	//update the DB with the new quotas
	onlyQuota := func(c *gorp.ColumnMap) bool {
		return c.ColumnName == "quota"
	}
	_, err = tx.UpdateColumns(onlyQuota, resourcesToUpdateAsUntyped...)
	if ReturnError(w, err) {
		return
	}
	err = tx.Commit()
	if ReturnError(w, err) {
		return
	}
	auditTrail.Commit()

	//otherwise, report success
	domains, err := reports.GetDomains(p.Cluster, &dbDomain.ID, db.DB, reports.ReadFilter(r))
	if ReturnError(w, err) {
		return
	}
	if len(domains) == 0 {
		http.Error(w, "no resource data found for domain", 500)
		return
	}

	ReturnJSON(w, 200, map[string]interface{}{"domain": domains[0]})
}

func checkDomainQuotaUpdate(srv db.DomainService, res db.DomainResource, domain *reports.Domain, newQuota uint64, canRaise, canLower bool) error {
	//if quota is being raised, only permission is required (overprovisioning of
	//domain quota over the cluster capacity is explicitly allowed because
	//capacity measurements are usually to be taken with a grain of salt)
	if res.Quota < newQuota {
		if canRaise {
			return nil
		}
		return fmt.Errorf("cannot change %s/%s quota: user is not allowed to raise quotas in this project", srv.Type, res.Name)
	}

	//if quota is being lowered, permission is required and the domain quota may
	//not be less than the sum of quotas that the domain gives out to projects
	if !canLower {
		return fmt.Errorf("cannot change %s/%s quota: user is not allowed to lower quotas in this project", srv.Type, res.Name)
	}
	projectsQuota := uint64(0)
	var unit limes.Unit
	if domainService, exists := domain.Services[srv.Type]; exists {
		if domainResource, exists := domainService.Resources[res.Name]; exists {
			projectsQuota = domainResource.ProjectsQuota
			unit = domainResource.Unit
		}
	}
	if newQuota < projectsQuota {
		return fmt.Errorf(
			"cannot change %s/%s quota: domain quota may not be smaller than sum of project quotas in that domain (%s)",
			srv.Type, res.Name,
			unit.Format(projectsQuota),
		)
	}

	return nil
}
<commit_msg>api.PutDomain: create missing domain_resources<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
	"fmt"
	"net/http"
	"strings"

	gorp "gopkg.in/gorp.v2"

	"github.com/sapcc/limes/pkg/collector"
	"github.com/sapcc/limes/pkg/db"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/reports"
	"github.com/sapcc/limes/pkg/util"
)

//ListDomains handles GET /v1/domains.
func (p *v1Provider) ListDomains(w http.ResponseWriter, r *http.Request) {
	if !p.CheckToken(r).Require(w, "domain:list") {
		return
	}

	domains, err := reports.GetDomains(p.Cluster, nil, db.DB, reports.ReadFilter(r))
	if ReturnError(w, err) {
		return
	}

	ReturnJSON(w, 200, map[string]interface{}{"domains": domains})
}

//GetDomain handles GET /v1/domains/:domain_id.
func (p *v1Provider) GetDomain(w http.ResponseWriter, r *http.Request) {
	if !p.CheckToken(r).Require(w, "domain:show") {
		return
	}
	dbDomain := p.FindDomainFromRequest(w, r)
	if dbDomain == nil {
		return
	}

	domains, err := reports.GetDomains(p.Cluster, &dbDomain.ID, db.DB, reports.ReadFilter(r))
	if ReturnError(w, err) {
		return
	}
	if len(domains) == 0 {
		http.Error(w, "no resource data found for domain", 500)
		return
	}

	ReturnJSON(w, 200, map[string]interface{}{"domain": domains[0]})
}

//DiscoverDomains handles POST /v1/domains/discover.
func (p *v1Provider) DiscoverDomains(w http.ResponseWriter, r *http.Request) {
	if !p.CheckToken(r).Require(w, "domain:discover") {
		return
	}

	newDomainUUIDs, err := collector.ScanDomains(p.Cluster, collector.ScanDomainsOpts{})
	if ReturnError(w, err) {
		return
	}

	if len(newDomainUUIDs) == 0 {
		w.WriteHeader(204)
		return
	}
	ReturnJSON(w, 202, map[string]interface{}{"new_domains": util.IDsToJSON(newDomainUUIDs)})
}

//PutDomain handles PUT /v1/domains/:domain_id.
func (p *v1Provider) PutDomain(w http.ResponseWriter, r *http.Request) {
	token := p.CheckToken(r)
	canRaise := token.Check("domain:raise")
	canLower := token.Check("domain:lower")
	if !canRaise && !canLower {
		token.Require(w, "domain:raise") //produce standard Unauthorized response
		return
	}

	dbDomain := p.FindDomainFromRequest(w, r)
	if dbDomain == nil {
		return
	}

	//parse request body
	var parseTarget struct {
		Domain struct {
			Services ServiceQuotas `json:"services"`
		} `json:"domain"`
	}
	parseTarget.Domain.Services = make(ServiceQuotas)
	if !RequireJSON(w, r, &parseTarget) {
		return
	}
	serviceQuotas := parseTarget.Domain.Services

	//start a transaction for the quota updates
	tx, err := db.DB.Begin()
	if ReturnError(w, err) {
		return
	}
	defer db.RollbackUnlessCommitted(tx)

	//gather a report on the domain's quotas to decide whether a quota update is legal
	domainReports, err := reports.GetDomains(p.Cluster, &dbDomain.ID, db.DB, reports.Filter{})
	if ReturnError(w, err) {
		return
	}
	if len(domainReports) == 0 {
		http.Error(w, "no resource data found for domain", 500)
		return
	}
	domainReport := domainReports[0]

	//check all services for resources to update
	var services []db.DomainService
	_, err = tx.Select(&services,
		`SELECT * FROM domain_services WHERE domain_id = $1 ORDER BY type`, dbDomain.ID)
	if ReturnError(w, err) {
		return
	}
	var resourcesToUpdate []db.DomainResource
	var resourcesToUpdateAsUntyped []interface{}
	var errors []string

	var auditTrail util.AuditTrail
	for _, srv := range services {
		resourceQuotas, exists := serviceQuotas[srv.Type]
		if !exists {
			continue
		}
		isExistingResource := make(map[string]bool)

		//check all existing resources
		var resources []db.DomainResource
		_, err = tx.Select(&resources,
			`SELECT * FROM domain_resources WHERE service_id = $1 ORDER BY name`, srv.ID)
		if ReturnError(w, err) {
			return
		}
		for _, res := range resources {
			isExistingResource[res.Name] = true
			newQuota, exists := resourceQuotas[res.Name]
			if !exists {
				continue
			}
			if res.Quota == newQuota {
				continue //nothing to do
			}

			err := checkDomainQuotaUpdate(srv, res, domainReport, newQuota, canRaise, canLower)
			if err != nil {
				errors = append(errors, err.Error())
				continue
			}

			//take a copy of the loop variable (it will be updated by the loop, so if
			//we didn't take a copy manually, the resourcesToUpdateAsUntyped list
			//would contain only identical pointers)
			res := res
			auditTrail.Add("set quota %s.%s = %d -> %d for domain %s by user %s (%s)",
				srv.Type, res.Name, res.Quota, newQuota,
				dbDomain.UUID, token.UserUUID, token.UserName,
			)
			res.Quota = newQuota
			resourcesToUpdate = append(resourcesToUpdate, res)
			resourcesToUpdateAsUntyped = append(resourcesToUpdateAsUntyped, &res)
		}

		//check resources that need to be created
		for resourceName, newQuota := range resourceQuotas {
			if isExistingResource[resourceName] {
				continue
			}
			if !p.Cluster.HasResource(srv.Type, resourceName) {
				errors = append(errors,
					fmt.Sprintf("cannot set %s/%s quota: no such resource", srv.Type, resourceName),
				)
				continue
			}

			res := db.DomainResource{
				ServiceID: srv.ID,
				Name:      resourceName,
				Quota:     0, //start with 0 because the previous value is taken into account by checkDomainQuotaUpdate
			}
			err := checkDomainQuotaUpdate(srv, res, domainReport, newQuota, canRaise, canLower)
			if err != nil {
				errors = append(errors, err.Error())
				continue
			}

			auditTrail.Add("set quota %s.%s = %d -> %d for domain %s by user %s (%s)",
				srv.Type, res.Name, res.Quota, newQuota,
				dbDomain.UUID, token.UserUUID, token.UserName,
			)
			res.Quota = newQuota
			err = tx.Insert(&res)
			if ReturnError(w, err) {
				return
			}
		}
	}

	//if not legal, report errors to the user
	if len(errors) > 0 {
		http.Error(w, strings.Join(errors, "\n"), 422)
		return
	}

	//update the DB with the new quotas
	onlyQuota := func(c *gorp.ColumnMap) bool {
		return c.ColumnName == "quota"
	}
	_, err = tx.UpdateColumns(onlyQuota, resourcesToUpdateAsUntyped...)
	if ReturnError(w, err) {
		return
	}
	err = tx.Commit()
	if ReturnError(w, err) {
		return
	}
	auditTrail.Commit()

	//otherwise, report success
	domains, err := reports.GetDomains(p.Cluster, &dbDomain.ID, db.DB, reports.ReadFilter(r))
	if ReturnError(w, err) {
		return
	}
	if len(domains) == 0 {
		http.Error(w, "no resource data found for domain", 500)
		return
	}

	ReturnJSON(w, 200, map[string]interface{}{"domain": domains[0]})
}

func checkDomainQuotaUpdate(srv db.DomainService, res db.DomainResource, domain *reports.Domain, newQuota uint64, canRaise, canLower bool) error {
	//if quota is being raised, only permission is required (overprovisioning of
	//domain quota over the cluster capacity is explicitly allowed because
	//capacity measurements are usually to be taken with a grain of salt)
	if res.Quota < newQuota {
		if canRaise {
			return nil
		}
		return fmt.Errorf("cannot change %s/%s quota: user is not allowed to raise quotas in this project", srv.Type, res.Name)
	}

	//if quota is being lowered, permission is required and the domain quota may
	//not be less than the sum of quotas that the domain gives out to projects
	if !canLower {
		return fmt.Errorf("cannot change %s/%s quota: user is not allowed to lower quotas in this project", srv.Type, res.Name)
	}
	projectsQuota := uint64(0)
	var unit limes.Unit
	if domainService, exists := domain.Services[srv.Type]; exists {
		if domainResource, exists := domainService.Resources[res.Name]; exists {
			projectsQuota = domainResource.ProjectsQuota
			unit = domainResource.Unit
		}
	}
	if newQuota < projectsQuota {
		return fmt.Errorf(
			"cannot change %s/%s quota: domain quota may not be smaller than sum of project quotas in that domain (%s)",
			srv.Type, res.Name,
			unit.Format(projectsQuota),
		)
	}

	return nil
}
<|endoftext|>