<commit_before>package config

import (
	"os"
	"log"
	"os/signal"
	"syscall"
	"bufio"
	"bytes"
	"reflect"
	"errors"
	"strings"
	"strconv"
	"fmt"
)

type Config struct {
	filename string
	data interface{}
	queue []reflect.Value
	current reflect.Value
	searchVal bool
	searchKey bool
	inBlock int
	inInclude int
	canSkip bool
	skip bool
	bkMulti int
	mapKey reflect.Value
}

func New(filename string) *Config {
	return &Config{filename:filename}
}

func (this *Config) Unmarshal(v interface{}) error{
	rev := reflect.ValueOf(v)
	if rev.Kind() != reflect.Ptr {
		err := errors.New("non-pointer passed to Unmarshal")
		return err
	}
	this.data = v
	this.current = rev.Elem()
	this.inSearchKey()
	this.inBlock = 0
	this.inInclude = 0
	return this.parse()
}

func (this *Config) Reload() error {
	return this.parse()
}

func (this *Config) watch() {
	l := log.New(os.Stderr, "", 0)
	sighup := make(chan os.Signal, 1)
	signal.Notify(sighup, syscall.SIGHUP)
	for {
		<-sighup
		l.Println("Caught SIGHUP, reloading config...")
		this.parse()
	}
}

func (this *Config) parse() error {
	var err error
	var s bytes.Buffer
	if _, err = os.Stat(this.filename);os.IsNotExist(err) {
		return err
	}

	var fp *os.File
	fp, err = os.Open(this.filename)
	if err != nil {
		return err
	}
	defer fp.Close()

	reader := bufio.NewReader(fp)
	var b byte
	for err == nil {
		b, err = reader.ReadByte()
		if err == bufio.ErrBufferFull {
			return nil
		}

		if this.canSkip && b == '#' {
			reader.ReadLine()
			continue
		}
		if this.canSkip && b == '/' {
			if this.skip {
				reader.ReadLine()
				this.skip = false
				continue
			}
			this.skip = true
			continue
		}
		if this.searchKey {
			if b == ' ' || b == '\r' || b == '\n' || b == '\t' {
				if s.Len() > 0 {
					this.inSearchVal()
					if strings.Compare(s.String(), "include") == 0 {
						s.Reset()
						this.inInclude++
						if this.inInclude > 100 {
							return errors.New("too many include, exceeds 100 limit!")
						}
						continue
					}
					this.getElement(s.String())
					s.Reset()
				}
				continue
			}
		}

		if b == '{' {
			// fixed { be close to key like server{
			if this.searchKey && s.Len() > 0 {
				this.getElement(s.String())
				s.Reset()
				this.inSearchVal()
			}

			this.inBlock++
			//	这里说明了可能是切片或者map
			if this.searchVal && s.Len() > 0 && this.current.Kind() == reflect.Map {
				this.bkMulti++
				if this.current.IsNil() {
					this.current.Set(reflect.MakeMap(this.current.Type()))
				}
				var v reflect.Value
				v = reflect.New(this.current.Type().Key())
				this.pushElement(v)
				err := this.set(s.String())
				if err != nil {
					return err
				}
				this.mapKey = this.current
				this.popElement()
				val := reflect.New(this.current.Type().Elem())
				this.pushElement(val)
			}

			if this.current.Kind() == reflect.Slice {
				this.bkMulti++
				n := this.current.Len()
				this.current.Set(reflect.Append(this.current, reflect.Zero(this.current.Type().Elem())))
				this.pushElement(this.current.Index(n))
			}
			this.inSearchKey()
			s.Reset()
			continue
		}

		if b == '}' && this.inBlock > 0 {
			if this.bkMulti > 0{
				this.bkMulti--
				val := this.current
				this.popElement()
				if this.current.Kind() == reflect.Map {
					this.current.SetMapIndex(this.mapKey, val)
				}
			}
			this.inBlock--
			this.popElement()
			this.inSearchKey()
			continue
		}

		if this.searchVal {
			if b == ';' {
				//	这里是要处理数据到this.current
				this.inSearchKey()

				if this.inInclude > 0 {
					this.filename = strings.TrimSpace(s.String())
					s.Reset()
					this.inInclude--
					if err := this.parse(); err != nil {
						return err
					}
					continue
				}

				err := this.set(s.String())
				cs := s.String()
				log.Println(cs)
				if err != nil {
					return err
				}

				s.Reset()
				this.popElement()
				continue
			}

		}

		s.WriteByte(b)
	}

	return nil
}

func (this *Config) set(s string) error {
	s = strings.TrimSpace(s)
	if this.current.Kind() == reflect.Ptr {
		if this.current.IsNil() {
			this.current.Set(reflect.New(this.current.Type().Elem()))
		}
		this.current = this.current.Elem()
	}

	switch this.current.Kind() {
	case reflect.String:
		this.current.SetString(s)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		itmp,err := strconv.ParseInt(s, 10, this.current.Type().Bits())
		if  err != nil {
			return err
		}
		this.current.SetInt(itmp)
	case  reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		itmp,err := strconv.ParseUint(s, 10, this.current.Type().Bits())
		if  err != nil {
			return err
		}
		this.current.SetUint(itmp)
	case reflect.Float32, reflect.Float64:
		ftmp, err := strconv.ParseFloat(s, this.current.Type().Bits())
		if err != nil {
			return err
		}
		this.current.SetFloat(ftmp)
	case reflect.Bool:
		if s == "yes" {
			this.current.SetBool(true)
		}else if s == "no" {
			this.current.SetBool(false)
		}else{
			btmp, err := strconv.ParseBool(s)
			if err != nil {
				return err
			}
			this.current.SetBool(btmp)
		}
	case reflect.Slice:
		sf := strings.Fields(s)
		for _,sv := range sf {
			n := this.current.Len()
			this.current.Set(reflect.Append(this.current, reflect.Zero(this.current.Type().Elem())))
			this.pushElement(this.current.Index(n))
			this.set(sv)
			this.popElement()
		}
	case reflect.Map:
		if this.current.IsNil() {
			this.current.Set(reflect.MakeMap(this.current.Type()))
		}

		sf := strings.Fields(s)
		if len(sf) != 2 {
			return errors.New("Invalid Config")
		}
		var v reflect.Value
		v=reflect.New(this.current.Type().Key())
		this.pushElement(v)
		this.set(sf[0])
		key := this.current
		this.popElement()
		v = reflect.New(this.current.Type().Elem())
		this.pushElement(v)
		this.set(sf[1])
		val := this.current
		this.popElement()

		this.current.SetMapIndex(key, val)
	default:
		return errors.New(fmt.Sprintf("Invalid Type:%s", this.current.Kind()))
	}
	return nil
}

func (this *Config) inSearchKey() {
	this.searchVal = false
	this.searchKey = true
	this.canSkip = true
}

func (this *Config) inSearchVal() {
	this.searchKey = false
	this.searchVal = true
	this.canSkip = false
}

func (this *Config) getElement(s string){
	s = strings.TrimSpace(s)
	if this.current.Kind() == reflect.Ptr {
		this.current = this.current.Elem()
	}
	this.queue = append(this.queue, this.current)
	this.current = this.current.FieldByName(strings.Title(s))
}

func (this *Config) pushElement(v reflect.Value) {
	this.queue = append(this.queue, this.current)
	this.current = v
}

func (this *Config) popElement() {
	this.current = this.queue[len(this.queue) - 1]
	this.queue = this.queue[:len(this.queue) - 1]
}
<commit_msg>opt:include support regular<commit_after>package config

import (
	"os"
	"log"
	"os/signal"
	"syscall"
	"bufio"
	"bytes"
	"reflect"
	"errors"
	"strings"
	"strconv"
	"fmt"
	"path/filepath"
)

type Config struct {
	filename string
	data interface{}
	queue []reflect.Value
	current reflect.Value
	searchVal bool
	searchKey bool
	inBlock int
	inInclude int
	canSkip bool
	skip bool
	bkMulti int
	mapKey reflect.Value
}

func New(filename string) *Config {
	return &Config{filename:filename}
}

func (this *Config) Unmarshal(v interface{}) error{
	rev := reflect.ValueOf(v)
	if rev.Kind() != reflect.Ptr {
		err := errors.New("non-pointer passed to Unmarshal")
		return err
	}
	this.data = v
	this.current = rev.Elem()
	this.inSearchKey()
	this.inBlock = 0
	this.inInclude = 0
	return this.parse()
}

func (this *Config) Reload() error {
	return this.parse()
}

func (this *Config) watch() {
	l := log.New(os.Stderr, "", 0)
	sighup := make(chan os.Signal, 1)
	signal.Notify(sighup, syscall.SIGHUP)
	for {
		<-sighup
		l.Println("Caught SIGHUP, reloading config...")
		this.parse()
	}
}

func (this *Config) parse() error {
	var err error
	var s bytes.Buffer
	if _, err = os.Stat(this.filename);os.IsNotExist(err) {
		return err
	}

	var fp *os.File
	fp, err = os.Open(this.filename)
	if err != nil {
		return err
	}
	defer fp.Close()

	reader := bufio.NewReader(fp)
	var b byte
	for err == nil {
		b, err = reader.ReadByte()
		if err == bufio.ErrBufferFull {
			return nil
		}

		if this.canSkip && b == '#' {
			reader.ReadLine()
			continue
		}
		if this.canSkip && b == '/' {
			if this.skip {
				reader.ReadLine()
				this.skip = false
				continue
			}
			this.skip = true
			continue
		}
		if this.searchKey {
			if b == ' ' || b == '\r' || b == '\n' || b == '\t' {
				if s.Len() > 0 {
					this.inSearchVal()
					if strings.Compare(s.String(), "include") == 0 {
						s.Reset()
						this.inInclude++
						if this.inInclude > 100 {
							return errors.New("too many include, exceeds 100 limit!")
						}
						continue
					}
					this.getElement(s.String())
					s.Reset()
				}
				continue
			}
		}

		if b == '{' {
			// fixed { be close to key like server{
			if this.searchKey && s.Len() > 0 {
				this.getElement(s.String())
				s.Reset()
				this.inSearchVal()
			}

			this.inBlock++
			//	这里说明了可能是切片或者map
			if this.searchVal && s.Len() > 0 && this.current.Kind() == reflect.Map {
				this.bkMulti++
				if this.current.IsNil() {
					this.current.Set(reflect.MakeMap(this.current.Type()))
				}
				var v reflect.Value
				v = reflect.New(this.current.Type().Key())
				this.pushElement(v)
				err := this.set(s.String())
				if err != nil {
					return err
				}
				this.mapKey = this.current
				this.popElement()
				val := reflect.New(this.current.Type().Elem())
				this.pushElement(val)
			}

			if this.current.Kind() == reflect.Slice {
				this.bkMulti++
				n := this.current.Len()
				this.current.Set(reflect.Append(this.current, reflect.Zero(this.current.Type().Elem())))
				this.pushElement(this.current.Index(n))
			}
			this.inSearchKey()
			s.Reset()
			continue
		}

		if b == '}' && this.inBlock > 0 {
			if this.bkMulti > 0{
				this.bkMulti--
				val := this.current
				this.popElement()
				if this.current.Kind() == reflect.Map {
					this.current.SetMapIndex(this.mapKey, val)
				}
			}
			this.inBlock--
			this.popElement()
			this.inSearchKey()
			continue
		}

		if this.searchVal {
			if b == ';' {
				//	这里是要处理数据到this.current
				this.inSearchKey()

				if this.inInclude > 0 {
					this.filename = strings.TrimSpace(s.String())
					s.Reset()
					this.inInclude--
					files, err := filepath.Glob(this.filename)
					if err != nil{
						return err
					}
					for _,file := range files {
						this.filename = file
						if err := this.parse(); err != nil {
							return err
						}
					}
					continue
				}

				err := this.set(s.String())
				cs := s.String()
				log.Println(cs)
				if err != nil {
					return err
				}

				s.Reset()
				this.popElement()
				continue
			}

		}

		s.WriteByte(b)
	}

	return nil
}

func (this *Config) set(s string) error {
	s = strings.TrimSpace(s)
	if this.current.Kind() == reflect.Ptr {
		if this.current.IsNil() {
			this.current.Set(reflect.New(this.current.Type().Elem()))
		}
		this.current = this.current.Elem()
	}

	switch this.current.Kind() {
	case reflect.String:
		this.current.SetString(s)
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		itmp,err := strconv.ParseInt(s, 10, this.current.Type().Bits())
		if  err != nil {
			return err
		}
		this.current.SetInt(itmp)
	case  reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		itmp,err := strconv.ParseUint(s, 10, this.current.Type().Bits())
		if  err != nil {
			return err
		}
		this.current.SetUint(itmp)
	case reflect.Float32, reflect.Float64:
		ftmp, err := strconv.ParseFloat(s, this.current.Type().Bits())
		if err != nil {
			return err
		}
		this.current.SetFloat(ftmp)
	case reflect.Bool:
		if s == "yes" {
			this.current.SetBool(true)
		}else if s == "no" {
			this.current.SetBool(false)
		}else{
			btmp, err := strconv.ParseBool(s)
			if err != nil {
				return err
			}
			this.current.SetBool(btmp)
		}
	case reflect.Slice:
		sf := strings.Fields(s)
		for _,sv := range sf {
			n := this.current.Len()
			this.current.Set(reflect.Append(this.current, reflect.Zero(this.current.Type().Elem())))
			this.pushElement(this.current.Index(n))
			this.set(sv)
			this.popElement()
		}
	case reflect.Map:
		if this.current.IsNil() {
			this.current.Set(reflect.MakeMap(this.current.Type()))
		}

		sf := strings.Fields(s)
		if len(sf) != 2 {
			return errors.New("Invalid Config")
		}
		var v reflect.Value
		v=reflect.New(this.current.Type().Key())
		this.pushElement(v)
		this.set(sf[0])
		key := this.current
		this.popElement()
		v = reflect.New(this.current.Type().Elem())
		this.pushElement(v)
		this.set(sf[1])
		val := this.current
		this.popElement()

		this.current.SetMapIndex(key, val)
	default:
		return errors.New(fmt.Sprintf("Invalid Type:%s", this.current.Kind()))
	}
	return nil
}

func (this *Config) inSearchKey() {
	this.searchVal = false
	this.searchKey = true
	this.canSkip = true
}

func (this *Config) inSearchVal() {
	this.searchKey = false
	this.searchVal = true
	this.canSkip = false
}

func (this *Config) getElement(s string){
	s = strings.TrimSpace(s)
	if this.current.Kind() == reflect.Ptr {
		this.current = this.current.Elem()
	}
	this.queue = append(this.queue, this.current)
	this.current = this.current.FieldByName(strings.Title(s))
}

func (this *Config) pushElement(v reflect.Value) {
	this.queue = append(this.queue, this.current)
	this.current = v
}

func (this *Config) popElement() {
	this.current = this.queue[len(this.queue) - 1]
	this.queue = this.queue[:len(this.queue) - 1]
}
<|endoftext|>


<commit_before>// Package go.config provides support for loading config options from
// a file, leveraging configuration defined using the `flag' package.
//
// Config files should consist of lines of the form
//
//   <key> = <value>
//
// or
//
//   # this is a comment
//
// Configuration options are loaded by passing in a flag.FlagSet; keys
// and values are looked up, parsed, and stored using the FlagSet. For
// many applications, you can just pass in flag.CommandLine to parse
// your explose your application's default set of command-line options
// as configuration frobs.
package config

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
)

// Load configuration from a dotfile. Looks for $HOME/.basename, and,
// if it exists, opens it and calls ParseConfig. Returns silently if
// no such file exists.
func LoadConfig(flags *flag.FlagSet, basename string) error {
	path := os.ExpandEnv(fmt.Sprintf("${HOME}/.%s", basename))
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer f.Close()
	return ParseConfig(flags, f)
}

// Parse a config file, using the provided FlagSet to look up, parse,
// and store values.
func ParseConfig(flags *flag.FlagSet, f io.Reader) error {
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}

		bits := strings.SplitN(line, "=", 2)
		if len(bits) != 2 {
			return fmt.Errorf("Illegal config line: `%s'", line)
		}

		key := strings.TrimSpace(bits[0])
		value := strings.TrimSpace(bits[1])

		if flag := flags.Lookup(key); flag == nil {
			return fmt.Errorf("Unknown option `%s'", bits[0])
		}

		if err := flags.Set(key, value); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Fix golint errors.<commit_after>// Package config provides support for loading config options from a
// file, leveraging configuration defined using the `flag' package.
//
// Config files should consist of lines of the form
//
//   <key> = <value>
//
// or
//
//   # this is a comment
//
// Configuration options are loaded by passing in a flag.FlagSet; keys
// and values are looked up, parsed, and stored using the FlagSet. For
// many applications, you can just pass in flag.CommandLine to parse
// your explose your application's default set of command-line options
// as configuration frobs.
package config

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
)

// LoadConfig loads configuration from a dotfile. It looks for
// $HOME/.basename, and, if it exists, opens it and calls
// ParseConfig. Returns silently if no such file exists.
func LoadConfig(flags *flag.FlagSet, basename string) error {
	path := os.ExpandEnv(fmt.Sprintf("${HOME}/.%s", basename))
	f, err := os.Open(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer f.Close()
	return ParseConfig(flags, f)
}

// ParseConfig parses a config file, using the provided FlagSet to
// look up, parse, and store values.
func ParseConfig(flags *flag.FlagSet, f io.Reader) error {
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}

		bits := strings.SplitN(line, "=", 2)
		if len(bits) != 2 {
			return fmt.Errorf("illegal config line: `%s'", line)
		}

		key := strings.TrimSpace(bits[0])
		value := strings.TrimSpace(bits[1])

		if flag := flags.Lookup(key); flag == nil {
			return fmt.Errorf("unknown option `%s'", bits[0])
		}

		if err := flags.Set(key, value); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

func CreateVars(systemDomain, mysqlHost string) (DeploymentVars, error) {
	o := DeploymentVars{}
	o["system_domain"] = systemDomain
	o["app_domain"] = systemDomain
	o.AddSystemComponent("uaa", CfgWithSubdomainURI|CfgWithHTTPSURL)
	o["uaa_token_url"] = fmt.Sprintf("https://%s/oauth/token", o["uaa_uri"])

	o.AddSystemComponent("login", CfgWithSubdomainURI)
	o.AddSystemComponent("api", CfgWithHTTPSURL)
	o.AddSystemComponent("loggregator", CfgNone)
	o.AddSystemComponent("doppler", CfgWithSubdomainURI)
	o.AddSystemComponent("blobstore", CfgNone)
	o["blobstore_public_url"] = fmt.Sprintf("http://%s", o["blobstore_uri"])
	o["blobstore_private_url"] = "https://blobstore.service.cf.internal:4443"
	o["metron_agent_deployment_name"] = systemDomain

	o.GeneratePasswords(
		"blobstore_admin_users_password",
		"blobstore_secure_link_secret",
		"cc_bulk_api_password",
		"cc_db_encryption_key",
		"cc_internal_api_password",
		"cc_staging_upload_password",
		"cf_mysql_mysql_admin_password",
		"cf_mysql_mysql_cluster_health_password",
		"cf_mysql_mysql_galera_healthcheck_endpoint_password",
		"cf_mysql_mysql_galera_healthcheck_password",
		"cf_mysql_mysql_roadmin_password",
		"cf_mysql_mysql_seeded_databases_cc_password",
		"cf_mysql_mysql_seeded_databases_diego_password",
		"cf_mysql_mysql_seeded_databases_uaa_password",
		"nats_password",
		"router_status_password",
		"uaa_scim_users_admin_password",
		"dropsonde_shared_secret",
		"router_route_services_secret",
		"uaa_admin_client_secret",
		"uaa_clients_cc-routing_secret",
		"uaa_clients_cc-service-dashboards_secret",
		"uaa_clients_cloud_controller_username_lookup_secret",
		"uaa_clients_doppler_secret",
		"uaa_clients_gorouter_secret",
		"uaa_clients_ssh-proxy_secret",
		"uaa_clients_tcp_emitter_secret",
		"uaa_clients_tcp_router_secret",
		"uaa_login_client_secret",
		"diego_bbs_encryption_keys_passphrase",
	)

	o.GeneratePasswordArray("consul_encrypt_keys", 1)

	o["uaa_scim_users_admin_name"] = "admin"
	o["blobstore_admin_users_username"] = "blobstore-user"
	o["cc_staging_upload_user"] = "staging_user"
	o["cf_mysql_mysql_galera_healthcheck_endpoint_username"] = "galera_healthcheck"
	o["cf_mysql_mysql_seeded_databases_cc_username"] = "cloud_controller"
	o["cf_mysql_mysql_seeded_databases_diego_username"] = "diego"
	o["cf_mysql_mysql_seeded_databases_uaa_username"] = "uaa"
	o["nats_user"] = "nats"
	o["router_status_user"] = "router-status"

	o["diego_bbs_sql_db_connection_string"] = fmt.Sprintf("%s:%s@tcp(%s:3306)/diego",
		o["cf_mysql_mysql_seeded_databases_diego_username"],
		o["cf_mysql_mysql_seeded_databases_diego_password"],
		mysqlHost)

	for setName, certSet := range certSets {
		if err := o.GenerateCerts(certSet); err != nil {
			return o, fmt.Errorf("generate cert set '%s': %s", setName, err)
		}
	}

	err := o.GeneratePlainKeyPair(&PlainKeyPair{
		VarName_PrivateKey: "uaa_jwt_signing_key",
		VarName_PublicKey:  "uaa_jwt_verification_key",
	})
	if err != nil {
		return o, fmt.Errorf("generate uaa jwt key pair: %s", err)
	}

	err = o.GenerateSSHKeyAndFingerprint("diego_ssh_proxy_host_key", "diego_ssh_proxy_host_key_fingerprint")
	if err != nil {
		return o, fmt.Errorf("generate ssh creds: %s", err)
	}

	return o, nil
}

var certSets = map[string]*CertSet{
	"etcd_servers": &CertSet{
		CA: &CA{
			VarName_CA: "etcd_ca_cert",
			CommonName: "etcdCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "etcd_server_cert",
				VarName_Key:  "etcd_server_key",
				CommonName:   "etcd.service.cf.internal",
				Domains: []string{
					"*.etcd.service.cf.internal",
					"etcd.service.cf.internal",
				},
			},
			&CertKeyPair{
				VarName_Cert: "etcd_client_cert",
				VarName_Key:  "etcd_client_key",
				CommonName:   "clientName",
			},
		},
	},

	"etcd_peers": &CertSet{
		CA: &CA{
			VarName_CA: "etcd_peer_ca_cert",
			CommonName: "peerCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "etcd_peer_cert",
				VarName_Key:  "etcd_peer_key",
				CommonName:   "etcd.service.cf.internal",
				Domains: []string{
					"*.etcd.service.cf.internal",
					"etcd.service.cf.internal",
				},
			},
		},
	},

	"blobstore": &CertSet{
		CA: &CA{
			VarName_CA: "blobstore_tls_ca_cert",
			CommonName: "blobstore_ca",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "blobstore_tls_cert",
				VarName_Key:  "blobstore_tls_private_key",
				CommonName:   "blobstore.service.cf.internal",
			},
		},
	},

	"consul_agent": &CertSet{
		CA: &CA{
			VarName_CA: "consul_agent_ca_cert",
			CommonName: "consulCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "consul_agent_cert",
				VarName_Key:  "consul_agent_agent_key",
				CommonName:   "consul_agent",
			},
			&CertKeyPair{
				VarName_Cert: "consul_agent_server_cert",
				VarName_Key:  "consul_agent_server_key",
				CommonName:   "server.dc1.cf.internal",
			},
		},
	},

	"diego_bbs": &CertSet{
		CA: &CA{
			VarName_CA: "diego_bbs_ca_cert",
			CommonName: "diegoCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "diego_bbs_client_cert",
				VarName_Key:  "diego_bbs_client_key",
				CommonName:   "bbs_client",
			},
			&CertKeyPair{
				VarName_Cert: "diego_bbs_server_cert",
				VarName_Key:  "diego_bbs_server_key",
				CommonName:   "bbs.service.cf.internal",
			},
		},
	},

	"loggregator": &CertSet{
		CA: &CA{
			VarName_CA: "loggregator_tls_ca_cert",
			CommonName: "loggregatorCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "doppler_tls_server_cert",
				VarName_Key:  "doppler_tls_server_key",
				CommonName:   "doppler",
			},
			&CertKeyPair{
				VarName_Cert: "metron_metron_agent_tls_client_cert",
				VarName_Key:  "metron_metron_agent_tls_client_key",
				CommonName:   "metron_agent",
			},
			&CertKeyPair{
				VarName_Cert: "loggregator_tls_doppler_cert",
				VarName_Key:  "loggregator_tls_doppler_key",
				CommonName:   "loggregator",
			},
			&CertKeyPair{
				VarName_Cert: "loggregator_tls_tc_cert",
				VarName_Key:  "loggregator_tls_tc_key",
				CommonName:   "loggregator",
			},
		},
	},

	"uaa": &CertSet{
		CA: &CA{CommonName: "uaaCA"},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "uaa_sslCertificate",
				VarName_Key:  "uaa_sslPrivateKey",
				CommonName:   "uaa.service.cf.internal",
			},
		},
	},
}
<commit_msg>Use recommended common names for loggregator certs<commit_after>package main

import "fmt"

func CreateVars(systemDomain, mysqlHost string) (DeploymentVars, error) {
	o := DeploymentVars{}
	o["system_domain"] = systemDomain
	o["app_domain"] = systemDomain
	o.AddSystemComponent("uaa", CfgWithSubdomainURI|CfgWithHTTPSURL)
	o["uaa_token_url"] = fmt.Sprintf("https://%s/oauth/token", o["uaa_uri"])

	o.AddSystemComponent("login", CfgWithSubdomainURI)
	o.AddSystemComponent("api", CfgWithHTTPSURL)
	o.AddSystemComponent("loggregator", CfgNone)
	o.AddSystemComponent("doppler", CfgWithSubdomainURI)
	o.AddSystemComponent("blobstore", CfgNone)
	o["blobstore_public_url"] = fmt.Sprintf("http://%s", o["blobstore_uri"])
	o["blobstore_private_url"] = "https://blobstore.service.cf.internal:4443"
	o["metron_agent_deployment_name"] = systemDomain

	o.GeneratePasswords(
		"blobstore_admin_users_password",
		"blobstore_secure_link_secret",
		"cc_bulk_api_password",
		"cc_db_encryption_key",
		"cc_internal_api_password",
		"cc_staging_upload_password",
		"cf_mysql_mysql_admin_password",
		"cf_mysql_mysql_cluster_health_password",
		"cf_mysql_mysql_galera_healthcheck_endpoint_password",
		"cf_mysql_mysql_galera_healthcheck_password",
		"cf_mysql_mysql_roadmin_password",
		"cf_mysql_mysql_seeded_databases_cc_password",
		"cf_mysql_mysql_seeded_databases_diego_password",
		"cf_mysql_mysql_seeded_databases_uaa_password",
		"nats_password",
		"router_status_password",
		"uaa_scim_users_admin_password",
		"dropsonde_shared_secret",
		"router_route_services_secret",
		"uaa_admin_client_secret",
		"uaa_clients_cc-routing_secret",
		"uaa_clients_cc-service-dashboards_secret",
		"uaa_clients_cloud_controller_username_lookup_secret",
		"uaa_clients_doppler_secret",
		"uaa_clients_gorouter_secret",
		"uaa_clients_ssh-proxy_secret",
		"uaa_clients_tcp_emitter_secret",
		"uaa_clients_tcp_router_secret",
		"uaa_login_client_secret",
		"diego_bbs_encryption_keys_passphrase",
	)

	o.GeneratePasswordArray("consul_encrypt_keys", 1)

	o["uaa_scim_users_admin_name"] = "admin"
	o["blobstore_admin_users_username"] = "blobstore-user"
	o["cc_staging_upload_user"] = "staging_user"
	o["cf_mysql_mysql_galera_healthcheck_endpoint_username"] = "galera_healthcheck"
	o["cf_mysql_mysql_seeded_databases_cc_username"] = "cloud_controller"
	o["cf_mysql_mysql_seeded_databases_diego_username"] = "diego"
	o["cf_mysql_mysql_seeded_databases_uaa_username"] = "uaa"
	o["nats_user"] = "nats"
	o["router_status_user"] = "router-status"

	o["diego_bbs_sql_db_connection_string"] = fmt.Sprintf("%s:%s@tcp(%s:3306)/diego",
		o["cf_mysql_mysql_seeded_databases_diego_username"],
		o["cf_mysql_mysql_seeded_databases_diego_password"],
		mysqlHost)

	for setName, certSet := range certSets {
		if err := o.GenerateCerts(certSet); err != nil {
			return o, fmt.Errorf("generate cert set '%s': %s", setName, err)
		}
	}

	err := o.GeneratePlainKeyPair(&PlainKeyPair{
		VarName_PrivateKey: "uaa_jwt_signing_key",
		VarName_PublicKey:  "uaa_jwt_verification_key",
	})
	if err != nil {
		return o, fmt.Errorf("generate uaa jwt key pair: %s", err)
	}

	err = o.GenerateSSHKeyAndFingerprint("diego_ssh_proxy_host_key", "diego_ssh_proxy_host_key_fingerprint")
	if err != nil {
		return o, fmt.Errorf("generate ssh creds: %s", err)
	}

	return o, nil
}

var certSets = map[string]*CertSet{
	"etcd_servers": &CertSet{
		CA: &CA{
			VarName_CA: "etcd_ca_cert",
			CommonName: "etcdCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "etcd_server_cert",
				VarName_Key:  "etcd_server_key",
				CommonName:   "etcd.service.cf.internal",
				Domains: []string{
					"*.etcd.service.cf.internal",
					"etcd.service.cf.internal",
				},
			},
			&CertKeyPair{
				VarName_Cert: "etcd_client_cert",
				VarName_Key:  "etcd_client_key",
				CommonName:   "clientName",
			},
		},
	},

	"etcd_peers": &CertSet{
		CA: &CA{
			VarName_CA: "etcd_peer_ca_cert",
			CommonName: "peerCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "etcd_peer_cert",
				VarName_Key:  "etcd_peer_key",
				CommonName:   "etcd.service.cf.internal",
				Domains: []string{
					"*.etcd.service.cf.internal",
					"etcd.service.cf.internal",
				},
			},
		},
	},

	"blobstore": &CertSet{
		CA: &CA{
			VarName_CA: "blobstore_tls_ca_cert",
			CommonName: "blobstore_ca",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "blobstore_tls_cert",
				VarName_Key:  "blobstore_tls_private_key",
				CommonName:   "blobstore.service.cf.internal",
			},
		},
	},

	"consul_agent": &CertSet{
		CA: &CA{
			VarName_CA: "consul_agent_ca_cert",
			CommonName: "consulCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "consul_agent_cert",
				VarName_Key:  "consul_agent_agent_key",
				CommonName:   "consul_agent",
			},
			&CertKeyPair{
				VarName_Cert: "consul_agent_server_cert",
				VarName_Key:  "consul_agent_server_key",
				CommonName:   "server.dc1.cf.internal",
			},
		},
	},

	"diego_bbs": &CertSet{
		CA: &CA{
			VarName_CA: "diego_bbs_ca_cert",
			CommonName: "diegoCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "diego_bbs_client_cert",
				VarName_Key:  "diego_bbs_client_key",
				CommonName:   "bbs_client",
			},
			&CertKeyPair{
				VarName_Cert: "diego_bbs_server_cert",
				VarName_Key:  "diego_bbs_server_key",
				CommonName:   "bbs.service.cf.internal",
			},
		},
	},

	"loggregator": &CertSet{
		CA: &CA{
			VarName_CA: "loggregator_tls_ca_cert",
			CommonName: "loggregatorCA",
		},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "doppler_tls_server_cert",
				VarName_Key:  "doppler_tls_server_key",
				CommonName:   "doppler",
			},
			&CertKeyPair{
				VarName_Cert: "metron_metron_agent_tls_client_cert",
				VarName_Key:  "metron_metron_agent_tls_client_key",
				CommonName:   "metron_agent",
			},
			&CertKeyPair{
				VarName_Cert: "loggregator_tls_doppler_cert",
				VarName_Key:  "loggregator_tls_doppler_key",
				CommonName:   "doppler",
			},
			&CertKeyPair{
				VarName_Cert: "loggregator_tls_tc_cert",
				VarName_Key:  "loggregator_tls_tc_key",
				CommonName:   "trafficcontroller",
			},
		},
	},

	"uaa": &CertSet{
		CA: &CA{CommonName: "uaaCA"},
		CertKeyPairs: []*CertKeyPair{
			&CertKeyPair{
				VarName_Cert: "uaa_sslCertificate",
				VarName_Key:  "uaa_sslPrivateKey",
				CommonName:   "uaa.service.cf.internal",
			},
		},
	},
}
<|endoftext|>


<commit_before>package golagraphite

import (
	"io/ioutil"

	"log"

	"gopkg.in/yaml.v2"
)

type Config struct {
	Graphite_settings    Graphite_settings
	Performance_counters Performance_counters
	Sql_metrics          Sql_metrics
}

type Graphite_settings struct {
	Udp   bool
	Hosts []string
}

type Performance_counters struct {
	Metric_prefix string
	Interval      int
	Counters      []string
}

type Sql_metrics struct {
	Sql_servers []Sql_server
}

type Sql_server struct {
	Connection_string string
	Queries           []Query
}

type Query struct {
	Interval      int
	Tsql_table    string
	Tsql_row      string
	Metric_prefix string
}

func NewConfig(config_path string) (config Config) {

	source, readfile_err := ioutil.ReadFile(config_path)
	if readfile_err != nil {
		log.Fatal(readfile_err)
	}

	readyaml_err := yaml.Unmarshal(source, &config)
	if readyaml_err != nil {
		log.Fatal(readyaml_err)
	}

	return
}
<commit_msg>add config<commit_after>package golagraphite

import (
	"io/ioutil"

	"log"

	"gopkg.in/yaml.v2"
)

type Config struct {
	Graphite_settings    Graphite_settings
	Performance_counters Performance_counters
	Sql_metrics          Sql_metrics
}

type Graphite_settings struct {
	Udp   bool
	Hosts []string
}

type Performance_counters struct {
	Metric_prefix string
	Interval      int
	Counters      []string
}

type Sql_metrics struct {
	Sql_servers []Sql_server
}

type Sql_server struct {
	Connection_string string
	Queries           []Query
}

type Query struct {
	Interval      int
	Tsql_table    string
	Tsql_row      string
	Timestamp     string
	Metric_prefix string
}

func NewConfig(config_path string) (config Config) {

	source, readfile_err := ioutil.ReadFile(config_path)
	if readfile_err != nil {
		log.Fatal(readfile_err)
	}

	readyaml_err := yaml.Unmarshal(source, &config)
	if readyaml_err != nil {
		log.Fatal(readyaml_err)
	}

	return
}
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

// ConfigNamespace holds the pieces joined together to form the
// directory namespacing for config.
type ConfigNamespace struct {
	Organization string // optional additional namespace for orgs.
	Namespace    string // usually project name.
}

// The Config interface implements config.
type Config interface {
	Load(src string, dst interface{}) error
}

// UserBase and SystemBase are the prefixes for the user and system config
// paths, respectively.
const (
	UserBase   string = "~/.config/"
	SystemBase string = "/etc/"
)

// Load expands the provided src path using config.ExpandUser, then reads
// the file and unmarshals into dst using go-yaml.
func Load(src string, dst interface{}) (err error) {
	dstv := reflect.ValueOf(dst)

	if dstv.Kind() != reflect.Ptr {
		err = errors.New("config: not a pointer")
	} else if dstv.IsNil() {
		err = fmt.Errorf("nil %s", reflect.TypeOf(dstv).String())
	}
	if err != nil {
		return
	}

	path, err := ExpandUser(src)
	if err != nil {
		return
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	err = yaml.Unmarshal(data, dst)
	return
}

// ExpandUser acts kind of like os.path.expanduser in Python, except only
// supports expanding "~/" or "$HOME"
func ExpandUser(path string) (exPath string, err error) {
	usr, err := user.Current()
	if err != nil {
		return
	}

	dir := fmt.Sprintf("%s/", usr.HomeDir)

	exPath = path
	if len(path) > 2 && path[:2] == "~/" {
		exPath = strings.Replace(exPath, "~/", dir, 1)
	} else if len(path) > 5 && path[:5] == "$HOME" {
		exPath = strings.Replace(exPath, "$HOME", dir, 1)
	}

	if err != nil {
		return "", err
	}

	exPath, err = filepath.Abs(filepath.Clean(exPath))

	if err != nil {
		return "", err
	}

	return
}

// Path() returns path to config, chosen by hierarchy and checked for
// existence:
//
// 1. User config (~/.config/podhub/canary/config.yaml)
//
// 2. System config (/etc/podhub/canary/config.yaml)
func (c ConfigNamespace) Path() (path string, err error) {
	systemPath, _ := c.systemPath()
	if _, err := os.Stat(systemPath); err == nil {
		path, _ = c.systemPath()
	}

	userPath, _ := c.userPath()
	if _, err := os.Stat(userPath); err == nil {
		path, _ = c.userPath()
	}
	return
}

func (c ConfigNamespace) systemPath() (path string, err error) {
	path = filepath.Join(SystemBase, c.Organization, c.Namespace, "config.yaml")
	return
}

func (c ConfigNamespace) userPath() (path string, err error) {
	userBase, err := ExpandUser(UserBase)
	if err != nil {
		return "", err
	}

	path = filepath.Join(userBase, c.Organization, c.Namespace, "config.yaml")
	return
}

// Load() is a convenience function registered to config.ConfigNamespace to
// implement Config.Load().
func (c ConfigNamespace) Load(dst interface{}) (err error) {
	cfgPath, err := c.Path()
	if err != nil {
		return
	}

	err = Load(cfgPath, dst)
	return
}
<commit_msg>fix golint<commit_after>package config

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"reflect"
	"strings"

	"gopkg.in/yaml.v2"
)

// ConfigNamespace holds the pieces joined together to form the
// directory namespacing for config.
type ConfigNamespace struct {
	Organization string // optional additional namespace for orgs.
	Namespace    string // usually project name.
}

// The Config interface implements config.
type Config interface {
	Load(src string, dst interface{}) error
}

// UserBase and SystemBase are the prefixes for the user and system config
// paths, respectively.
const (
	UserBase   string = "~/.config/"
	SystemBase string = "/etc/"
)

// Load expands the provided src path using config.ExpandUser, then reads
// the file and unmarshals into dst using go-yaml.
func Load(src string, dst interface{}) (err error) {
	dstv := reflect.ValueOf(dst)

	if dstv.Kind() != reflect.Ptr {
		err = errors.New("config: not a pointer")
	} else if dstv.IsNil() {
		err = fmt.Errorf("nil %s", reflect.TypeOf(dstv).String())
	}
	if err != nil {
		return
	}

	path, err := ExpandUser(src)
	if err != nil {
		return
	}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return
	}

	err = yaml.Unmarshal(data, dst)
	return
}

// ExpandUser acts kind of like os.path.expanduser in Python, except only
// supports expanding "~/" or "$HOME"
func ExpandUser(path string) (exPath string, err error) {
	usr, err := user.Current()
	if err != nil {
		return
	}

	dir := fmt.Sprintf("%s/", usr.HomeDir)

	exPath = path
	if len(path) > 2 && path[:2] == "~/" {
		exPath = strings.Replace(exPath, "~/", dir, 1)
	} else if len(path) > 5 && path[:5] == "$HOME" {
		exPath = strings.Replace(exPath, "$HOME", dir, 1)
	}

	if err != nil {
		return "", err
	}

	exPath, err = filepath.Abs(filepath.Clean(exPath))

	if err != nil {
		return "", err
	}

	return
}

// Path returns path to config, chosen by hierarchy and checked for
// existence:
//
// 1. User config (~/.config/podhub/canary/config.yaml)
//
// 2. System config (/etc/podhub/canary/config.yaml)
func (c ConfigNamespace) Path() (path string, err error) {
	systemPath, _ := c.systemPath()
	if _, err := os.Stat(systemPath); err == nil {
		path, _ = c.systemPath()
	}

	userPath, _ := c.userPath()
	if _, err := os.Stat(userPath); err == nil {
		path, _ = c.userPath()
	}
	return
}

func (c ConfigNamespace) systemPath() (path string, err error) {
	path = filepath.Join(SystemBase, c.Organization, c.Namespace, "config.yaml")
	return
}

func (c ConfigNamespace) userPath() (path string, err error) {
	userBase, err := ExpandUser(UserBase)
	if err != nil {
		return "", err
	}

	path = filepath.Join(userBase, c.Organization, c.Namespace, "config.yaml")
	return
}

// Load is a convenience function registered to config.ConfigNamespace to
// implement Config.Load().
func (c ConfigNamespace) Load(dst interface{}) (err error) {
	cfgPath, err := c.Path()
	if err != nil {
		return
	}

	err = Load(cfgPath, dst)
	return
}
<|endoftext|>


<commit_before>package keyring

type Config struct {
	// AllowedBackends is a whitelist of backend providers that can be used. Nil means all available.
	AllowedBackends []BackendType

	// ServiceName is a generic service name that is used by backends that support the concept
	ServiceName string

	// MacOSKeychainNameKeychainName is the name of the macOS keychain that is used
	KeychainName string

	// KeychainTrustApplication is whether the calling application should be trusted by default by items
	KeychainTrustApplication bool

	// KeychainSynchronizable is whether the item can be synchronized to iCloud
	KeychainSynchronizable bool

	// KeychainAccessibleWhenUnlocked is whether the item is accessible when the device is locked
	KeychainAccessibleWhenUnlocked bool

	// KeychainPasswordFunc is an optional function used to prompt the user for a password
	KeychainPasswordFunc PromptFunc

	// FilePasswordFunc is a required function used to prompt the user for a password
	FilePasswordFunc PromptFunc

	// FileDir is the directory that keyring files are stored in, ~ is resolved to home dir
	FileDir string

	// KWalletAppID is the application id for KWallet
	KWalletAppID string

	// KWalletFolder is the folder for KWallet
	KWalletFolder string

	// LibSecretCollectionName is the name collection in secret-service
	LibSecretCollectionName string

	PassDir    string
	PassCmd    string
	PassPrefix string
}
<commit_msg>Add comments for pass configuration options<commit_after>package keyring

type Config struct {
	// AllowedBackends is a whitelist of backend providers that can be used. Nil means all available.
	AllowedBackends []BackendType

	// ServiceName is a generic service name that is used by backends that support the concept
	ServiceName string

	// MacOSKeychainNameKeychainName is the name of the macOS keychain that is used
	KeychainName string

	// KeychainTrustApplication is whether the calling application should be trusted by default by items
	KeychainTrustApplication bool

	// KeychainSynchronizable is whether the item can be synchronized to iCloud
	KeychainSynchronizable bool

	// KeychainAccessibleWhenUnlocked is whether the item is accessible when the device is locked
	KeychainAccessibleWhenUnlocked bool

	// KeychainPasswordFunc is an optional function used to prompt the user for a password
	KeychainPasswordFunc PromptFunc

	// FilePasswordFunc is a required function used to prompt the user for a password
	FilePasswordFunc PromptFunc

	// FileDir is the directory that keyring files are stored in, ~ is resolved to home dir
	FileDir string

	// KWalletAppID is the application id for KWallet
	KWalletAppID string

	// KWalletFolder is the folder for KWallet
	KWalletFolder string

	// LibSecretCollectionName is the name collection in secret-service
	LibSecretCollectionName string

	// PassDir is the pass password-store directory
	PassDir    string

	// PassCmd is the name of the pass executable
	PassCmd    string

	// PassPrefix is a string prefix to prepend to the item path stored in pass
	PassPrefix string
}
<|endoftext|>


<commit_before>package solidproxy

const (
	ServerVersion = "v2.1.0"
	ServerName    = "SolidProxy"
)

type ServerConfig struct {
	Verbose            bool
	InsecureSkipVerify bool
	Version            string
	Agent              string
	EnableTLS          bool
	TLSKey             string
	TLSCert            string
	Port               string
}

func NewServerConfig() *ServerConfig {
	return &ServerConfig{
		Verbose:            false,
		InsecureSkipVerify: false,
		Port:               "3129",
		EnableTLS:          false,
		Version:            ServerVersion,
	}
}

func GetServerVersion() string {
	return ServerVersion
}

func GetServerName() string {
	return ServerName
}

func GetServerFullName() string {
	return ServerName + "-" + ServerVersion
}
<commit_msg>bumped version<commit_after>package solidproxy

const (
	ServerVersion = "v2.1.1"
	ServerName    = "SolidProxy"
)

type ServerConfig struct {
	Verbose            bool
	InsecureSkipVerify bool
	Version            string
	Agent              string
	EnableTLS          bool
	TLSKey             string
	TLSCert            string
	Port               string
}

func NewServerConfig() *ServerConfig {
	return &ServerConfig{
		Verbose:            false,
		InsecureSkipVerify: false,
		Port:               "3129",
		EnableTLS:          false,
		Version:            ServerVersion,
	}
}

func GetServerVersion() string {
	return ServerVersion
}

func GetServerName() string {
	return ServerName
}

func GetServerFullName() string {
	return ServerName + "-" + ServerVersion
}
<|endoftext|>


<commit_before>package main

import (
	"camlistore.org/pkg/errorutil"
	"encoding/json"
	"fmt"
	"github.com/miekg/dns"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path"
	"runtime"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"time"
)

var configLastRead = map[string]time.Time{}

func configReader(dirName string, Zones Zones) {
	for {
		configReadDir(dirName, Zones)
		time.Sleep(5 * time.Second)
	}
}

func configReadDir(dirName string, Zones Zones) error {
	dir, err := ioutil.ReadDir(dirName)
	if err != nil {
		log.Println("Could not read", dirName, ":", err)
		return err
	}

	seenFiles := map[string]bool{}

	var parse_err error

	for _, file := range dir {
		fileName := file.Name()
		if !strings.HasSuffix(strings.ToLower(fileName), ".json") {
			continue
		}

		seenFiles[fileName] = true

		if lastRead, ok := configLastRead[fileName]; !ok || file.ModTime().After(lastRead) {
			logPrintln("Updated file, going to read", fileName)
			configLastRead[fileName] = file.ModTime()
			zoneName := fileName[0:strings.LastIndex(fileName, ".")]
			//log.Println("FILE:", i, file, zoneName)
			runtime.GC()
			config, err := readZoneFile(zoneName, path.Join(dirName, fileName))
			if config == nil || err != nil {
				log.Println(err)
				parse_err = err
				continue
			}

			Zones[zoneName] = config
			dns.HandleFunc(zoneName, setupServerFunc(config))
			runtime.GC()
		}

		// TODO(ask) Disable zones not seen in two subsequent runs
	}

	return parse_err
}

func setupPgeodnsZone(Zones Zones) {
	zoneName := "pgeodns"
	Zone := new(Zone)
	Zone.Labels = make(labels)
	Zone.Origin = zoneName
	Zone.LenLabels = dns.LenLabels(Zone.Origin)
	label := new(Label)
	label.Records = make(map[uint16]Records)
	label.Weight = make(map[uint16]int)
	Zone.Labels[""] = label
	setupSOA(Zone)
	Zones[zoneName] = Zone
	dns.HandleFunc(zoneName, setupServerFunc(Zone))
}

func readZoneFile(zoneName, fileName string) (*Zone, error) {
	defer func() {
		if err := recover(); err != nil {
			log.Printf("reading %s failed: %s", zoneName, err)
			debug.PrintStack()
		}
	}()

	fh, err := os.Open(fileName)
	if err != nil {
		log.Println("Could not read ", fileName, ": ", err)
		panic(err)
	}

	Zone := new(Zone)
	Zone.Labels = make(labels)
	Zone.Origin = zoneName
	Zone.LenLabels = dns.LenLabels(Zone.Origin)
	Zone.Options.Ttl = 120
	Zone.Options.MaxHosts = 2
	Zone.Options.Contact = "support.bitnames.com"

	if err == nil {
		var objmap map[string]interface{}
		decoder := json.NewDecoder(fh)
		if err = decoder.Decode(&objmap); err != nil {
			extra := ""
			if serr, ok := err.(*json.SyntaxError); ok {
				if _, serr := fh.Seek(0, os.SEEK_SET); serr != nil {
					log.Fatalf("seek error: %v", serr)
				}
				line, col, highlight := errorutil.HighlightBytePosition(fh, serr.Offset)
				extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
					line, col, serr.Offset, highlight)
			}
			return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v",
				fh.Name(), extra, err)
		}

		if err != nil {
			panic(err)
		}
		//log.Println(objmap)

		var data map[string]interface{}

		for k, v := range objmap {
			//log.Printf("k: %s v: %#v, T: %T\n", k, v, v)

			switch k {
			case "ttl", "serial", "max_hosts", "contact":
				switch option := k; option {
				case "ttl":
					Zone.Options.Ttl = valueToInt(v)
				case "serial":
					Zone.Options.Serial = valueToInt(v)
				case "contact":
					Zone.Options.Contact = v.(string)
				case "max_hosts":
					Zone.Options.MaxHosts = valueToInt(v)
				}
				continue

			case "data":
				data = v.(map[string]interface{})
			}
		}

		setupZoneData(data, Zone)

	}

	//log.Printf("ZO T: %T %s\n", Zones["0.us"], Zones["0.us"])

	//log.Println("IP", string(Zone.Regions["0.us"].IPv4[0].ip))

	return Zone, nil
}

func setupZoneData(data map[string]interface{}, Zone *Zone) {

	recordTypes := map[string]uint16{
		"a":     dns.TypeA,
		"aaaa":  dns.TypeAAAA,
		"ns":    dns.TypeNS,
		"cname": dns.TypeCNAME,
		"mx":    dns.TypeMX,
		"alias": dns.TypeMF,
	}

	for dk, dv_inter := range data {

		dv := dv_inter.(map[string]interface{})

		//log.Printf("K %s V %s TYPE-V %T\n", dk, dv, dv)

		label := Zone.AddLabel(dk)

		if ttl, ok := dv["ttl"]; ok {
			label.Ttl = valueToInt(ttl)
		}

		if maxHosts, ok := dv["max_hosts"]; ok {
			label.MaxHosts = valueToInt(maxHosts)
		}

		for rType, dnsType := range recordTypes {

			rdata := dv[rType]

			if rdata == nil {
				//log.Printf("No %s records for label %s\n", rType, dk)
				continue
			}

			//log.Printf("rdata %s TYPE-R %T\n", rdata, rdata)

			records := make(map[string][]interface{})

			switch rdata.(type) {
			case map[string]interface{}:
				// Handle NS map syntax, map[ns2.example.net:<nil> ns1.example.net:<nil>]
				tmp := make([]interface{}, 0)
				for rdata_k, rdata_v := range rdata.(map[string]interface{}) {
					if rdata_v == nil {
						rdata_v = ""
					}
					tmp = append(tmp, []string{rdata_k, rdata_v.(string)})
				}
				records[rType] = tmp
			case string:
				// CNAME and alias
				tmp := make([]interface{}, 1)
				tmp[0] = rdata.(string)
				records[rType] = tmp
			default:
				records[rType] = rdata.([]interface{})
			}

			//log.Printf("RECORDS %s TYPE-REC %T\n", Records, Records)

			label.Records[dnsType] = make(Records, len(records[rType]))

			for i := 0; i < len(records[rType]); i++ {

				//log.Printf("RT %T %#v\n", records[rType][i], records[rType][i])

				record := new(Record)

				var h dns.RR_Header
				// log.Println("TTL OPTIONS", Zone.Options.Ttl)
				h.Ttl = uint32(label.Ttl)
				h.Class = dns.ClassINET
				h.Rrtype = dnsType
				h.Name = label.Label + "." + Zone.Origin + "."

				switch dnsType {
				case dns.TypeA, dns.TypeAAAA:
					rec := records[rType][i].([]interface{})
					ip := rec[0].(string)
					var err error

					if len(rec) > 1 {
						switch rec[1].(type) {
						case string:
							record.Weight, err = strconv.Atoi(rec[1].(string))
							if err != nil {
								panic("Error converting weight to integer")
							}
						case float64:
							record.Weight = int(rec[1].(float64))
						}
					}
					switch dnsType {
					case dns.TypeA:
						if x := net.ParseIP(ip); x != nil {
							record.RR = &dns.A{Hdr: h, A: x}
							break
						}
						panic("Bad A record")
					case dns.TypeAAAA:
						if x := net.ParseIP(ip); x != nil {
							record.RR = &dns.AAAA{Hdr: h, AAAA: x}
							break
						}
						panic("Bad AAAA record")
					}

				case dns.TypeMX:
					rec := records[rType][i].(map[string]interface{})
					pref := uint16(0)
					mx := rec["mx"].(string)
					if !strings.HasSuffix(mx, ".") {
						mx = mx + "."
					}
					if rec["weight"] != nil {
						record.Weight = valueToInt(rec["weight"])
					}
					if rec["preference"] != nil {
						pref = uint16(valueToInt(rec["preference"]))
					}
					record.RR = &dns.MX{
						Hdr:        h,
						Mx:         mx,
						Preference: pref}

				case dns.TypeCNAME:
					rec := records[rType][i]
					target := rec.(string)
					if !dns.IsFqdn(target) {
						target = target + "." + Zone.Origin
					}
					record.RR = &dns.CNAME{Hdr: h, Target: dns.Fqdn(target)}

				case dns.TypeMF:
					rec := records[rType][i]
					// MF records (how we store aliases) are not FQDNs
					record.RR = &dns.MF{Hdr: h, Mf: rec.(string)}

				case dns.TypeNS:
					rec := records[rType][i]
					if h.Ttl < 86400 {
						h.Ttl = 86400
					}

					var ns string

					switch rec.(type) {
					case string:
						ns = rec.(string)
					case []string:
						recl := rec.([]string)
						ns = recl[0]
						if len(recl[1]) > 0 {
							log.Println("NS records with names syntax not supported")
						}
					default:
						log.Printf("Data: %T %#v\n", rec, rec)
						panic("Unrecognized NS format/syntax")
					}

					rr := &dns.NS{Hdr: h, Ns: dns.Fqdn(ns)}

					record.RR = rr

				default:
					log.Println("type:", rType)
					panic("Don't know how to handle this type")
				}

				if record.RR == nil {
					panic("record.RR is nil")
				}

				label.Weight[dnsType] += record.Weight
				label.Records[dnsType][i] = *record
			}
			if label.Weight[dnsType] > 0 {
				sort.Sort(RecordsByWeight{label.Records[dnsType]})
			}
		}
	}

	setupSOA(Zone)

	//log.Println(Zones[k])
}

func setupSOA(Zone *Zone) {
	label := Zone.Labels[""]

	primaryNs := "ns"

	// log.Println("LABEL", label)

	if label == nil {
		log.Println(Zone.Origin, "doesn't have any 'root' records,",
			"you should probably add some NS records")
		label = Zone.AddLabel("")
	}

	if record, ok := label.Records[dns.TypeNS]; ok {
		primaryNs = record[0].RR.(*dns.NS).Ns
	}

	s := Zone.Origin + ". 3600 IN SOA " +
		primaryNs + " " + Zone.Options.Contact + " " +
		strconv.Itoa(Zone.Options.Serial) +
		" 5400 5400 2419200 " +
		strconv.Itoa(Zone.Options.Ttl)

	// log.Println("SOA: ", s)

	rr, err := dns.NewRR(s)

	if err != nil {
		log.Println("SOA Error", err)
		panic("Could not setup SOA")
	}

	record := Record{RR: rr}

	label.Records[dns.TypeSOA] = make([]Record, 1)
	label.Records[dns.TypeSOA][0] = record

}

func valueToInt(v interface{}) (rv int) {
	switch v.(type) {
	case string:
		i, err := strconv.Atoi(v.(string))
		if err != nil {
			panic("Error converting weight to integer")
		}
		rv = i
	case float64:
		rv = int(v.(float64))
	default:
		log.Println("Can't convert", v, "to integer")
		panic("Can't convert value")
	}
	return rv
}
<commit_msg>Always log when zones are re-read<commit_after>package main

import (
	"camlistore.org/pkg/errorutil"
	"encoding/json"
	"fmt"
	"github.com/miekg/dns"
	"io/ioutil"
	"log"
	"net"
	"os"
	"path"
	"runtime"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"time"
)

var configLastRead = map[string]time.Time{}

func configReader(dirName string, Zones Zones) {
	for {
		configReadDir(dirName, Zones)
		time.Sleep(5 * time.Second)
	}
}

func configReadDir(dirName string, Zones Zones) error {
	dir, err := ioutil.ReadDir(dirName)
	if err != nil {
		log.Println("Could not read", dirName, ":", err)
		return err
	}

	seenFiles := map[string]bool{}

	var parse_err error

	for _, file := range dir {
		fileName := file.Name()
		if !strings.HasSuffix(strings.ToLower(fileName), ".json") {
			continue
		}

		seenFiles[fileName] = true

		if lastRead, ok := configLastRead[fileName]; !ok || file.ModTime().After(lastRead) {
			if ok {
				log.Printf("Reloading %s\n", fileName)
			} else {
				logPrintf("Reading new file %s\n", fileName)
			}
			configLastRead[fileName] = file.ModTime()
			zoneName := fileName[0:strings.LastIndex(fileName, ".")]
			//log.Println("FILE:", i, file, zoneName)
			runtime.GC()
			config, err := readZoneFile(zoneName, path.Join(dirName, fileName))
			if config == nil || err != nil {
				log.Println(err)
				parse_err = err
				continue
			}

			Zones[zoneName] = config
			dns.HandleFunc(zoneName, setupServerFunc(config))
			runtime.GC()
		}

		// TODO(ask) Disable zones not seen in two subsequent runs
	}

	return parse_err
}

func setupPgeodnsZone(Zones Zones) {
	zoneName := "pgeodns"
	Zone := new(Zone)
	Zone.Labels = make(labels)
	Zone.Origin = zoneName
	Zone.LenLabels = dns.LenLabels(Zone.Origin)
	label := new(Label)
	label.Records = make(map[uint16]Records)
	label.Weight = make(map[uint16]int)
	Zone.Labels[""] = label
	setupSOA(Zone)
	Zones[zoneName] = Zone
	dns.HandleFunc(zoneName, setupServerFunc(Zone))
}

func readZoneFile(zoneName, fileName string) (*Zone, error) {
	defer func() {
		if err := recover(); err != nil {
			log.Printf("reading %s failed: %s", zoneName, err)
			debug.PrintStack()
		}
	}()

	fh, err := os.Open(fileName)
	if err != nil {
		log.Println("Could not read ", fileName, ": ", err)
		panic(err)
	}

	Zone := new(Zone)
	Zone.Labels = make(labels)
	Zone.Origin = zoneName
	Zone.LenLabels = dns.LenLabels(Zone.Origin)
	Zone.Options.Ttl = 120
	Zone.Options.MaxHosts = 2
	Zone.Options.Contact = "support.bitnames.com"

	if err == nil {
		var objmap map[string]interface{}
		decoder := json.NewDecoder(fh)
		if err = decoder.Decode(&objmap); err != nil {
			extra := ""
			if serr, ok := err.(*json.SyntaxError); ok {
				if _, serr := fh.Seek(0, os.SEEK_SET); serr != nil {
					log.Fatalf("seek error: %v", serr)
				}
				line, col, highlight := errorutil.HighlightBytePosition(fh, serr.Offset)
				extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
					line, col, serr.Offset, highlight)
			}
			return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v",
				fh.Name(), extra, err)
		}

		if err != nil {
			panic(err)
		}
		//log.Println(objmap)

		var data map[string]interface{}

		for k, v := range objmap {
			//log.Printf("k: %s v: %#v, T: %T\n", k, v, v)

			switch k {
			case "ttl", "serial", "max_hosts", "contact":
				switch option := k; option {
				case "ttl":
					Zone.Options.Ttl = valueToInt(v)
				case "serial":
					Zone.Options.Serial = valueToInt(v)
				case "contact":
					Zone.Options.Contact = v.(string)
				case "max_hosts":
					Zone.Options.MaxHosts = valueToInt(v)
				}
				continue

			case "data":
				data = v.(map[string]interface{})
			}
		}

		setupZoneData(data, Zone)

	}

	//log.Printf("ZO T: %T %s\n", Zones["0.us"], Zones["0.us"])

	//log.Println("IP", string(Zone.Regions["0.us"].IPv4[0].ip))

	return Zone, nil
}

func setupZoneData(data map[string]interface{}, Zone *Zone) {

	recordTypes := map[string]uint16{
		"a":     dns.TypeA,
		"aaaa":  dns.TypeAAAA,
		"ns":    dns.TypeNS,
		"cname": dns.TypeCNAME,
		"mx":    dns.TypeMX,
		"alias": dns.TypeMF,
	}

	for dk, dv_inter := range data {

		dv := dv_inter.(map[string]interface{})

		//log.Printf("K %s V %s TYPE-V %T\n", dk, dv, dv)

		label := Zone.AddLabel(dk)

		if ttl, ok := dv["ttl"]; ok {
			label.Ttl = valueToInt(ttl)
		}

		if maxHosts, ok := dv["max_hosts"]; ok {
			label.MaxHosts = valueToInt(maxHosts)
		}

		for rType, dnsType := range recordTypes {

			rdata := dv[rType]

			if rdata == nil {
				//log.Printf("No %s records for label %s\n", rType, dk)
				continue
			}

			//log.Printf("rdata %s TYPE-R %T\n", rdata, rdata)

			records := make(map[string][]interface{})

			switch rdata.(type) {
			case map[string]interface{}:
				// Handle NS map syntax, map[ns2.example.net:<nil> ns1.example.net:<nil>]
				tmp := make([]interface{}, 0)
				for rdata_k, rdata_v := range rdata.(map[string]interface{}) {
					if rdata_v == nil {
						rdata_v = ""
					}
					tmp = append(tmp, []string{rdata_k, rdata_v.(string)})
				}
				records[rType] = tmp
			case string:
				// CNAME and alias
				tmp := make([]interface{}, 1)
				tmp[0] = rdata.(string)
				records[rType] = tmp
			default:
				records[rType] = rdata.([]interface{})
			}

			//log.Printf("RECORDS %s TYPE-REC %T\n", Records, Records)

			label.Records[dnsType] = make(Records, len(records[rType]))

			for i := 0; i < len(records[rType]); i++ {

				//log.Printf("RT %T %#v\n", records[rType][i], records[rType][i])

				record := new(Record)

				var h dns.RR_Header
				// log.Println("TTL OPTIONS", Zone.Options.Ttl)
				h.Ttl = uint32(label.Ttl)
				h.Class = dns.ClassINET
				h.Rrtype = dnsType
				h.Name = label.Label + "." + Zone.Origin + "."

				switch dnsType {
				case dns.TypeA, dns.TypeAAAA:
					rec := records[rType][i].([]interface{})
					ip := rec[0].(string)
					var err error

					if len(rec) > 1 {
						switch rec[1].(type) {
						case string:
							record.Weight, err = strconv.Atoi(rec[1].(string))
							if err != nil {
								panic("Error converting weight to integer")
							}
						case float64:
							record.Weight = int(rec[1].(float64))
						}
					}
					switch dnsType {
					case dns.TypeA:
						if x := net.ParseIP(ip); x != nil {
							record.RR = &dns.A{Hdr: h, A: x}
							break
						}
						panic("Bad A record")
					case dns.TypeAAAA:
						if x := net.ParseIP(ip); x != nil {
							record.RR = &dns.AAAA{Hdr: h, AAAA: x}
							break
						}
						panic("Bad AAAA record")
					}

				case dns.TypeMX:
					rec := records[rType][i].(map[string]interface{})
					pref := uint16(0)
					mx := rec["mx"].(string)
					if !strings.HasSuffix(mx, ".") {
						mx = mx + "."
					}
					if rec["weight"] != nil {
						record.Weight = valueToInt(rec["weight"])
					}
					if rec["preference"] != nil {
						pref = uint16(valueToInt(rec["preference"]))
					}
					record.RR = &dns.MX{
						Hdr:        h,
						Mx:         mx,
						Preference: pref}

				case dns.TypeCNAME:
					rec := records[rType][i]
					target := rec.(string)
					if !dns.IsFqdn(target) {
						target = target + "." + Zone.Origin
					}
					record.RR = &dns.CNAME{Hdr: h, Target: dns.Fqdn(target)}

				case dns.TypeMF:
					rec := records[rType][i]
					// MF records (how we store aliases) are not FQDNs
					record.RR = &dns.MF{Hdr: h, Mf: rec.(string)}

				case dns.TypeNS:
					rec := records[rType][i]
					if h.Ttl < 86400 {
						h.Ttl = 86400
					}

					var ns string

					switch rec.(type) {
					case string:
						ns = rec.(string)
					case []string:
						recl := rec.([]string)
						ns = recl[0]
						if len(recl[1]) > 0 {
							log.Println("NS records with names syntax not supported")
						}
					default:
						log.Printf("Data: %T %#v\n", rec, rec)
						panic("Unrecognized NS format/syntax")
					}

					rr := &dns.NS{Hdr: h, Ns: dns.Fqdn(ns)}

					record.RR = rr

				default:
					log.Println("type:", rType)
					panic("Don't know how to handle this type")
				}

				if record.RR == nil {
					panic("record.RR is nil")
				}

				label.Weight[dnsType] += record.Weight
				label.Records[dnsType][i] = *record
			}
			if label.Weight[dnsType] > 0 {
				sort.Sort(RecordsByWeight{label.Records[dnsType]})
			}
		}
	}

	setupSOA(Zone)

	//log.Println(Zones[k])
}

func setupSOA(Zone *Zone) {
	label := Zone.Labels[""]

	primaryNs := "ns"

	// log.Println("LABEL", label)

	if label == nil {
		log.Println(Zone.Origin, "doesn't have any 'root' records,",
			"you should probably add some NS records")
		label = Zone.AddLabel("")
	}

	if record, ok := label.Records[dns.TypeNS]; ok {
		primaryNs = record[0].RR.(*dns.NS).Ns
	}

	s := Zone.Origin + ". 3600 IN SOA " +
		primaryNs + " " + Zone.Options.Contact + " " +
		strconv.Itoa(Zone.Options.Serial) +
		" 5400 5400 2419200 " +
		strconv.Itoa(Zone.Options.Ttl)

	// log.Println("SOA: ", s)

	rr, err := dns.NewRR(s)

	if err != nil {
		log.Println("SOA Error", err)
		panic("Could not setup SOA")
	}

	record := Record{RR: rr}

	label.Records[dns.TypeSOA] = make([]Record, 1)
	label.Records[dns.TypeSOA][0] = record

}

func valueToInt(v interface{}) (rv int) {
	switch v.(type) {
	case string:
		i, err := strconv.Atoi(v.(string))
		if err != nil {
			panic("Error converting weight to integer")
		}
		rv = i
	case float64:
		rv = int(v.(float64))
	default:
		log.Println("Can't convert", v, "to integer")
		panic("Can't convert value")
	}
	return rv
}
<|endoftext|>


<commit_before>package flag

import (
	"flag"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"unsafe"
)

func Configure(config interface{}) error {
	val, err := checkType(config)
	if err != nil {
		return err
	}
	structType := val.Type()
	for i := 0; i < structType.NumField(); i++ {
		f := (reflect.StructField)(structType.Field(i))
		t := f.Tag.Get("flag")
		if t == "" {
			// TODO no tag available
			fmt.Printf("Skipping '%s' ...\n", f.Name)
			continue
		}
		tag := parseTag(t)
		if tag.Name == "" {
			return fmt.Errorf("invalid flag name: empty string")
		}
		var fieldptr = unsafe.Pointer(val.UnsafeAddr() + f.Offset)
		switch f.Type.Kind() {
		case reflect.String:
			flag.StringVar((*string)(fieldptr), tag.Name, tag.DefaultValue, tag.Description)
		case reflect.Bool:
			defaultVal, err := strconv.ParseBool(tag.DefaultValue)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.BoolVar((*bool)(fieldptr), tag.Name, defaultVal, tag.Description)
		case reflect.Float64:
			defaultVal, err := strconv.ParseFloat(tag.DefaultValue, f.Type.Bits())
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.Float64Var((*float64)(fieldptr), tag.Name, defaultVal, tag.Description)
		case reflect.Int:
			defaultVal, err := strconv.ParseInt(tag.DefaultValue, 10, f.Type.Bits())
			// TODO distinguish between oct (0o1234567), dec (1234), hex (0x0123456789abcdef)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.IntVar((*int)(fieldptr), tag.Name, int(defaultVal), tag.Description)
		case reflect.Int64:
			defaultVal, err := strconv.ParseInt(tag.DefaultValue, 10, f.Type.Bits())
			// TODO distinguish between oct (0o1234567), dec (1234), hex (0x0123456789abcdef)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.Int64Var((*int64)(fieldptr), tag.Name, defaultVal, tag.Description)
		case reflect.Uint:
			defaultVal, err := strconv.ParseUint(tag.DefaultValue, 10, f.Type.Bits())
			// TODO distinguish between oct (0o1234567), dec (1234), hex (0x0123456789abcdef)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.UintVar((*uint)(fieldptr), tag.Name, uint(defaultVal), tag.Description)
		case reflect.Uint64:
			defaultVal, err := strconv.ParseUint(tag.DefaultValue, 10, 64)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.Uint64Var((*uint64)(fieldptr), tag.Name, defaultVal, tag.Description)
		// TODO support Duration
		// TODO support Var (any variable via flag.Value interface)
		// TODO support for smaller int, uint, float types?
		// TODO how to handle unsupported types (return error, leave message, etc.)
		}
	}
	return nil
}

func checkType(config interface{}) (reflect.Value, error) {
	var zero reflect.Value
	if config == nil {
		return zero, fmt.Errorf("config cannot be nil")
	}
	ptr := reflect.ValueOf(config)
	if ptr.IsNil() {
		return zero, fmt.Errorf("config cannot point to nil")
	}
	val := reflect.Indirect(ptr)
	if val.Kind() != reflect.Struct {
		return zero, fmt.Errorf("config instance is not a struct")
	}
	return val, nil
}

func parseTag(value string) flagTag {
	parts := strings.SplitN(value, ",", 3)
	for len(parts) < 3 {
		parts = append(parts, "")
	}
	return flagTag{Name: parts[0], DefaultValue: parts[1], Description: parts[2]}
}

type flagTag struct {
	Name         string
	DefaultValue string
	Description  string
}
<commit_msg>Use 0 for automatically detecting the base.<commit_after>package flag

import (
	"flag"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"unsafe"
)

func Configure(config interface{}) error {
	val, err := checkType(config)
	if err != nil {
		return err
	}
	structType := val.Type()
	for i := 0; i < structType.NumField(); i++ {
		f := (reflect.StructField)(structType.Field(i))
		t := f.Tag.Get("flag")
		if t == "" {
			// TODO no tag available
			fmt.Printf("Skipping '%s' ...\n", f.Name)
			continue
		}
		tag := parseTag(t)
		if tag.Name == "" {
			return fmt.Errorf("invalid flag name: empty string")
		}
		var fieldptr = unsafe.Pointer(val.UnsafeAddr() + f.Offset)
		switch f.Type.Kind() {
		case reflect.String:
			flag.StringVar((*string)(fieldptr), tag.Name, tag.DefaultValue, tag.Description)
		case reflect.Bool:
			defaultVal, err := strconv.ParseBool(tag.DefaultValue)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.BoolVar((*bool)(fieldptr), tag.Name, defaultVal, tag.Description)
		case reflect.Float64:
			defaultVal, err := strconv.ParseFloat(tag.DefaultValue, f.Type.Bits())
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.Float64Var((*float64)(fieldptr), tag.Name, defaultVal, tag.Description)
		case reflect.Int:
			defaultVal, err := strconv.ParseInt(tag.DefaultValue, 0, f.Type.Bits())
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.IntVar((*int)(fieldptr), tag.Name, int(defaultVal), tag.Description)
		case reflect.Int64:
			defaultVal, err := strconv.ParseInt(tag.DefaultValue, 0, f.Type.Bits())
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.Int64Var((*int64)(fieldptr), tag.Name, defaultVal, tag.Description)
		case reflect.Uint:
			defaultVal, err := strconv.ParseUint(tag.DefaultValue, 0, f.Type.Bits())
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.UintVar((*uint)(fieldptr), tag.Name, uint(defaultVal), tag.Description)
		case reflect.Uint64:
			defaultVal, err := strconv.ParseUint(tag.DefaultValue, 0, 64)
			if err != nil {
				// TODO invalid default, skipping
				fmt.Printf("Invalid value, skipping %s ...\n", f.Name)
				continue
			}
			flag.Uint64Var((*uint64)(fieldptr), tag.Name, defaultVal, tag.Description)
		// TODO support Duration
		// TODO support Var (any variable via flag.Value interface)
		// TODO support for smaller int, uint, float types?
		// TODO how to handle unsupported types (return error, leave message, etc.)
		}
	}
	return nil
}

func checkType(config interface{}) (reflect.Value, error) {
	var zero reflect.Value
	if config == nil {
		return zero, fmt.Errorf("config cannot be nil")
	}
	ptr := reflect.ValueOf(config)
	if ptr.IsNil() {
		return zero, fmt.Errorf("config cannot point to nil")
	}
	val := reflect.Indirect(ptr)
	if val.Kind() != reflect.Struct {
		return zero, fmt.Errorf("config instance is not a struct")
	}
	return val, nil
}

func parseTag(value string) flagTag {
	parts := strings.SplitN(value, ",", 3)
	for len(parts) < 3 {
		parts = append(parts, "")
	}
	return flagTag{Name: parts[0], DefaultValue: parts[1], Description: parts[2]}
}

type flagTag struct {
	Name         string
	DefaultValue string
	Description  string
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gnd.la/config"
	"gnd.la/log"
	"gnd.la/util/textutil"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"
)

type Config struct {
	File             string
	Command          string
	Name             string
	Dir              string
	Environment      map[string]string
	User             string
	Group            string
	Priority         int `default:"1000"`
	Watchdog         *Watchdog
	WatchdogInterval int `default:"300"`
	Log              *Logger
	Err              error
}

func (c *Config) Cmd() (*exec.Cmd, error) {
	if c.Err != nil {
		return nil, c.Err
	}
	if c.Command == "" {
		return nil, fmt.Errorf("no command")
	}
	fields, err := textutil.SplitFields(c.Command, " ")
	if err != nil {
		return nil, err
	}
	if !filepath.IsAbs(fields[0]) {
		p, err := exec.LookPath(fields[0])
		if err != nil {
			return nil, err
		}
		fields[0] = p
	}
	dir := c.Dir
	if dir == "" {
		dir = filepath.Dir(fields[0])
	}
	cmd := &exec.Cmd{Path: fields[0], Args: fields, Dir: dir}
	for k, v := range c.Environment {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}
	if _, ok := c.Environment["GOMAXPROCS"]; !ok {
		cmd.Env = append(cmd.Env, fmt.Sprintf("GOMAXPROCS=%d", runtime.NumCPU()))
	}
	for _, v := range os.Environ() {
		if p := strings.IndexByte(v, '='); p >= 0 {
			k := v[:p]
			if _, ok := c.Environment[k]; !ok {
				cmd.Env = append(cmd.Env, v)
			}
		}
	}
	info, err := os.Stat(fields[0])
	if err != nil {
		return nil, err
	}
	stat := info.Sys().(*syscall.Stat_t)
	uid := stat.Uid
	gid := stat.Gid
	if c.Group != "" {
		if g := getGroupId(c.Group); g > 0 {
			gid = uint32(g)
		} else {
			return nil, fmt.Errorf("invalid group %q", c.Group)
		}
	}
	if c.User != "" {
		u, err := user.Lookup(c.User)
		if err != nil {
			return nil, err
		}
		ui, _ := strconv.Atoi(u.Uid)
		uid = uint32(ui)
		if gid == 0 {
			gi, _ := strconv.Atoi(u.Gid)
			gid = uint32(gi)
		}
	}
	var cred *syscall.Credential
	if uid != 0 || gid != 0 {
		cred = &syscall.Credential{Uid: uid, Gid: gid}
		attr := &syscall.SysProcAttr{Credential: cred}
		cmd.SysProcAttr = attr
	}
	log.Debugf("%s wd: %s, env: %s, cred: %+v", c.ServiceName(), dir, cmd.Env, cred)
	return cmd, nil
}

func (c *Config) ServiceName() string {
	if c.Name != "" {
		return c.Name
	}
	return c.File
}

func ParseConfig(filename string) *Config {
	cfg := &Config{File: filename}
	err := config.ParseFile(filepath.Join(servicesDir(), filename), cfg)
	cfg.Err = err
	if cfg.Log == nil {
		cfg.Log = new(Logger)
		cfg.Log.Parse("")
	}
	cfg.Log.Name = cfg.ServiceName()
	return cfg
}

func ParseConfigs() ([]*Config, error) {
	dir := servicesDir()
	if configDirIsDefault() {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return nil, fmt.Errorf("error creating services directory %s: %s", dir, err)
		}
	}
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("error reading config directory %s: %s", dir, err)
	}
	var configs []*Config
	for _, v := range files {
		name := v.Name()
		if shouldIgnoreFile(name, false) {
			continue
		}
		cfg := ParseConfig(name)
		log.Debugf("Parsed config %s: %+v", name, cfg)
		configs = append(configs, cfg)
	}
	return configs, nil
}

func shouldIgnoreFile(name string, deleted bool) bool {
	if name == "" || name[0] == '.' || strings.HasSuffix(name, "~") {
		return true
	}
	if !deleted {
		info, err := os.Stat(filepath.Join(servicesDir(), name))
		if err != nil || info.Size() == 0 || info.IsDir() {
			return true
		}
	}
	return false
}
<commit_msg>Rename Environment to Env<commit_after>package main

import (
	"fmt"
	"gnd.la/config"
	"gnd.la/log"
	"gnd.la/util/textutil"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"syscall"
)

type Config struct {
	File             string
	Command          string
	Name             string
	Dir              string
	Env              map[string]string
	User             string
	Group            string
	Priority         int `default:"1000"`
	Watchdog         *Watchdog
	WatchdogInterval int `default:"300"`
	Log              *Logger
	Err              error
}

func (c *Config) Cmd() (*exec.Cmd, error) {
	if c.Err != nil {
		return nil, c.Err
	}
	if c.Command == "" {
		return nil, fmt.Errorf("no command")
	}
	fields, err := textutil.SplitFields(c.Command, " ")
	if err != nil {
		return nil, err
	}
	if !filepath.IsAbs(fields[0]) {
		p, err := exec.LookPath(fields[0])
		if err != nil {
			return nil, err
		}
		fields[0] = p
	}
	dir := c.Dir
	if dir == "" {
		dir = filepath.Dir(fields[0])
	}
	cmd := &exec.Cmd{Path: fields[0], Args: fields, Dir: dir}
	for k, v := range c.Env {
		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
	}
	if _, ok := c.Env["GOMAXPROCS"]; !ok {
		cmd.Env = append(cmd.Env, fmt.Sprintf("GOMAXPROCS=%d", runtime.NumCPU()))
	}
	for _, v := range os.Environ() {
		if p := strings.IndexByte(v, '='); p >= 0 {
			k := v[:p]
			if _, ok := c.Env[k]; !ok {
				cmd.Env = append(cmd.Env, v)
			}
		}
	}
	info, err := os.Stat(fields[0])
	if err != nil {
		return nil, err
	}
	stat := info.Sys().(*syscall.Stat_t)
	uid := stat.Uid
	gid := stat.Gid
	if c.Group != "" {
		if g := getGroupId(c.Group); g > 0 {
			gid = uint32(g)
		} else {
			return nil, fmt.Errorf("invalid group %q", c.Group)
		}
	}
	if c.User != "" {
		u, err := user.Lookup(c.User)
		if err != nil {
			return nil, err
		}
		ui, _ := strconv.Atoi(u.Uid)
		uid = uint32(ui)
		if gid == 0 {
			gi, _ := strconv.Atoi(u.Gid)
			gid = uint32(gi)
		}
	}
	var cred *syscall.Credential
	if uid != 0 || gid != 0 {
		cred = &syscall.Credential{Uid: uid, Gid: gid}
		attr := &syscall.SysProcAttr{Credential: cred}
		cmd.SysProcAttr = attr
	}
	log.Debugf("%s wd: %s, env: %s, cred: %+v", c.ServiceName(), dir, cmd.Env, cred)
	return cmd, nil
}

func (c *Config) ServiceName() string {
	if c.Name != "" {
		return c.Name
	}
	return c.File
}

func ParseConfig(filename string) *Config {
	cfg := &Config{File: filename}
	err := config.ParseFile(filepath.Join(servicesDir(), filename), cfg)
	cfg.Err = err
	if cfg.Log == nil {
		cfg.Log = new(Logger)
		cfg.Log.Parse("")
	}
	cfg.Log.Name = cfg.ServiceName()
	return cfg
}

func ParseConfigs() ([]*Config, error) {
	dir := servicesDir()
	if configDirIsDefault() {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return nil, fmt.Errorf("error creating services directory %s: %s", dir, err)
		}
	}
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("error reading config directory %s: %s", dir, err)
	}
	var configs []*Config
	for _, v := range files {
		name := v.Name()
		if shouldIgnoreFile(name, false) {
			continue
		}
		cfg := ParseConfig(name)
		log.Debugf("Parsed config %s: %+v", name, cfg)
		configs = append(configs, cfg)
	}
	return configs, nil
}

func shouldIgnoreFile(name string, deleted bool) bool {
	if name == "" || name[0] == '.' || strings.HasSuffix(name, "~") {
		return true
	}
	if !deleted {
		info, err := os.Stat(filepath.Join(servicesDir(), name))
		if err != nil || info.Size() == 0 || info.IsDir() {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Package models stores the structs for the objects we have & interacts with mongo
package models

import (
	"fmt"
	"time"

	"gopkg.in/mgo.v2/bson"

	"github.com/nicomo/EResourcesMetadataHub/logger"
)

type Ebook struct {
	Id                   bson.ObjectId `bson:"_id,omitempty"`
	DateCreated          time.Time
	DateUpdated          time.Time `bson:",omitempty"`
	Active               bool
	SfxId                string          `bson:",omitempty"`
	SFXLastHarvest       time.Time       `bson:",omitempty"`
	PublisherLastHarvest time.Time       `bson:",omitempty"`
	SudocLastHarvest     time.Time       `bson:",omitempty"`
	Authors              []string        `bson:",omitempty"`
	Title                string          `bson:",omitempty"`
	Publisher            string          `bson:",omitempty"`
	Pubdate              string          `bson:",omitempty"`
	Edition              int             `bson:",omitempty"`
	Lang                 string          `bson:",omitempty"`
	TargetService        []TargetService `bson:",omitempty"` // this is the name of the package in SFX, e.g. CAIRN QSJ
	OpenURL              string          `bson:",omitempty"`
	PackageURL           string          `bson:",omitempty"`
	Acquired             bool            `bson:",omitempty"`
	Isbns                []Isbn          `bson:",omitempty"`
	Ppns                 []PPN           `bson:",omitempty"`
	MarcRecords          []string        `bson:",omitempty"`
	Deleted              bool
}

type Isbn struct {
	Isbn       string
	Electronic bool
}

type PPN struct {
	Ppn        string
	Electronic bool
}

// EbookCreate saves a single ebook to mongo DB
func EbookCreate(ebk Ebook) error {

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()
	coll := getEbooksColl()

	// let's add the time and save
	ebk.DateCreated = time.Now()
	err := coll.Insert(ebk)
	if err != nil {
		logger.Error.Printf("could not save ebook with isbn %v in DB: %s", ebk.Isbns[0], err)
		return err
	}

	return nil
}

// EbookGetById retrieves an ebook given its mongodb id
func EbookGetById(id string) (Ebook, error) {
	ebk := Ebook{}

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()

	// collection ebooks
	coll := getEbooksColl()

	// cast id as ObjectId
	objectId := bson.ObjectIdHex(id)

	qry := bson.M{"_id": objectId}
	err := coll.Find(qry).One(&ebk)
	if err != nil {
		return ebk, err
	}

	return ebk, nil

}

// EbookGetByIsbn retrieves an ebook
func EbookGetByIsbns(isbns []string) (Ebook, error) {
	ebk := Ebook{}

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()

	// collection ebooks
	coll := getEbooksColl()

	// construct query

	var isbnQry []bson.M
	for i := 0; i < len(isbns); i++ {
		if isbns[i] != "" {
			sTerm := bson.M{"isbns.isbn": isbns[i]}
			isbnQry = append(isbnQry, sTerm)
		}
	}
	qry := bson.M{
		"$or": isbnQry,
	}

	// execute query
	err := coll.Find(qry).One(&ebk)
	if err != nil {
		return ebk, err
	}

	return ebk, nil
}

// EbooksGetByPackageName
func EbooksGetByPackageName(tsname string) ([]Ebook, error) {
	var result []Ebook

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()

	// collection ebooks
	coll := getEbooksColl()

	// FIXME: iterate over slices of 1000 ebooks
	// maybe use channels?
	// or else retrieve all then calculate result / 1000 and just manage display of chuncks of 1000 ebooks
	iter := coll.Find(bson.M{"targetservice.tsname": tsname}).Limit(1000).Iter()
	err := iter.All(&result)
	if err != nil {
		return result, err
	}

	return result, nil
}

// EbookUpdate saves an ebk struct to DB
func EbookUpdate(ebk Ebook) (Ebook, error) {
	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()
	coll := getEbooksColl()

	// let's add the time and save
	ebk.DateUpdated = time.Now()

	logger.Debug.Printf("EbookUpdate: %v", ebk)

	// we select on the ebook's id
	selector := bson.M{"_id": ebk.Id}
	logger.Debug.Println(selector)

	err := coll.Update(selector, &ebk)
	if err != nil {
		logger.Error.Printf("Couldn't update ebook: %v", err)
		return ebk, err
	}

	return ebk, nil
}

//TODO: EbookSoftDelete
func EbookSoftDelete(ebkId int) error {
	return nil
}

//TODO: EbookDelete
func EbookDelete(ebkId int) error {
	return nil
}

// EbooksCreateOrUpdate checks if ebook exists in DB, using ISBN, then routes to either create or update
func EbooksCreateOrUpdate(records []Ebook) (int, int, error) {

	var createdCounter, updatedCounter int

	for _, record := range records { // for each record

		// pull out the isbns
		isbnsToQuery := make([]string, 0)
		for _, isbn := range record.Isbns { // for each isbn
			if isbn.Isbn != "" {
				isbnsToQuery = append(isbnsToQuery, isbn.Isbn)
			}
		}

		// test if we already know this ebook
		existingRecord, err := EbookGetByIsbns(isbnsToQuery)
		if err != nil { // we don't: none of the isbns were found in DB
			// let's create a new record
			ebkCreateErr := EbookCreate(record)
			if ebkCreateErr != nil {
				logger.Error.Println(ebkCreateErr)
			}
			createdCounter++
			continue
		}

		// TODO: we've found the record, let's update.it
		fmt.Println(existingRecord)
		updatedCounter++
	}
	return createdCounter, updatedCounter, nil
}

// PPNCreate creates the PPN structs to be embedded in an ebook struct
func PPNCreate(ppns []string, electronic bool) []PPN {
	myPPNs := make([]PPN, 0)
	for _, v := range ppns {
		p := PPN{Ppn: v, Electronic: electronic}
		myPPNs = append(myPPNs, p)
	}
	return myPPNs
}
<commit_msg>update in EbooksCreateOrUpdate<commit_after>// Package models stores the structs for the objects we have & interacts with mongo
package models

import (
	"time"

	"gopkg.in/mgo.v2/bson"

	"github.com/nicomo/EResourcesMetadataHub/logger"
)

type Ebook struct {
	Id                   bson.ObjectId `bson:"_id,omitempty"`
	DateCreated          time.Time
	DateUpdated          time.Time `bson:",omitempty"`
	Active               bool
	SfxId                string          `bson:",omitempty"`
	SFXLastHarvest       time.Time       `bson:",omitempty"`
	PublisherLastHarvest time.Time       `bson:",omitempty"`
	SudocLastHarvest     time.Time       `bson:",omitempty"`
	Authors              []string        `bson:",omitempty"`
	Title                string          `bson:",omitempty"`
	Publisher            string          `bson:",omitempty"`
	Pubdate              string          `bson:",omitempty"`
	Edition              int             `bson:",omitempty"`
	Lang                 string          `bson:",omitempty"`
	TargetService        []TargetService `bson:",omitempty"` // this is the name of the package in SFX, e.g. CAIRN QSJ
	OpenURL              string          `bson:",omitempty"`
	PackageURL           string          `bson:",omitempty"`
	Acquired             bool            `bson:",omitempty"`
	Isbns                []Isbn          `bson:",omitempty"`
	Ppns                 []PPN           `bson:",omitempty"`
	MarcRecords          []string        `bson:",omitempty"`
	Deleted              bool
}

type Isbn struct {
	Isbn       string
	Electronic bool
}

type PPN struct {
	Ppn        string
	Electronic bool
}

// EbookCreate saves a single ebook to mongo DB
func EbookCreate(ebk Ebook) error {

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()
	coll := getEbooksColl()

	// let's add the time and save
	ebk.DateCreated = time.Now()
	err := coll.Insert(ebk)
	if err != nil {
		logger.Error.Printf("could not save ebook with isbn %v in DB: %s", ebk.Isbns[0], err)
		return err
	}

	return nil
}

// EbookGetById retrieves an ebook given its mongodb id
func EbookGetById(id string) (Ebook, error) {
	ebk := Ebook{}

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()

	// collection ebooks
	coll := getEbooksColl()

	// cast id as ObjectId
	objectId := bson.ObjectIdHex(id)

	qry := bson.M{"_id": objectId}
	err := coll.Find(qry).One(&ebk)
	if err != nil {
		return ebk, err
	}

	return ebk, nil

}

// EbookGetByIsbn retrieves an ebook
func EbookGetByIsbns(isbns []string) (Ebook, error) {
	ebk := Ebook{}

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()

	// collection ebooks
	coll := getEbooksColl()

	// construct query

	var isbnQry []bson.M
	for i := 0; i < len(isbns); i++ {
		if isbns[i] != "" {
			sTerm := bson.M{"isbns.isbn": isbns[i]}
			isbnQry = append(isbnQry, sTerm)
		}
	}
	qry := bson.M{
		"$or": isbnQry,
	}

	// execute query
	err := coll.Find(qry).One(&ebk)
	if err != nil {
		return ebk, err
	}

	return ebk, nil
}

// EbooksGetByPackageName
func EbooksGetByPackageName(tsname string) ([]Ebook, error) {
	var result []Ebook

	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()

	// collection ebooks
	coll := getEbooksColl()

	// FIXME: iterate over slices of 1000 ebooks
	// maybe use channels?
	// or else retrieve all then calculate result / 1000 and just manage display of chuncks of 1000 ebooks
	iter := coll.Find(bson.M{"targetservice.tsname": tsname}).Limit(1000).Iter()
	err := iter.All(&result)
	if err != nil {
		return result, err
	}

	return result, nil
}

// EbookUpdate saves an ebk struct to DB
func EbookUpdate(ebk Ebook) (Ebook, error) {
	// Request a socket connection from the session to process our query.
	mgoSession := mgoSession.Copy()
	defer mgoSession.Close()
	coll := getEbooksColl()

	// let's add the time and save
	ebk.DateUpdated = time.Now()

	logger.Debug.Printf("EbookUpdate: %v", ebk)

	// we select on the ebook's id
	selector := bson.M{"_id": ebk.Id}
	logger.Debug.Println(selector)

	err := coll.Update(selector, &ebk)
	if err != nil {
		logger.Error.Printf("Couldn't update ebook: %v", err)
		return ebk, err
	}

	return ebk, nil
}

//TODO: EbookSoftDelete
func EbookSoftDelete(ebkId int) error {
	return nil
}

//TODO: EbookDelete
func EbookDelete(ebkId int) error {
	return nil
}

// EbooksCreateOrUpdate checks if ebook exists in DB, using ISBN, then routes to either create or update
func EbooksCreateOrUpdate(records []Ebook) (int, int, error) {

	var createdCounter, updatedCounter int

	for _, record := range records { // for each record

		// pull out the isbns
		isbnsToQuery := make([]string, 0)
		for _, isbn := range record.Isbns { // for each isbn
			if isbn.Isbn != "" {
				isbnsToQuery = append(isbnsToQuery, isbn.Isbn)
			}
		}

		// test if we already know this ebook
		existingRecord, err := EbookGetByIsbns(isbnsToQuery)
		if err != nil { // we don't: none of the isbns were found in DB
			// let's create a new record
			ebkCreateErr := EbookCreate(record)
			if ebkCreateErr != nil {
				logger.Error.Println(ebkCreateErr)
			}
			createdCounter++
			continue
		}

		// we did find the record, let's update it
		record.Id = existingRecord.Id
		updatedRecord, err := EbookUpdate(record)
		if err != nil {
			logger.Error.Println(err)
		}
		updatedCounter++
	}
	return createdCounter, updatedCounter, nil
}

// PPNCreate creates the PPN structs to be embedded in an ebook struct
func PPNCreate(ppns []string, electronic bool) []PPN {
	myPPNs := make([]PPN, 0)
	for _, v := range ppns {
		p := PPN{Ppn: v, Electronic: electronic}
		myPPNs = append(myPPNs, p)
	}
	return myPPNs
}
<|endoftext|>


<commit_before>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ccd

import (
	"context"
	"time"

	"github.com/corestoreio/errors"
	"github.com/corestoreio/log"
	"github.com/corestoreio/pkg/config"
	"github.com/corestoreio/pkg/sql/ddl"
	"github.com/corestoreio/pkg/sql/dml"
	"github.com/corestoreio/pkg/store/scope"
)

// WithCoreConfigData reads the table core_config_data into the Service and
// overrides existing values. Stops on errors.
func WithCoreConfigData(tbls *ddl.Tables, o Options) config.Option {
	return func(s *config.Service) error {
		var writtenRows int
		if s.Log.IsDebug() {
			defer log.WhenDone(s.Log).Debug("ccd.WithCoreConfigData.Written", log.Int("writtenRows", writtenRows))
		}

		tn := o.TableName
		if tn == "" {
			tn = TableNameCoreConfigData
		}

		tbl, err := tbls.Table(tn)
		if err != nil {
			return errors.WithStack(err)
		}

		if o.ContextTimeoutAllKeys == 0 {
			o.ContextTimeoutAllKeys = time.Second * 10 // just a guess
		}

		ctx, cancel := context.WithTimeout(context.Background(), o.ContextTimeoutAllKeys)
		defer cancel()

		return tbl.SelectAll().WithArgs().IterateSerial(ctx, func(cm *dml.ColumnMap) error {
			var ccd TableCoreConfigData
			if err := ccd.MapColumns(cm); err != nil {
				return errors.Wrapf(err, "[ccd] dbs.stmtAll.IterateSerial at row %d", cm.Count)
			}

			var v []byte
			if ccd.Value.Valid {
				v = []byte(ccd.Value.String)
			}
			scp := scope.FromString(ccd.Scope).Pack(ccd.ScopeID)
			p, err := config.MakePathWithScope(scp, ccd.Path)
			if err != nil {
				return errors.Wrapf(err, "[ccd] WithCoreConfigData.config.MakePathWithScope Path %q Scope: %q ID: %d", ccd.Path, scp, ccd.ConfigID)
			}
			if err = s.Write(p, v); err != nil {
				return errors.Wrapf(err, "[ccd] WithCoreConfigData.Service.Write Path %q Scope: %q ID: %d", ccd.Path, scp, ccd.ConfigID)
			}

			return nil
		})
	}
}
<commit_msg>config/storage/ccd: Remove log in WithCoreConfigData<commit_after>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ccd

import (
	"context"
	"time"

	"github.com/corestoreio/errors"
	"github.com/corestoreio/pkg/config"
	"github.com/corestoreio/pkg/sql/ddl"
	"github.com/corestoreio/pkg/sql/dml"
	"github.com/corestoreio/pkg/store/scope"
)

// WithCoreConfigData reads the table core_config_data into the Service and
// overrides existing values. Stops on errors.
func WithCoreConfigData(tbls *ddl.Tables, o Options) config.Option {
	return func(s *config.Service) error {

		tn := o.TableName
		if tn == "" {
			tn = TableNameCoreConfigData
		}

		tbl, err := tbls.Table(tn)
		if err != nil {
			return errors.WithStack(err)
		}

		if o.ContextTimeoutAllKeys == 0 {
			o.ContextTimeoutAllKeys = time.Second * 10 // just a guess
		}

		ctx, cancel := context.WithTimeout(context.Background(), o.ContextTimeoutAllKeys)
		defer cancel()

		return tbl.SelectAll().WithArgs().IterateSerial(ctx, func(cm *dml.ColumnMap) error {
			var ccd TableCoreConfigData
			if err := ccd.MapColumns(cm); err != nil {
				return errors.Wrapf(err, "[ccd] dbs.stmtAll.IterateSerial at row %d", cm.Count)
			}

			var v []byte
			if ccd.Value.Valid {
				v = []byte(ccd.Value.String)
			}
			scp := scope.FromString(ccd.Scope).Pack(ccd.ScopeID)
			p, err := config.MakePathWithScope(scp, ccd.Path)
			if err != nil {
				return errors.Wrapf(err, "[ccd] WithCoreConfigData.config.MakePathWithScope Path %q Scope: %q ID: %d", ccd.Path, scp, ccd.ConfigID)
			}
			if err = s.Write(p, v); err != nil {
				return errors.Wrapf(err, "[ccd] WithCoreConfigData.Service.Write Path %q Scope: %q ID: %d", ccd.Path, scp, ccd.ConfigID)
			}

			return nil
		})
	}
}
<|endoftext|>