<commit_before>// Package conf parses start up args and config file
package conf

import (
	//"flag"
	"errors"
	"fmt"
	"github.com/MG-RAST/golib/goconfig/config"
	"os"
	"strconv"
	"strings"
)

type idxOpts struct {
	unique   bool
	dropDups bool
	sparse   bool
}

var (
	// Admin
	ADMIN_EMAIL string
	ADMIN_USERS string
	AdminUsers  []string

	// Permissions for anonymous user
	ANON_READ   bool
	ANON_WRITE  bool
	ANON_DELETE bool

	// Address
	API_IP   string
	API_PORT int
	API_URL  string // for external address only

	// Auth
	AUTH_BASIC              bool
	AUTH_GLOBUS_TOKEN_URL   string
	AUTH_GLOBUS_PROFILE_URL string
	AUTH_MGRAST_OAUTH_URL   string
	AUTH_CACHE_TIMEOUT      int

	// Default Chunksize for size virtual index
	CHUNK_SIZE int64 = 1048576

	// Config File
	CONFIG_FILE string
	LOG_OUTPUT  string

	// Runtime

	EXPIRE_WAIT int // wait time for reaper in minutes
	GOMAXPROCS  string
	MAX_REVISIONS = 3  // max number of node revisions to keep; values < 0 mean keep all
	EXPIRE_WAIT   = 60 // wait time for reaper in minutes
	GOMAXPROCS    = ""

	// Logs
	LOG_PERF   bool // Indicates whether performance logs should be stored
	LOG_ROTATE bool // Indicates whether logs should be rotated daily

	// Mongo information
	MONGODB_HOSTS             string
	MONGODB_DATABASE          string
	MONGODB_USER              string
	MONGODB_PASSWORD          string
	MONGODB_ATTRIBUTE_INDEXES string

	// Node Indices
	NODE_IDXS map[string]idxOpts = nil

	// Paths
	PATH_SITE    string
	PATH_DATA    string
	PATH_LOGS    string
	PATH_LOCAL   string
	PATH_PIDFILE string

	// Reload
	RELOAD string

	// SSL
	SSL      bool
	SSL_KEY  string
	SSL_CERT string

	// Versions
	VERSIONS = make(map[string]int)

	PRINT_HELP   bool // full usage
	SHOW_HELP    bool // simple usage
	SHOW_VERSION bool

	// internal config control
	FAKE_VAR = false
)

// Initialize is an explicit init. Enables outside use
// of shock-server packages. Parses config and populates
// the conf variables.
func Initialize() (err error) {

	for i, elem := range os.Args {
		if strings.HasPrefix(elem, "-conf") || strings.HasPrefix(elem, "--conf") {
			parts := strings.SplitN(elem, "=", 2)
			if len(parts) == 2 {
				CONFIG_FILE = parts[1]
			} else if i+1 < len(os.Args) {
				CONFIG_FILE = os.Args[i+1]
			} else {
				err = errors.New("ERROR: parsing command options, missing conf file")
				return
			}
		}
	}

	var c *config.Config = nil
	if CONFIG_FILE != "" {
		c, err = config.ReadDefault(CONFIG_FILE)
		if err != nil {
			err = errors.New("ERROR: error reading conf file: " + err.Error())
			return
		}
		fmt.Printf("read %s\n", CONFIG_FILE)
	} else {
		fmt.Printf("No config file specified.\n")
		c = config.NewDefault()
	}

	c_store, err := getConfiguration(c) // from config file and command line arguments
	if err != nil {
		err = fmt.Errorf("ERROR: error reading conf file: %v\n", err)
		return
	}

	// ####### at this point configuration variables are set ########

	if FAKE_VAR == false {
		err = fmt.Errorf("ERROR: config was not parsed\n")
		return
	}
	if PRINT_HELP || SHOW_HELP {
		c_store.PrintHelp()
		os.Exit(0)
	}

	return
}

// Bool is a convenience wrapper around strconv.ParseBool
func Bool(s string) bool {
	b, _ := strconv.ParseBool(s)
	return b
}

// Print prints the configuration loads to stdout
func Print() {
	fmt.Printf("####### Anonymous ######\nread:\t%v\nwrite:\t%v\ndelete:\t%v\n\n", ANON_READ, ANON_WRITE, ANON_DELETE)
	if (AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "") || AUTH_MGRAST_OAUTH_URL != "" {
		fmt.Printf("##### Auth #####\n")
		if AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "" {
			fmt.Printf("type:\tglobus\ntoken_url:\t%s\nprofile_url:\t%s\n\n", AUTH_GLOBUS_TOKEN_URL, AUTH_GLOBUS_PROFILE_URL)
		}
		if AUTH_MGRAST_OAUTH_URL != "" {
			fmt.Printf("type:\tmgrast\noauth_url:\t%s\n\n", AUTH_MGRAST_OAUTH_URL)
		}
	}
	fmt.Printf("##### Admin #####\nusers:\t%s\n\n", ADMIN_USERS)
	fmt.Printf("##### Paths #####\nsite:\t%s\ndata:\t%s\nlogs:\t%s\nlocal_paths:\t%s\n\n", PATH_SITE, PATH_DATA, PATH_LOGS, PATH_LOCAL)
	if SSL {
		fmt.Printf("##### SSL enabled #####\n")
		fmt.Printf("##### SSL key:\t%s\n##### SSL cert:\t%s\n\n", SSL_KEY, SSL_CERT)
	} else {
		fmt.Printf("##### SSL disabled #####\n\n")
	}
	fmt.Printf("##### Mongodb #####\nhost(s):\t%s\ndatabase:\t%s\n\n", MONGODB_HOSTS, MONGODB_DATABASE)
	fmt.Printf("##### Address #####\nip:\t%s\nport:\t%d\n\n", API_IP, API_PORT)
	if LOG_PERF {
		fmt.Printf("##### PerfLog enabled #####\n\n")
	}
	if LOG_ROTATE {
		fmt.Printf("##### Log rotation enabled #####\n\n")
	} else {
		fmt.Printf("##### Log rotation disabled #####\n\n")
	}
       fmt.Printf("##### Expiration #####\nexpire_wait:\t%d minutes\n\n", EXPIRE_WAIT)
	fmt.Printf("##### Max Revisions #####\nmax_revisions:\t%d\n\n", MAX_REVISIONS)
	fmt.Printf("API_PORT: %d\n", API_PORT)
}

func getConfiguration(c *config.Config) (c_store *Config_store, err error) {
	c_store = NewCS(c)

	// Admin
	//ADMIN_EMAIL, _ = c.String("Admin", "email")
	c_store.AddString(&ADMIN_EMAIL, "", "Admin", "email", "", "")
	//ADMIN_USERS, _ = c.String("Admin", "users")
	c_store.AddString(&ADMIN_USERS, "", "Admin", "users", "", "")
	if ADMIN_USERS != "" {
		for _, name := range strings.Split(ADMIN_USERS, ",") {
			AdminUsers = append(AdminUsers, strings.TrimSpace(name))
		}
	}

	// Access-Control
	c_store.AddBool(&ANON_READ, true, "Anonymous", "read", "", "")
	c_store.AddBool(&ANON_WRITE, true, "Anonymous", "write", "", "")
	c_store.AddBool(&ANON_DELETE, true, "Anonymous", "delete", "", "")

	// Address
	c_store.AddString(&API_IP, "0.0.0.0", "Address", "api-ip", "", "")
	c_store.AddInt(&API_PORT, 7445, "Address", "api-port", "", "")

	// URLs
	c_store.AddString(&API_URL, "http://localhost", "External", "api-url", "", "")

	// Auth
	c_store.AddBool(&AUTH_BASIC, false, "Auth", "basic", "", "")
	c_store.AddString(&AUTH_GLOBUS_TOKEN_URL, "", "Auth", "globus_token_url", "", "")
	c_store.AddString(&AUTH_GLOBUS_PROFILE_URL, "", "Auth", "globus_profile_url", "", "")
	c_store.AddString(&AUTH_MGRAST_OAUTH_URL, "", "Auth", "mgrast_oauth_url", "", "")
	c_store.AddInt(&AUTH_CACHE_TIMEOUT, 60, "Auth", "cache_timeout", "", "")

	// Runtime
	c_store.AddInt(&EXPIRE_WAIT, 60, "Runtime", "expire_wait", "", "")
	c_store.AddString(&GOMAXPROCS, "", "Runtime", "GOMAXPROCS", "", "")
	c_store.AddInt(&MAX_REVISIONS, 3, "Runtime", "max_revisions", "", "")

	c_store.AddBool(&LOG_PERF, false, "Log", "perf_log", "", "")
	c_store.AddBool(&LOG_ROTATE, true, "Log", "rotate", "", "")

	// Mongodb
	c_store.AddString(&MONGODB_ATTRIBUTE_INDEXES, "", "Mongodb", "attribute_indexes", "", "")
	c_store.AddString(&MONGODB_DATABASE, "ShockDB", "Mongodb", "database", "", "")

	//MONGODB_HOSTS, _ = c.String("Mongodb", "hosts")
	//MONGODB_PASSWORD, _ = c.String("Mongodb", "password")
	//MONGODB_USER, _ = c.String("Mongodb", "user")
	c_store.AddString(&MONGODB_HOSTS, "mongo", "Mongodb", "hosts", "", "")
	c_store.AddString(&MONGODB_PASSWORD, "", "Mongodb", "password", "", "")
	c_store.AddString(&MONGODB_USER, "", "Mongodb", "user", "", "")

	// parse Node-Indices
	NODE_IDXS = map[string]idxOpts{}
	nodeIdx, _ := c.Options("Node-Indices")
	for _, opt := range nodeIdx {
		val, _ := c.String("Node-Indices", opt)
		opts := idxOpts{}
		for _, parts := range strings.Split(val, ",") {
			p := strings.Split(parts, ":")
			if p[0] == "unique" {
				if p[1] == "true" {
					opts.unique = true
				} else {
					opts.unique = false
				}
			} else if p[0] == "dropDups" {
				if p[1] == "true" {
					opts.dropDups = true
				} else {
					opts.dropDups = false
				}
			} else if p[0] == "sparse" {
				if p[1] == "true" {
					opts.sparse = true
				} else {
					opts.sparse = false
				}
			}
		}
		NODE_IDXS[opt] = opts
	}

	// Paths
	//PATH_SITE, _ = c.String("Paths", "site")
	//PATH_DATA, _ = c.String("Paths", "data")
	//PATH_LOGS, _ = c.String("Paths", "logs")
	//PATH_LOCAL, _ = c.String("Paths", "local_paths")
	//PATH_PIDFILE, _ = c.String("Paths", "pidfile")

	c_store.AddString(&PATH_SITE, "/usr/local/shock/site", "Paths", "site", "", "")
	c_store.AddString(&PATH_DATA, "/usr/local/shock", "Paths", "data", "", "")
	c_store.AddString(&PATH_LOGS, "/var/log/shock", "Paths", "logs", "", "")
	c_store.AddString(&PATH_LOCAL, "", "Paths", "local_paths", "", "")
	c_store.AddString(&PATH_PIDFILE, "", "Paths", "pidfile", "", "")

	// SSL
	//SSL, _ = c.Bool("SSL", "enable")
	c_store.AddBool(&SSL, false, "SSL", "enable", "", "")
	if SSL {
		//SSL_KEY, _ = c.String("SSL", "key")
		//SSL_CERT, _ = c.String("SSL", "cert")
		c_store.AddString(&SSL_KEY, "", "SSL", "key", "", "")
		c_store.AddString(&SSL_CERT, "", "SSL", "cert", "", "")
	}

	// Log
	c_store.AddString(&LOG_OUTPUT, "console", "Log", "logoutput", "console, file or both", "")

	//Other
	c_store.AddString(&RELOAD, "", "Other", "reload", "path or url to shock data. WARNING this will drop all current data.", "")
	gopath := os.Getenv("GOPATH")
	c_store.AddString(&CONFIG_FILE, gopath+"/src/github.com/MG-RAST/Shock/shock-server.conf.template", "Other", "conf", "path to config file", "")
	c_store.AddBool(&SHOW_VERSION, false, "Other", "version", "show version", "")
	c_store.AddBool(&PRINT_HELP, false, "Other", "fullhelp", "show detailed usage without \"--\"-prefixes", "")
	c_store.AddBool(&SHOW_HELP, false, "Other", "help", "show usage", "")

	VERSIONS["ACL"] = 2
	VERSIONS["Auth"] = 1
	VERSIONS["Node"] = 4

	c_store.Parse()

	return

}

<commit_msg>fix conflict<commit_after>// Package conf parses start up args and config file
package conf

import (
	//"flag"
	"errors"
	"fmt"
	"github.com/MG-RAST/golib/goconfig/config"
	"os"
	"strconv"
	"strings"
)

type idxOpts struct {
	unique   bool
	dropDups bool
	sparse   bool
}

var (
	// Admin
	ADMIN_EMAIL string
	ADMIN_USERS string
	AdminUsers  []string

	// Permissions for anonymous user
	ANON_READ   bool
	ANON_WRITE  bool
	ANON_DELETE bool

	// Address
	API_IP   string
	API_PORT int
	API_URL  string // for external address only

	// Auth
	AUTH_BASIC              bool
	AUTH_GLOBUS_TOKEN_URL   string
	AUTH_GLOBUS_PROFILE_URL string
	AUTH_MGRAST_OAUTH_URL   string
	AUTH_CACHE_TIMEOUT      int

	// Default Chunksize for size virtual index
	CHUNK_SIZE int64 = 1048576

	// Config File
	CONFIG_FILE string
	LOG_OUTPUT  string

	// Runtime

	EXPIRE_WAIT int // wait time for reaper in minutes
	GOMAXPROCS  string
	MAX_REVISIONS int  // max number of node revisions to keep; values < 0 mean keep all

	// Logs
	LOG_PERF   bool // Indicates whether performance logs should be stored
	LOG_ROTATE bool // Indicates whether logs should be rotated daily

	// Mongo information
	MONGODB_HOSTS             string
	MONGODB_DATABASE          string
	MONGODB_USER              string
	MONGODB_PASSWORD          string
	MONGODB_ATTRIBUTE_INDEXES string

	// Node Indices
	NODE_IDXS map[string]idxOpts = nil

	// Paths
	PATH_SITE    string
	PATH_DATA    string
	PATH_LOGS    string
	PATH_LOCAL   string
	PATH_PIDFILE string

	// Reload
	RELOAD string

	// SSL
	SSL      bool
	SSL_KEY  string
	SSL_CERT string

	// Versions
	VERSIONS = make(map[string]int)

	PRINT_HELP   bool // full usage
	SHOW_HELP    bool // simple usage
	SHOW_VERSION bool

	// internal config control
	FAKE_VAR = false
)

// Initialize is an explicit init. Enables outside use
// of shock-server packages. Parses config and populates
// the conf variables.
func Initialize() (err error) {

	for i, elem := range os.Args {
		if strings.HasPrefix(elem, "-conf") || strings.HasPrefix(elem, "--conf") {
			parts := strings.SplitN(elem, "=", 2)
			if len(parts) == 2 {
				CONFIG_FILE = parts[1]
			} else if i+1 < len(os.Args) {
				CONFIG_FILE = os.Args[i+1]
			} else {
				err = errors.New("ERROR: parsing command options, missing conf file")
				return
			}
		}
	}

	var c *config.Config = nil
	if CONFIG_FILE != "" {
		c, err = config.ReadDefault(CONFIG_FILE)
		if err != nil {
			err = errors.New("ERROR: error reading conf file: " + err.Error())
			return
		}
		fmt.Printf("read %s\n", CONFIG_FILE)
	} else {
		fmt.Printf("No config file specified.\n")
		c = config.NewDefault()
	}

	c_store, err := getConfiguration(c) // from config file and command line arguments
	if err != nil {
		err = fmt.Errorf("ERROR: error reading conf file: %v\n", err)
		return
	}

	// ####### at this point configuration variables are set ########

	if FAKE_VAR == false {
		err = fmt.Errorf("ERROR: config was not parsed\n")
		return
	}
	if PRINT_HELP || SHOW_HELP {
		c_store.PrintHelp()
		os.Exit(0)
	}

	return
}

// Bool is a convenience wrapper around strconv.ParseBool
func Bool(s string) bool {
	b, _ := strconv.ParseBool(s)
	return b
}

// Print prints the configuration loads to stdout
func Print() {
	fmt.Printf("####### Anonymous ######\nread:\t%v\nwrite:\t%v\ndelete:\t%v\n\n", ANON_READ, ANON_WRITE, ANON_DELETE)
	if (AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "") || AUTH_MGRAST_OAUTH_URL != "" {
		fmt.Printf("##### Auth #####\n")
		if AUTH_GLOBUS_TOKEN_URL != "" && AUTH_GLOBUS_PROFILE_URL != "" {
			fmt.Printf("type:\tglobus\ntoken_url:\t%s\nprofile_url:\t%s\n\n", AUTH_GLOBUS_TOKEN_URL, AUTH_GLOBUS_PROFILE_URL)
		}
		if AUTH_MGRAST_OAUTH_URL != "" {
			fmt.Printf("type:\tmgrast\noauth_url:\t%s\n\n", AUTH_MGRAST_OAUTH_URL)
		}
	}
	fmt.Printf("##### Admin #####\nusers:\t%s\n\n", ADMIN_USERS)
	fmt.Printf("##### Paths #####\nsite:\t%s\ndata:\t%s\nlogs:\t%s\nlocal_paths:\t%s\n\n", PATH_SITE, PATH_DATA, PATH_LOGS, PATH_LOCAL)
	if SSL {
		fmt.Printf("##### SSL enabled #####\n")
		fmt.Printf("##### SSL key:\t%s\n##### SSL cert:\t%s\n\n", SSL_KEY, SSL_CERT)
	} else {
		fmt.Printf("##### SSL disabled #####\n\n")
	}
	fmt.Printf("##### Mongodb #####\nhost(s):\t%s\ndatabase:\t%s\n\n", MONGODB_HOSTS, MONGODB_DATABASE)
	fmt.Printf("##### Address #####\nip:\t%s\nport:\t%d\n\n", API_IP, API_PORT)
	if LOG_PERF {
		fmt.Printf("##### PerfLog enabled #####\n\n")
	}
	if LOG_ROTATE {
		fmt.Printf("##### Log rotation enabled #####\n\n")
	} else {
		fmt.Printf("##### Log rotation disabled #####\n\n")
	}
       fmt.Printf("##### Expiration #####\nexpire_wait:\t%d minutes\n\n", EXPIRE_WAIT)
	fmt.Printf("##### Max Revisions #####\nmax_revisions:\t%d\n\n", MAX_REVISIONS)
	fmt.Printf("API_PORT: %d\n", API_PORT)
}

func getConfiguration(c *config.Config) (c_store *Config_store, err error) {
	c_store = NewCS(c)

	// Admin
	//ADMIN_EMAIL, _ = c.String("Admin", "email")
	c_store.AddString(&ADMIN_EMAIL, "", "Admin", "email", "", "")
	//ADMIN_USERS, _ = c.String("Admin", "users")
	c_store.AddString(&ADMIN_USERS, "", "Admin", "users", "", "")
	if ADMIN_USERS != "" {
		for _, name := range strings.Split(ADMIN_USERS, ",") {
			AdminUsers = append(AdminUsers, strings.TrimSpace(name))
		}
	}

	// Access-Control
	c_store.AddBool(&ANON_READ, true, "Anonymous", "read", "", "")
	c_store.AddBool(&ANON_WRITE, true, "Anonymous", "write", "", "")
	c_store.AddBool(&ANON_DELETE, true, "Anonymous", "delete", "", "")

	// Address
	c_store.AddString(&API_IP, "0.0.0.0", "Address", "api-ip", "", "")
	c_store.AddInt(&API_PORT, 7445, "Address", "api-port", "", "")

	// URLs
	c_store.AddString(&API_URL, "http://localhost", "External", "api-url", "", "")

	// Auth
	c_store.AddBool(&AUTH_BASIC, false, "Auth", "basic", "", "")
	c_store.AddString(&AUTH_GLOBUS_TOKEN_URL, "", "Auth", "globus_token_url", "", "")
	c_store.AddString(&AUTH_GLOBUS_PROFILE_URL, "", "Auth", "globus_profile_url", "", "")
	c_store.AddString(&AUTH_MGRAST_OAUTH_URL, "", "Auth", "mgrast_oauth_url", "", "")
	c_store.AddInt(&AUTH_CACHE_TIMEOUT, 60, "Auth", "cache_timeout", "", "")

	// Runtime
	c_store.AddInt(&EXPIRE_WAIT, 60, "Runtime", "expire_wait", "", "")
	c_store.AddString(&GOMAXPROCS, "", "Runtime", "GOMAXPROCS", "", "")
	c_store.AddInt(&MAX_REVISIONS, 3, "Runtime", "max_revisions", "", "")

	c_store.AddBool(&LOG_PERF, false, "Log", "perf_log", "", "")
	c_store.AddBool(&LOG_ROTATE, true, "Log", "rotate", "", "")

	// Mongodb
	c_store.AddString(&MONGODB_ATTRIBUTE_INDEXES, "", "Mongodb", "attribute_indexes", "", "")
	c_store.AddString(&MONGODB_DATABASE, "ShockDB", "Mongodb", "database", "", "")

	//MONGODB_HOSTS, _ = c.String("Mongodb", "hosts")
	//MONGODB_PASSWORD, _ = c.String("Mongodb", "password")
	//MONGODB_USER, _ = c.String("Mongodb", "user")
	c_store.AddString(&MONGODB_HOSTS, "mongo", "Mongodb", "hosts", "", "")
	c_store.AddString(&MONGODB_PASSWORD, "", "Mongodb", "password", "", "")
	c_store.AddString(&MONGODB_USER, "", "Mongodb", "user", "", "")

	// parse Node-Indices
	NODE_IDXS = map[string]idxOpts{}
	nodeIdx, _ := c.Options("Node-Indices")
	for _, opt := range nodeIdx {
		val, _ := c.String("Node-Indices", opt)
		opts := idxOpts{}
		for _, parts := range strings.Split(val, ",") {
			p := strings.Split(parts, ":")
			if p[0] == "unique" {
				if p[1] == "true" {
					opts.unique = true
				} else {
					opts.unique = false
				}
			} else if p[0] == "dropDups" {
				if p[1] == "true" {
					opts.dropDups = true
				} else {
					opts.dropDups = false
				}
			} else if p[0] == "sparse" {
				if p[1] == "true" {
					opts.sparse = true
				} else {
					opts.sparse = false
				}
			}
		}
		NODE_IDXS[opt] = opts
	}

	// Paths
	//PATH_SITE, _ = c.String("Paths", "site")
	//PATH_DATA, _ = c.String("Paths", "data")
	//PATH_LOGS, _ = c.String("Paths", "logs")
	//PATH_LOCAL, _ = c.String("Paths", "local_paths")
	//PATH_PIDFILE, _ = c.String("Paths", "pidfile")

	c_store.AddString(&PATH_SITE, "/usr/local/shock/site", "Paths", "site", "", "")
	c_store.AddString(&PATH_DATA, "/usr/local/shock", "Paths", "data", "", "")
	c_store.AddString(&PATH_LOGS, "/var/log/shock", "Paths", "logs", "", "")
	c_store.AddString(&PATH_LOCAL, "", "Paths", "local_paths", "", "")
	c_store.AddString(&PATH_PIDFILE, "", "Paths", "pidfile", "", "")

	// SSL
	//SSL, _ = c.Bool("SSL", "enable")
	c_store.AddBool(&SSL, false, "SSL", "enable", "", "")
	if SSL {
		//SSL_KEY, _ = c.String("SSL", "key")
		//SSL_CERT, _ = c.String("SSL", "cert")
		c_store.AddString(&SSL_KEY, "", "SSL", "key", "", "")
		c_store.AddString(&SSL_CERT, "", "SSL", "cert", "", "")
	}

	// Log
	c_store.AddString(&LOG_OUTPUT, "console", "Log", "logoutput", "console, file or both", "")

	//Other
	c_store.AddString(&RELOAD, "", "Other", "reload", "path or url to shock data. WARNING this will drop all current data.", "")
	gopath := os.Getenv("GOPATH")
	c_store.AddString(&CONFIG_FILE, gopath+"/src/github.com/MG-RAST/Shock/shock-server.conf.template", "Other", "conf", "path to config file", "")
	c_store.AddBool(&SHOW_VERSION, false, "Other", "version", "show version", "")
	c_store.AddBool(&PRINT_HELP, false, "Other", "fullhelp", "show detailed usage without \"--\"-prefixes", "")
	c_store.AddBool(&SHOW_HELP, false, "Other", "help", "show usage", "")

	VERSIONS["ACL"] = 2
	VERSIONS["Auth"] = 1
	VERSIONS["Node"] = 4

	c_store.Parse()

	return

}

<|endoftext|>


<commit_before>package httputil

import (
	"fmt"
	"koding/kites/common"
	"net"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/koding/logging"
)

var defaultLog = common.NewLogger("dialer", false)

type Dialer struct {
	*net.Dialer
	Log   logging.Logger
	Debug bool

	mu    sync.Mutex // protects conns
	once  sync.Once
	conns map[*Conn]struct{}
	tick  *time.Ticker
}

func NewDialer(cfg *ClientConfig) *Dialer {
	return &Dialer{
		Dialer: &net.Dialer{
			Timeout:   cfg.DialTimeout,
			KeepAlive: cfg.KeepAlive,
		},
		Log:   cfg.Log,
		Debug: cfg.DebugTCP,
	}
}

func (d *Dialer) Dial(network, addr string) (net.Conn, error) {
	if !d.Debug {
		return d.Dialer.Dial(network, addr)
	}

	return d.dial(network, addr)
}

func (d *Dialer) init() {
	d.conns = make(map[*Conn]struct{})
	d.tick = time.NewTicker(5 * time.Minute)
	go d.process()
}

func (d *Dialer) log() logging.Logger {
	if d.Log != nil {
		return d.Log
	}

	return defaultLog
}

func (d *Dialer) dial(network, addr string) (net.Conn, error) {
	d.once.Do(d.init)

	conn, err := d.Dialer.Dial(network, addr)
	if err != nil {
		return nil, err
	}

	c := &Conn{
		Connected:  time.Now(),
		Stacktrace: stacktrace(10),
		Conn:       conn,
	}

	c.close = func() {
		d.mu.Lock()
		delete(d.conns, c)
		d.mu.Unlock()

		d.log().Debug("connection closed: %s", c)
	}

	d.mu.Lock()
	d.conns[c] = struct{}{}
	d.mu.Unlock()

	return c, nil
}

func (d *Dialer) process() {
	d.log().Debug("starting processing goroutine")

	for range d.tick.C {
		d.mu.Lock()
		conns := make([]Conn, 0, len(d.conns)) // stores copies of all Conns
		for c := range d.conns {
			c.mu.Lock()
			conns = append(conns, *c) // shallow copy of Conn
			c.mu.Unlock()
		}
		d.mu.Unlock()

		now := time.Now()

		for i, c := range conns {
			c := &c

			d.log().Debug("(%d/%d) active connection: %s", i+1, len(conns), c)

			last := c.LastRead
			if c.LastWrite.After(last) {
				last = c.LastWrite
			}

			if dur := now.Sub(last); dur > 10*time.Minute {
				// To be accurate each HTTP client can hold multiple idle
				// conections per host (2 by default); if number of idle
				// connections per host greatly exceeds that number then
				// it may be a leak.
				d.log().Error("(%d/%d) possible leak, idle connection was active %s ago: %s", i+1, len(conns), dur, c)
			}

			if dur := now.Sub(c.Connected); dur > 15*time.Minute {
				d.log().Warning("(%d/%d) long-running connection %s: %s", i+1, len(conns), dur, c)
			}
		}
	}
}

type Conn struct {
	Connected  time.Time
	LastRead   time.Time
	LastWrite  time.Time
	Stacktrace []string

	mu    sync.Mutex // protects Last{Read,Write}
	close func()
	net.Conn
}

func (c *Conn) String() string {
	return fmt.Sprintf("%s->%s: Connected=%s, LastRead=%s, LastWrite=%s, Stacktrace=%v",
		c.Conn.LocalAddr(), c.Conn.RemoteAddr(), c.Connected, c.LastRead,
		c.LastWrite, c.Stacktrace)
}

func (c *Conn) Read(p []byte) (int, error) {
	c.mu.Lock()
	c.LastRead = time.Now()
	c.mu.Unlock()

	return c.Conn.Read(p)
}

func (c *Conn) Write(p []byte) (int, error) {
	c.mu.Lock()
	c.LastWrite = time.Now()
	c.mu.Unlock()

	return c.Conn.Write(p)
}

func (c *Conn) Close() error {
	c.close()

	return c.Conn.Close()
}

func stacktrace(max int) []string {
	pc, stack := make([]uintptr, max), make([]string, 0, max)
	runtime.Callers(2, pc)
	for _, pc := range pc {
		if f := runtime.FuncForPC(pc); f != nil {
			fname := f.Name()
			idx := strings.LastIndex(fname, string(os.PathSeparator))
			if idx != -1 {
				stack = append(stack, fname[idx+1:])
			} else {
				stack = append(stack, fname)
			}
		}
	}
	return stack
}
<commit_msg>httputil: make Dialer log bytes read/written<commit_after>package httputil

import (
	"fmt"
	"koding/kites/common"
	"net"
	"runtime/debug"
	"sync"
	"time"

	"github.com/koding/logging"
)

var defaultLog = common.NewLogger("dialer", false)

type Dialer struct {
	*net.Dialer
	Log   logging.Logger
	Debug bool

	mu    sync.Mutex // protects conns
	once  sync.Once
	conns map[*Conn]struct{}
	tick  *time.Ticker
}

func NewDialer(cfg *ClientConfig) *Dialer {
	return &Dialer{
		Dialer: &net.Dialer{
			Timeout:   cfg.DialTimeout,
			KeepAlive: cfg.KeepAlive,
		},
		Log:   cfg.Log,
		Debug: cfg.DebugTCP,
	}
}

func (d *Dialer) Dial(network, addr string) (net.Conn, error) {
	if !d.Debug {
		return d.Dialer.Dial(network, addr)
	}

	return d.dial(network, addr)
}

func (d *Dialer) init() {
	d.conns = make(map[*Conn]struct{})
	d.tick = time.NewTicker(5 * time.Minute)
	go d.process()
}

func (d *Dialer) log() logging.Logger {
	if d.Log != nil {
		return d.Log
	}

	return defaultLog
}

func (d *Dialer) dial(network, addr string) (net.Conn, error) {
	d.once.Do(d.init)

	conn, err := d.Dialer.Dial(network, addr)
	if err != nil {
		return nil, err
	}

	c := &Conn{
		Connected:  time.Now(),
		Stacktrace: string(debug.Stack()),
		Conn:       conn,
		network:    network,
		addr:       addr,
	}

	c.close = func() {
		d.mu.Lock()
		delete(d.conns, c)
		d.mu.Unlock()

		d.log().Debug("connection closed: %s", c.ShortString())
	}

	d.mu.Lock()
	d.conns[c] = struct{}{}
	d.mu.Unlock()

	return c, nil
}

func (d *Dialer) process() {
	d.log().Debug("starting processing goroutine (%p)", d)

	for range d.tick.C {
		d.mu.Lock()
		conns := make([]Conn, 0, len(d.conns)) // stores copies of all Conns
		for c := range d.conns {
			c.mu.Lock()
			conns = append(conns, *c) // shallow copy of Conn
			c.mu.Unlock()
		}
		d.mu.Unlock()

		now := time.Now()

		for i, c := range conns {
			c := &c

			d.log().Debug("(%d/%d) active connection: %s", i+1, len(conns), c.ShortString())

			if dur := c.Since(now); dur > 10*time.Minute {
				// To be accurate each HTTP client can hold multiple idle
				// conections per host (2 by default); if number of idle
				// connections per host greatly exceeds that number then
				// it may be a leak.
				d.log().Error("(%d/%d) possible leak, idle connection was active %s ago: %s", i+1, len(conns), dur, c)
			}

			if dur := now.Sub(c.Connected); dur > 15*time.Minute {
				d.log().Warning("(%d/%d) long-running connection %s: %s", i+1, len(conns), dur, c)
			}
		}
	}
}

type Conn struct {
	Connected    time.Time
	LastRead     time.Time
	LastWrite    time.Time
	BytesRead    int64
	BytesWritten int64
	Stacktrace   string

	network string
	addr    string
	mu      sync.Mutex // protects Last{Read,Write} and Bytes{Read,Written}
	close   func()
	net.Conn
}

func (c *Conn) Since(now time.Time) time.Duration {
	if now.IsZero() {
		now = time.Now()
	}

	last := c.LastRead
	if c.LastWrite.After(last) {
		last = c.LastWrite
	}

	return now.Sub(last)
}

func (c *Conn) ShortString() string {
	return fmt.Sprintf("%s->%s (%q, %q): BytesRead=%d, BytesWritten=%d, Duration=%s",
		c.Conn.LocalAddr(), c.Conn.RemoteAddr(), c.network, c.addr, c.BytesRead,
		c.BytesWritten, c.Since(time.Now()))
}

func (c *Conn) String() string {
	return fmt.Sprintf("%s->%s (%q, %q): Connected=%s, BytesRead=%d, LastRead=%s, BytesWritten=%d, LastWrite=%s, Stacktrace=%s",
		c.Conn.LocalAddr(), c.Conn.RemoteAddr(), c.network, c.addr, c.Connected, c.BytesRead, c.LastRead, c.BytesWritten,
		c.LastWrite, c.Stacktrace)
}

func (c *Conn) Read(p []byte) (int, error) {
	n, err := c.Conn.Read(p)

	c.mu.Lock()
	c.LastRead = time.Now()
	c.BytesRead += int64(n)
	c.mu.Unlock()

	return n, err
}

func (c *Conn) Write(p []byte) (int, error) {
	n, err := c.Conn.Write(p)

	c.mu.Lock()
	c.LastWrite = time.Now()
	c.BytesWritten += int64(n)
	c.mu.Unlock()

	return n, err
}

func (c *Conn) Close() error {
	c.close()

	return c.Conn.Close()
}
<|endoftext|>


<commit_before>package sequential

import (
	"math/rand"
)

func (model *AvigdorModel) initBoardStrain() {
	// init on the metal
	model.BoardStrain = make([][]int, model.Parameters.BoardSize)
	for row_i := range model.BoardStrain {
		model.BoardStrain[row_i] = make([]int, model.Parameters.BoardSize)
	}

	// init in the model
	for row_i := range model.BoardStrain {
		for col_i := range model.BoardStrain[row_i] {
			if rand.Float64() < model.Parameters.RInitOdds {
				model.BoardStrain[row_i][col_i] += 1
			}
			if rand.Float64() < model.Parameters.SInitOdds {
				model.BoardStrain[row_i][col_i] += 2
			}
			// In Avigdor's model, all public goods are wt.
			model.BoardStrain[row_i][col_i] += 4
		}
	}
}

func (model *AvigdorModel) initBoardProd() {
	// init on the metal
	model.BoardProd = make([][]bool, model.Parameters.BoardSize)
	for i0 := range model.BoardProd {
		model.BoardProd[i0] = make([]bool, model.Parameters.BoardSize)
	}

	// init in the model
	center_coord := Coordinate{}
	for center_coord.r = range model.BoardProd {
		for center_coord.c = range model.BoardProd[center_coord.r] {
			strain_at_center_coord := model.CellStrain(center_coord)
			receptor_allele_at_center_coord := r4strain[strain_at_center_coord]
			if model.CellSignalNum(center_coord, receptor_allele_at_center_coord) >= model.Parameters.SignalThreshold {
				model.SetCellProd(center_coord, 1 > 0)
			}
		}
	}
}

func (model *AvigdorModel) initBoardPGNum() {
	model.BoardPGNum = make([][]int, model.Parameters.BoardSize)
	for row_i := range model.BoardPGNum {
		model.BoardPGNum[row_i] = make([]int, model.Parameters.BoardSize)
	}

	center_coord := Coordinate{}
	for center_coord.r = range model.BoardPGNum {
		for center_coord.c = range model.BoardPGNum[center_coord.r] {
			rad_coord := Coordinate{}
			for rad_coord.r = center_coord.r - model.Parameters.PGRadius; rad_coord.r < center_coord.r+model.Parameters.PGRadius+1; rad_coord.r++ {
				for rad_coord.c = center_coord.c - model.Parameters.PGRadius; rad_coord.c < center_coord.c+model.Parameters.PGRadius+1; rad_coord.c++ {
					rad_coord_t := rad_coord.toroidCoordinates(model.Parameters.BoardSize)
					if model.CellProd(rad_coord_t) {
						model.AddToCellPGNum(center_coord, 1)
					}
				}
			}
		}
	}
}

func (model *AvigdorModel) initBoards() {
	model.initBoardStrain()
	model.InitBoardSignalNum()
	model.initBoardProd()
	model.initBoardPGNum()
}
<commit_msg>Capitalizing some method's name.<commit_after>package sequential

import (
	"math/rand"
)

func (model *AvigdorModel) initBoardStrain() {
	// init on the metal
	model.BoardStrain = make([][]int, model.Parameters.BoardSize)
	for row_i := range model.BoardStrain {
		model.BoardStrain[row_i] = make([]int, model.Parameters.BoardSize)
	}

	// init in the model
	for row_i := range model.BoardStrain {
		for col_i := range model.BoardStrain[row_i] {
			if rand.Float64() < model.Parameters.RInitOdds {
				model.BoardStrain[row_i][col_i] += 1
			}
			if rand.Float64() < model.Parameters.SInitOdds {
				model.BoardStrain[row_i][col_i] += 2
			}
			// In Avigdor's model, all public goods are wt.
			model.BoardStrain[row_i][col_i] += 4
		}
	}
}

func (model *AvigdorModel) initBoardProd() {
	// init on the metal
	model.BoardProd = make([][]bool, model.Parameters.BoardSize)
	for i0 := range model.BoardProd {
		model.BoardProd[i0] = make([]bool, model.Parameters.BoardSize)
	}

	// init in the model
	center_coord := Coordinate{}
	for center_coord.r = range model.BoardProd {
		for center_coord.c = range model.BoardProd[center_coord.r] {
			strain_at_center_coord := model.CellStrain(center_coord)
			receptor_allele_at_center_coord := r4strain[strain_at_center_coord]
			if model.CellSignalNum(center_coord, receptor_allele_at_center_coord) >= model.Parameters.SignalThreshold {
				model.SetCellProd(center_coord, 1 > 0)
			}
		}
	}
}

func (model *AvigdorModel) initBoardPGNum() {
	model.BoardPGNum = make([][]int, model.Parameters.BoardSize)
	for row_i := range model.BoardPGNum {
		model.BoardPGNum[row_i] = make([]int, model.Parameters.BoardSize)
	}

	center_coord := Coordinate{}
	for center_coord.r = range model.BoardPGNum {
		for center_coord.c = range model.BoardPGNum[center_coord.r] {
			rad_coord := Coordinate{}
			for rad_coord.r = center_coord.r - model.Parameters.PGRadius; rad_coord.r < center_coord.r+model.Parameters.PGRadius+1; rad_coord.r++ {
				for rad_coord.c = center_coord.c - model.Parameters.PGRadius; rad_coord.c < center_coord.c+model.Parameters.PGRadius+1; rad_coord.c++ {
					rad_coord_t := rad_coord.ToroidCoordinates(model.Parameters.BoardSize)
					if model.CellProd(rad_coord_t) {
						model.AddToCellPGNum(center_coord, 1)
					}
				}
			}
		}
	}
}

func (model *AvigdorModel) initBoards() {
	model.initBoardStrain()
	model.InitBoardSignalNum()
	model.initBoardProd()
	model.initBoardPGNum()
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
)

func resourceGoogleProjectIamMember() *schema.Resource {
	return &schema.Resource{
		Create: resourceGoogleProjectIamMemberCreate,
		Read:   resourceGoogleProjectIamMemberRead,
		Delete: resourceGoogleProjectIamMemberDelete,

		Schema: map[string]*schema.Schema{
			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"member": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"etag": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceGoogleProjectIamMemberCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Get the binding in the template
	log.Println("[DEBUG]: Reading google_project_iam_member")
	p := getResourceIamMember(d)
	mutexKV.Lock(projectIamMemberMutexKey(pid, p.Role, p.Members[0]))
	defer mutexKV.Unlock(projectIamMemberMutexKey(pid, p.Role, p.Members[0]))

	for {
		backoff := time.Second
		// Get the existing bindings
		log.Println("[DEBUG]: Retrieving policy for project", pid)
		ep, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return err
		}
		log.Printf("[DEBUG]: Retrieved policy for project %q: %+v\n", pid, ep)

		// find the binding
		var binding *cloudresourcemanager.Binding
		for _, b := range ep.Bindings {
			if b.Role != p.Role {
				continue
			}
			binding = b
			break
		}
		if binding == nil {
			binding = &cloudresourcemanager.Binding{
				Role:    p.Role,
				Members: p.Members,
			}
		}

		// Merge the bindings together
		ep.Bindings = mergeBindings(append(ep.Bindings, p))
		log.Printf("[DEBUG]: Setting policy for project %q to %+v\n", pid, ep)
		err = setProjectIamPolicy(ep, config, pid)
		if err != nil && isConflictError(err) {
			log.Printf("[DEBUG]: Concurrent policy changes, restarting read-modify-write after %s\n", backoff)
			time.Sleep(backoff)
			backoff = backoff * 2
			if backoff > 30*time.Second {
				return fmt.Errorf("Error applying IAM policy to project %q: too many concurrent policy changes.\n", pid)
			}
			continue
		} else if err != nil {
			return fmt.Errorf("Error applying IAM policy to project: %v", err)
		}
		break
	}
	log.Printf("[DEBUG]: Set policy for project %q", pid)
	d.SetId(pid + ":" + p.Role + ":" + p.Members[0])
	return resourceGoogleProjectIamMemberRead(d, meta)
}

func resourceGoogleProjectIamMemberRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid, err := getProject(d, config)
	if err != nil {
		return err
	}

	eMember := getResourceIamMember(d)

	log.Println("[DEBUG]: Retrieving policy for project", pid)
	p, err := getProjectIamPolicy(pid, config)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG]: Retrieved policy for project %q: %+v\n", pid, p)

	var binding *cloudresourcemanager.Binding
	for _, b := range p.Bindings {
		if b.Role != eMember.Role {
			continue
		}
		binding = b
		break
	}
	if binding == nil {
		d.SetId("")
		return nil
	}
	var member string
	for _, m := range binding.Members {
		if m == eMember.Members[0] {
			member = m
		}
	}
	d.Set("etag", p.Etag)
	d.Set("member", member)
	d.Set("role", binding.Role)
	return nil
}

func resourceGoogleProjectIamMemberDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid, err := getProject(d, config)
	if err != nil {
		return err
	}

	member := getResourceIamMember(d)
	mutexKV.Lock(projectIamMemberMutexKey(pid, member.Role, member.Members[0]))
	defer mutexKV.Unlock(projectIamMemberMutexKey(pid, member.Role, member.Members[0]))

	for {
		backoff := time.Second
		log.Println("[DEBUG]: Retrieving policy for project", pid)
		p, err := getProjectIamPolicy(pid, config)
		if err != nil {
			return err
		}
		log.Printf("[DEBUG]: Retrieved policy for project %q: %+v\n", pid, p)

		bindingToRemove := -1
		for pos, b := range p.Bindings {
			if b.Role != member.Role {
				continue
			}
			bindingToRemove = pos
			break
		}
		if bindingToRemove < 0 {
			return resourceGoogleProjectIamMemberRead(d, meta)
		}
		binding := p.Bindings[bindingToRemove]
		memberToRemove := -1
		for pos, m := range binding.Members {
			if m != member.Members[0] {
				continue
			}
			memberToRemove = pos
			break
		}
		if memberToRemove < 0 {
			return resourceGoogleProjectIamMemberRead(d, meta)
		}
		binding.Members = append(binding.Members[:memberToRemove], binding.Members[memberToRemove+1:]...)
		p.Bindings[bindingToRemove] = binding

		log.Printf("[DEBUG]: Setting policy for project %q to %+v\n", pid, p)
		err = setProjectIamPolicy(p, config, pid)
		if err != nil && isConflictError(err) {
			log.Printf("[DEBUG]: Concurrent policy changes, restarting read-modify-write after %s\n", backoff)
			time.Sleep(backoff)
			backoff = backoff * 2
			if backoff > 30*time.Second {
				return fmt.Errorf("Error applying IAM policy to project %q: too many concurrent policy changes.\n", pid)
			}
			continue
		} else if err != nil {
			return fmt.Errorf("Error applying IAM policy to project: %v", err)
		}
		break
	}
	log.Printf("[DEBUG]: Set policy for project %q\n", pid)

	return resourceGoogleProjectIamMemberRead(d, meta)
}

// Get a cloudresourcemanager.Binding from a schema.ResourceData
func getResourceIamMember(d *schema.ResourceData) *cloudresourcemanager.Binding {
	return &cloudresourcemanager.Binding{
		Members: []string{d.Get("member").(string)},
		Role:    d.Get("role").(string),
	}
}

func projectIamMemberMutexKey(pid, role, member string) string {
	return fmt.Sprintf("google-project-iam-member-%s-%s-%s", pid, role, member)
}
<commit_msg>Use the policy r/m/w helper and handle edge case.<commit_after>package google

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
)

func resourceGoogleProjectIamMember() *schema.Resource {
	return &schema.Resource{
		Create: resourceGoogleProjectIamMemberCreate,
		Read:   resourceGoogleProjectIamMemberRead,
		Delete: resourceGoogleProjectIamMemberDelete,

		Schema: map[string]*schema.Schema{
			"project": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},
			"role": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"member": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"etag": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceGoogleProjectIamMemberCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid, err := getProject(d, config)
	if err != nil {
		return err
	}

	// Get the binding in the template
	log.Println("[DEBUG]: Reading google_project_iam_member")
	p := getResourceIamMember(d)
	mutexKV.Lock(projectIamMemberMutexKey(pid, p.Role, p.Members[0]))
	defer mutexKV.Unlock(projectIamMemberMutexKey(pid, p.Role, p.Members[0]))

	err = projectIamPolicyReadModifyWrite(d, config, pid, func(ep *cloudresourcemanager.Policy) error {
		// find the binding
		var binding *cloudresourcemanager.Binding
		for _, b := range ep.Bindings {
			if b.Role != p.Role {
				continue
			}
			binding = b
			break
		}
		if binding == nil {
			binding = &cloudresourcemanager.Binding{
				Role:    p.Role,
				Members: p.Members,
			}
		}

		// Merge the bindings together
		ep.Bindings = mergeBindings(append(ep.Bindings, p))
		return nil
	})
	if err != nil {
		return err
	}
	d.SetId(pid + ":" + p.Role + ":" + p.Members[0])
	return resourceGoogleProjectIamMemberRead(d, meta)
}

func resourceGoogleProjectIamMemberRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid, err := getProject(d, config)
	if err != nil {
		return err
	}

	eMember := getResourceIamMember(d)

	log.Println("[DEBUG]: Retrieving policy for project", pid)
	p, err := getProjectIamPolicy(pid, config)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG]: Retrieved policy for project %q: %+v\n", pid, p)

	var binding *cloudresourcemanager.Binding
	for _, b := range p.Bindings {
		if b.Role != eMember.Role {
			continue
		}
		binding = b
		break
	}
	if binding == nil {
		d.SetId("")
		return nil
	}
	var member string
	for _, m := range binding.Members {
		if m == eMember.Members[0] {
			member = m
		}
	}
	if member == "" {
		d.SetId("")
		return nil
	}
	d.Set("etag", p.Etag)
	d.Set("member", member)
	d.Set("role", binding.Role)
	return nil
}

func resourceGoogleProjectIamMemberDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid, err := getProject(d, config)
	if err != nil {
		return err
	}

	member := getResourceIamMember(d)
	mutexKV.Lock(projectIamMemberMutexKey(pid, member.Role, member.Members[0]))
	defer mutexKV.Unlock(projectIamMemberMutexKey(pid, member.Role, member.Members[0]))

	err = projectIamPolicyReadModifyWrite(d, config, pid, func(p *cloudresourcemanager.Policy) error {
		bindingToRemove := -1
		for pos, b := range p.Bindings {
			if b.Role != member.Role {
				continue
			}
			bindingToRemove = pos
			break
		}
		if bindingToRemove < 0 {
			return nil
		}
		binding := p.Bindings[bindingToRemove]
		memberToRemove := -1
		for pos, m := range binding.Members {
			if m != member.Members[0] {
				continue
			}
			memberToRemove = pos
			break
		}
		if memberToRemove < 0 {
			return nil
		}
		binding.Members = append(binding.Members[:memberToRemove], binding.Members[memberToRemove+1:]...)
		p.Bindings[bindingToRemove] = binding
		return nil
	})
	if err != nil {
		return err
	}

	return resourceGoogleProjectIamMemberRead(d, meta)
}

// Get a cloudresourcemanager.Binding from a schema.ResourceData
func getResourceIamMember(d *schema.ResourceData) *cloudresourcemanager.Binding {
	return &cloudresourcemanager.Binding{
		Members: []string{d.Get("member").(string)},
		Role:    d.Get("role").(string),
	}
}

func projectIamMemberMutexKey(pid, role, member string) string {
	return fmt.Sprintf("google-project-iam-member-%s-%s-%s", pid, role, member)
}
<|endoftext|>


<commit_before>package stats

import (
	"regexp"
	"strings"
)

var wordRegex = regexp.MustCompile("[ \t\r\n\v\f]+")
var wordChars = " \t\r\n\v\f.?!"

var sentenceRegex = regexp.MustCompile("[ \t\r\n\v\f]*[.?!]+[ \t\r\n\v\f]*")

var endingRegex = regexp.MustCompile("([^laeiouy]es|ed|[^laeoiuy]e)$")
var staringYRegex = regexp.MustCompile("^y")
var vowelRegex = regexp.MustCompile("[aeiouy]{1,2}")

// Words returns a slice of the words in a given string.
func Words(text string) []string {
	return wordRegex.Split(text, -1)
}

// Sentences returns a slice of the sentences in a given string.
func Sentences(text string) []string {
	sentences := sentenceRegex.Split(text, -1)

	if sentences[len(sentences)-1] == "" {
		return sentences[0 : len(sentences)-1]
	}

	return sentences
}

// SyllableCount returns the number of syllables in a given string.
// Dipthongs are not taken into account.
func SyllableCount(word string) int {
	if len(word) <= 3 {
		return 1
	}

	word = endingRegex.ReplaceAllString(word, "")
	word = staringYRegex.ReplaceAllString(word, "")

	return len(vowelRegex.FindAllString(word, -1))
}

// FleschKincaidEase computes the ease of reading a given text.
// The algorithm is explained in detail here:
// http://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests
func FleschKincaidEase(text string) float64 {
	words := Words(text)
	numWords := float64(len(words))
	numSentences := float64(len(Sentences(text)))
	numSyllables := 0

	for _, word := range words {
		numSyllables += SyllableCount(word)
	}

	return 206.835 - 1.015*(numWords/numSentences) - 84.6*(float64(numSyllables)/numWords)

}

func cleanWord(word string) string {
	return strings.Trim(word, wordChars)
}
<commit_msg>fixes stats.Words()<commit_after>package stats

import (
	"regexp"
)

var wordRegex = regexp.MustCompile("[A-Za-z0-9_']+")

var sentenceRegex = regexp.MustCompile("[ \t\r\n\v\f]*[.?!]+[ \t\r\n\v\f]*")

var endingRegex = regexp.MustCompile("([^laeiouy]es|ed|[^laeoiuy]e)$")
var staringYRegex = regexp.MustCompile("^y")
var vowelRegex = regexp.MustCompile("[aeiouy]{1,2}")

// Words returns a slice of the words in a given string.
func Words(text string) []string {
	return wordRegex.FindAllString(text, -1)
}

// Sentences returns a slice of the sentences in a given string.
func Sentences(text string) []string {
	sentences := sentenceRegex.Split(text, -1)

	if sentences[len(sentences)-1] == "" {
		return sentences[0 : len(sentences)-1]
	}

	return sentences
}

// SyllableCount returns the number of syllables in a given string.
// Dipthongs are not taken into account.
func SyllableCount(word string) int {
	if len(word) <= 3 {
		return 1
	}

	word = endingRegex.ReplaceAllString(word, "")
	word = staringYRegex.ReplaceAllString(word, "")

	return len(vowelRegex.FindAllString(word, -1))
}

// FleschKincaidEase computes the ease of reading a given text.
// The algorithm is explained in detail here:
// http://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests
func FleschKincaidEase(text string) float64 {
	words := Words(text)
	numWords := float64(len(words))
	numSentences := float64(len(Sentences(text)))
	numSyllables := 0

	for _, word := range words {
		numSyllables += SyllableCount(word)
	}

	return 206.835 - 1.015*(numWords/numSentences) - 84.6*(float64(numSyllables)/numWords)

}
<|endoftext|>


<commit_before>package streamtools

import (
	"github.com/bitly/go-simplejson"
	"github.com/bitly/go-nsq"
	"log"
)

var (
	lookupdHTTPAddrs = "127.0.0.1:4161"
	nsqdHTTPAddrs    = "127.0.0.1:4150"
	nsqdTCPAddrs     = "127.0.0.1:4150"
)

type SyncHandler struct {
	msgChan chan *simplejson.Json
}

func (self *SyncHandler) HandleMessage(m *nsq.Message) error {
	blob, err := simplejson.NewJson(m.Body)
	if err != nil {
		log.Fatal(err.Error())
	}
	self.msgChan <- blob
	return nil
}

func nsqReader(topic string, channel string, writeChan chan *simplejson.Json) {
	r, err := nsq.NewReader(topic, channel)
	if err != nil {
		log.Fatal(err.Error())
	}
	sh := SyncHandler{
		msgChan: writeChan,
	}
	r.AddHandler(&sh)
	_ = r.ConnectToLookupd(lookupdHTTPAddrs)
	<-r.ExitChan
}

func nsqWriter(topic string, readChan chan *simplejson.Json) {

	w := nsq.NewWriter(0)
	err := w.ConnectToNSQ(nsqdHTTPAddrs)
	if err != nil {
		log.Fatal(err.Error())
	}
	for {
		select {
		case msg := <-readChan:
			outMsg, _ := msg.Encode()
			frameType, data, err := w.Publish(topic, outMsg)
			if err != nil {
				log.Fatalf("frametype %d data %s error %s", frameType, string(data), err.Error())
			}
		}
	}
}

func deMuxWriter(readChan chan *simplejson.Json) {
	w := nsq.NewWriter(0)
	err := w.ConnectToNSQ(nsqdHTTPAddrs)
	if err != nil {
		log.Fatal(err.Error())
	}
	for {
		select {
		case msg := <-readChan:
			log.Println(msg)
			topic, err := msg.Get("_StreamtoolsTopic").String()
			if err != nil {
				log.Fatal(err.Error())
			}
			origMsg := msg.Get("_StreamtoolsData")
			log.Println("origMsg:", origMsg)
			outMsg, err := origMsg.Encode()
			if err != nil {
				log.Fatal(err.Error())
			}
			log.Println("outMsg:", string(outMsg))
			frameType, data, err := w.Publish(topic, outMsg)
			if err != nil {
				log.Fatalf("frametype %d data %s error %s", frameType, string(data), err.Error())
			}
		}
	}
}
<commit_msg>updated writer code to use new writer API<commit_after>package streamtools

import (
	"github.com/bitly/go-nsq"
	"github.com/bitly/go-simplejson"
	"log"
)

var (
	lookupdHTTPAddrs = "127.0.0.1:4161"
	nsqdHTTPAddrs    = "127.0.0.1:4150"
	nsqdTCPAddrs     = "127.0.0.1:4150"
)

type SyncHandler struct {
	msgChan chan *simplejson.Json
}

func (self *SyncHandler) HandleMessage(m *nsq.Message) error {
	blob, err := simplejson.NewJson(m.Body)
	if err != nil {
		log.Fatal(err.Error())
	}
	self.msgChan <- blob
	return nil
}

func nsqReader(topic string, channel string, writeChan chan *simplejson.Json) {
	r, err := nsq.NewReader(topic, channel)
	if err != nil {
		log.Fatal(err.Error())
	}
	sh := SyncHandler{
		msgChan: writeChan,
	}
	r.AddHandler(&sh)
	_ = r.ConnectToLookupd(lookupdHTTPAddrs)
	<-r.ExitChan
}

func nsqWriter(topic string, readChan chan *simplejson.Json) {

	w := nsq.NewWriter(nsqdHTTPAddrs)
	if err != nil {
		log.Fatal(err.Error())
	}
	for {
		select {
		case msg := <-readChan:
			outMsg, _ := msg.Encode()
			frameType, data, err := w.Publish(topic, outMsg)
			if err != nil {
				log.Fatalf("frametype %d data %s error %s", frameType, string(data), err.Error())
			}
		}
	}
}

func deMuxWriter(readChan chan *simplejson.Json) {
	w := nsq.NewWriter(0)
	err := w.ConnectToNSQ(nsqdHTTPAddrs)
	if err != nil {
		log.Fatal(err.Error())
	}
	for {
		select {
		case msg := <-readChan:
			log.Println(msg)
			topic, err := msg.Get("_StreamtoolsTopic").String()
			if err != nil {
				log.Fatal(err.Error())
			}
			origMsg := msg.Get("_StreamtoolsData")
			log.Println("origMsg:", origMsg)
			outMsg, err := origMsg.Encode()
			if err != nil {
				log.Fatal(err.Error())
			}
			log.Println("outMsg:", string(outMsg))
			frameType, data, err := w.Publish(topic, outMsg)
			if err != nil {
				log.Fatalf("frametype %d data %s error %s", frameType, string(data), err.Error())
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"errors"
	"math/rand"
	"sort"
)

var allZones = []string{
	"ap-northeast-1a",
	"ap-northeast-1c",
	"ap-northeast-1d",
	"ap-northeast-2a",
	//"ap-northeast-2b" - AZ does not exist, so we're breaking the 3 AZs per region target here
	"ap-northeast-2c",
	"ap-northeast-2d",
	//"ap-northeast-3a", - Disabled until etcd-manager supports the region and the AMIs used in testing are present
	//"ap-northeast-3b",
	//"ap-northeast-3c",
	"ap-south-1a",
	"ap-south-1b",
	"ap-south-1c",
	"ap-southeast-1a",
	"ap-southeast-1b",
	"ap-southeast-1c",
	"ap-southeast-2a",
	"ap-southeast-2b",
	"ap-southeast-2c",
	"ca-central-1a",
	"ca-central-1b",
	"ca-central-1d",
	"eu-central-1a",
	"eu-central-1b",
	"eu-central-1c",
	"eu-north-1a",
	"eu-north-1b",
	"eu-north-1c",
	"eu-west-1a",
	"eu-west-1b",
	"eu-west-1c",
	"eu-west-2a",
	"eu-west-2b",
	"eu-west-2c",
	"eu-west-3a",
	"eu-west-3b",
	"eu-west-3c",
	"sa-east-1a",
	"sa-east-1b",
	"sa-east-1c",
	"us-east-1a",
	"us-east-1b",
	"us-east-1c",
	"us-east-1d",
	"us-east-1e",
	"us-east-1f",
	"us-east-2a",
	"us-east-2b",
	"us-east-2c",
	"us-west-1a",
	"us-west-1b",
	//"us-west-1c", AZ does not exist, so we're breaking the 3 AZs per region target here
	"us-west-2a",
	"us-west-2b",
	"us-west-2c",
	"us-west-2d",
}

// ErrNoEligibleRegion indicates the requested number of zones is not available in any region
var ErrNoEligibleRegion = errors.New("No eligible AWS region found with enough zones")

// RandomZones returns a random set of availability zones within a region
func RandomZones(count int) ([]string, error) {
	regions := make(map[string][]string)
	for _, zone := range allZones {
		region := zone[:len(zone)-1]
		regions[region] = append(regions[region], zone)
	}
	eligibleRegions := make([][]string, 0)
	for _, zones := range regions {
		if len(zones) >= count {
			eligibleRegions = append(eligibleRegions, zones)
		}
	}
	if len(eligibleRegions) == 0 {
		return nil, ErrNoEligibleRegion
	}
	chosenRegion := eligibleRegions[rand.Int()%len(eligibleRegions)]

	chosenZones := make([]string, 0)
	randIndexes := rand.Perm(len(chosenRegion))
	for i := 0; i < count; i++ {
		chosenZones = append(chosenZones, chosenRegion[randIndexes[i]])
	}
	sort.Strings(chosenZones)
	return chosenZones, nil
}
<commit_msg>Run tests only on accounts with increased limits<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"errors"
	"math/rand"
	"sort"
)

var allZones = []string{
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"ap-northeast-1a",
	//"ap-northeast-1c",
	//"ap-northeast-1d",
	// AZ does not exist, so we're breaking the 3 AZs per region target here
	//"ap-northeast-2b",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"ap-northeast-2a",
	//"ap-northeast-2c",
	//"ap-northeast-2d",
	// Disabled until etcd-manager supports the region and the AMIs used in testing are present
	//"ap-northeast-3a",
	//"ap-northeast-3b",
	//"ap-northeast-3c",
	"ap-south-1a",
	"ap-south-1b",
	"ap-south-1c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"ap-southeast-1a",
	//"ap-southeast-1b",
	//"ap-southeast-1c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"ap-southeast-2a",
	//"ap-southeast-2b",
	//"ap-southeast-2c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"ca-central-1a",
	//"ca-central-1b",
	//"ca-central-1d",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"eu-central-1a",
	//"eu-central-1b",
	//"eu-central-1c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"eu-north-1a",
	//"eu-north-1b",
	//"eu-north-1c",
	"eu-west-1a",
	"eu-west-1b",
	"eu-west-1c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"eu-west-2a",
	//"eu-west-2b",
	//"eu-west-2c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"eu-west-3a",
	//"eu-west-3b",
	//"eu-west-3c",
	// Disabled until region limits are increased https://github.com/kubernetes/k8s.io/issues/1921
	//"sa-east-1a",
	//"sa-east-1b",
	//"sa-east-1c",
	"us-east-1a",
	"us-east-1b",
	"us-east-1c",
	"us-east-1d",
	"us-east-1e",
	"us-east-1f",
	"us-east-2a",
	"us-east-2b",
	"us-east-2c",
	"us-west-1a",
	"us-west-1b",
	//"us-west-1c", AZ does not exist, so we're breaking the 3 AZs per region target here
	"us-west-2a",
	"us-west-2b",
	"us-west-2c",
	"us-west-2d",
}

// ErrNoEligibleRegion indicates the requested number of zones is not available in any region
var ErrNoEligibleRegion = errors.New("No eligible AWS region found with enough zones")

// RandomZones returns a random set of availability zones within a region
func RandomZones(count int) ([]string, error) {
	regions := make(map[string][]string)
	for _, zone := range allZones {
		region := zone[:len(zone)-1]
		regions[region] = append(regions[region], zone)
	}
	eligibleRegions := make([][]string, 0)
	for _, zones := range regions {
		if len(zones) >= count {
			eligibleRegions = append(eligibleRegions, zones)
		}
	}
	if len(eligibleRegions) == 0 {
		return nil, ErrNoEligibleRegion
	}
	chosenRegion := eligibleRegions[rand.Int()%len(eligibleRegions)]

	chosenZones := make([]string, 0)
	randIndexes := rand.Perm(len(chosenRegion))
	for i := 0; i < count; i++ {
		chosenZones = append(chosenZones, chosenRegion[randIndexes[i]])
	}
	sort.Strings(chosenZones)
	return chosenZones, nil
}
<|endoftext|>


<commit_before>package netscaler

import (
	"github.com/chiradeep/go-nitro/config/cs"
	"github.com/chiradeep/go-nitro/netscaler"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"fmt"
	"log"
)

func resourceNetScalerCspolicy() *schema.Resource {
	return &schema.Resource{
		SchemaVersion: 1,
		Create:        createCspolicyFunc,
		Read:          readCspolicyFunc,
		Update:        updateCspolicyFunc,
		Delete:        deleteCspolicyFunc,
		Schema: map[string]*schema.Schema{
			"action": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"domain": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"logaction": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"newname": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"policyname": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"rule": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"url": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"csvserver": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"targetlbvserver": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"priority": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
			},
		},
	}
}

func createCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In createCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client

	csvserver := d.Get("csvserver").(string)
	targetlbvserver, lbok := d.GetOk("targetlbvserver")
	priority, pok := d.GetOk("priority")
	action, aok := d.GetOk("action")
	_, dok := d.GetOk("domain")
	_, uok := d.GetOk("url")
	_, rok := d.GetOk("rule")

	if lbok && rok && !pok {
		return fmt.Errorf("[ERROR] netscaler-provider: Priority needs to be specified if target lb vserver and rule is specified")
	}
	if !lbok && pok {
		return fmt.Errorf("[ERROR] netscaler-provider: Priority needs to be specified if target lb vserver is not specified")
	}
	if !lbok && !aok {
		return fmt.Errorf("[ERROR] netscaler-provider: Action or targetlbvserver needs to be specified")
	}
	if aok {
		actionExists := client.ResourceExists(netscaler.Csaction.Type(), action.(string))
		if !actionExists {
			return fmt.Errorf("[ERROR] netscaler-provider: Specified Action %s does not exist", action.(string))
		}
		if !rok {
			return fmt.Errorf("[ERROR] netscaler-provider: Action  %s specified without rule", action.(string))
		}
		if dok || uok {
			return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify url or domain when action  %s is specified", action.(string))
		}
	}
	if uok && dok {
		return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify both url and domain ")
	}
	if rok && (uok || dok) {
		return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify both rule and domain or url ")
	}
	if (uok || dok) && pok {
		return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify both priority and domain or url ")
	}

	var cspolicyName string
	if v, ok := d.GetOk("policyname"); ok {
		cspolicyName = v.(string)
	} else {
		cspolicyName = resource.PrefixedUniqueId("tf-cspolicy-")
		d.Set("name", cspolicyName)
	}
	cspolicy := cs.Cspolicy{
		Policyname: d.Get("policyname").(string),
		Action:     d.Get("action").(string),
		Domain:     d.Get("domain").(string),
		Logaction:  d.Get("logaction").(string),
		Rule:       d.Get("rule").(string),
		Url:        d.Get("url").(string),
	}

	_, err := client.AddResource(netscaler.Cspolicy.Type(), cspolicyName, &cspolicy)
	if err != nil {
		return err
	}

	binding := cs.Csvservercspolicybinding{
		Name:            csvserver,
		Policyname:      cspolicyName,
		Targetlbvserver: targetlbvserver.(string),
		Priority:        priority.(int),
	}

	if !lbok {
		binding = cs.Csvservercspolicybinding{
			Name:       csvserver,
			Policyname: cspolicyName,
		}
	}

	err = client.BindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, &binding)
	if err != nil {
		d.SetId("")
		err2 := client.DeleteResource(netscaler.Cspolicy.Type(), cspolicyName)
		if err2 != nil {
			return fmt.Errorf("[ERROR] netscaler-provider:  Failed to undo add cspolicy after bind cspolicy %s to Csvserver failed", cspolicyName, err2)
		}
		return fmt.Errorf("[ERROR] netscaler-provider:  Failed to bind cspolicy %s to Csvserver", cspolicyName, err)
	}
	d.SetId(cspolicyName)
	err = readCspolicyFunc(d, meta)
	if err != nil {
		log.Printf("[ERROR] netscaler-provider:  ?? we just created this cspolicy but we can't read it ?? %s", cspolicyName)
		return nil
	}
	return nil
}

func readCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In readCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client
	cspolicyName := d.Id()
	log.Printf("[DEBUG] netscaler-provider:  Reading cspolicy state %s", cspolicyName)
	data, err := client.FindResource(netscaler.Cspolicy.Type(), cspolicyName)
	if err != nil {
		log.Printf("[WARN] netscaler-provider:  Clearing cspolicy state %s", cspolicyName)
		d.SetId("")
		return nil
	}
	d.Set("name", data["name"])
	d.Set("action", data["action"])
	d.Set("domain", data["domain"])
	d.Set("logaction", data["logaction"])
	d.Set("rule", data["rule"])
	d.Set("url", data["url"])

	return nil

}

func updateCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In updateCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client
	cspolicyName := d.Get("policyname").(string)
	csvserver := d.Get("csvserver").(string)

	cspolicy := cs.Cspolicy{
		Policyname: d.Get("policyname").(string),
	}
	hasChange := false
	lbvserverChanged := false
	priorityChanged := false

	if d.HasChange("action") {
		log.Printf("[DEBUG] netscaler-provider: Action has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Action = d.Get("action").(string)
		hasChange = true
	}
	if d.HasChange("domain") {
		log.Printf("[DEBUG] netscaler-provider: Domain has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Domain = d.Get("domain").(string)
		hasChange = true
	}
	if d.HasChange("logaction") {
		log.Printf("[DEBUG] netscaler-provider: Logaction has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Logaction = d.Get("logaction").(string)
		hasChange = true
	}
	if d.HasChange("newname") {
		log.Printf("[DEBUG] netscaler-provider: Newname has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Newname = d.Get("newname").(string)
		hasChange = true
	}
	if d.HasChange("rule") {
		log.Printf("[DEBUG] netscaler-provider: Rule has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Rule = d.Get("rule").(string)
		hasChange = true
	}
	if d.HasChange("url") {
		log.Printf("[DEBUG] netscaler-provider: Url has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Url = d.Get("url").(string)
		hasChange = true
	}
	if d.HasChange("priority") {
		log.Printf("[DEBUG] netscaler-provider: Priority has changed for cspolicy %s, starting update", cspolicyName)
		priorityChanged = true
	}

	if d.HasChange("targetlbvserver") {
		log.Printf("[DEBUG] netscaler-provider: targetlbvserver has changed for cspolicy %s, starting update", cspolicyName)
		lbvserverChanged = true
	}

	if lbvserverChanged || priorityChanged {
		//Binding has to be updated
		//First we unbind from cs vserver
		err := client.UnbindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, "policyname")
		if err != nil {
			return fmt.Errorf("[ERROR] netscaler-provider: Error unbinding cspolicy from csvserver %s", cspolicyName)
		}
		log.Printf("[DEBUG] netscaler-provider: cspolicy has been unbound from csvserver for cspolicy %s ", cspolicyName)
	}

	if hasChange {
		_, err := client.UpdateResource(netscaler.Cspolicy.Type(), cspolicyName, &cspolicy)
		if err != nil {
			return fmt.Errorf("[ERROR] netscaler-provider: Error updating cspolicy %s", cspolicyName)
		}
		log.Printf("[DEBUG] netscaler-provider: cspolicy has been updated  cspolicy %s ", cspolicyName)
	}

	if lbvserverChanged || priorityChanged {
		//Binding has to be updated
		//rebind
		targetlbvserver, lbok := d.GetOk("targetlbvserver")
		priority, pok := d.GetOk("priority")

		if !pok && lbok {
			return fmt.Errorf("[ERROR] netscaler-provider: Need to specify priority if lbvserver is specified")
		}

		binding := cs.Csvservercspolicybinding{
			Name:            csvserver,
			Policyname:      cspolicyName,
			Targetlbvserver: targetlbvserver.(string),
			Priority:        priority.(int),
		}
		err := client.BindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, &binding)
		if err != nil {
			return fmt.Errorf("[ERROR] netscaler-provider: Failed to bind new cspolicy to Csvserver")
		}
		log.Printf("[DEBUG] netscaler-provider: cspolicy has been bound to csvserver  cspolicy %s csvserver %s", cspolicyName, csvserver)
	}

	return readCspolicyFunc(d, meta)
}

func deleteCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In deleteCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client
	cspolicyName := d.Id()
	csvserver := d.Get("csvserver").(string)

	//First we unbind from cs vserver if necessary
	err := client.UnbindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, "policyname")
	if err != nil {
		return fmt.Errorf("[ERROR] netscaler-provider: Error unbinding cspolicy from csvserver %s", cspolicyName)
	}
	err = client.DeleteResource(netscaler.Cspolicy.Type(), cspolicyName)
	if err != nil {
		return fmt.Errorf("[ERROR] netscaler-provider: Error  deleting cspolicy %s", cspolicyName)
	}

	d.SetId("")

	return nil
}
<commit_msg>Read in csvserver binding during refresh<commit_after>package netscaler

import (
	"github.com/chiradeep/go-nitro/config/cs"
	"github.com/chiradeep/go-nitro/netscaler"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"fmt"
	"log"
)

func resourceNetScalerCspolicy() *schema.Resource {
	return &schema.Resource{
		SchemaVersion: 1,
		Create:        createCspolicyFunc,
		Read:          readCspolicyFunc,
		Update:        updateCspolicyFunc,
		Delete:        deleteCspolicyFunc,
		Schema: map[string]*schema.Schema{
			"action": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"domain": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"logaction": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"newname": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"policyname": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"rule": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"url": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},

			"csvserver": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"targetlbvserver": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
			},
			"priority": &schema.Schema{
				Type:     schema.TypeInt,
				Optional: true,
			},
		},
	}
}

func createCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In createCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client

	csvserver := d.Get("csvserver").(string)
	targetlbvserver, lbok := d.GetOk("targetlbvserver")
	priority, pok := d.GetOk("priority")
	action, aok := d.GetOk("action")
	_, dok := d.GetOk("domain")
	_, uok := d.GetOk("url")
	_, rok := d.GetOk("rule")

	if lbok && rok && !pok {
		return fmt.Errorf("[ERROR] netscaler-provider: Priority needs to be specified if target lb vserver and rule is specified")
	}
	if !lbok && pok {
		return fmt.Errorf("[ERROR] netscaler-provider: Priority needs to be specified if target lb vserver is not specified")
	}
	if !lbok && !aok {
		return fmt.Errorf("[ERROR] netscaler-provider: Action or targetlbvserver needs to be specified")
	}
	if aok {
		actionExists := client.ResourceExists(netscaler.Csaction.Type(), action.(string))
		if !actionExists {
			return fmt.Errorf("[ERROR] netscaler-provider: Specified Action %s does not exist", action.(string))
		}
		if !rok {
			return fmt.Errorf("[ERROR] netscaler-provider: Action  %s specified without rule", action.(string))
		}
		if dok || uok {
			return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify url or domain when action  %s is specified", action.(string))
		}
	}
	if uok && dok {
		return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify both url and domain ")
	}
	if rok && (uok || dok) {
		return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify both rule and domain or url ")
	}
	if (uok || dok) && pok {
		return fmt.Errorf("[ERROR] netscaler-provider: Cannot specify both priority and domain or url ")
	}

	var cspolicyName string
	if v, ok := d.GetOk("policyname"); ok {
		cspolicyName = v.(string)
	} else {
		cspolicyName = resource.PrefixedUniqueId("tf-cspolicy-")
		d.Set("name", cspolicyName)
	}
	cspolicy := cs.Cspolicy{
		Policyname: d.Get("policyname").(string),
		Action:     d.Get("action").(string),
		Domain:     d.Get("domain").(string),
		Logaction:  d.Get("logaction").(string),
		Rule:       d.Get("rule").(string),
		Url:        d.Get("url").(string),
	}

	_, err := client.AddResource(netscaler.Cspolicy.Type(), cspolicyName, &cspolicy)
	if err != nil {
		return err
	}

	binding := cs.Csvservercspolicybinding{
		Name:            csvserver,
		Policyname:      cspolicyName,
		Targetlbvserver: targetlbvserver.(string),
		Priority:        priority.(int),
	}

	if !lbok {
		binding = cs.Csvservercspolicybinding{
			Name:       csvserver,
			Policyname: cspolicyName,
		}
	}

	err = client.BindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, &binding)
	if err != nil {
		d.SetId("")
		err2 := client.DeleteResource(netscaler.Cspolicy.Type(), cspolicyName)
		if err2 != nil {
			return fmt.Errorf("[ERROR] netscaler-provider:  Failed to undo add cspolicy after bind cspolicy %s to Csvserver failed", cspolicyName, err2)
		}
		return fmt.Errorf("[ERROR] netscaler-provider:  Failed to bind cspolicy %s to Csvserver", cspolicyName, err)
	}
	d.SetId(cspolicyName)
	err = readCspolicyFunc(d, meta)
	if err != nil {
		log.Printf("[ERROR] netscaler-provider:  ?? we just created this cspolicy but we can't read it ?? %s", cspolicyName)
		return nil
	}
	return nil
}

func readCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In readCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client
	cspolicyName := d.Id()
	log.Printf("[DEBUG] netscaler-provider:  Reading cspolicy state %s", cspolicyName)
	data, err := client.FindResource(netscaler.Cspolicy.Type(), cspolicyName)
	if err != nil {
		log.Printf("[WARN] netscaler-provider:  Clearing cspolicy state %s", cspolicyName)
		d.SetId("")
		return nil
	}
	d.Set("name", data["name"])
	d.Set("action", data["action"])
	d.Set("domain", data["domain"])
	d.Set("logaction", data["logaction"])
	d.Set("rule", data["rule"])
	d.Set("url", data["url"])

	//read the csvserver binding and update
	bindings, err := client.FindAllBoundResources(netscaler.Cspolicy.Type(), cspolicyName, netscaler.Csvserver.Type())
	if err != nil {
		log.Printf("[WARN] netscaler-provider: cspolicy binding to csvserver error %s", cspolicyName)
		return nil
	}
	var boundCsvserver string
	for _, binding := range bindings {
		csv, ok := binding["domain"]
		if ok {
			boundCsvserver = csv.(string)
			break
		}
	}
	d.Set("csvserver", boundCsvserver)

	return nil

}

func updateCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In updateCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client
	cspolicyName := d.Get("policyname").(string)
	csvserver := d.Get("csvserver").(string)

	cspolicy := cs.Cspolicy{
		Policyname: d.Get("policyname").(string),
	}
	hasChange := false
	lbvserverChanged := false
	priorityChanged := false

	if d.HasChange("action") {
		log.Printf("[DEBUG] netscaler-provider: Action has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Action = d.Get("action").(string)
		hasChange = true
	}
	if d.HasChange("domain") {
		log.Printf("[DEBUG] netscaler-provider: Domain has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Domain = d.Get("domain").(string)
		hasChange = true
	}
	if d.HasChange("logaction") {
		log.Printf("[DEBUG] netscaler-provider: Logaction has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Logaction = d.Get("logaction").(string)
		hasChange = true
	}
	if d.HasChange("newname") {
		log.Printf("[DEBUG] netscaler-provider: Newname has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Newname = d.Get("newname").(string)
		hasChange = true
	}
	if d.HasChange("rule") {
		log.Printf("[DEBUG] netscaler-provider: Rule has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Rule = d.Get("rule").(string)
		hasChange = true
	}
	if d.HasChange("url") {
		log.Printf("[DEBUG] netscaler-provider: Url has changed for cspolicy %s, starting update", cspolicyName)
		cspolicy.Url = d.Get("url").(string)
		hasChange = true
	}
	if d.HasChange("priority") {
		log.Printf("[DEBUG] netscaler-provider: Priority has changed for cspolicy %s, starting update", cspolicyName)
		priorityChanged = true
	}

	if d.HasChange("targetlbvserver") {
		log.Printf("[DEBUG] netscaler-provider: targetlbvserver has changed for cspolicy %s, starting update", cspolicyName)
		lbvserverChanged = true
	}

	if lbvserverChanged || priorityChanged {
		//Binding has to be updated
		//First we unbind from cs vserver
		err := client.UnbindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, "policyname")
		if err != nil {
			return fmt.Errorf("[ERROR] netscaler-provider: Error unbinding cspolicy from csvserver %s", cspolicyName)
		}
		log.Printf("[DEBUG] netscaler-provider: cspolicy has been unbound from csvserver for cspolicy %s ", cspolicyName)
	}

	if hasChange {
		_, err := client.UpdateResource(netscaler.Cspolicy.Type(), cspolicyName, &cspolicy)
		if err != nil {
			return fmt.Errorf("[ERROR] netscaler-provider: Error updating cspolicy %s", cspolicyName)
		}
		log.Printf("[DEBUG] netscaler-provider: cspolicy has been updated  cspolicy %s ", cspolicyName)
	}

	if lbvserverChanged || priorityChanged {
		//Binding has to be updated
		//rebind
		targetlbvserver, lbok := d.GetOk("targetlbvserver")
		priority, pok := d.GetOk("priority")

		if !pok && lbok {
			return fmt.Errorf("[ERROR] netscaler-provider: Need to specify priority if lbvserver is specified")
		}

		binding := cs.Csvservercspolicybinding{
			Name:            csvserver,
			Policyname:      cspolicyName,
			Targetlbvserver: targetlbvserver.(string),
			Priority:        priority.(int),
		}
		err := client.BindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, &binding)
		if err != nil {
			return fmt.Errorf("[ERROR] netscaler-provider: Failed to bind new cspolicy to Csvserver")
		}
		log.Printf("[DEBUG] netscaler-provider: cspolicy has been bound to csvserver  cspolicy %s csvserver %s", cspolicyName, csvserver)
	}

	return readCspolicyFunc(d, meta)
}

func deleteCspolicyFunc(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[DEBUG] netscaler-provider: In deleteCspolicyFunc")
	client := meta.(*NetScalerNitroClient).client
	cspolicyName := d.Id()
	csvserver := d.Get("csvserver").(string)

	//First we unbind from cs vserver if necessary
	err := client.UnbindResource(netscaler.Csvserver.Type(), csvserver, netscaler.Cspolicy.Type(), cspolicyName, "policyname")
	if err != nil {
		return fmt.Errorf("[ERROR] netscaler-provider: Error unbinding cspolicy from csvserver %s", cspolicyName)
	}
	err = client.DeleteResource(netscaler.Cspolicy.Type(), cspolicyName)
	if err != nil {
		return fmt.Errorf("[ERROR] netscaler-provider: Error  deleting cspolicy %s", cspolicyName)
	}

	d.SetId("")

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/hashicorp/consul/api"
	"github.com/jeromer/syslogparser/rfc3164"
	"github.com/nsqio/go-nsq"
	"log"
	"net"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)

type StreamServer struct {
	*Setting
	exitChan      chan int
	msgChan       chan [][]byte
	CurrentConfig map[string][]*regexp.Regexp
	wg            sync.WaitGroup
	client        *api.Client
	sync.Mutex
}

func (s *StreamServer) Run() {
	cfg := nsq.NewConfig()
	hostname, _ := os.Hostname()
	cfg.Set("user_agent", fmt.Sprintf("netstream/%s", hostname))
	cfg.Set("snappy", true)
	for i := 0; i < s.WritePoolSize; i++ {
		w, _ := nsq.NewProducer(s.NsqdAddr, cfg)
		go s.writeLoop(w)
	}
	ticker := time.Tick(time.Second * 600)
	go s.readUDP()
	go s.readTCP()
	var err error
	config := api.DefaultConfig()
	config.Address = s.ConsulAddress
	config.Datacenter = s.Datacenter
	config.Token = s.Token
	s.client, err = api.NewClient(config)
	if err != nil {
		fmt.Println("reload consul setting failed", err)
	}
	s.CurrentConfig, err = s.GetRegexp()
	for {
		select {
		case <-ticker:
			s.Lock()
			s.CurrentConfig, err = s.GetRegexp()
			s.Unlock()
			if err != nil {
				fmt.Println("reload consul setting failed", err)
			}
		case <-s.exitChan:
			return
		}
	}
}

func (s *StreamServer) writeLoop(w *nsq.Producer) {
	for {
		select {
		case msg := <-s.msgChan:
			w.MultiPublish(s.Topic, msg)
		case <-s.exitChan:
			return
		}
	}
}
func (s *StreamServer) Stop() {
	close(s.exitChan)
	s.wg.Wait()
}

func (s *StreamServer) readUDP() {
	udpAddr, err := net.ResolveUDPAddr("udp", s.UdpPort)
	if err != nil {
		log.Fatal("udp:", err)
	}
	server, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Fatal("server bind failed:", err)
	}
	defer server.Close()
	buf := make([]byte, 8192*8)
	var bodies [][]byte
	for {
		select {
		case <-s.exitChan:
			return
		default:
			size, addr, err := server.ReadFromUDP(buf)
			if err != nil {
				log.Println("read log failed", err)
				continue
			}
			if s.IsIgnoreLog(buf[:size]) {
				continue
			}
			if len(buf[:size]) < 1 {
				continue
			}
			logFormat := &LogFormat{
				From:   proto.String(addr.String()),
				Rawmsg: proto.String(string(buf[:size])),
			}
			record, err := proto.Marshal(logFormat)
			if err != nil {
				continue
			}
			bodies = append(bodies, record)
			if len(bodies) > 100 {
				s.msgChan <- bodies
				bodies = bodies[:0]
			}
		}
	}
}
func (s *StreamServer) readTCP() {
	server, err := net.Listen("tcp", s.TcpPort)
	if err != nil {
		log.Fatal("server bind failed:", err)
	}
	defer server.Close()
	for {
		select {
		case <-s.exitChan:
			return
		default:
			fd, err := server.Accept()
			if err != nil {
				log.Fatal("accept error", err)
				time.Sleep(time.Second)
			} else {
				go s.loghandle(fd)
			}
		}
	}
}

// receive log from tcp socket, encode json and send to msg_chan
func (s *StreamServer) loghandle(fd net.Conn) {
	defer fd.Close()
	scanner := bufio.NewScanner(fd)
	scanner.Split(bufio.ScanLines)
	addr := fd.RemoteAddr()
	s.wg.Add(1)
	defer s.wg.Done()
	var bodies [][]byte
	var err error
	for {
		select {
		case <-s.exitChan:
			return
		default:
			if scanner.Scan() == false {
				err = scanner.Err()
				if err == nil {
					return
				}
			}
			if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
				log.Println(err)
				return
			}
			msg := scanner.Text()
			if s.IsIgnoreLog([]byte(msg)) {
				continue
			}
			if len(msg) < 1 {
				continue
			}
			logFormat := &LogFormat{
				From:   proto.String(addr.String()),
				Rawmsg: proto.String(msg),
			}
			record, err := proto.Marshal(logFormat)
			if err != nil {
				continue
			}
			bodies = append(bodies, record)
			if len(bodies) > 100 {
				s.msgChan <- bodies
				bodies = bodies[:0]
			}

		}
	}
}

func (s *StreamServer) IsIgnoreLog(buf []byte) bool {
	p := rfc3164.NewParser(buf)
	if err := p.Parse(); err != nil {
		return false
	}
	data := p.Dump()
	tag := data["tag"].(string)
	if len(tag) == 0 {
		return false
	}
	s.Lock()
	rgs, ok := s.CurrentConfig[tag]
	s.Unlock()
	if ok {
		for _, r := range rgs {
			if r.MatchString(data["content"].(string)) {
				return true
			}
		}
	}
	return false
}
func (s *StreamServer) GetRegexp() (map[string][]*regexp.Regexp, error) {
	consulSetting := make(map[string][]*regexp.Regexp)
	kv := s.client.KV()
	pairs, _, err := kv.List(s.ConsulKey, nil)
	if err != nil {
		return consulSetting, err
	}
	size := len(s.ConsulKey) + 1
	for _, value := range pairs {
		if len(value.Key) > size {
			var regs []string
			if err := json.Unmarshal(value.Value, &regs); err == nil {
				var rs []*regexp.Regexp
				for _, v := range regs {
					x, e := regexp.CompilePOSIX(v)
					if e != nil {
						log.Println("get regexp", e)
						continue
					}
					rs = append(rs, x)
				}
				consulSetting[value.Key[size:]] = rs
			}
		}
	}
	return consulSetting, err
}
<commit_msg>avoid check if null<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/golang/protobuf/proto"
	"github.com/hashicorp/consul/api"
	"github.com/jeromer/syslogparser/rfc3164"
	"github.com/nsqio/go-nsq"
	"log"
	"net"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)

type StreamServer struct {
	*Setting
	exitChan      chan int
	msgChan       chan [][]byte
	CurrentConfig map[string][]*regexp.Regexp
	wg            sync.WaitGroup
	client        *api.Client
	sync.Mutex
}

func (s *StreamServer) Run() {
	cfg := nsq.NewConfig()
	hostname, _ := os.Hostname()
	cfg.Set("user_agent", fmt.Sprintf("netstream/%s", hostname))
	cfg.Set("snappy", true)
	for i := 0; i < s.WritePoolSize; i++ {
		w, _ := nsq.NewProducer(s.NsqdAddr, cfg)
		go s.writeLoop(w)
	}
	ticker := time.Tick(time.Second * 600)
	go s.readUDP()
	go s.readTCP()
	var err error
	config := api.DefaultConfig()
	config.Address = s.ConsulAddress
	config.Datacenter = s.Datacenter
	config.Token = s.Token
	s.client, err = api.NewClient(config)
	if err != nil {
		fmt.Println("reload consul setting failed", err)
	}
	s.CurrentConfig, err = s.GetRegexp()
	for {
		select {
		case <-ticker:
			s.Lock()
			s.CurrentConfig, err = s.GetRegexp()
			s.Unlock()
			if err != nil {
				fmt.Println("reload consul setting failed", err)
			}
		case <-s.exitChan:
			return
		}
	}
}

func (s *StreamServer) writeLoop(w *nsq.Producer) {
	for {
		select {
		case msg := <-s.msgChan:
			w.MultiPublish(s.Topic, msg)
		case <-s.exitChan:
			return
		}
	}
}
func (s *StreamServer) Stop() {
	close(s.exitChan)
	s.wg.Wait()
}

func (s *StreamServer) readUDP() {
	udpAddr, err := net.ResolveUDPAddr("udp", s.UdpPort)
	if err != nil {
		log.Fatal("udp:", err)
	}
	server, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Fatal("server bind failed:", err)
	}
	defer server.Close()
	buf := make([]byte, 8192*8)
	var bodies [][]byte
	for {
		select {
		case <-s.exitChan:
			return
		default:
			size, addr, err := server.ReadFromUDP(buf)
			if err != nil {
				log.Println("read log failed", err)
				continue
			}
			if s.IsIgnoreLog(buf[:size]) {
				continue
			}
			logFormat := &LogFormat{
				From:   proto.String(addr.String()),
				Rawmsg: proto.String(string(buf[:size])),
			}
			record, err := proto.Marshal(logFormat)
			if err != nil {
				continue
			}
			bodies = append(bodies, record)
			if len(bodies) > 100 {
				s.msgChan <- bodies
				bodies = bodies[:0]
			}
		}
	}
}
func (s *StreamServer) readTCP() {
	server, err := net.Listen("tcp", s.TcpPort)
	if err != nil {
		log.Fatal("server bind failed:", err)
	}
	defer server.Close()
	for {
		select {
		case <-s.exitChan:
			return
		default:
			fd, err := server.Accept()
			if err != nil {
				log.Fatal("accept error", err)
				time.Sleep(time.Second)
			} else {
				go s.loghandle(fd)
			}
		}
	}
}

// receive log from tcp socket, encode json and send to msg_chan
func (s *StreamServer) loghandle(fd net.Conn) {
	defer fd.Close()
	scanner := bufio.NewScanner(fd)
	scanner.Split(bufio.ScanLines)
	addr := fd.RemoteAddr()
	s.wg.Add(1)
	defer s.wg.Done()
	var bodies [][]byte
	var err error
	for {
		select {
		case <-s.exitChan:
			return
		default:
			if scanner.Scan() == false {
				err = scanner.Err()
				if err == nil {
					return
				}
			}
			if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
				log.Println(err)
				return
			}
			msg := scanner.Text()
			if s.IsIgnoreLog([]byte(msg)) {
				continue
			}
			if len(msg) < 1 {
				continue
			}
			logFormat := &LogFormat{
				From:   proto.String(addr.String()),
				Rawmsg: proto.String(msg),
			}
			record, err := proto.Marshal(logFormat)
			if err != nil {
				continue
			}
			bodies = append(bodies, record)
			if len(bodies) > 100 {
				s.msgChan <- bodies
				bodies = bodies[:0]
			}

		}
	}
}

func (s *StreamServer) IsIgnoreLog(buf []byte) bool {
	if len(buf) < 1 {
		return true
	}
	p := rfc3164.NewParser(buf)
	if err := p.Parse(); err != nil {
		return false
	}
	data := p.Dump()
	tag := data["tag"].(string)
	if len(tag) == 0 {
		return false
	}
	s.Lock()
	rgs, ok := s.CurrentConfig[tag]
	s.Unlock()
	if ok {
		for _, r := range rgs {
			if r.MatchString(data["content"].(string)) {
				return true
			}
		}
	}
	return false
}
func (s *StreamServer) GetRegexp() (map[string][]*regexp.Regexp, error) {
	consulSetting := make(map[string][]*regexp.Regexp)
	kv := s.client.KV()
	pairs, _, err := kv.List(s.ConsulKey, nil)
	if err != nil {
		return consulSetting, err
	}
	size := len(s.ConsulKey) + 1
	for _, value := range pairs {
		if len(value.Key) > size {
			var regs []string
			if err := json.Unmarshal(value.Value, &regs); err == nil {
				var rs []*regexp.Regexp
				for _, v := range regs {
					x, e := regexp.CompilePOSIX(v)
					if e != nil {
						log.Println("get regexp", e)
						continue
					}
					rs = append(rs, x)
				}
				consulSetting[value.Key[size:]] = rs
			}
		}
	}
	return consulSetting, err
}
<|endoftext|>


<commit_before>package blockcreator

import (
	"encoding/json"
	"math/big"
	"time"

	"github.com/rivine/rivine/crypto"
	"github.com/rivine/rivine/types"
)

// SolveBlocks participates in the Proof Of Block Stake protocol by continuously checking if
// unspent block stake outputs make a solution for the current unsolved block.
// If a match is found, the block is submitted to the consensus set.
// This function does not return until the blockcreator threadgroup is stopped.
func (bc *BlockCreator) SolveBlocks() {
	for {

		// Bail if 'Stop' has been called.
		select {
		case <-bc.tg.StopChan():
			return
		default:
		}

		// This is mainly here to avoid the creation of useless blocks during IBD and when a node comes back online
		// after some downtime
		if !bc.csSynced {
			if !bc.cs.Synced() {
				bc.log.Debugln("Consensus set is not synced, don't create blocks")
				time.Sleep(8 * time.Second)
				continue
			}
			bc.csSynced = true
		}

		// Try to solve a block for blocktimes of the next 10 seconds
		now := time.Now().Unix()
		bc.log.Debugln("[BC] Attempting to solve blocks")
		b := bc.solveBlock(uint64(now), 10)
		if b != nil {
			bjson, _ := json.Marshal(b)
			bc.log.Debugln("Solved block:", string(bjson))

			err := bc.submitBlock(*b)
			if err != nil {
				bc.log.Println("ERROR: An error occurred while submitting a solved block:", err)
			}
		}
		//sleep a while before recalculating
		time.Sleep(8 * time.Second)
	}
}

func (bc *BlockCreator) solveBlock(startTime uint64, secondsInTheFuture uint64) (b *types.Block) {

	bc.mu.RLock()
	defer bc.mu.RUnlock()

	currentBlock := bc.cs.CurrentBlock()
	stakemodifier := bc.cs.CalculateStakeModifier(bc.persist.Height+1, currentBlock, bc.chainCts.StakeModifierDelay-1)
	cbid := bc.cs.CurrentBlock().ID()
	target, _ := bc.cs.ChildTarget(cbid)

	// Try all unspent blockstake outputs
	unspentBlockStakeOutputs := bc.wallet.GetUnspentBlockStakeOutputs()
	for _, ubso := range unspentBlockStakeOutputs {
		BlockStakeAge := types.Timestamp(0)
		// Filter all unspent block stakes for aging. If the index of the unspent
		// block stake output is not the first transaction with the first index,
		// then block stake can only be used to solve blocks after its aging is
		// older than types.BlockStakeAging (more than 1 day)
		if ubso.Indexes.TransactionIndex != 0 || ubso.Indexes.OutputIndex != 0 {
			blockatheigh, _ := bc.cs.BlockAtHeight(ubso.Indexes.BlockHeight)
			BlockStakeAge = blockatheigh.Header().Timestamp + types.Timestamp(bc.chainCts.BlockStakeAging)
		}
		// Try all timestamps for this timerange
		for blocktime := startTime; blocktime < startTime+secondsInTheFuture; blocktime++ {
			if BlockStakeAge > types.Timestamp(blocktime) {
				continue
			}
			// Calculate the hash for the given unspent output and timestamp
			pobshash := crypto.HashAll(stakemodifier.Bytes(), ubso.Indexes.BlockHeight, ubso.Indexes.TransactionIndex, ubso.Indexes.OutputIndex, blocktime)
			// Check if it meets the difficulty
			pobshashvalue := big.NewInt(0).SetBytes(pobshash[:])
			pobshashvalue.Div(pobshashvalue, ubso.Value.Big()) //TODO rivine : this div can be mul on the other side of the compare

			if pobshashvalue.Cmp(target.Int()) == -1 {
				bc.log.Debugln("\nSolved block with target", target)
				blockToSubmit := types.Block{
					ParentID:   bc.unsolvedBlock.ParentID,
					Timestamp:  types.Timestamp(blocktime),
					POBSOutput: ubso.Indexes,
				}

				bc.RespentBlockStake(ubso)

				// Block is going to be passed to external memory, but the memory pointed
				// to by the transactions slice is still being modified - needs to be
				// copied.
				txns := make([]types.Transaction, len(bc.unsolvedBlock.Transactions))
				copy(txns, bc.unsolvedBlock.Transactions)
				blockToSubmit.Transactions = txns
				// Collect the block creation fee
				if !bc.chainCts.BlockCreatorFee.IsZero() {
					blockToSubmit.MinerPayouts = append(blockToSubmit.MinerPayouts, types.MinerPayout{
						Value: bc.chainCts.BlockCreatorFee, UnlockHash: ubso.Condition.UnlockHash()})
				}
				collectedMinerFees := blockToSubmit.CalculateTotalMinerFees()
				if !collectedMinerFees.IsZero() {
					condition := bc.chainCts.TransactionFeeCondition
					if condition.ConditionType() == types.ConditionTypeNil {
						condition = ubso.Condition
					}
					blockToSubmit.MinerPayouts = append(blockToSubmit.MinerPayouts, types.MinerPayout{
						Value: collectedMinerFees, UnlockHash: condition.UnlockHash()})
				}

				return &blockToSubmit
			}
		}
	}
	return
}

// RespentBlockStake will spent the unspent block stake output which is needed
// for the POBS algorithm. The transaction created will be the first transaction
// in the block to avoid the BlockStakeAging for later use of this block stake.
func (bc *BlockCreator) RespentBlockStake(ubso types.UnspentBlockStakeOutput) {

	// There is a special case: When the unspent block stake output is already
	// used in another transaction in this unsolved block, this extra transaction
	// is obsolete
	for _, ubstr := range bc.unsolvedBlock.Transactions {
		for _, ubstrinput := range ubstr.BlockStakeInputs {
			if ubstrinput.ParentID == ubso.BlockStakeOutputID {
				return
			}
		}
	}

	//otherwise the blockstake is not yet spent in this block, spent it now
	t := bc.wallet.StartTransaction()
	t.SpendBlockStake(ubso.BlockStakeOutputID) // link the input of this transaction
	// to the used BlockStake output

	bso := types.BlockStakeOutput{
		Value:     ubso.Value,     //use the same amount of BlockStake
		Condition: ubso.Condition, //use the same condition.
	}
	ind := t.AddBlockStakeOutput(bso)
	if ind != 0 {
		// should not happen //TODO: not right error
	}
	txnSet, err := t.Sign()
	if err != nil {
		// should not happen //TODO: not right error
	}
	//Only one transaction is generated for this.
	if len(txnSet) > 1 {
		// should not happen //TODO: not right error
	}
	//add this transaction in front of the list of unsolved block transactions
	bc.unsolvedBlock.Transactions = append(txnSet, bc.unsolvedBlock.Transactions...)

	return
}
<commit_msg>ensure solveBlocks fails if respending a blockstake failed<commit_after>package blockcreator

import (
	"encoding/json"
	"math/big"
	"time"

	"github.com/rivine/rivine/crypto"
	"github.com/rivine/rivine/types"
)

// SolveBlocks participates in the Proof Of Block Stake protocol by continuously checking if
// unspent block stake outputs make a solution for the current unsolved block.
// If a match is found, the block is submitted to the consensus set.
// This function does not return until the blockcreator threadgroup is stopped.
func (bc *BlockCreator) SolveBlocks() {
	for {

		// Bail if 'Stop' has been called.
		select {
		case <-bc.tg.StopChan():
			return
		default:
		}

		// This is mainly here to avoid the creation of useless blocks during IBD and when a node comes back online
		// after some downtime
		if !bc.csSynced {
			if !bc.cs.Synced() {
				bc.log.Debugln("Consensus set is not synced, don't create blocks")
				time.Sleep(8 * time.Second)
				continue
			}
			bc.csSynced = true
		}

		// Try to solve a block for blocktimes of the next 10 seconds
		now := time.Now().Unix()
		bc.log.Debugln("[BC] Attempting to solve blocks")
		b := bc.solveBlock(uint64(now), 10)
		if b != nil {
			bjson, _ := json.Marshal(b)
			bc.log.Debugln("Solved block:", string(bjson))

			err := bc.submitBlock(*b)
			if err != nil {
				bc.log.Println("ERROR: An error occurred while submitting a solved block:", err)
			}
		}
		//sleep a while before recalculating
		time.Sleep(8 * time.Second)
	}
}

func (bc *BlockCreator) solveBlock(startTime uint64, secondsInTheFuture uint64) (b *types.Block) {
	bc.mu.RLock()
	defer bc.mu.RUnlock()

	currentBlock := bc.cs.CurrentBlock()
	stakemodifier := bc.cs.CalculateStakeModifier(bc.persist.Height+1, currentBlock, bc.chainCts.StakeModifierDelay-1)
	cbid := bc.cs.CurrentBlock().ID()
	target, _ := bc.cs.ChildTarget(cbid)

	// Try all unspent blockstake outputs
	unspentBlockStakeOutputs := bc.wallet.GetUnspentBlockStakeOutputs()
	for _, ubso := range unspentBlockStakeOutputs {
		BlockStakeAge := types.Timestamp(0)
		// Filter all unspent block stakes for aging. If the index of the unspent
		// block stake output is not the first transaction with the first index,
		// then block stake can only be used to solve blocks after its aging is
		// older than types.BlockStakeAging (more than 1 day)
		if ubso.Indexes.TransactionIndex != 0 || ubso.Indexes.OutputIndex != 0 {
			blockatheigh, _ := bc.cs.BlockAtHeight(ubso.Indexes.BlockHeight)
			BlockStakeAge = blockatheigh.Header().Timestamp + types.Timestamp(bc.chainCts.BlockStakeAging)
		}
		// Try all timestamps for this timerange
		for blocktime := startTime; blocktime < startTime+secondsInTheFuture; blocktime++ {
			if BlockStakeAge > types.Timestamp(blocktime) {
				continue
			}
			// Calculate the hash for the given unspent output and timestamp
			pobshash := crypto.HashAll(stakemodifier.Bytes(), ubso.Indexes.BlockHeight, ubso.Indexes.TransactionIndex, ubso.Indexes.OutputIndex, blocktime)
			// Check if it meets the difficulty
			pobshashvalue := big.NewInt(0).SetBytes(pobshash[:])
			pobshashvalue.Div(pobshashvalue, ubso.Value.Big()) //TODO rivine : this div can be mul on the other side of the compare

			if pobshashvalue.Cmp(target.Int()) == -1 {
				err := bc.RespentBlockStake(ubso)
				if err != nil {
					bc.log.Printf("failed to respond block stake %q: %v", ubso.BlockStakeOutputID.String(), err)
					return nil
				}

				bc.log.Debugln("\nSolved block with target", target)
				blockToSubmit := types.Block{
					ParentID:   bc.unsolvedBlock.ParentID,
					Timestamp:  types.Timestamp(blocktime),
					POBSOutput: ubso.Indexes,
				}

				// Block is going to be passed to external memory, but the memory pointed
				// to by the transactions slice is still being modified - needs to be
				// copied.
				txns := make([]types.Transaction, len(bc.unsolvedBlock.Transactions))
				copy(txns, bc.unsolvedBlock.Transactions)
				blockToSubmit.Transactions = txns
				// Collect the block creation fee
				if !bc.chainCts.BlockCreatorFee.IsZero() {
					blockToSubmit.MinerPayouts = append(blockToSubmit.MinerPayouts, types.MinerPayout{
						Value: bc.chainCts.BlockCreatorFee, UnlockHash: ubso.Condition.UnlockHash()})
				}
				collectedMinerFees := blockToSubmit.CalculateTotalMinerFees()
				if !collectedMinerFees.IsZero() {
					condition := bc.chainCts.TransactionFeeCondition
					if condition.ConditionType() == types.ConditionTypeNil {
						condition = ubso.Condition
					}
					blockToSubmit.MinerPayouts = append(blockToSubmit.MinerPayouts, types.MinerPayout{
						Value: collectedMinerFees, UnlockHash: condition.UnlockHash()})
				}

				return &blockToSubmit
			}
		}
	}
	return
}

// RespentBlockStake will spent the unspent block stake output which is needed
// for the POBS algorithm. The transaction created will be the first transaction
// in the block to avoid the BlockStakeAging for later use of this block stake.
func (bc *BlockCreator) RespentBlockStake(ubso types.UnspentBlockStakeOutput) error {
	// There is a special case: When the unspent block stake output is already
	// used in another transaction in this unsolved block, this extra transaction
	// is obsolete
	for _, ubstr := range bc.unsolvedBlock.Transactions {
		for _, ubstrinput := range ubstr.BlockStakeInputs {
			if ubstrinput.ParentID == ubso.BlockStakeOutputID {
				return nil
			}
		}
	}

	//otherwise the blockstake is not yet spent in this block, spent it now
	t := bc.wallet.StartTransaction()
	err := t.SpendBlockStake(ubso.BlockStakeOutputID) // link the input of this transaction
	// to the used BlockStake output
	if err != nil {
		return err
	}

	bso := types.BlockStakeOutput{
		Value:     ubso.Value,     //use the same amount of BlockStake
		Condition: ubso.Condition, //use the same condition.
	}
	t.AddBlockStakeOutput(bso)
	txnSet, err := t.Sign()
	if err != nil {
		return err
	}
	//add this transaction in front of the list of unsolved block transactions
	bc.unsolvedBlock.Transactions = append(txnSet, bc.unsolvedBlock.Transactions...)
	return nil
}
<|endoftext|>