<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/appcelerator/amp/api/server"
	"github.com/appcelerator/amp/config"
	flag "github.com/spf13/pflag"
	"os"
)

const (
	defaultPort         = ":50101"
	defaultClientID     = ""
	defaultClientSecret = ""
)

// build vars
var (
	// Version is set with a linker flag (see Makefile)
	Version string

	// Build is set with a linker flag (see Makefile)
	Build string
)

// config vars - used for generating a config from command line flags
var (
	config           server.Config
	port             string
	etcdEndpoints    string
	elasticsearchURL string
	clientID         string
	clientSecret     string
	natsURL          string
	influxURL        string
	dockerURL        string
	dockerVersion    string
)

func parseFlags() {
	var displayVersion bool

	//
	// set up flags
	flag.StringVarP(&port, "port", "p", defaultPort, "server port (default '"+defaultPort+"')")
	flag.StringVarP(&etcdEndpoints, "endpoints", "e", amp.EtcdDefaultEndpoint, "etcd comma-separated endpoints")
	flag.StringVarP(&elasticsearchURL, "elasticsearchURL", "s", amp.ElasticsearchDefaultURL, "elasticsearch URL (default '"+amp.ElasticsearchDefaultURL+"')")
	flag.StringVarP(&clientID, "clientid", "i", defaultClientID, "github app clientid (default '"+defaultClientID+"')")
	flag.StringVarP(&clientSecret, "clientsecret", "c", defaultClientSecret, "github app clientsecret (default '"+defaultClientSecret+"')")
	flag.StringVarP(&natsURL, "natsURL", "", amp.NatsDefaultURL, "Nats URL (default '"+amp.NatsDefaultURL+"')")
	flag.StringVarP(&influxURL, "influxURL", "", amp.InfluxDefaultURL, "InfluxDB URL (default '"+amp.InfluxDefaultURL+"')")
	flag.StringVar(&dockerURL, "dockerURL", amp.DockerDefaultURL, "Docker URL (default '"+amp.DockerDefaultURL+"')")
	flag.BoolVarP(&displayVersion, "version", "v", false, "Print version information and quit")

	// parse command line flags
	flag.Parse()

	// Check if command if version
	for _, arg := range os.Args {
		if arg == "version" {
			displayVersion = true
			break
		}
	}

	if displayVersion {
		os.Exit(0)
	}

	// update config
	config.Port = port
	config.ClientID = clientID
	config.ClientSecret = clientSecret
	for _, s := range strings.Split(etcdEndpoints, ",") {
		config.EtcdEndpoints = append(config.EtcdEndpoints, s)
	}
	config.ElasticsearchURL = elasticsearchURL
	config.NatsURL = natsURL
	config.InfluxURL = influxURL
	config.DockerURL = dockerURL
	config.DockerVersion = amp.DockerDefaultVersion
}

func main() {
	fmt.Printf("amplifier (server version: %s, build: %s)\n", Version, Build)
	parseFlags()
	server.Start(config)
}
<commit_msg>Using capital V shortcut for version (#593)<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/appcelerator/amp/api/server"
	"github.com/appcelerator/amp/config"
	flag "github.com/spf13/pflag"
	"os"
)

const (
	defaultPort         = ":50101"
	defaultClientID     = ""
	defaultClientSecret = ""
)

// build vars
var (
	// Version is set with a linker flag (see Makefile)
	Version string

	// Build is set with a linker flag (see Makefile)
	Build string
)

// config vars - used for generating a config from command line flags
var (
	config           server.Config
	port             string
	etcdEndpoints    string
	elasticsearchURL string
	clientID         string
	clientSecret     string
	natsURL          string
	influxURL        string
	dockerURL        string
	dockerVersion    string
)

func parseFlags() {
	var displayVersion bool

	//
	// set up flags
	flag.StringVarP(&port, "port", "p", defaultPort, "server port (default '"+defaultPort+"')")
	flag.StringVarP(&etcdEndpoints, "endpoints", "e", amp.EtcdDefaultEndpoint, "etcd comma-separated endpoints")
	flag.StringVarP(&elasticsearchURL, "elasticsearchURL", "s", amp.ElasticsearchDefaultURL, "elasticsearch URL (default '"+amp.ElasticsearchDefaultURL+"')")
	flag.StringVarP(&clientID, "clientid", "i", defaultClientID, "github app clientid (default '"+defaultClientID+"')")
	flag.StringVarP(&clientSecret, "clientsecret", "c", defaultClientSecret, "github app clientsecret (default '"+defaultClientSecret+"')")
	flag.StringVarP(&natsURL, "natsURL", "", amp.NatsDefaultURL, "Nats URL (default '"+amp.NatsDefaultURL+"')")
	flag.StringVarP(&influxURL, "influxURL", "", amp.InfluxDefaultURL, "InfluxDB URL (default '"+amp.InfluxDefaultURL+"')")
	flag.StringVar(&dockerURL, "dockerURL", amp.DockerDefaultURL, "Docker URL (default '"+amp.DockerDefaultURL+"')")
	flag.BoolVarP(&displayVersion, "version", "V", false, "Print version information and quit")

	// parse command line flags
	flag.Parse()

	// Check if command if version
	for _, arg := range os.Args {
		if arg == "version" {
			displayVersion = true
			break
		}
	}

	if displayVersion {
		os.Exit(0)
	}

	// update config
	config.Port = port
	config.ClientID = clientID
	config.ClientSecret = clientSecret
	for _, s := range strings.Split(etcdEndpoints, ",") {
		config.EtcdEndpoints = append(config.EtcdEndpoints, s)
	}
	config.ElasticsearchURL = elasticsearchURL
	config.NatsURL = natsURL
	config.InfluxURL = influxURL
	config.DockerURL = dockerURL
	config.DockerVersion = amp.DockerDefaultVersion
}

func main() {
	fmt.Printf("amplifier (server version: %s, build: %s)\n", Version, Build)
	parseFlags()
	server.Start(config)
}
<|endoftext|>


<commit_before>// -*- tab-width: 4 -*-
package couch

import (
	"bytes"
	"fmt"
	"os"
	"json"
	"http"
	"net"
	"io/ioutil"
)

var def_hdrs = map[string]string{}

type buffer struct {
	b *bytes.Buffer
}

func (b *buffer) Read(out []byte) (int, os.Error) {
	return b.b.Read(out)
}

func (b *buffer) Close() os.Error { return nil }

// Converts given URL to string containing the body of the response.
func url_to_buf(url string) []byte {
	if r, _, err := http.Get(url); err == nil {
		b, err := ioutil.ReadAll(r.Body)
		r.Body.Close()
		if err == nil {
			return b
		}
	}
	return make([]byte, 0)
}

type IdAndRev struct {
	Id  string "_id"
	Rev string "_rev"
}

// Sends a query to CouchDB and parses the response back.
// method: the name of the HTTP method (POST, PUT,...)
// url: the URL to interact with
// headers: additional headers to pass to the request
// in: body of the request
// out: a structure to fill in with the returned JSON document
func (p Database) interact(method, url string, headers map[string]string, in []byte, out interface{}) (int, os.Error) {
	fullHeaders := map[string]string{}
	for k, v := range headers {
		fullHeaders[k] = v
	}
	bodyLength := 0
	if in != nil {
		bodyLength = len(in)
		fullHeaders["Content-Type"] = "application/json"
	}
	req := http.Request{
		Method:        method,
		ProtoMajor:    1,
		ProtoMinor:    1,
		Close:         true,
		ContentLength: int64(bodyLength),
		Header:        fullHeaders,
	}
	req.TransferEncoding = []string{"chunked"}
	req.URL, _ = http.ParseURL(url)
	if in != nil {
		req.Body = &buffer{bytes.NewBuffer(in)}
	}
	conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
	if err != nil {
		return 0, err
	}
	http_conn := http.NewClientConn(conn, nil)
	defer http_conn.Close()
	if err := http_conn.Write(&req); err != nil {
		return 0, err
	}
	r, err := http_conn.Read()
	if err != nil {
		return 0, err
	}
	if r.StatusCode < 200 || r.StatusCode >= 300 {
		b := []byte{}
		r.Body.Read(b)
		return r.StatusCode, os.NewError("server said: " + r.Status)
	}
	decoder := json.NewDecoder(r.Body)
	if err = decoder.Decode(out); err != nil {
		return 0, err
	}
	r.Body.Close()
	return r.StatusCode, nil
}

type Database struct {
	Host string
	Port string
	Name string
}

func (p Database) BaseURL() string {
	return fmt.Sprintf("http://%s:%s", p.Host, p.Port)
}

func (p Database) DBURL() string {
	return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Test whether CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
	url := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs")
	s := url_to_buf(url)
	if len(s) > 0 {
		return true
	}
	return false
}

type database_info struct {
	Db_name string
	// other stuff too, ignore for now
}

// Test whether specified database exists in specified CouchDB instance
func (p Database) Exists() bool {
	di := new(database_info)
	if err := json.Unmarshal(url_to_buf(p.DBURL()), di); err != nil {
		return false
	}
	if di.Db_name != p.Name {
		return false
	}
	return true
}

func (p Database) create_database() os.Error {
	ir := response{}
	if _, err := p.interact("PUT", p.DBURL(), def_hdrs, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return os.NewError("Create database operation returned not-OK")
	}
	return nil
}

// Deletes the given database and all documents
func (p Database) DeleteDatabase() os.Error {
	ir := response{}
	if _, err := p.interact("DELETE", p.DBURL(), def_hdrs, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return os.NewError("Delete database operation returned not-OK")
	}
	return nil
}

func NewDatabase(host, port, name string) (Database, os.Error) {
	db := Database{host, port, name}
	if !db.Running() {
		return db, os.NewError("CouchDB not running")
	}
	if !db.Exists() {
		if err := db.create_database(); err != nil {
			return db, err
		}
	}
	return db, nil
}

// Strip _id and _rev from d, returning them separately if they exist
func clean_JSON(d interface{}) (json_buf []byte, id, rev string, err os.Error) {
	json_buf, err = json.Marshal(d)
	if err != nil {
		return
	}
	m := map[string]interface{}{}
	err = json.Unmarshal(json_buf, &m)
	if err != nil {
		return
	}
	id_rev := new(IdAndRev)
	err = json.Unmarshal(json_buf, &id_rev)
	if err != nil {
		return
	}
	if _, ok := m["_id"]; ok {
		id = id_rev.Id
		m["_id"] = nil, false
	}
	if _, ok := m["_rev"]; ok {
		rev = id_rev.Rev
		m["_rev"] = nil, false
	}
	json_buf, err = json.Marshal(m)
	return
}

type response struct {
	Ok     bool
	Id     string
	Rev    string
	Error  string
	Reason string
}

// Inserts document to CouchDB, returning id and rev on success.
// Document may specify both "_id" and "_rev" fields (will overwrite existing)
//   or just "_id" (will use that id, but not overwrite existing)
//   or neither (will use autogenerated id)
func (p Database) Insert(d interface{}) (string, string, os.Error) {
	json_buf, id, rev, err := clean_JSON(d)
	if err != nil {
		return "", "", err
	}
	if id != "" && rev != "" {
		new_rev, err2 := p.Edit(d)
		return id, new_rev, err2
	} else if id != "" {
		return p.insert_with(json_buf, id)
	} else if id == "" {
		return p.insert(json_buf)
	}
	return "", "", os.NewError("invalid Document")
}

// Private implementation of simple autogenerated-id insert
func (p Database) insert(json_buf []byte) (string, string, os.Error) {
	ir := response{}
	if _, err := p.interact("POST", p.DBURL(), def_hdrs, json_buf, &ir); err != nil {
		return "", "", err
	}
	if !ir.Ok {
		return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
	}
	return ir.Id, ir.Rev, nil
}

// Inserts the given document (shouldn't contain "_id" or "_rev" tagged fields)
// using the passed 'id' as the _id. Will fail if the id already exists.
func (p Database) InsertWith(d interface{}, id string) (string, string, os.Error) {
	json_buf, err := json.Marshal(d)
	if err != nil {
		return "", "", err
	}
	return p.insert_with(json_buf, id)
}

// Private implementation of insert with given id
func (p Database) insert_with(json_buf []byte, id string) (string, string, os.Error) {
	url := fmt.Sprintf("%s/%s", p.DBURL(), http.URLEscape(id))
	ir := response{}
	if _, err := p.interact("PUT", url, def_hdrs, json_buf, &ir); err != nil {
		return "", "", err
	}
	if !ir.Ok {
		return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
	}
	return ir.Id, ir.Rev, nil
}

// Edits the given document, returning the new revision.
// d must contain "_id" and "_rev" tagged fields.
func (p Database) Edit(d interface{}) (string, os.Error) {
	json_buf, err := json.Marshal(d)
	if err != nil {
		return "", err
	}
	id_rev := new(IdAndRev)
	err = json.Unmarshal(json_buf, id_rev)
	if err != nil {
		return "", err
	}
	if id_rev.Id == "" {
		return "", os.NewError("Id not specified in interface")
	}
	if id_rev.Rev == "" {
		return "", os.NewError("Rev not specified in interface (try InsertWith)")
	}
	url := fmt.Sprintf("%s/%s", p.DBURL(), http.URLEscape(id_rev.Id))
	ir := response{}
	if _, err = p.interact("PUT", url, def_hdrs, json_buf, &ir); err != nil {
		return "", err
	}
	return ir.Rev, nil
}

// Edits the given document, returning the new revision.
// d should not contain "_id" or "_rev" tagged fields. If it does, they will
// be overwritten with the passed values.
func (p Database) EditWith(d interface{}, id, rev string) (string, os.Error) {
	if id == "" || rev == "" {
		return "", os.NewError("EditWith: must specify both id and rev")
	}
	json_buf, err := json.Marshal(d)
	if err != nil {
		return "", err
	}
	m := map[string]interface{}{}
	err = json.Unmarshal(json_buf, &m)
	if err != nil {
		return "", err
	}
	m["_id"] = id
	m["_rev"] = rev
	return p.Edit(m)
}

// Unmarshals the document matching id to the given interface, returning rev.
func (p Database) Retrieve(id string, d interface{}) (string, os.Error) {
	if id == "" {
		return "", os.NewError("no id specified")
	}
	json_buf := url_to_buf(fmt.Sprintf("%s/%s", p.DBURL(), id))
	id_rev := new(IdAndRev)
	if err := json.Unmarshal(json_buf, &id_rev); err != nil {
		return "", err
	}
	if id_rev.Id != id {
		return "", os.NewError("invalid id specified")
	}
	return id_rev.Rev, json.Unmarshal(json_buf, d)
}

// Deletes document given by id and rev.
func (p Database) Delete(id, rev string) os.Error {
	headers := map[string]string{
		"If-Match": rev,
	}
	url := fmt.Sprintf("%s/%s", p.DBURL(), id)
	ir := response{}
	if _, err := p.interact("DELETE", url, headers, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
	}
	return nil
}

type Row struct {
	Id  string
	Key string
}

type keyed_view_response struct {
	Total_rows uint64
	Offset     uint64
	Rows       []Row
}

// Return array of document ids as returned by the given view/options combo.
// view should be eg. "_design/my_foo/_view/my_bar"
// options should be eg. { "limit": 10, "key": "baz" }
func (p Database) Query(view string, options map[string]interface{}) ([]string, os.Error) {
	if view == "" {
		return make([]string, 0), os.NewError("empty view")
	}
	parameters := ""
	for k, v := range options {
		switch t := v.(type) {
		case string:
			parameters += fmt.Sprintf(`%s="%s"&`, k, http.URLEscape(t))
		case int:
			parameters += fmt.Sprintf(`%s=%d&`, k, t)
		case bool:
			parameters += fmt.Sprintf(`%s=%v&`, k, t)
		default:
			// TODO more types are supported
			panic(fmt.Sprintf("unsupported value-type %T in Query", t))
		}
	}
	full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters)
	json_buf := url_to_buf(full_url)
	kvr := new(keyed_view_response)
	if err := json.Unmarshal(json_buf, kvr); err != nil {
		return make([]string, 0), err
	}
	ids := make([]string, len(kvr.Rows))
	for i, row := range kvr.Rows {
		ids[i] = row.Id
	}
	return ids, nil
}
<commit_msg>Couchdb views may return null for key & id.<commit_after>// -*- tab-width: 4 -*-
package couch

import (
	"bytes"
	"fmt"
	"os"
	"json"
	"http"
	"net"
	"io/ioutil"
)

var def_hdrs = map[string]string{}

type buffer struct {
	b *bytes.Buffer
}

func (b *buffer) Read(out []byte) (int, os.Error) {
	return b.b.Read(out)
}

func (b *buffer) Close() os.Error { return nil }

// Converts given URL to string containing the body of the response.
func url_to_buf(url string) []byte {
	if r, _, err := http.Get(url); err == nil {
		b, err := ioutil.ReadAll(r.Body)
		r.Body.Close()
		if err == nil {
			return b
		}
	}
	return make([]byte, 0)
}

type IdAndRev struct {
	Id  string "_id"
	Rev string "_rev"
}

// Sends a query to CouchDB and parses the response back.
// method: the name of the HTTP method (POST, PUT,...)
// url: the URL to interact with
// headers: additional headers to pass to the request
// in: body of the request
// out: a structure to fill in with the returned JSON document
func (p Database) interact(method, url string, headers map[string]string, in []byte, out interface{}) (int, os.Error) {
	fullHeaders := map[string]string{}
	for k, v := range headers {
		fullHeaders[k] = v
	}
	bodyLength := 0
	if in != nil {
		bodyLength = len(in)
		fullHeaders["Content-Type"] = "application/json"
	}
	req := http.Request{
		Method:        method,
		ProtoMajor:    1,
		ProtoMinor:    1,
		Close:         true,
		ContentLength: int64(bodyLength),
		Header:        fullHeaders,
	}
	req.TransferEncoding = []string{"chunked"}
	req.URL, _ = http.ParseURL(url)
	if in != nil {
		req.Body = &buffer{bytes.NewBuffer(in)}
	}
	conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
	if err != nil {
		return 0, err
	}
	http_conn := http.NewClientConn(conn, nil)
	defer http_conn.Close()
	if err := http_conn.Write(&req); err != nil {
		return 0, err
	}
	r, err := http_conn.Read()
	if err != nil {
		return 0, err
	}
	if r.StatusCode < 200 || r.StatusCode >= 300 {
		b := []byte{}
		r.Body.Read(b)
		return r.StatusCode, os.NewError("server said: " + r.Status)
	}
	decoder := json.NewDecoder(r.Body)
	if err = decoder.Decode(out); err != nil {
		return 0, err
	}
	r.Body.Close()
	return r.StatusCode, nil
}

type Database struct {
	Host string
	Port string
	Name string
}

func (p Database) BaseURL() string {
	return fmt.Sprintf("http://%s:%s", p.Host, p.Port)
}

func (p Database) DBURL() string {
	return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Test whether CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
	url := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs")
	s := url_to_buf(url)
	if len(s) > 0 {
		return true
	}
	return false
}

type database_info struct {
	Db_name string
	// other stuff too, ignore for now
}

// Test whether specified database exists in specified CouchDB instance
func (p Database) Exists() bool {
	di := new(database_info)
	if err := json.Unmarshal(url_to_buf(p.DBURL()), di); err != nil {
		return false
	}
	if di.Db_name != p.Name {
		return false
	}
	return true
}

func (p Database) create_database() os.Error {
	ir := response{}
	if _, err := p.interact("PUT", p.DBURL(), def_hdrs, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return os.NewError("Create database operation returned not-OK")
	}
	return nil
}

// Deletes the given database and all documents
func (p Database) DeleteDatabase() os.Error {
	ir := response{}
	if _, err := p.interact("DELETE", p.DBURL(), def_hdrs, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return os.NewError("Delete database operation returned not-OK")
	}
	return nil
}

func NewDatabase(host, port, name string) (Database, os.Error) {
	db := Database{host, port, name}
	if !db.Running() {
		return db, os.NewError("CouchDB not running")
	}
	if !db.Exists() {
		if err := db.create_database(); err != nil {
			return db, err
		}
	}
	return db, nil
}

// Strip _id and _rev from d, returning them separately if they exist
func clean_JSON(d interface{}) (json_buf []byte, id, rev string, err os.Error) {
	json_buf, err = json.Marshal(d)
	if err != nil {
		return
	}
	m := map[string]interface{}{}
	err = json.Unmarshal(json_buf, &m)
	if err != nil {
		return
	}
	id_rev := new(IdAndRev)
	err = json.Unmarshal(json_buf, &id_rev)
	if err != nil {
		return
	}
	if _, ok := m["_id"]; ok {
		id = id_rev.Id
		m["_id"] = nil, false
	}
	if _, ok := m["_rev"]; ok {
		rev = id_rev.Rev
		m["_rev"] = nil, false
	}
	json_buf, err = json.Marshal(m)
	return
}

type response struct {
	Ok     bool
	Id     string
	Rev    string
	Error  string
	Reason string
}

// Inserts document to CouchDB, returning id and rev on success.
// Document may specify both "_id" and "_rev" fields (will overwrite existing)
//   or just "_id" (will use that id, but not overwrite existing)
//   or neither (will use autogenerated id)
func (p Database) Insert(d interface{}) (string, string, os.Error) {
	json_buf, id, rev, err := clean_JSON(d)
	if err != nil {
		return "", "", err
	}
	if id != "" && rev != "" {
		new_rev, err2 := p.Edit(d)
		return id, new_rev, err2
	} else if id != "" {
		return p.insert_with(json_buf, id)
	} else if id == "" {
		return p.insert(json_buf)
	}
	return "", "", os.NewError("invalid Document")
}

// Private implementation of simple autogenerated-id insert
func (p Database) insert(json_buf []byte) (string, string, os.Error) {
	ir := response{}
	if _, err := p.interact("POST", p.DBURL(), def_hdrs, json_buf, &ir); err != nil {
		return "", "", err
	}
	if !ir.Ok {
		return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
	}
	return ir.Id, ir.Rev, nil
}

// Inserts the given document (shouldn't contain "_id" or "_rev" tagged fields)
// using the passed 'id' as the _id. Will fail if the id already exists.
func (p Database) InsertWith(d interface{}, id string) (string, string, os.Error) {
	json_buf, err := json.Marshal(d)
	if err != nil {
		return "", "", err
	}
	return p.insert_with(json_buf, id)
}

// Private implementation of insert with given id
func (p Database) insert_with(json_buf []byte, id string) (string, string, os.Error) {
	url := fmt.Sprintf("%s/%s", p.DBURL(), http.URLEscape(id))
	ir := response{}
	if _, err := p.interact("PUT", url, def_hdrs, json_buf, &ir); err != nil {
		return "", "", err
	}
	if !ir.Ok {
		return "", "", os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
	}
	return ir.Id, ir.Rev, nil
}

// Edits the given document, returning the new revision.
// d must contain "_id" and "_rev" tagged fields.
func (p Database) Edit(d interface{}) (string, os.Error) {
	json_buf, err := json.Marshal(d)
	if err != nil {
		return "", err
	}
	id_rev := new(IdAndRev)
	err = json.Unmarshal(json_buf, id_rev)
	if err != nil {
		return "", err
	}
	if id_rev.Id == "" {
		return "", os.NewError("Id not specified in interface")
	}
	if id_rev.Rev == "" {
		return "", os.NewError("Rev not specified in interface (try InsertWith)")
	}
	url := fmt.Sprintf("%s/%s", p.DBURL(), http.URLEscape(id_rev.Id))
	ir := response{}
	if _, err = p.interact("PUT", url, def_hdrs, json_buf, &ir); err != nil {
		return "", err
	}
	return ir.Rev, nil
}

// Edits the given document, returning the new revision.
// d should not contain "_id" or "_rev" tagged fields. If it does, they will
// be overwritten with the passed values.
func (p Database) EditWith(d interface{}, id, rev string) (string, os.Error) {
	if id == "" || rev == "" {
		return "", os.NewError("EditWith: must specify both id and rev")
	}
	json_buf, err := json.Marshal(d)
	if err != nil {
		return "", err
	}
	m := map[string]interface{}{}
	err = json.Unmarshal(json_buf, &m)
	if err != nil {
		return "", err
	}
	m["_id"] = id
	m["_rev"] = rev
	return p.Edit(m)
}

// Unmarshals the document matching id to the given interface, returning rev.
func (p Database) Retrieve(id string, d interface{}) (string, os.Error) {
	if id == "" {
		return "", os.NewError("no id specified")
	}
	json_buf := url_to_buf(fmt.Sprintf("%s/%s", p.DBURL(), id))
	id_rev := new(IdAndRev)
	if err := json.Unmarshal(json_buf, &id_rev); err != nil {
		return "", err
	}
	if id_rev.Id != id {
		return "", os.NewError("invalid id specified")
	}
	return id_rev.Rev, json.Unmarshal(json_buf, d)
}

// Deletes document given by id and rev.
func (p Database) Delete(id, rev string) os.Error {
	headers := map[string]string{
		"If-Match": rev,
	}
	url := fmt.Sprintf("%s/%s", p.DBURL(), id)
	ir := response{}
	if _, err := p.interact("DELETE", url, headers, nil, &ir); err != nil {
		return err
	}
	if !ir.Ok {
		return os.NewError(fmt.Sprintf("%s: %s", ir.Error, ir.Reason))
	}
	return nil
}

type Row struct {
	Id  *string
	Key *string
}

type keyed_view_response struct {
	Total_rows uint64
	Offset     uint64
	Rows       []Row
}

// Return array of document ids as returned by the given view/options combo.
// view should be eg. "_design/my_foo/_view/my_bar"
// options should be eg. { "limit": 10, "key": "baz" }
func (p Database) Query(view string, options map[string]interface{}) ([]*string, os.Error) {
	if view == "" {
		return make([]*string, 0), os.NewError("empty view")
	}
	parameters := ""
	for k, v := range options {
		switch t := v.(type) {
		case string:
			parameters += fmt.Sprintf(`%s="%s"&`, k, http.URLEscape(t))
		case int:
			parameters += fmt.Sprintf(`%s=%d&`, k, t)
		case bool:
			parameters += fmt.Sprintf(`%s=%v&`, k, t)
		default:
			// TODO more types are supported
			panic(fmt.Sprintf("unsupported value-type %T in Query", t))
		}
	}
	full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters)
	json_buf := url_to_buf(full_url)
	kvr := new(keyed_view_response)
	if err := json.Unmarshal(json_buf, kvr); err != nil {
		return make([]*string, 0), err
	}
	ids := make([]*string, len(kvr.Rows))
	for i, row := range kvr.Rows {
		ids[i] = row.Id
	}
	return ids, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/moul/bolosseum"
	"github.com/moul/bolosseum/bots"
	"github.com/moul/bolosseum/bots/filebot"
	"github.com/moul/bolosseum/bots/httpbot"
	"github.com/moul/bolosseum/bots/stupidbot"
	"github.com/moul/bolosseum/games"
	"github.com/moul/bolosseum/games/coinflip"
	"github.com/moul/bolosseum/games/connectfour"
	"github.com/moul/bolosseum/games/russianbullet"
	"github.com/moul/bolosseum/games/tictactoe"
	"github.com/moul/bolosseum/stupid-ias"
	"github.com/moul/bolosseum/stupid-ias/coinflip"
	"github.com/moul/bolosseum/stupid-ias/connectfour"
	"github.com/moul/bolosseum/stupid-ias/tictactoe"
	"github.com/urfave/cli"
)

var availableGames = []string{
	"coinflip",
	"connectfour",
}

func getGame(gameName string) (games.Game, error) {
	switch gameName {
	case "coinflip":
		return coinflip.NewGame()
	case "connectfour":
		return connectfour.NewGame()
	case "russianbullet":
		return russianbullet.NewGame()
	case "tictactoe":
		return tictactoe.NewGame()
	default:
		return nil, fmt.Errorf("unknown game %q", gameName)
	}
}

func getStupidIA(iaPath string) (stupidias.StupidIA, error) {
	logrus.Warnf("Getting stupid IA %q", iaPath)
	switch iaPath {
	case "connectfour":
		return stupidconnectfour.NewIA()
	case "coinflip":
		return stupidcoinflip.NewIA()
	case "tictactoe":
		return stupidtictactoe.NewIA()
	default:
		return nil, fmt.Errorf("unknown stupid IA %q", iaPath)
	}
}

func getBot(botPath string) (bots.Bot, error) {
	logrus.Warnf("Getting bot %q", botPath)
	splt := strings.Split(botPath, "://")
	if len(splt) != 2 {
		return nil, fmt.Errorf("invalid bot path")
	}

	scheme := splt[0]
	path := splt[1]

	switch scheme {
	case "file":
		return filebot.NewBot(path)
	case "http+get":
		return httpbot.NewBot(path, "GET", "http")
	case "http+post", "http":
		return httpbot.NewBot(path, "POST", "http")
	case "https+get":
		return httpbot.NewBot(path, "GET", "https")
	case "https+post", "https":
		return httpbot.NewBot(path, "POST", "https")
	case "stupid":
		ia, err := getStupidIA(path)
		if err != nil {
			return nil, err
		}
		return stupidbot.NewStupidBot(path, ia)
	default:
		return nil, fmt.Errorf("invalid bot scheme: %q (%q)", scheme, path)
	}
}

func main() {
	// seed random
	rand.Seed(time.Now().UTC().UnixNano())

	// configure CLI
	app := cli.NewApp()
	app.Name = "bolosseum"
	app.Usage = "colosseum for bots"
	app.Version = bolosseum.VERSION

	app.Commands = []cli.Command{
		{
			Name:   "run",
			Usage:  "Start a battle",
			Action: run,
		},
	}
	app.Run(os.Args)
}

func run(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return cli.NewExitError("You need to specify the game", -1)
	}

	// initialize game
	logrus.Warnf("Initializing game %q", args[0])
	game, err := getGame(args[0])
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("No such game %q", args[0]), -1)
	}
	logrus.Warnf("Game: %q: %q", game.Name(), game)

	if err = game.CheckArgs(args[1:]); err != nil {
		return cli.NewExitError(fmt.Sprintf("%v", err), -1)
	}

	// initialize bots
	hasError := false
	for _, botPath := range args[1:] {
		bot, err := getBot(botPath)
		if err != nil {
			hasError = true
			logrus.Errorf("Failed to initialize bot %q", bot)
		} else {
			logrus.Warnf("Registering bot %q", bot.Path())
			game.RegisterBot(bot)
		}
	}
	if hasError {
		return cli.NewExitError("Invalid bots", -1)
	}

	// run
	if err = game.Run("gameid"); err != nil {
		logrus.Errorf("Run error: %v", err)
	}

	return nil
}
<commit_msg>Can now ust call 'stupid'<commit_after>package main

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/moul/bolosseum"
	"github.com/moul/bolosseum/bots"
	"github.com/moul/bolosseum/bots/filebot"
	"github.com/moul/bolosseum/bots/httpbot"
	"github.com/moul/bolosseum/bots/stupidbot"
	"github.com/moul/bolosseum/games"
	"github.com/moul/bolosseum/games/coinflip"
	"github.com/moul/bolosseum/games/connectfour"
	"github.com/moul/bolosseum/games/russianbullet"
	"github.com/moul/bolosseum/games/tictactoe"
	"github.com/moul/bolosseum/stupid-ias"
	"github.com/moul/bolosseum/stupid-ias/coinflip"
	"github.com/moul/bolosseum/stupid-ias/connectfour"
	"github.com/moul/bolosseum/stupid-ias/tictactoe"
	"github.com/urfave/cli"
)

var availableGames = []string{
	"coinflip",
	"connectfour",
}

func getGame(gameName string) (games.Game, error) {
	switch gameName {
	case "coinflip":
		return coinflip.NewGame()
	case "connectfour":
		return connectfour.NewGame()
	case "russianbullet":
		return russianbullet.NewGame()
	case "tictactoe":
		return tictactoe.NewGame()
	default:
		return nil, fmt.Errorf("unknown game %q", gameName)
	}
}

func getStupidIA(iaPath string) (stupidias.StupidIA, error) {
	logrus.Warnf("Getting stupid IA %q", iaPath)
	switch iaPath {
	case "connectfour":
		return stupidconnectfour.NewIA()
	case "coinflip":
		return stupidcoinflip.NewIA()
	case "tictactoe":
		return stupidtictactoe.NewIA()
	default:
		return nil, fmt.Errorf("unknown stupid IA %q", iaPath)
	}
}

func getBot(botPath string, game games.Game) (bots.Bot, error) {
	logrus.Warnf("Getting bot %q", botPath)

	if botPath == "stupid" {
		botPath = fmt.Sprintf("stupid://%s", game.Name())
	}

	splt := strings.Split(botPath, "://")
	if len(splt) != 2 {
		return nil, fmt.Errorf("invalid bot path")
	}

	scheme := splt[0]
	path := splt[1]

	switch scheme {
	case "file":
		return filebot.NewBot(path)
	case "http+get":
		return httpbot.NewBot(path, "GET", "http")
	case "http+post", "http":
		return httpbot.NewBot(path, "POST", "http")
	case "https+get":
		return httpbot.NewBot(path, "GET", "https")
	case "https+post", "https":
		return httpbot.NewBot(path, "POST", "https")
	case "stupid":
		ia, err := getStupidIA(path)
		if err != nil {
			return nil, err
		}
		return stupidbot.NewStupidBot(path, ia)
	default:
		return nil, fmt.Errorf("invalid bot scheme: %q (%q)", scheme, path)
	}
}

func main() {
	// seed random
	rand.Seed(time.Now().UTC().UnixNano())

	// configure CLI
	app := cli.NewApp()
	app.Name = "bolosseum"
	app.Usage = "colosseum for bots"
	app.Version = bolosseum.VERSION

	app.Commands = []cli.Command{
		{
			Name:   "run",
			Usage:  "Start a battle",
			Action: run,
		},
	}
	app.Run(os.Args)
}

func run(c *cli.Context) error {
	args := c.Args()
	if len(args) < 1 {
		return cli.NewExitError("You need to specify the game", -1)
	}

	// initialize game
	logrus.Warnf("Initializing game %q", args[0])
	game, err := getGame(args[0])
	if err != nil {
		return cli.NewExitError(fmt.Sprintf("No such game %q", args[0]), -1)
	}
	logrus.Warnf("Game: %q: %q", game.Name(), game)

	if err = game.CheckArgs(args[1:]); err != nil {
		return cli.NewExitError(fmt.Sprintf("%v", err), -1)
	}

	// initialize bots
	hasError := false
	for _, botPath := range args[1:] {
		bot, err := getBot(botPath, game)
		if err != nil {
			hasError = true
			logrus.Errorf("Failed to initialize bot %q", bot)
		} else {
			logrus.Warnf("Registering bot %q", bot.Path())
			game.RegisterBot(bot)
		}
	}
	if hasError {
		return cli.NewExitError("Invalid bots", -1)
	}

	// run
	if err = game.Run("gameid"); err != nil {
		logrus.Errorf("Run error: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * fs_keyring.go - Add/remove encryption policy keys to/from filesystem
 *
 * Copyright 2019 Google LLC
 * Author: Eric Biggers (ebiggers@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package keyring

/*
#include <string.h>
*/
import "C"

import (
	"encoding/hex"
	"log"
	"os"
	"os/user"
	"sync"
	"unsafe"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/crypto"
	"github.com/google/fscrypt/filesystem"
	"github.com/google/fscrypt/security"
	"github.com/google/fscrypt/util"
)

var (
	fsKeyringSupported      bool
	fsKeyringSupportedKnown bool
	fsKeyringSupportedLock  sync.Mutex
)

func checkForFsKeyringSupport(mount *filesystem.Mount) bool {
	dir, err := os.Open(mount.Path)
	if err != nil {
		log.Printf("Unexpected error opening %q. Assuming filesystem keyring is unsupported.",
			mount.Path)
		return false
	}
	defer dir.Close()

	// FS_IOC_ADD_ENCRYPTION_KEY with a NULL argument will fail with ENOTTY
	// if the ioctl isn't supported.  Otherwise it should fail with EFAULT.
	//
	// Note that there's no need to check for FS_IOC_REMOVE_ENCRYPTION_KEY
	// support separately, since it's guaranteed to be available if
	// FS_IOC_ADD_ENCRYPTION_KEY is.  There's also no need to check for
	// support on every filesystem separately, since either the kernel
	// supports the ioctls on all fscrypt-capable filesystems or it doesn't.
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(), unix.FS_IOC_ADD_ENCRYPTION_KEY, 0)
	if errno == unix.ENOTTY {
		log.Printf("Kernel doesn't support filesystem keyring. Falling back to user keyring.")
		return false
	}
	if errno == unix.EFAULT {
		log.Printf("Detected support for filesystem keyring")
	} else {
		// EFAULT is expected, but as long as we didn't get ENOTTY the
		// ioctl should be available.
		log.Printf("Unexpected error from FS_IOC_ADD_ENCRYPTION_KEY(%q, NULL): %v", mount.Path, errno)
	}
	return true
}

// IsFsKeyringSupported returns true if the kernel supports the ioctls to
// add/remove fscrypt keys directly to/from the filesystem.  For support to be
// detected, the given Mount must be for a filesystem that supports fscrypt.
func IsFsKeyringSupported(mount *filesystem.Mount) bool {
	fsKeyringSupportedLock.Lock()
	defer fsKeyringSupportedLock.Unlock()
	if !fsKeyringSupportedKnown {
		fsKeyringSupported = checkForFsKeyringSupport(mount)
		fsKeyringSupportedKnown = true
	}
	return fsKeyringSupported
}

// buildKeySpecifier converts the key descriptor string to an FscryptKeySpecifier.
func buildKeySpecifier(spec *unix.FscryptKeySpecifier, descriptor string) error {
	descriptorBytes, err := hex.DecodeString(descriptor)
	if err != nil {
		return errors.Errorf("key descriptor %q is invalid", descriptor)
	}
	switch len(descriptorBytes) {
	case unix.FSCRYPT_KEY_DESCRIPTOR_SIZE:
		spec.Type = unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR
	case unix.FSCRYPT_KEY_IDENTIFIER_SIZE:
		spec.Type = unix.FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER
	default:
		return errors.Errorf("key descriptor %q has unknown length", descriptor)
	}
	copy(spec.U[:], descriptorBytes)
	return nil
}

type savedPrivs struct {
	ruid, euid, suid int
}

// dropPrivsIfNeeded drops privileges (UIDs only) to the given user if we're
// working with a v2 policy key, and if the user is different from the user the
// process is currently running as.
//
// This is needed to change the effective UID so that FS_IOC_ADD_ENCRYPTION_KEY
// and FS_IOC_REMOVE_ENCRYPTION_KEY will add/remove a claim to the key for the
// intended user, and so that FS_IOC_GET_ENCRYPTION_KEY_STATUS will return the
// correct status flags for the user.
func dropPrivsIfNeeded(user *user.User, spec *unix.FscryptKeySpecifier) (*savedPrivs, error) {
	if spec.Type == unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR {
		// v1 policy keys don't have any concept of user claims.
		return nil, nil
	}
	targetUID := util.AtoiOrPanic(user.Uid)
	ruid, euid, suid := security.GetUids()
	if euid == targetUID {
		return nil, nil
	}
	if err := security.SetUids(targetUID, targetUID, euid); err != nil {
		return nil, err
	}
	return &savedPrivs{ruid, euid, suid}, nil
}

// restorePrivs restores root privileges if needed.
func restorePrivs(privs *savedPrivs) error {
	if privs != nil {
		return security.SetUids(privs.ruid, privs.euid, privs.suid)
	}
	return nil
}

// validateKeyDescriptor validates that the correct key descriptor was provided.
// This isn't really necessary; this is just an extra sanity check.
func validateKeyDescriptor(spec *unix.FscryptKeySpecifier, descriptor string) (string, error) {
	if spec.Type != unix.FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER {
		// v1 policy key: the descriptor is chosen arbitrarily by
		// userspace, so there's nothing to validate.
		return descriptor, nil
	}
	// v2 policy key.  The descriptor ("identifier" in the kernel UAPI) is
	// calculated as a cryptographic hash of the key itself.  The kernel
	// ignores the provided value, and calculates and returns it itself.  So
	// verify that the returned value is as expected.  If it's not, the key
	// doesn't actually match the encryption policy we thought it was for.
	actual := hex.EncodeToString(spec.U[:unix.FSCRYPT_KEY_IDENTIFIER_SIZE])
	if descriptor == actual {
		return descriptor, nil
	}
	return actual,
		errors.Errorf("provided and actual key descriptors differ (%q != %q)",
			descriptor, actual)
}

// fsAddEncryptionKey adds the specified encryption key to the specified filesystem.
func fsAddEncryptionKey(key *crypto.Key, descriptor string,
	mount *filesystem.Mount, user *user.User) error {

	dir, err := os.Open(mount.Path)
	if err != nil {
		return err
	}
	defer dir.Close()

	argKey, err := crypto.NewBlankKey(int(unsafe.Sizeof(unix.FscryptAddKeyArg{})) + key.Len())
	if err != nil {
		return err
	}
	defer argKey.Wipe()
	arg := (*unix.FscryptAddKeyArg)(argKey.UnsafePtr())

	if err = buildKeySpecifier(&arg.Key_spec, descriptor); err != nil {
		return err
	}

	raw := unsafe.Pointer(uintptr(argKey.UnsafePtr()) + unsafe.Sizeof(*arg))
	arg.Raw_size = uint32(key.Len())
	C.memcpy(raw, key.UnsafePtr(), C.size_t(key.Len()))

	savedPrivs, err := dropPrivsIfNeeded(user, &arg.Key_spec)
	if err != nil {
		return err
	}
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
		unix.FS_IOC_ADD_ENCRYPTION_KEY, uintptr(argKey.UnsafePtr()))
	restorePrivs(savedPrivs)

	log.Printf("FS_IOC_ADD_ENCRYPTION_KEY(%q, %s, <raw>) = %v", mount.Path, descriptor, errno)
	if errno != 0 {
		return errors.Wrap(ErrKeyAdd, errno.Error())
	}
	if descriptor, err = validateKeyDescriptor(&arg.Key_spec, descriptor); err != nil {
		fsRemoveEncryptionKey(descriptor, mount, user)
		return err
	}
	return nil
}

// fsRemoveEncryptionKey removes the specified encryption key from the specified
// filesystem.
func fsRemoveEncryptionKey(descriptor string, mount *filesystem.Mount,
	user *user.User) error {

	dir, err := os.Open(mount.Path)
	if err != nil {
		return err
	}
	defer dir.Close()

	var arg unix.FscryptRemoveKeyArg
	if err = buildKeySpecifier(&arg.Key_spec, descriptor); err != nil {
		return err
	}

	ioc := unix.FS_IOC_REMOVE_ENCRYPTION_KEY
	iocName := "FS_IOC_REMOVE_ENCRYPTION_KEY"
	var savedPrivs *savedPrivs
	if user == nil {
		ioc = unix.FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS
		iocName = "FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS"
	} else {
		savedPrivs, err = dropPrivsIfNeeded(user, &arg.Key_spec)
		if err != nil {
			return err
		}
	}
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(), uintptr(ioc), uintptr(unsafe.Pointer(&arg)))
	restorePrivs(savedPrivs)

	log.Printf("%s(%q, %s) = %v, removal_status_flags=0x%x",
		iocName, mount.Path, descriptor, errno, arg.Removal_status_flags)
	switch errno {
	case 0:
		switch {
		case arg.Removal_status_flags&unix.FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS != 0:
			return ErrKeyAddedByOtherUsers
		case arg.Removal_status_flags&unix.FSCRYPT_KEY_REMOVAL_STATUS_FLAG_FILES_BUSY != 0:
			return ErrKeyFilesOpen
		}
		return nil
	case unix.ENOKEY:
		// ENOKEY means either the key is completely missing or that the
		// current user doesn't have a claim to it.  Distinguish between
		// these two cases by getting the key status.
		if user != nil {
			status, _ := fsGetEncryptionKeyStatus(descriptor, mount, user)
			if status == KeyPresentButOnlyOtherUsers {
				return ErrKeyAddedByOtherUsers
			}
		}
		return ErrKeyNotPresent
	default:
		return errors.Wrap(ErrKeyRemove, errno.Error())
	}
}

// fsGetEncryptionKeyStatus gets the status of the specified encryption key on
// the specified filesystem.
func fsGetEncryptionKeyStatus(descriptor string, mount *filesystem.Mount,
	user *user.User) (KeyStatus, error) {

	dir, err := os.Open(mount.Path)
	if err != nil {
		return KeyStatusUnknown, err
	}
	defer dir.Close()

	var arg unix.FscryptGetKeyStatusArg
	err = buildKeySpecifier(&arg.Key_spec, descriptor)
	if err != nil {
		return KeyStatusUnknown, err
	}

	savedPrivs, err := dropPrivsIfNeeded(user, &arg.Key_spec)
	if err != nil {
		return KeyStatusUnknown, err
	}
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
		unix.FS_IOC_GET_ENCRYPTION_KEY_STATUS, uintptr(unsafe.Pointer(&arg)))
	restorePrivs(savedPrivs)

	log.Printf("FS_IOC_GET_ENCRYPTION_KEY_STATUS(%q, %s) = %v, status=%d, status_flags=0x%x",
		mount.Path, descriptor, errno, arg.Status, arg.Status_flags)
	if errno != 0 {
		return KeyStatusUnknown, errors.Wrap(ErrKeySearch, errno.Error())
	}
	switch arg.Status {
	case unix.FSCRYPT_KEY_STATUS_ABSENT:
		return KeyAbsent, nil
	case unix.FSCRYPT_KEY_STATUS_PRESENT:
		if arg.Key_spec.Type != unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR &&
			(arg.Status_flags&unix.FSCRYPT_KEY_STATUS_FLAG_ADDED_BY_SELF) == 0 {
			return KeyPresentButOnlyOtherUsers, nil
		}
		return KeyPresent, nil
	case unix.FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED:
		return KeyAbsentButFilesBusy, nil
	default:
		return KeyStatusUnknown,
			errors.Wrapf(ErrKeySearch, "unknown key status (%d)", arg.Status)
	}
}
<commit_msg>keyring: cast FS_IOC_REMOVE_ENCRYPTION_KEY to uintptr (#221)<commit_after>/*
 * fs_keyring.go - Add/remove encryption policy keys to/from filesystem
 *
 * Copyright 2019 Google LLC
 * Author: Eric Biggers (ebiggers@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package keyring

/*
#include <string.h>
*/
import "C"

import (
	"encoding/hex"
	"log"
	"os"
	"os/user"
	"sync"
	"unsafe"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/crypto"
	"github.com/google/fscrypt/filesystem"
	"github.com/google/fscrypt/security"
	"github.com/google/fscrypt/util"
)

var (
	fsKeyringSupported      bool
	fsKeyringSupportedKnown bool
	fsKeyringSupportedLock  sync.Mutex
)

func checkForFsKeyringSupport(mount *filesystem.Mount) bool {
	dir, err := os.Open(mount.Path)
	if err != nil {
		log.Printf("Unexpected error opening %q. Assuming filesystem keyring is unsupported.",
			mount.Path)
		return false
	}
	defer dir.Close()

	// FS_IOC_ADD_ENCRYPTION_KEY with a NULL argument will fail with ENOTTY
	// if the ioctl isn't supported.  Otherwise it should fail with EFAULT.
	//
	// Note that there's no need to check for FS_IOC_REMOVE_ENCRYPTION_KEY
	// support separately, since it's guaranteed to be available if
	// FS_IOC_ADD_ENCRYPTION_KEY is.  There's also no need to check for
	// support on every filesystem separately, since either the kernel
	// supports the ioctls on all fscrypt-capable filesystems or it doesn't.
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(), unix.FS_IOC_ADD_ENCRYPTION_KEY, 0)
	if errno == unix.ENOTTY {
		log.Printf("Kernel doesn't support filesystem keyring. Falling back to user keyring.")
		return false
	}
	if errno == unix.EFAULT {
		log.Printf("Detected support for filesystem keyring")
	} else {
		// EFAULT is expected, but as long as we didn't get ENOTTY the
		// ioctl should be available.
		log.Printf("Unexpected error from FS_IOC_ADD_ENCRYPTION_KEY(%q, NULL): %v", mount.Path, errno)
	}
	return true
}

// IsFsKeyringSupported returns true if the kernel supports the ioctls to
// add/remove fscrypt keys directly to/from the filesystem.  For support to be
// detected, the given Mount must be for a filesystem that supports fscrypt.
func IsFsKeyringSupported(mount *filesystem.Mount) bool {
	fsKeyringSupportedLock.Lock()
	defer fsKeyringSupportedLock.Unlock()
	if !fsKeyringSupportedKnown {
		fsKeyringSupported = checkForFsKeyringSupport(mount)
		fsKeyringSupportedKnown = true
	}
	return fsKeyringSupported
}

// buildKeySpecifier converts the key descriptor string to an FscryptKeySpecifier.
func buildKeySpecifier(spec *unix.FscryptKeySpecifier, descriptor string) error {
	descriptorBytes, err := hex.DecodeString(descriptor)
	if err != nil {
		return errors.Errorf("key descriptor %q is invalid", descriptor)
	}
	switch len(descriptorBytes) {
	case unix.FSCRYPT_KEY_DESCRIPTOR_SIZE:
		spec.Type = unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR
	case unix.FSCRYPT_KEY_IDENTIFIER_SIZE:
		spec.Type = unix.FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER
	default:
		return errors.Errorf("key descriptor %q has unknown length", descriptor)
	}
	copy(spec.U[:], descriptorBytes)
	return nil
}

type savedPrivs struct {
	ruid, euid, suid int
}

// dropPrivsIfNeeded drops privileges (UIDs only) to the given user if we're
// working with a v2 policy key, and if the user is different from the user the
// process is currently running as.
//
// This is needed to change the effective UID so that FS_IOC_ADD_ENCRYPTION_KEY
// and FS_IOC_REMOVE_ENCRYPTION_KEY will add/remove a claim to the key for the
// intended user, and so that FS_IOC_GET_ENCRYPTION_KEY_STATUS will return the
// correct status flags for the user.
func dropPrivsIfNeeded(user *user.User, spec *unix.FscryptKeySpecifier) (*savedPrivs, error) {
	if spec.Type == unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR {
		// v1 policy keys don't have any concept of user claims.
		return nil, nil
	}
	targetUID := util.AtoiOrPanic(user.Uid)
	ruid, euid, suid := security.GetUids()
	if euid == targetUID {
		return nil, nil
	}
	if err := security.SetUids(targetUID, targetUID, euid); err != nil {
		return nil, err
	}
	return &savedPrivs{ruid, euid, suid}, nil
}

// restorePrivs restores root privileges if needed.
func restorePrivs(privs *savedPrivs) error {
	if privs != nil {
		return security.SetUids(privs.ruid, privs.euid, privs.suid)
	}
	return nil
}

// validateKeyDescriptor validates that the correct key descriptor was provided.
// This isn't really necessary; this is just an extra sanity check.
func validateKeyDescriptor(spec *unix.FscryptKeySpecifier, descriptor string) (string, error) {
	if spec.Type != unix.FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER {
		// v1 policy key: the descriptor is chosen arbitrarily by
		// userspace, so there's nothing to validate.
		return descriptor, nil
	}
	// v2 policy key.  The descriptor ("identifier" in the kernel UAPI) is
	// calculated as a cryptographic hash of the key itself.  The kernel
	// ignores the provided value, and calculates and returns it itself.  So
	// verify that the returned value is as expected.  If it's not, the key
	// doesn't actually match the encryption policy we thought it was for.
	actual := hex.EncodeToString(spec.U[:unix.FSCRYPT_KEY_IDENTIFIER_SIZE])
	if descriptor == actual {
		return descriptor, nil
	}
	return actual,
		errors.Errorf("provided and actual key descriptors differ (%q != %q)",
			descriptor, actual)
}

// fsAddEncryptionKey adds the specified encryption key to the specified filesystem.
func fsAddEncryptionKey(key *crypto.Key, descriptor string,
	mount *filesystem.Mount, user *user.User) error {

	dir, err := os.Open(mount.Path)
	if err != nil {
		return err
	}
	defer dir.Close()

	argKey, err := crypto.NewBlankKey(int(unsafe.Sizeof(unix.FscryptAddKeyArg{})) + key.Len())
	if err != nil {
		return err
	}
	defer argKey.Wipe()
	arg := (*unix.FscryptAddKeyArg)(argKey.UnsafePtr())

	if err = buildKeySpecifier(&arg.Key_spec, descriptor); err != nil {
		return err
	}

	raw := unsafe.Pointer(uintptr(argKey.UnsafePtr()) + unsafe.Sizeof(*arg))
	arg.Raw_size = uint32(key.Len())
	C.memcpy(raw, key.UnsafePtr(), C.size_t(key.Len()))

	savedPrivs, err := dropPrivsIfNeeded(user, &arg.Key_spec)
	if err != nil {
		return err
	}
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
		unix.FS_IOC_ADD_ENCRYPTION_KEY, uintptr(argKey.UnsafePtr()))
	restorePrivs(savedPrivs)

	log.Printf("FS_IOC_ADD_ENCRYPTION_KEY(%q, %s, <raw>) = %v", mount.Path, descriptor, errno)
	if errno != 0 {
		return errors.Wrap(ErrKeyAdd, errno.Error())
	}
	if descriptor, err = validateKeyDescriptor(&arg.Key_spec, descriptor); err != nil {
		fsRemoveEncryptionKey(descriptor, mount, user)
		return err
	}
	return nil
}

// fsRemoveEncryptionKey removes the specified encryption key from the specified
// filesystem.
func fsRemoveEncryptionKey(descriptor string, mount *filesystem.Mount,
	user *user.User) error {

	dir, err := os.Open(mount.Path)
	if err != nil {
		return err
	}
	defer dir.Close()

	var arg unix.FscryptRemoveKeyArg
	if err = buildKeySpecifier(&arg.Key_spec, descriptor); err != nil {
		return err
	}

	ioc := uintptr(unix.FS_IOC_REMOVE_ENCRYPTION_KEY)
	iocName := "FS_IOC_REMOVE_ENCRYPTION_KEY"
	var savedPrivs *savedPrivs
	if user == nil {
		ioc = unix.FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS
		iocName = "FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS"
	} else {
		savedPrivs, err = dropPrivsIfNeeded(user, &arg.Key_spec)
		if err != nil {
			return err
		}
	}
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(), ioc, uintptr(unsafe.Pointer(&arg)))
	restorePrivs(savedPrivs)

	log.Printf("%s(%q, %s) = %v, removal_status_flags=0x%x",
		iocName, mount.Path, descriptor, errno, arg.Removal_status_flags)
	switch errno {
	case 0:
		switch {
		case arg.Removal_status_flags&unix.FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS != 0:
			return ErrKeyAddedByOtherUsers
		case arg.Removal_status_flags&unix.FSCRYPT_KEY_REMOVAL_STATUS_FLAG_FILES_BUSY != 0:
			return ErrKeyFilesOpen
		}
		return nil
	case unix.ENOKEY:
		// ENOKEY means either the key is completely missing or that the
		// current user doesn't have a claim to it.  Distinguish between
		// these two cases by getting the key status.
		if user != nil {
			status, _ := fsGetEncryptionKeyStatus(descriptor, mount, user)
			if status == KeyPresentButOnlyOtherUsers {
				return ErrKeyAddedByOtherUsers
			}
		}
		return ErrKeyNotPresent
	default:
		return errors.Wrap(ErrKeyRemove, errno.Error())
	}
}

// fsGetEncryptionKeyStatus gets the status of the specified encryption key on
// the specified filesystem.
func fsGetEncryptionKeyStatus(descriptor string, mount *filesystem.Mount,
	user *user.User) (KeyStatus, error) {

	dir, err := os.Open(mount.Path)
	if err != nil {
		return KeyStatusUnknown, err
	}
	defer dir.Close()

	var arg unix.FscryptGetKeyStatusArg
	err = buildKeySpecifier(&arg.Key_spec, descriptor)
	if err != nil {
		return KeyStatusUnknown, err
	}

	savedPrivs, err := dropPrivsIfNeeded(user, &arg.Key_spec)
	if err != nil {
		return KeyStatusUnknown, err
	}
	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
		unix.FS_IOC_GET_ENCRYPTION_KEY_STATUS, uintptr(unsafe.Pointer(&arg)))
	restorePrivs(savedPrivs)

	log.Printf("FS_IOC_GET_ENCRYPTION_KEY_STATUS(%q, %s) = %v, status=%d, status_flags=0x%x",
		mount.Path, descriptor, errno, arg.Status, arg.Status_flags)
	if errno != 0 {
		return KeyStatusUnknown, errors.Wrap(ErrKeySearch, errno.Error())
	}
	switch arg.Status {
	case unix.FSCRYPT_KEY_STATUS_ABSENT:
		return KeyAbsent, nil
	case unix.FSCRYPT_KEY_STATUS_PRESENT:
		if arg.Key_spec.Type != unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR &&
			(arg.Status_flags&unix.FSCRYPT_KEY_STATUS_FLAG_ADDED_BY_SELF) == 0 {
			return KeyPresentButOnlyOtherUsers, nil
		}
		return KeyPresent, nil
	case unix.FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED:
		return KeyAbsentButFilesBusy, nil
	default:
		return KeyStatusUnknown,
			errors.Wrapf(ErrKeySearch, "unknown key status (%d)", arg.Status)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/converger/converger_process"
	oldBbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/bbs/lock_bbs"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/cloudfoundry/storeadapter/etcdstoreadapter"
	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server URLs (scheme://ip:port)",
)

var lockTTL = flag.Duration(
	"lockTTL",
	lock_bbs.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	lock_bbs.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var convergeRepeatInterval = flag.Duration(
	"convergeRepeatInterval",
	30*time.Second,
	"the interval between runs of the converge process",
)

var kickTaskDuration = flag.Duration(
	"kickTaskDuration",
	30*time.Second,
	"the interval, in seconds, between kicks to tasks",
)

var expireCompletedTaskDuration = flag.Duration(
	"expireCompletedTaskDuration",
	120*time.Second,
	"completed, unresolved tasks are deleted after this duration",
)

var expirePendingTaskDuration = flag.Duration(
	"expirePendingTaskDuration",
	30*time.Minute,
	"unclaimed tasks are marked as failed, after this duration",
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	1*time.Minute,
	"Timeout applied to all HTTP requests.",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

const (
	dropsondeOrigin      = "converger"
	dropsondeDestination = "localhost:3457"
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	etcdFlags := etcdstoreadapter.AddFlags(flag.CommandLine)
	flag.Parse()

	cf_http.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cf_lager.New("converger")

	etcdOptions, err := etcdFlags.Validate()
	if err != nil {
		logger.Fatal("etcd-validation-failed", err)
	}

	initializeDropsonde(logger)

	client, err := consuladapter.NewClient(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	sessionMgr := consuladapter.NewSessionManager(client)
	consulSession, err := consuladapter.NewSession("converger", *lockTTL, client, sessionMgr)
	if err != nil {
		logger.Fatal("consul-session-failed", err)
	}

	convergerBBS := initializeConvergerBBS(etcdOptions, logger, consulSession)

	uuid, err := uuid.NewV4()
	if err != nil {
		logger.Fatal("Couldn't generate uuid", err)
	}

	lockMaintainer := convergerBBS.NewConvergeLock(uuid.String(), *lockRetryInterval)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	bbsClient := bbs.NewClient(*bbsAddress)

	converger := converger_process.New(
		convergerBBS,
		bbsClient,
		consulSession,
		logger,
		clock.NewClock(),
		*convergeRepeatInterval,
		*kickTaskDuration,
		*expirePendingTaskDuration,
		*expireCompletedTaskDuration,
	)

	members := grouper.Members{
		{"lock-maintainer", lockMaintainer},
		{"converger", converger},
	}

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	process := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-process.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeConvergerBBS(etcdOptions *etcdstoreadapter.ETCDOptions, logger lager.Logger, session *consuladapter.Session) oldBbs.ConvergerBBS {
	workPool, err := workpool.NewWorkPool(oldBbs.ConvergerBBSWorkPoolSize)
	if err != nil {
		logger.Fatal("failed-to-construct-etcd-adapter-workpool", err, lager.Data{"num-workers": oldBbs.ConvergerBBSWorkPoolSize}) // should never happen
	}

	etcdAdapter, err := etcdstoreadapter.New(etcdOptions, workPool)

	if err != nil {
		logger.Fatal("failed-to-construct-etcd-tls-client", err)
	}

	return oldBbs.NewConvergerBBS(etcdAdapter, session, clock.NewClock(), logger)
}

func initializeDropsonde(logger lager.Logger) {
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}
<commit_msg>Use locket for Lock Management<commit_after>package main

import (
	"errors"
	"flag"
	"os"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/converger/converger_process"
	"github.com/cloudfoundry-incubator/locket"
	oldBbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry/dropsonde"
	"github.com/cloudfoundry/gunk/workpool"
	"github.com/cloudfoundry/storeadapter/etcdstoreadapter"
	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server URLs (scheme://ip:port)",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var convergeRepeatInterval = flag.Duration(
	"convergeRepeatInterval",
	30*time.Second,
	"the interval between runs of the converge process",
)

var kickTaskDuration = flag.Duration(
	"kickTaskDuration",
	30*time.Second,
	"the interval, in seconds, between kicks to tasks",
)

var expireCompletedTaskDuration = flag.Duration(
	"expireCompletedTaskDuration",
	120*time.Second,
	"completed, unresolved tasks are deleted after this duration",
)

var expirePendingTaskDuration = flag.Duration(
	"expirePendingTaskDuration",
	30*time.Minute,
	"unclaimed tasks are marked as failed, after this duration",
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	1*time.Minute,
	"Timeout applied to all HTTP requests.",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

const (
	dropsondeOrigin      = "converger"
	dropsondeDestination = "localhost:3457"
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	etcdFlags := etcdstoreadapter.AddFlags(flag.CommandLine)
	flag.Parse()

	cf_http.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cf_lager.New("converger")

	etcdOptions, err := etcdFlags.Validate()
	if err != nil {
		logger.Fatal("etcd-validation-failed", err)
	}

	initializeDropsonde(logger)

	client, err := consuladapter.NewClient(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	sessionMgr := consuladapter.NewSessionManager(client)
	consulSession, err := consuladapter.NewSession("converger", *lockTTL, client, sessionMgr)
	if err != nil {
		logger.Fatal("consul-session-failed", err)
	}

	convergerBBS := initializeConvergerBBS(etcdOptions, logger, consulSession)

	convergeClock := clock.NewClock()
	locket := locket.New(consulSession, convergeClock, logger)

	uuid, err := uuid.NewV4()
	if err != nil {
		logger.Fatal("Couldn't generate uuid", err)
	}

	lockMaintainer := locket.NewConvergeLock(uuid.String(), *lockRetryInterval)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	bbsClient := bbs.NewClient(*bbsAddress)

	converger := converger_process.New(
		convergerBBS,
		bbsClient,
		consulSession,
		logger,
		convergeClock,
		*convergeRepeatInterval,
		*kickTaskDuration,
		*expirePendingTaskDuration,
		*expireCompletedTaskDuration,
	)

	members := grouper.Members{
		{"lock-maintainer", lockMaintainer},
		{"converger", converger},
	}

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	process := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-process.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeConvergerBBS(etcdOptions *etcdstoreadapter.ETCDOptions, logger lager.Logger, session *consuladapter.Session) oldBbs.ConvergerBBS {
	workPool, err := workpool.NewWorkPool(oldBbs.ConvergerBBSWorkPoolSize)
	if err != nil {
		logger.Fatal("failed-to-construct-etcd-adapter-workpool", err, lager.Data{"num-workers": oldBbs.ConvergerBBSWorkPoolSize}) // should never happen
	}

	etcdAdapter, err := etcdstoreadapter.New(etcdOptions, workPool)

	if err != nil {
		logger.Fatal("failed-to-construct-etcd-tls-client", err)
	}

	return oldBbs.NewConvergerBBS(etcdAdapter, session, clock.NewClock(), logger)
}

func initializeDropsonde(logger lager.Logger) {
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/dom/herd"
	"github.com/Symantec/Dominator/dom/mdb"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/logbuf"
	"log"
	"os"
	"path"
	"runtime"
	"time"
)

var (
	debug = flag.Bool("debug", false,
		"If true, show debugging output")
	imageServerHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of image server")
	imageServerPortNum = flag.Uint("imageServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	logbufLines = flag.Uint("logbufLines", 1024,
		"Number of lines to store in the log buffer")
	minInterval = flag.Uint("minInterval", 1,
		"Minimum interval between loops (in seconds)")
	portNum = flag.Uint("portNum", constants.DomPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
	stateDir = flag.String("stateDir", "/var/lib/Dominator",
		"Name of dominator state directory.")
)

func showMdb(mdb *mdb.Mdb) {
	fmt.Println()
	mdb.DebugWrite(os.Stdout)
	fmt.Println()
}

func main() {
	flag.Parse()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the Dominator as root")
		os.Exit(1)
	}
	fi, err := os.Lstat(*stateDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
		os.Exit(1)
	}
	if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
		os.Exit(1)
	}
	interval := time.Duration(*minInterval) * time.Second
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	mdbChannel := mdb.StartMdbDaemon(path.Join(*stateDir, "mdb"), logger)
	herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
		*imageServerPortNum), logger)
	herd.AddHtmlWriter(circularBuffer)
	if err = herd.StartServer(*portNum, true); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	nextCycleStopTime := time.Now().Add(interval)
	for {
		select {
		case mdb := <-mdbChannel:
			herd.MdbUpdate(mdb)
			if *debug {
				showMdb(mdb)
			}
			runtime.GC() // An opportune time to take out the garbage.
		default:
			// Do work.
			if herd.PollNextSub() {
				if *debug {
					fmt.Print(".")
				}
				time.Sleep(nextCycleStopTime.Sub(time.Now()))
				nextCycleStopTime = time.Now().Add(interval)
				runtime.GC() // An opportune time to take out the garbage.
			}
		}
	}
}
<commit_msg>Do not force a garbage collection in Dominator main loop if it was idle.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/dom/herd"
	"github.com/Symantec/Dominator/dom/mdb"
	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/logbuf"
	"log"
	"os"
	"path"
	"runtime"
	"time"
)

var (
	debug = flag.Bool("debug", false,
		"If true, show debugging output")
	imageServerHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of image server")
	imageServerPortNum = flag.Uint("imageServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	logbufLines = flag.Uint("logbufLines", 1024,
		"Number of lines to store in the log buffer")
	minInterval = flag.Uint("minInterval", 1,
		"Minimum interval between loops (in seconds)")
	portNum = flag.Uint("portNum", constants.DomPortNumber,
		"Port number to allocate and listen on for HTTP/RPC")
	stateDir = flag.String("stateDir", "/var/lib/Dominator",
		"Name of dominator state directory.")
)

func showMdb(mdb *mdb.Mdb) {
	fmt.Println()
	mdb.DebugWrite(os.Stdout)
	fmt.Println()
}

func main() {
	flag.Parse()
	if os.Geteuid() == 0 {
		fmt.Fprintln(os.Stderr, "Do not run the Dominator as root")
		os.Exit(1)
	}
	fi, err := os.Lstat(*stateDir)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot stat: %s\t%s\n", *stateDir, err)
		os.Exit(1)
	}
	if !fi.IsDir() {
		fmt.Fprintf(os.Stderr, "%s is not a directory\n", *stateDir)
		os.Exit(1)
	}
	interval := time.Duration(*minInterval) * time.Second
	circularBuffer := logbuf.New(*logbufLines)
	logger := log.New(circularBuffer, "", log.LstdFlags)
	mdbChannel := mdb.StartMdbDaemon(path.Join(*stateDir, "mdb"), logger)
	herd := herd.NewHerd(fmt.Sprintf("%s:%d", *imageServerHostname,
		*imageServerPortNum), logger)
	herd.AddHtmlWriter(circularBuffer)
	if err = herd.StartServer(*portNum, true); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to create http server\t%s\n", err)
		os.Exit(1)
	}
	nextCycleStopTime := time.Now().Add(interval)
	for {
		select {
		case mdb := <-mdbChannel:
			herd.MdbUpdate(mdb)
			if *debug {
				showMdb(mdb)
			}
			runtime.GC() // An opportune time to take out the garbage.
		default:
			// Do work.
			if herd.PollNextSub() {
				if *debug {
					fmt.Print(".")
				}
				sleepTime := nextCycleStopTime.Sub(time.Now())
				time.Sleep(sleepTime)
				nextCycleStopTime = time.Now().Add(interval)
				if sleepTime < 0 { // There was no time to rest.
					runtime.GC() // An opportune time to take out the garbage.
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2014-2017 Erlend Johannessen.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Exclusions contains the excluded directories and files
type Exclusions struct {
	ExcludeDirectories []string
	ExcludeFiles       []string
}

// forEachFileSystemEntry
func forEachFileSystemEntry(filename string, f os.FileInfo, err error) error {
	countExtension(filename, f)
	return nil
}

// isExcluded
func isExcluded(filename string) bool {
	// Get full path of file
	var fulldir, _ = filepath.Abs(filename)

	var excluded = isInString(fulldir+pathSeparator, exclusions.ExcludeDirectories)

	if !excluded {
		excluded = isInSlice(exclusions.ExcludeFiles, filepath.Base(filename))
	}

	return excluded
}

// showDirectoriesOrFile
func showDirectoriesOrFile(isDir bool, filename string, excluded bool) {
	var status string

	if *showDirectories && isDir {
		if excluded {
			status = " EXCLUDED"
		} else {
			status = ""
		}

		if (*showOnlyIncluded && !excluded) || (*showOnlyExcluded && excluded) || (!*showOnlyIncluded && !*showOnlyExcluded) {
			fmt.Printf("Directory %s%s\n", strings.Replace(filename, root+pathSeparator, "", 1), status)
		}
	}

	if *showFiles && !isDir {
		var indent = "   "
		if !*showDirectories {
			indent = "File "
		}

		if excluded {
			status = " EXCLUDED"
		} else {
			status = ""
		}

		if (*showOnlyIncluded && !excluded) || (*showOnlyExcluded && excluded) || (!*showOnlyIncluded && !*showOnlyExcluded) {
			fmt.Printf("%s %s%s\n", indent, strings.Replace(filename, root+pathSeparator, "", 1), status)
		}
	}
}

// countExtension
func countExtension(filename string, f os.FileInfo) {
	if f == nil {
		return
	}

	// Default excluded if it is a directory
	// If not, check for exclusions
	//var excluded = f.IsDir() || isExcluded(filename)
	var excluded = isExcluded(filename)

	showDirectoriesOrFile(f.IsDir(), filename, excluded)

	if !f.IsDir() && !excluded {
		// Extension for the entry we're looking at
		var ext = filepath.Ext(filename)

		// Is the extension one of the relevant ones?
		var _, willBeCounted = countResult.Extensions[ext]

		// If yes, proceed with counting
		if willBeCounted {
			countResult.Extensions[ext].NumberOfFiles++
			countResult.TotalNumberOfFiles++

			var size = f.Size()
			countResult.Extensions[ext].Filesize += size
			countResult.TotalSize += size

			// Slurp the whole file into memory
			var contents, err = ioutil.ReadFile(filename)

			if err != nil {
				fmt.Printf("Problem reading inputfile %s, error:%v\n", filename, err)
				return
			}

			var isBinary = isBinaryFormat(contents)

			// Binary files will not have "number of lines"
			// but might need to have the binary flag set
			if isBinary && !countResult.Extensions[ext].IsBinary {
				countResult.Extensions[ext].IsBinary = true
			} else {
				var stringContents = string(contents)
				var newline = determineNewline(stringContents)

				var numberOfLines = len(strings.Split(stringContents, newline))
				countResult.Extensions[ext].NumberOfLines += numberOfLines
				countResult.TotalNumberOfLines += numberOfLines
				bigFiles = append(bigFiles, fileSize{f.Name(), size, numberOfLines})
			}
		}
	}
}
<commit_msg>Removing indent<commit_after>// Copyright 2014-2017 Erlend Johannessen.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// Exclusions contains the excluded directories and files
type Exclusions struct {
	ExcludeDirectories []string
	ExcludeFiles       []string
}

// forEachFileSystemEntry
func forEachFileSystemEntry(filename string, f os.FileInfo, err error) error {
	countExtension(filename, f)
	return nil
}

// isExcluded
func isExcluded(filename string) bool {
	// Get full path of file
	var fulldir, _ = filepath.Abs(filename)

	var excluded = isInString(fulldir+pathSeparator, exclusions.ExcludeDirectories)

	if !excluded {
		excluded = isInSlice(exclusions.ExcludeFiles, filepath.Base(filename))
	}

	return excluded
}

// showDirectoriesOrFile
func showDirectoriesOrFile(isDir bool, filename string, excluded bool) {
	var status string

	if *showDirectories && isDir {
		if excluded {
			status = " EXCLUDED"
		} else {
			status = ""
		}

		if (*showOnlyIncluded && !excluded) || (*showOnlyExcluded && excluded) || (!*showOnlyIncluded && !*showOnlyExcluded) {
			fmt.Printf("Directory %s%s\n", strings.Replace(filename, root+pathSeparator, "", 1), status)
		}
	}

	if *showFiles && !isDir {
		var indent = "   "
		if !*showDirectories {
			indent = "File "
		}

		if excluded {
			status = " EXCLUDED"
		} else {
			status = ""
		}

		if (*showOnlyIncluded && !excluded) || (*showOnlyExcluded && excluded) || (!*showOnlyIncluded && !*showOnlyExcluded) {
			fmt.Printf("%s %s%s\n", indent, strings.Replace(filename, root+pathSeparator, "", 1), status)
		}
	}
}

// countExtension
func countExtension(filename string, f os.FileInfo) {
	if f == nil {
		return
	}

	// Default excluded if it is a directory
	// If not, check for exclusions
	//var excluded = f.IsDir() || isExcluded(filename)
	var excluded = isExcluded(filename)

	showDirectoriesOrFile(f.IsDir(), filename, excluded)

	if !f.IsDir() && !excluded {
		// Extension for the entry we're looking at
		var ext = filepath.Ext(filename)

		// Is the extension one of the relevant ones?
		var _, willBeCounted = countResult.Extensions[ext]

		// If no, exit
		if !willBeCounted {
			return
		}

		countResult.Extensions[ext].NumberOfFiles++
		countResult.TotalNumberOfFiles++

		var size = f.Size()
		countResult.Extensions[ext].Filesize += size
		countResult.TotalSize += size

		// Slurp the whole file into memory
		var contents, err = ioutil.ReadFile(filename)

		if err != nil {
			fmt.Printf("Problem reading inputfile %s, error:%v\n", filename, err)
			return
		}

		var isBinary = isBinaryFormat(contents)

		// Binary files will not have "number of lines"
		// but might need to have the binary flag set
		if isBinary && !countResult.Extensions[ext].IsBinary {
			countResult.Extensions[ext].IsBinary = true
		} else {
			var stringContents = string(contents)
			var newline = determineNewline(stringContents)

			var numberOfLines = len(strings.Split(stringContents, newline))
			countResult.Extensions[ext].NumberOfLines += numberOfLines
			countResult.TotalNumberOfLines += numberOfLines
			bigFiles = append(bigFiles, fileSize{f.Name(), size, numberOfLines})
		}
	}
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
	"reflect"
	"strings"
	"testing"

	"github.com/elliotchance/c2go/util"
)

func formatMultiLine(o interface{}) string {
	s := fmt.Sprintf("%#v", o)
	s = strings.Replace(s, "{", "{\n", -1)
	s = strings.Replace(s, ", ", "\n", -1)

	return s
}

func runNodeTests(t *testing.T, tests map[string]Node) {
	i := 1
	for line, expected := range tests {
		testName := fmt.Sprintf("Example%d", i)
		i++

		t.Run(testName, func(t *testing.T) {
			// Append the name of the struct onto the front. This would make the
			// complete line it would normally be parsing.
			name := reflect.TypeOf(expected).Elem().Name()
			actual := Parse(name + " " + line)

			if !reflect.DeepEqual(expected, actual) {
				t.Errorf("%s", util.ShowDiff(formatMultiLine(expected),
					formatMultiLine(actual)))
			}
		})
	}
}

func TestPrint(t *testing.T) {
	cond := &ConditionalOperator{}
	cond.AddChild(&ImplicitCastExpr{})
	cond.AddChild(&ImplicitCastExpr{})
	s := Atos(cond)
	if len(s) == 0 {
		t.Fatalf("Cannot convert AST tree : %#v", cond)
	}
	lines := strings.Split(s, "\n")
	var amount int
	for _, l := range lines {
		if strings.Contains(l, "ImplicitCastExpr") {
			amount++
		}
	}
	if amount != 2 {
		t.Error("Not correct design of output")
	}
}

var lines = []string{
// c2go ast sqlite3.c | head -5000 | sed 's/^[ |`-]*//' | sed 's/<<<NULL>>>/NullStmt/g' | gawk 'length > 0 {print "`" $0 "`,"}'
}

func BenchmarkParse(b *testing.B) {
	for n := 0; n < b.N; n++ {
		for _, line := range lines {
			Parse(line)
		}
	}
}
<commit_msg>gofmt (#751)<commit_after>package ast

import (
	"fmt"
	"reflect"
	"strings"
	"testing"

	"github.com/elliotchance/c2go/util"
)

func formatMultiLine(o interface{}) string {
	s := fmt.Sprintf("%#v", o)
	s = strings.Replace(s, "{", "{\n", -1)
	s = strings.Replace(s, ", ", "\n", -1)

	return s
}

func runNodeTests(t *testing.T, tests map[string]Node) {
	i := 1
	for line, expected := range tests {
		testName := fmt.Sprintf("Example%d", i)
		i++

		t.Run(testName, func(t *testing.T) {
			// Append the name of the struct onto the front. This would make the
			// complete line it would normally be parsing.
			name := reflect.TypeOf(expected).Elem().Name()
			actual := Parse(name + " " + line)

			if !reflect.DeepEqual(expected, actual) {
				t.Errorf("%s", util.ShowDiff(formatMultiLine(expected),
					formatMultiLine(actual)))
			}
		})
	}
}

func TestPrint(t *testing.T) {
	cond := &ConditionalOperator{}
	cond.AddChild(&ImplicitCastExpr{})
	cond.AddChild(&ImplicitCastExpr{})
	s := Atos(cond)
	if len(s) == 0 {
		t.Fatalf("Cannot convert AST tree : %#v", cond)
	}
	lines := strings.Split(s, "\n")
	var amount int
	for _, l := range lines {
		if strings.Contains(l, "ImplicitCastExpr") {
			amount++
		}
	}
	if amount != 2 {
		t.Error("Not correct design of output")
	}
}

var lines = []string{
	// c2go ast sqlite3.c | head -5000 | sed 's/^[ |`-]*//' | sed 's/<<<NULL>>>/NullStmt/g' | gawk 'length > 0 {print "`" $0 "`,"}'
}

func BenchmarkParse(b *testing.B) {
	for n := 0; n < b.N; n++ {
		for _, line := range lines {
			Parse(line)
		}
	}
}
<|endoftext|>