<commit_before>/* bogomilter is a milter service for postfix */
package main

import (
	"bytes"
	"fmt"
	"sync"
	"time"
)

/* Mailbox object to keep outgoing rate */
type Mailbox struct {
	Name    string
	Blocked bool
	SentLog []time.Time
}

/* Size returns approximate size of memory consumed by Mailbox object */
func (m Mailbox) Size() uint64 {
	return uint64(len(m.Name) + len(m.SentLog)*24 + 1)
}

/* MarshalJSON implements the json.Marshaller interface */
func (m Mailbox) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	fmt.Fprintf(buffer,
		`{"name":"%s","blocked":%v,"count":%d}`,
		m.Name,
		m.Blocked,
		len(m.SentLog),
	)
	return buffer.Bytes(), nil
}

/* MailboxMemoryCache  */
type MailboxMemoryCache struct {
	Data  map[string]Mailbox
	Mutex sync.Mutex
}

/* IsBlocked returns true if mailbox is blocked from sending emails */
func (m *MailboxMemoryCache) IsBlocked(name string, RateLimit int, Duration time.Duration) bool {
	// acquire mutex lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// get mailbox object
	mailbox, ok := m.Data[name]
	if !ok {
		// mailbox object does not exist, create a new one
		mailbox = Mailbox{
			Name:    name,
			Blocked: false,
		}
	}
	// check if mailbox is already blocked
	if mailbox.Blocked {
		return true
	}
	// clean up sent log
	threshold := time.Now().Add(-Duration)
	var SentLog []time.Time
	for _, ts := range mailbox.SentLog {
		if threshold.Before(ts) {
			SentLog = append(SentLog, ts)
		}
	}
	// add current timestamp and replace SentLog
	SentLog = append(SentLog, time.Now())
	mailbox.SentLog = SentLog
	// check if RateLimit is exceeded
	if len(mailbox.SentLog) >= RateLimit {
		mailbox.Blocked = true
		// TODO: sent notification to administrator
		// run in a goroutine to avoid excessive lock
	}
	m.Data[name] = mailbox
	return mailbox.Blocked
}

/* CleanUp unused memory */
func (m *MailboxMemoryCache) CleanUp(Duration time.Duration) {
	// acquire lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// walk all mailboxes
	threshold := time.Now().Add(-Duration)
	for name, mailbox := range m.Data {
		// blocked mailboxes should stay blocked
		if mailbox.Blocked {
			continue
		}
		// clean up expired records
		var SentLog []time.Time
		for _, ts := range mailbox.SentLog {
			if threshold.Before(ts) {
				SentLog = append(SentLog, ts)
			}
		}
		// remove cache for expired records
		if len(SentLog) == 0 {
			delete(m.Data, name)
			continue
		}
		// set new log
		mailbox.SentLog = SentLog
		m.Data[name] = mailbox
	}
}

func (m *MailboxMemoryCache) Size() uint64 {
	var size uint64
	for _, mailbox := range m.Data {
		size += mailbox.Size()
	}
	return size
}

/* MarshalJSON implements json.Marshaler interface */
func (m *MailboxMemoryCache) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	// write opening bracket
	if _, err := buffer.WriteString(fmt.Sprintf(`{"cache":%d,"mailboxes":[`, m.Size())); err != nil {
		return nil, err
	}
	// walk mailboxes
	first := true
	for _, mailbox := range m.Data {
		if b, err := mailbox.MarshalJSON(); err != nil {
			return nil, err
		} else {
			// do not write delimiter for first item in the list
			if first {
				first = false
			} else {
				// write delimiter
				if _, err := buffer.WriteRune(','); err != nil {
					return nil, err
				}
			}
			// write data
			if _, err := buffer.Write(b); err != nil {
				return nil, err
			}
		}
	}
	// write closing bracket
	if _, err := buffer.WriteString("]}"); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}

/* GetBlocked returs list of blocked mailboxes */
func (m *MailboxMemoryCache) GetBlocked() ([]Mailbox) {
	var result []Mailbox
	for _, mailbox := range m.Data {
		if mailbox.Blocked {
			result = append(result, mailbox)
		}
	}
	return result
}

func NewMailboxMemoryCache() *MailboxMemoryCache {
	return &MailboxMemoryCache{Data: make(map[string]Mailbox)}
}
<commit_msg>add missing comment<commit_after>/* bogomilter is a milter service for postfix */
package main

import (
	"bytes"
	"fmt"
	"sync"
	"time"
)

/* Mailbox object to keep outgoing rate */
type Mailbox struct {
	Name    string
	Blocked bool
	SentLog []time.Time
}

/* Size returns approximate size of memory consumed by Mailbox object */
func (m Mailbox) Size() uint64 {
	return uint64(len(m.Name) + len(m.SentLog)*24 + 1)
}

/* MarshalJSON implements the json.Marshaller interface */
func (m Mailbox) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	fmt.Fprintf(buffer,
		`{"name":"%s","blocked":%v,"count":%d}`,
		m.Name,
		m.Blocked,
		len(m.SentLog),
	)
	return buffer.Bytes(), nil
}

/* MailboxMemoryCache  */
type MailboxMemoryCache struct {
	Data  map[string]Mailbox
	Mutex sync.Mutex
}

/* IsBlocked returns true if mailbox is blocked from sending emails */
func (m *MailboxMemoryCache) IsBlocked(name string, RateLimit int, Duration time.Duration) bool {
	// acquire mutex lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// get mailbox object
	mailbox, ok := m.Data[name]
	if !ok {
		// mailbox object does not exist, create a new one
		mailbox = Mailbox{
			Name:    name,
			Blocked: false,
		}
	}
	// check if mailbox is already blocked
	if mailbox.Blocked {
		return true
	}
	// clean up sent log
	threshold := time.Now().Add(-Duration)
	var SentLog []time.Time
	for _, ts := range mailbox.SentLog {
		if threshold.Before(ts) {
			SentLog = append(SentLog, ts)
		}
	}
	// add current timestamp and replace SentLog
	SentLog = append(SentLog, time.Now())
	mailbox.SentLog = SentLog
	// check if RateLimit is exceeded
	if len(mailbox.SentLog) >= RateLimit {
		mailbox.Blocked = true
		// TODO: sent notification to administrator
		// run in a goroutine to avoid excessive lock
	}
	m.Data[name] = mailbox
	return mailbox.Blocked
}

/* CleanUp unused memory */
func (m *MailboxMemoryCache) CleanUp(Duration time.Duration) {
	// acquire lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// walk all mailboxes
	threshold := time.Now().Add(-Duration)
	for name, mailbox := range m.Data {
		// blocked mailboxes should stay blocked
		if mailbox.Blocked {
			continue
		}
		// clean up expired records
		var SentLog []time.Time
		for _, ts := range mailbox.SentLog {
			if threshold.Before(ts) {
				SentLog = append(SentLog, ts)
			}
		}
		// remove cache for expired records
		if len(SentLog) == 0 {
			delete(m.Data, name)
			continue
		}
		// set new log
		mailbox.SentLog = SentLog
		m.Data[name] = mailbox
	}
}

/* Size returns approximate size in bytes used in memory cache */
func (m *MailboxMemoryCache) Size() uint64 {
	var size uint64
	for _, mailbox := range m.Data {
		size += mailbox.Size()
	}
	return size
}

/* MarshalJSON implements json.Marshaler interface */
func (m *MailboxMemoryCache) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	// write opening bracket
	if _, err := buffer.WriteString(fmt.Sprintf(`{"cache":%d,"mailboxes":[`, m.Size())); err != nil {
		return nil, err
	}
	// walk mailboxes
	first := true
	for _, mailbox := range m.Data {
		if b, err := mailbox.MarshalJSON(); err != nil {
			return nil, err
		} else {
			// do not write delimiter for first item in the list
			if first {
				first = false
			} else {
				// write delimiter
				if _, err := buffer.WriteRune(','); err != nil {
					return nil, err
				}
			}
			// write data
			if _, err := buffer.Write(b); err != nil {
				return nil, err
			}
		}
	}
	// write closing bracket
	if _, err := buffer.WriteString("]}"); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}

/* GetBlocked returs list of blocked mailboxes */
func (m *MailboxMemoryCache) GetBlocked() ([]Mailbox) {
	var result []Mailbox
	for _, mailbox := range m.Data {
		if mailbox.Blocked {
			result = append(result, mailbox)
		}
	}
	return result
}

func NewMailboxMemoryCache() *MailboxMemoryCache {
	return &MailboxMemoryCache{Data: make(map[string]Mailbox)}
}
<|endoftext|>


<commit_before>package outreach

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

type Prospect struct {
	Data   ProspectData `json:"data"`
	Errors []map[string]interface{}
}

type ProspectData struct {
	Attributes ProspectAttributes `json:"attributes"`
}

type ProspectAttributes struct {
	Address  ProspectAddress  `json:"address,omitempty"`
	Company  ProspectCompany  `json:"company,omitempty"`
	Contact  ProspectContact  `json:"contact"`
	Personal ProspectPersonal `json:"personal"`
	Social   ProspectSocial   `json:"social,omitempty"`
	Meta     ProspectMeta     `json:"metadata,omitempty"`
}

type ProspectAddress struct {
	City    string   `json:"city,omitempty"`
	State   string   `json:"state,omitempty"`
	Country string   `json:"country,omitempty"`
	Street  []string `json:"street,omitempty"`
	Zip     string   `json:"zip,omitempty"`
}

type ProspectCompany struct {
	Name     string `json:"name,omitempty"`
	Type     string `json:"type,omitempty"`
	Industry string `json:"industry,omitempty"`
	Size     string `json:"size,omitempty"`
	Locality string `json:"locality,omitempty"`
}

type ProspectContact struct {
	Timezone string        `json:"timezone,omitempty"`
	Email    string        `json:"email"`
	Phone    ProspectPhone `json:"phone,omitempty"`
}

type ProspectPhone struct {
	Personal string `json:"personal,omitempty"`
	Work     string `json:"work,omitempty"`
}

type ProspectPersonal struct {
	Name       ProspectName `json:"name"`
	Gender     string       `json:"gender,omitempty"`
	Occupation string       `json:"occupation,omitempty"`
	Title      string       `json:"title,omitempty"`
}

type ProspectName struct {
	First string `json:"first"`
	Last  string `json:"last"`
}

type ProspectSocial struct {
	Website  string `json:"website,omitempty"`
	Facebook string `json:"facebook,omitempty"`
	LinkedIn string `json:"linkedin,omitempty"`
	Plus     string `json:"plus,omitempty"`
	Quora    string `json:"quora,omitempty"`
	Twitter  string `json:"twitter,omitempty"`
}

type ProspectMeta struct {
	OptOut bool     `json:"opted_out"`
	Source string   `json:"source,omitempty"`
	Notes  []string `json:"notes,omitempty"`
	Tags   []string `json:"tags,omitempty"`
	Custom []string `json:"custom,omitempty"`
}

type ProspectResponse struct {
	Data   ProspectResponseData
	Errors []map[string]interface{}
}

type ProspectResponseData struct {
	ID int
}

type ProspectInstance struct {
	Client *http.Client
}

func (i *ProspectInstance) Get(id int) (Prospect, error) {
	p := Prospect{}

	if i.Client == nil {
		return p, errors.New("You must assign a HTTP client.")
	}

	resp, e := i.Client.Get("https://api.outreach.io/1.0/prospect/" + strconv.Itoa(id))
	if e != nil {
		return p, e
	}

	body, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return p, e
	}

	e = json.Unmarshal(body, &p)
	if e != nil {
		return p, e
	}

	if len(p.Errors) > 0 {
		return p, fmt.Errorf("Got error response: %+v\n", p.Errors)
	}

	return p, nil
}

func (i *ProspectInstance) Post(p Prospect) (ProspectResponse, error) {
	pr := ProspectResponse{}

	if i.Client == nil {
		return pr, errors.New("You must assign a HTTP client.")
	}

	j, e := json.Marshal(p)
	if e != nil {
		return pr, e
	}

	resp, e := i.Client.Post("https://api.outreach.io/1.0/prospect", "application/json", bytes.NewReader(j))
	if e != nil {
		return pr, e
	}

	body, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return pr, e
	}

	e = json.Unmarshal(body, &pr)
	if e != nil {
		return pr, e
	}

	if len(pr.Errors) > 0 {
		return pr, fmt.Errorf("Got error response: %+v\n", pr.Errors)
	}

	return pr, nil
}
<commit_msg>Post to prospects, not prospect.<commit_after>package outreach

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

type Prospect struct {
	Data   ProspectData `json:"data"`
	Errors []map[string]interface{}
}

type ProspectData struct {
	Attributes ProspectAttributes `json:"attributes"`
}

type ProspectAttributes struct {
	Address  ProspectAddress  `json:"address,omitempty"`
	Company  ProspectCompany  `json:"company,omitempty"`
	Contact  ProspectContact  `json:"contact"`
	Personal ProspectPersonal `json:"personal"`
	Social   ProspectSocial   `json:"social,omitempty"`
	Meta     ProspectMeta     `json:"metadata,omitempty"`
}

type ProspectAddress struct {
	City    string   `json:"city,omitempty"`
	State   string   `json:"state,omitempty"`
	Country string   `json:"country,omitempty"`
	Street  []string `json:"street,omitempty"`
	Zip     string   `json:"zip,omitempty"`
}

type ProspectCompany struct {
	Name     string `json:"name,omitempty"`
	Type     string `json:"type,omitempty"`
	Industry string `json:"industry,omitempty"`
	Size     string `json:"size,omitempty"`
	Locality string `json:"locality,omitempty"`
}

type ProspectContact struct {
	Timezone string        `json:"timezone,omitempty"`
	Email    string        `json:"email"`
	Phone    ProspectPhone `json:"phone,omitempty"`
}

type ProspectPhone struct {
	Personal string `json:"personal,omitempty"`
	Work     string `json:"work,omitempty"`
}

type ProspectPersonal struct {
	Name       ProspectName `json:"name"`
	Gender     string       `json:"gender,omitempty"`
	Occupation string       `json:"occupation,omitempty"`
	Title      string       `json:"title,omitempty"`
}

type ProspectName struct {
	First string `json:"first"`
	Last  string `json:"last"`
}

type ProspectSocial struct {
	Website  string `json:"website,omitempty"`
	Facebook string `json:"facebook,omitempty"`
	LinkedIn string `json:"linkedin,omitempty"`
	Plus     string `json:"plus,omitempty"`
	Quora    string `json:"quora,omitempty"`
	Twitter  string `json:"twitter,omitempty"`
}

type ProspectMeta struct {
	OptOut bool     `json:"opted_out"`
	Source string   `json:"source,omitempty"`
	Notes  []string `json:"notes,omitempty"`
	Tags   []string `json:"tags,omitempty"`
	Custom []string `json:"custom,omitempty"`
}

type ProspectResponse struct {
	Data   ProspectResponseData
	Errors []map[string]interface{}
}

type ProspectResponseData struct {
	ID int
}

type ProspectInstance struct {
	Client *http.Client
}

func (i *ProspectInstance) Get(id int) (Prospect, error) {
	p := Prospect{}

	if i.Client == nil {
		return p, errors.New("You must assign a HTTP client.")
	}

	resp, e := i.Client.Get("https://api.outreach.io/1.0/prospects/" + strconv.Itoa(id))
	if e != nil {
		return p, e
	}

	body, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return p, e
	}

	e = json.Unmarshal(body, &p)
	if e != nil {
		return p, e
	}

	if len(p.Errors) > 0 {
		return p, fmt.Errorf("Got error response: %+v\n", p.Errors)
	}

	return p, nil
}

func (i *ProspectInstance) Post(p Prospect) (ProspectResponse, error) {
	pr := ProspectResponse{}

	if i.Client == nil {
		return pr, errors.New("You must assign a HTTP client.")
	}

	j, e := json.Marshal(p)
	if e != nil {
		return pr, e
	}

	resp, e := i.Client.Post("https://api.outreach.io/1.0/prospects", "application/json", bytes.NewReader(j))
	if e != nil {
		return pr, e
	}

	body, e := ioutil.ReadAll(resp.Body)
	if e != nil {
		return pr, e
	}

	e = json.Unmarshal(body, &pr)
	if e != nil {
		return pr, e
	}

	if len(pr.Errors) > 0 {
		return pr, fmt.Errorf("Got error response: %+v\n", pr.Errors)
	}

	return pr, nil
}
<|endoftext|>


<commit_before>package zk

import (
	"errors"
	"fmt"
	"log"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/samuel/go-zookeeper/zk"
)

const (
	coordinationVersion = "v1"
	defaultZKPort       = 2181
	maxCreateRetries    = 5
)

var defaultZkACL = zk.WorldACL(zk.PermAll)

// A Watcher manages a single connection to zookeeper, watching for changes to
// directories and managing ephemeral nodes. It lazily connects and reconnects
// to zookeeper, and tries its best to be resilient to failures, but defaults to
// silently not providing updates.
type Watcher struct {
	lock           sync.RWMutex
	zkServers      []string
	connectTimeout time.Duration
	sessionTimeout time.Duration
	prefix         string
	conn           *zk.Conn
	errs           chan error
	shutdown       chan bool

	hooksLock      sync.Mutex
	ephemeralNodes map[string]bool
	watchedNodes   map[string]watchedNode
}

type watchedNode struct {
	updates      chan []string
	disconnected chan bool
	cancel       chan bool
}

// Connect connects to the zookeeper cluster specified by zkServers, and returns
// a Watcher. All future operations on the Watcher are rooted at the given
// prefix.
func Connect(zkServers []string, prefix string, connectTimeout, sessionTimeout time.Duration) (*Watcher, error) {
	w := &Watcher{
		zkServers:      zkServers,
		connectTimeout: connectTimeout,
		sessionTimeout: sessionTimeout,
		prefix:         path.Join(prefix, coordinationVersion),
		errs:           make(chan error, 1),
		shutdown:       make(chan bool),
		ephemeralNodes: make(map[string]bool),
		watchedNodes:   make(map[string]watchedNode),
	}

	err := w.reconnect()
	if err != nil {
		return nil, fmt.Errorf("Zookeeper error: %s", err)
	}

	go w.run()
	return w, nil
}

func (w *Watcher) reconnect() error {
	var conn *zk.Conn
	var events <-chan zk.Event
	var err error

	for i, s := range w.zkServers {
		if strings.Index(s, ":") < 0 {
			w.zkServers[i] = fmt.Sprintf("%s:%d", s, defaultZKPort)
		}
	}

	w.lock.Lock()
	defer w.lock.Unlock()

	servers := strings.Join(w.zkServers, ",")
	log.Println("Connecting to zookeeper at", servers)
	conn, events, err = zk.Connect(w.zkServers, w.sessionTimeout)
	if err != nil {
		return err
	}

	if w.conn != nil {
		w.conn.Close()
	}
	w.conn = conn

	connectTimeout := time.NewTimer(w.connectTimeout)
	select {
	case <-connectTimeout.C:
		return errors.New("connection timeout")
	case event := <-events:
		if event.State != zk.StateConnected && event.State != zk.StateConnecting {
			return fmt.Errorf("connection error: %s", event)
		}
	}

	err = w.createAll(w.prefix)
	if err != nil {
		return fmt.Errorf("creating base path: %s", err)
	}

	go func() {
		for ev := range events {
			if ev.State != zk.StateConnected && ev.State != zk.StateConnecting {
				if ev.Err != nil {
					sendErr(w.errs, ev.Err, ev.Path, ev.Server)
					return
				}
			}
		}
	}()

	// Every time we connect, reset watches and recreate ephemeral nodes.
	err = w.runHooks()
	if err != nil {
		return fmt.Errorf("zookeeper hooks error: %v", err)
	}

	return nil
}

// Must be called with `w.lock` locked for write.
func (w *Watcher) runHooks() error {
	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	for node := range w.ephemeralNodes {
		err := w.createEphemeral(node)
		if err != nil {
			return err
		}
	}

	for node, wn := range w.watchedNodes {
		err := w.watchChildren(node, wn)
		if err != nil {
			return err
		}
	}

	return nil
}

func (w *Watcher) notifyDisconnected() {
	for _, wn := range w.watchedNodes {
		select {
		case wn.disconnected <- true:
		default:
		}
	}
}

func (w *Watcher) cancelWatches() {
	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	w.notifyDisconnected()

	for _, wn := range w.watchedNodes {
		wn.cancel <- true
	}
}

// run runs the main loop. On any errors, it resets the connection.
func (w *Watcher) run() {
	first := true

Reconnect:
	for {
		if !first {
			// Wait before trying to reconnect again.
			// Let's wait longer than the ZK client.
			wait := time.NewTimer(w.sessionTimeout * 2)
			select {
			case <-w.shutdown:
				break Reconnect
			case <-wait.C:
			}

			err := w.reconnect()
			if err != nil && w.conn.State() == zk.StateDisconnected {
				log.Println("Error reconnecting to zookeeper:", err)
				continue Reconnect
			}
		} else {
			first = false
		}

		select {
		case <-w.shutdown:
			break Reconnect
		case err := <-w.errs:
			log.Println("Disconnecting from zookeeper because of error:", err)
			w.cancelWatches()
			continue Reconnect
		}
	}

	w.cancelWatches()
}

// CreateEphemeral creates an ephemeral node on the zookeeper cluster. Any
// needed parent nodes are also created, as permanent nodes. The ephemeral is
// then recreated any time the Watcher reconnects.
func (w *Watcher) CreateEphemeral(node string) {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	w.ephemeralNodes[node] = true
	err := w.createEphemeral(node)
	// It's fine if we're currently reconnecting to zookeeper, since `runHooks`
	// will create the node for us.
	if err != nil && err != zk.ErrClosing {
		sendErr(w.errs, err, node, "")
	}
}

// Must be called with `w.lock` locked (either for read or write).
func (w *Watcher) createEphemeral(node string) error {
	// Retry a few times, in case the node is removed in between the two following
	// steps.
	for i := 0; i < maxCreateRetries; i++ {
		_, err := w.conn.Create(node, []byte{}, zk.FlagEphemeral, defaultZkACL)
		if err == nil {
			break
		} else if err != nil && err != zk.ErrNoNode {
			return err
		}

		// Create the parent nodes.
		parent, _ := path.Split(node)
		err = w.createAll(parent)
		if err != nil {
			return fmt.Errorf("create %s: %s", node, err)
		}
	}

	return nil
}

// RemoveEphemeral removes the given ephemeral node from the zookeeper cluster.
// The node is then no longer recreated when the Watcher reconnects.
func (w *Watcher) RemoveEphemeral(node string) {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	w.conn.Delete(node, -1)
	delete(w.ephemeralNodes, node)
}

// WatchChildren sets up a watch on the given node, and returns two channels.
// The first is updated whenever the list of children for the node changes; the
// second, whenever the cluster is disconnected. The watch persists through
// reconnects. Before the watch is set, WatchChildren creates the node and any
// parents.
func (w *Watcher) WatchChildren(node string) (chan []string, chan bool) {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	updates := make(chan []string)
	disconnected := make(chan bool)
	cancel := make(chan bool)

	wn := watchedNode{updates: updates, disconnected: disconnected, cancel: cancel}
	w.watchedNodes[node] = wn
	err := w.watchChildren(node, wn)
	// It's fine if we're currently reconnecting to zookeeper, since `runHooks`
	// will set up the watch for us.
	if err != nil && err != zk.ErrClosing {
		sendErr(w.errs, err, node, "")
		go func() {
			<-cancel
		}()
	}

	return updates, disconnected
}

// Must be called with `w.lock` locked (either for read or write).
func (w *Watcher) watchChildren(node string, wn watchedNode) error {
	children, _, events, err := w.childrenW(node)
	if err != nil {
		return err
	}

	go func() {
		// Normally, a watchChildren loop closes just so it can be reestablished
		// once we're reconnected to zookeeper. In that case wn.cancel just gets an
		// update, rather than being closed. If wn.cancel is closed, then
		// reconnecting gets set to false below, and we also close wn.updates and
		// wn.disconnected on our way out.
		reconnecting := true
		defer func() {
			if !reconnecting {
				close(wn.updates)
				close(wn.disconnected)
			}
		}()

		for {
			select {
			case reconnecting = <-wn.cancel:
				return
			case wn.updates <- children:
			}

			select {
			case reconnecting = <-wn.cancel:
				return
			case ev := <-events:

				if ev.Err != nil {
					log.Println("DEBUG: callsite=watcher.WatchChildren")
					sendErr(w.errs, ev.Err, ev.Path, ev.Server)
					<-wn.cancel
					return
				}
			}

			w.lock.RLock()
			children, _, events, err = w.childrenW(node)
			w.lock.RUnlock()

			if err != nil {
				sendErr(w.errs, err, node, "")
				reconnecting = <-wn.cancel
				return
			}
		}
	}()

	return nil
}

func (w *Watcher) childrenW(node string) (children []string, stat *zk.Stat, events <-chan zk.Event, err error) {
	// Retry a few times, in case the node is removed in between the two following
	// steps.
	for i := 0; i < maxCreateRetries; i++ {
		children, stat, events, err = w.conn.ChildrenW(node)
		if err != zk.ErrNoNode {
			return
		}

		// Create the node so we can watch it.
		err = w.createAll(node)
		if err != nil {
			err = fmt.Errorf("create %s: %s", node, err)
			return
		}
	}

	return
}

// RemoveWatch removes a watch previously set by WatchChildren.
func (w *Watcher) RemoveWatch(node string) {
	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	if wn, ok := w.watchedNodes[node]; ok {
		delete(w.watchedNodes, node)
		close(wn.cancel)
	}
}

func (w *Watcher) createAll(node string) error {
	base, _ := path.Split(path.Clean(node))
	if base != "" && base != "/" {
		err := w.createAll(base)
		if err != nil {
			return err
		}
	}

	_, err := w.conn.Create(path.Clean(node), []byte{}, 0, defaultZkACL)
	if err != nil && err != zk.ErrNodeExists {
		return err
	}

	return nil
}

// TriggerCleanup walks the prefix and deletes any non-ephemeral, empty
// nodes under it. It ignores any errors encountered.
func (w *Watcher) TriggerCleanup() {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.cleanupTree(w.prefix)
}

func (w *Watcher) cleanupTree(node string) {
	children, stat, err := w.conn.Children(node)
	if err != nil {
		return
	} else if stat.EphemeralOwner != 0 {
		return
	}

	for _, child := range children {
		w.cleanupTree(path.Join(node, child))
	}

	w.conn.Delete(node, -1)
}

func (w *Watcher) Close() {
	w.lock.Lock()
	defer w.lock.Unlock()

	w.shutdown <- true
	w.conn.Close()
}

// sendErr sends the error over the channel, or discards it if the error is full.
func sendErr(errs chan error, err error, path string, server string) {
	log.Printf("Zookeeper error: err=%q, path=%q, server=%q", err, path, server)

	select {
	case errs <- err:
	default:
	}
}
<commit_msg>pass-if-node-exists<commit_after>package zk

import (
	"errors"
	"fmt"
	"log"
	"path"
	"strings"
	"sync"
	"time"

	"github.com/samuel/go-zookeeper/zk"
)

const (
	coordinationVersion = "v1"
	defaultZKPort       = 2181
	maxCreateRetries    = 5
)

var defaultZkACL = zk.WorldACL(zk.PermAll)

// A Watcher manages a single connection to zookeeper, watching for changes to
// directories and managing ephemeral nodes. It lazily connects and reconnects
// to zookeeper, and tries its best to be resilient to failures, but defaults to
// silently not providing updates.
type Watcher struct {
	lock           sync.RWMutex
	zkServers      []string
	connectTimeout time.Duration
	sessionTimeout time.Duration
	prefix         string
	conn           *zk.Conn
	errs           chan error
	shutdown       chan bool

	hooksLock      sync.Mutex
	ephemeralNodes map[string]bool
	watchedNodes   map[string]watchedNode
}

type watchedNode struct {
	updates      chan []string
	disconnected chan bool
	cancel       chan bool
}

// Connect connects to the zookeeper cluster specified by zkServers, and returns
// a Watcher. All future operations on the Watcher are rooted at the given
// prefix.
func Connect(zkServers []string, prefix string, connectTimeout, sessionTimeout time.Duration) (*Watcher, error) {
	w := &Watcher{
		zkServers:      zkServers,
		connectTimeout: connectTimeout,
		sessionTimeout: sessionTimeout,
		prefix:         path.Join(prefix, coordinationVersion),
		errs:           make(chan error, 1),
		shutdown:       make(chan bool),
		ephemeralNodes: make(map[string]bool),
		watchedNodes:   make(map[string]watchedNode),
	}

	err := w.reconnect()
	if err != nil {
		return nil, fmt.Errorf("Zookeeper error: %s", err)
	}

	go w.run()
	return w, nil
}

func (w *Watcher) reconnect() error {
	var conn *zk.Conn
	var events <-chan zk.Event
	var err error

	for i, s := range w.zkServers {
		if strings.Index(s, ":") < 0 {
			w.zkServers[i] = fmt.Sprintf("%s:%d", s, defaultZKPort)
		}
	}

	w.lock.Lock()
	defer w.lock.Unlock()

	servers := strings.Join(w.zkServers, ",")
	log.Println("Connecting to zookeeper at", servers)
	conn, events, err = zk.Connect(w.zkServers, w.sessionTimeout)
	if err != nil {
		return err
	}

	if w.conn != nil {
		w.conn.Close()
	}
	w.conn = conn

	connectTimeout := time.NewTimer(w.connectTimeout)
	select {
	case <-connectTimeout.C:
		return errors.New("connection timeout")
	case event := <-events:
		if event.State != zk.StateConnected && event.State != zk.StateConnecting {
			return fmt.Errorf("connection error: %s", event)
		}
	}

	err = w.createAll(w.prefix)
	if err != nil {
		return fmt.Errorf("creating base path: %s", err)
	}

	go func() {
		for ev := range events {
			if ev.State != zk.StateConnected && ev.State != zk.StateConnecting {
				if ev.Err != nil {
					sendErr(w.errs, ev.Err, ev.Path, ev.Server)
					return
				}
			}
		}
	}()

	// Every time we connect, reset watches and recreate ephemeral nodes.
	err = w.runHooks()
	if err != nil {
		return fmt.Errorf("zookeeper hooks error: %v", err)
	}

	return nil
}

// Must be called with `w.lock` locked for write.
func (w *Watcher) runHooks() error {
	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	for node := range w.ephemeralNodes {
		err := w.createEphemeral(node)
		if err != nil {
			return err
		}
	}

	for node, wn := range w.watchedNodes {
		err := w.watchChildren(node, wn)
		if err != nil {
			return err
		}
	}

	return nil
}

func (w *Watcher) notifyDisconnected() {
	for _, wn := range w.watchedNodes {
		select {
		case wn.disconnected <- true:
		default:
		}
	}
}

func (w *Watcher) cancelWatches() {
	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	w.notifyDisconnected()

	for _, wn := range w.watchedNodes {
		wn.cancel <- true
	}
}

// run runs the main loop. On any errors, it resets the connection.
func (w *Watcher) run() {
	first := true

Reconnect:
	for {
		if !first {
			// Wait before trying to reconnect again.
			// Let's wait longer than the ZK client.
			wait := time.NewTimer(w.sessionTimeout * 2)
			select {
			case <-w.shutdown:
				break Reconnect
			case <-wait.C:
			}

			err := w.reconnect()
			if err != nil && w.conn.State() == zk.StateDisconnected {
				log.Println("Error reconnecting to zookeeper:", err)
				continue Reconnect
			}
		} else {
			first = false
		}

		select {
		case <-w.shutdown:
			break Reconnect
		case err := <-w.errs:
			log.Println("Disconnecting from zookeeper because of error:", err)
			w.cancelWatches()
			continue Reconnect
		}
	}

	w.cancelWatches()
}

// CreateEphemeral creates an ephemeral node on the zookeeper cluster. Any
// needed parent nodes are also created, as permanent nodes. The ephemeral is
// then recreated any time the Watcher reconnects.
func (w *Watcher) CreateEphemeral(node string) {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	w.ephemeralNodes[node] = true
	err := w.createEphemeral(node)
	// It's fine if we're currently reconnecting to zookeeper, since `runHooks`
	// will create the node for us.
	if err != nil && err != zk.ErrClosing {
		sendErr(w.errs, err, node, "")
	}
}

// Must be called with `w.lock` locked (either for read or write).
func (w *Watcher) createEphemeral(node string) error {
	// Retry a few times, in case the node is removed in between the two following
	// steps.
	for i := 0; i < maxCreateRetries; i++ {
		_, err := w.conn.Create(node, []byte{}, zk.FlagEphemeral, defaultZkACL)
		if err == nil {
			break
		} else if err != nil && err != zk.ErrNoNode && err != zk.ErrNodeExists {
			return err
		}

		// Create the parent nodes.
		parent, _ := path.Split(node)
		err = w.createAll(parent)
		if err != nil {
			return fmt.Errorf("create %s: %s", node, err)
		}
	}

	return nil
}

// RemoveEphemeral removes the given ephemeral node from the zookeeper cluster.
// The node is then no longer recreated when the Watcher reconnects.
func (w *Watcher) RemoveEphemeral(node string) {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	w.conn.Delete(node, -1)
	delete(w.ephemeralNodes, node)
}

// WatchChildren sets up a watch on the given node, and returns two channels.
// The first is updated whenever the list of children for the node changes; the
// second, whenever the cluster is disconnected. The watch persists through
// reconnects. Before the watch is set, WatchChildren creates the node and any
// parents.
func (w *Watcher) WatchChildren(node string) (chan []string, chan bool) {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	updates := make(chan []string)
	disconnected := make(chan bool)
	cancel := make(chan bool)

	wn := watchedNode{updates: updates, disconnected: disconnected, cancel: cancel}
	w.watchedNodes[node] = wn
	err := w.watchChildren(node, wn)
	// It's fine if we're currently reconnecting to zookeeper, since `runHooks`
	// will set up the watch for us.
	if err != nil && err != zk.ErrClosing {
		sendErr(w.errs, err, node, "")
		go func() {
			<-cancel
		}()
	}

	return updates, disconnected
}

// Must be called with `w.lock` locked (either for read or write).
func (w *Watcher) watchChildren(node string, wn watchedNode) error {
	children, _, events, err := w.childrenW(node)
	if err != nil {
		return err
	}

	go func() {
		// Normally, a watchChildren loop closes just so it can be reestablished
		// once we're reconnected to zookeeper. In that case wn.cancel just gets an
		// update, rather than being closed. If wn.cancel is closed, then
		// reconnecting gets set to false below, and we also close wn.updates and
		// wn.disconnected on our way out.
		reconnecting := true
		defer func() {
			if !reconnecting {
				close(wn.updates)
				close(wn.disconnected)
			}
		}()

		for {
			select {
			case reconnecting = <-wn.cancel:
				return
			case wn.updates <- children:
			}

			select {
			case reconnecting = <-wn.cancel:
				return
			case ev := <-events:

				if ev.Err != nil {
					log.Println("DEBUG: callsite=watcher.WatchChildren")
					sendErr(w.errs, ev.Err, ev.Path, ev.Server)
					<-wn.cancel
					return
				}
			}

			w.lock.RLock()
			children, _, events, err = w.childrenW(node)
			w.lock.RUnlock()

			if err != nil {
				sendErr(w.errs, err, node, "")
				reconnecting = <-wn.cancel
				return
			}
		}
	}()

	return nil
}

func (w *Watcher) childrenW(node string) (children []string, stat *zk.Stat, events <-chan zk.Event, err error) {
	// Retry a few times, in case the node is removed in between the two following
	// steps.
	for i := 0; i < maxCreateRetries; i++ {
		children, stat, events, err = w.conn.ChildrenW(node)
		if err != zk.ErrNoNode {
			return
		}

		// Create the node so we can watch it.
		err = w.createAll(node)
		if err != nil {
			err = fmt.Errorf("create %s: %s", node, err)
			return
		}
	}

	return
}

// RemoveWatch removes a watch previously set by WatchChildren.
func (w *Watcher) RemoveWatch(node string) {
	w.hooksLock.Lock()
	defer w.hooksLock.Unlock()

	node = path.Join(w.prefix, node)
	if wn, ok := w.watchedNodes[node]; ok {
		delete(w.watchedNodes, node)
		close(wn.cancel)
	}
}

func (w *Watcher) createAll(node string) error {
	base, _ := path.Split(path.Clean(node))
	if base != "" && base != "/" {
		err := w.createAll(base)
		if err != nil {
			return err
		}
	}

	_, err := w.conn.Create(path.Clean(node), []byte{}, 0, defaultZkACL)
	if err != nil && err != zk.ErrNodeExists {
		return err
	}

	return nil
}

// TriggerCleanup walks the prefix and deletes any non-ephemeral, empty
// nodes under it. It ignores any errors encountered.
func (w *Watcher) TriggerCleanup() {
	w.lock.RLock()
	defer w.lock.RUnlock()

	w.cleanupTree(w.prefix)
}

func (w *Watcher) cleanupTree(node string) {
	children, stat, err := w.conn.Children(node)
	if err != nil {
		return
	} else if stat.EphemeralOwner != 0 {
		return
	}

	for _, child := range children {
		w.cleanupTree(path.Join(node, child))
	}

	w.conn.Delete(node, -1)
}

func (w *Watcher) Close() {
	w.lock.Lock()
	defer w.lock.Unlock()

	w.shutdown <- true
	w.conn.Close()
}

// sendErr sends the error over the channel, or discards it if the error is full.
func sendErr(errs chan error, err error, path string, server string) {
	log.Printf("Zookeeper error: err=%q, path=%q, server=%q", err, path, server)

	select {
	case errs <- err:
	default:
	}
}
<|endoftext|>


<commit_before>package node

import (
	"errors"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/db"
	"github.com/MG-RAST/golib/mgo"
	"github.com/MG-RAST/golib/mgo/bson"
	"io/ioutil"
	"path/filepath"
	"strings"
)

// Initialize creates a copy of the mongodb connection and then uses that connection to
// create the Nodes collection in mongodb. Then, it ensures that there is a unique index
// on the id key in this collection, creating the index if necessary.
func Initialize() {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	c.EnsureIndex(mgo.Index{Key: []string{"id"}, Unique: true})
	c.EnsureIndex(mgo.Index{Key: []string{"file.path"}, Background: true})
	c.EnsureIndex(mgo.Index{Key: []string{"file.virtual_parts"}, Background: true})
	if conf.Conf["mongodb-indexes"] != "" {
		for _, v := range strings.Split(conf.Conf["mongodb-attribute-indexes"], ",") {
			c.EnsureIndex(mgo.Index{Key: []string{strings.TrimSpace(v)}, Background: true})
		}
	}
}

func dbDelete(q bson.M) (err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	_, err = c.RemoveAll(q)
	return
}

func dbUpsert(n *Node) (err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	_, err = c.Upsert(bson.M{"id": n.Id}, &n)
	return
}

func dbFind(q bson.M, results *Nodes, options map[string]int) (count int, err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	if limit, has := options["limit"]; has {
		if offset, has := options["offset"]; has {
			query := c.Find(q)
			if count, err = query.Count(); err != nil {
				return 0, err
			}
			err = query.Limit(limit).Skip(offset).All(results)
			return
		} else {
			return 0, errors.New("store.db.Find options limit and offset must be used together")
		}
	}
	err = c.Find(q).All(results)
	return
}

func Load(id string, uuid string) (n *Node, err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	n = new(Node)
	if err = c.Find(bson.M{"id": id}).One(&n); err == nil {
		rights := n.Acl.Check(uuid)
		if !rights["read"] {
			return nil, errors.New("User Unauthorized")
		}
		return n, nil
	} else {
		return nil, err
	}
	return
}

func LoadUnauth(id string) (n *Node, err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	n = new(Node)
	if err = c.Find(bson.M{"id": id}).One(&n); err == nil {
		return n, nil
	} else {
		return nil, err
	}
	return
}

func LoadNodes(ids []string) (n Nodes, err error) {
	if _, err = dbFind(bson.M{"id": bson.M{"$in": ids}}, &n, nil); err == nil {
		return n, err
	}
	return nil, err
}

func ReloadFromDisk(path string) (err error) {
	id := filepath.Base(path)
	nbson, err := ioutil.ReadFile(path + "/" + id + ".bson")
	if err != nil {
		return
	}
	node := new(Node)
	if err = bson.Unmarshal(nbson, &node); err == nil {
		if err = dbUpsert(node); err != nil {
			return err
		}
	}
	return
}
<commit_msg>Added indexes for acl queries<commit_after>package node

import (
	"errors"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/db"
	"github.com/MG-RAST/golib/mgo"
	"github.com/MG-RAST/golib/mgo/bson"
	"io/ioutil"
	"path/filepath"
	"strings"
)

// Initialize creates a copy of the mongodb connection and then uses that connection to
// create the Nodes collection in mongodb. Then, it ensures that there is a unique index
// on the id key in this collection, creating the index if necessary.
func Initialize() {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	c.EnsureIndex(mgo.Index{Key: []string{"id"}, Unique: true})
	c.EnsureIndex(mgo.Index{Key: []string{"file.path"}, Background: true})
	c.EnsureIndex(mgo.Index{Key: []string{"file.virtual_parts"}, Background: true})
	c.EnsureIndex(mgo.Index{Key: []string{"acl.owner"}, Background: true})
	c.EnsureIndex(mgo.Index{Key: []string{"acl.read"}, Background: true})
	c.EnsureIndex(mgo.Index{Key: []string{"acl.write"}, Background: true})
	c.EnsureIndex(mgo.Index{Key: []string{"acl.delete"}, Background: true})
	if conf.Conf["mongodb-indexes"] != "" {
		for _, v := range strings.Split(conf.Conf["mongodb-attribute-indexes"], ",") {
			c.EnsureIndex(mgo.Index{Key: []string{strings.TrimSpace(v)}, Background: true})
		}
	}
}

func dbDelete(q bson.M) (err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	_, err = c.RemoveAll(q)
	return
}

func dbUpsert(n *Node) (err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	_, err = c.Upsert(bson.M{"id": n.Id}, &n)
	return
}

func dbFind(q bson.M, results *Nodes, options map[string]int) (count int, err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	if limit, has := options["limit"]; has {
		if offset, has := options["offset"]; has {
			query := c.Find(q)
			if count, err = query.Count(); err != nil {
				return 0, err
			}
			err = query.Limit(limit).Skip(offset).All(results)
			return
		} else {
			return 0, errors.New("store.db.Find options limit and offset must be used together")
		}
	}
	err = c.Find(q).All(results)
	return
}

func Load(id string, uuid string) (n *Node, err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	n = new(Node)
	if err = c.Find(bson.M{"id": id}).One(&n); err == nil {
		rights := n.Acl.Check(uuid)
		if !rights["read"] {
			return nil, errors.New("User Unauthorized")
		}
		return n, nil
	} else {
		return nil, err
	}
	return
}

func LoadUnauth(id string) (n *Node, err error) {
	session := db.Connection.Session.Copy()
	defer session.Close()
	c := session.DB(conf.Conf["mongodb-database"]).C("Nodes")
	n = new(Node)
	if err = c.Find(bson.M{"id": id}).One(&n); err == nil {
		return n, nil
	} else {
		return nil, err
	}
	return
}

func LoadNodes(ids []string) (n Nodes, err error) {
	if _, err = dbFind(bson.M{"id": bson.M{"$in": ids}}, &n, nil); err == nil {
		return n, err
	}
	return nil, err
}

func ReloadFromDisk(path string) (err error) {
	id := filepath.Base(path)
	nbson, err := ioutil.ReadFile(path + "/" + id + ".bson")
	if err != nil {
		return
	}
	node := new(Node)
	if err = bson.Unmarshal(nbson, &node); err == nil {
		if err = dbUpsert(node); err != nil {
			return err
		}
	}
	return
}
<|endoftext|>


<commit_before>package runner

import (
	"os/exec"
	"strings"

	"github.com/go-errors/errors"
)

type appRunner struct {
	params *RunnerParams
}

var _ Runner = (*appRunner)(nil)

func newAppRunner(params *RunnerParams) (Runner, error) {
	ar := &appRunner{
		params: params,
	}
	return ar, nil
}

func (ar *appRunner) Prepare() error {
	// nothing to prepare
	return nil
}

func (ar *appRunner) Run() error {
	params := ar.params
	consumer := params.Consumer

	fullLowerPath := strings.ToLower(params.FullTargetPath)
	if !strings.HasSuffix(fullLowerPath, ".app") {
		// TODO: relax that a bit at some point
		return errors.New("itch only supports launching .app bundles")
	}

	var args = []string{
		"-W",
		params.FullTargetPath,
		"--args",
	}
	args = append(args, params.Args...)

	consumer.Infof("Opening (%s)", params.FullTargetPath)

	cmd := exec.CommandContext(params.Ctx, "open", args...)
	// I doubt this matters
	cmd.Dir = params.Dir
	cmd.Env = params.Env
	// 'open' does not relay stdout or stderr, so we don't
	// even bother setting them

	err := cmd.Run()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	return nil
}
<commit_msg>Install signal handler when running app on macOS<commit_after>package runner

import (
	"os"
	"os/exec"
	"os/signal"
	"strings"

	"github.com/go-errors/errors"
)

type appRunner struct {
	params *RunnerParams
}

var _ Runner = (*appRunner)(nil)

func newAppRunner(params *RunnerParams) (Runner, error) {
	ar := &appRunner{
		params: params,
	}
	return ar, nil
}

func (ar *appRunner) Prepare() error {
	// nothing to prepare
	return nil
}

func (ar *appRunner) Run() error {
	params := ar.params
	consumer := params.Consumer

	fullLowerPath := strings.ToLower(params.FullTargetPath)
	if !strings.HasSuffix(fullLowerPath, ".app") {
		// TODO: relax that a bit at some point
		return errors.New("itch only supports launching .app bundles")
	}

	var args = []string{
		"-W",
		params.FullTargetPath,
		"--args",
	}
	args = append(args, params.Args...)

	consumer.Infof("Opening (%s)", params.FullTargetPath)

	cmd := exec.CommandContext(params.Ctx, "open", args...)
	// I doubt this matters
	cmd.Dir = params.Dir
	cmd.Env = params.Env
	// 'open' does not relay stdout or stderr, so we don't
	// even bother setting them

	go func() {
		// catch SIGINT and kill the child if needed
		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)

		consumer.Infof("Signal handler installed...")

		// Block until a signal is received.
		s := <-c
		consumer.Warnf("Got signal:", s)

		consumer.Warnf("Killing app...")
		// TODO: kill the actual binary, not the app
		cmd := exec.Command("pkill", "-f", params.FullTargetPath)
		err := cmd.Run()
		if err != nil {
			consumer.Errorf("While killing: ", err.Error())
		}
		os.Exit(0)
	}()

	err := cmd.Run()
	if err != nil {
		return errors.Wrap(err, 0)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lifecycle defines an event for an app's lifecycle.
//
// The app lifecycle consists of moving back and forth between an ordered
// sequence of stages. For example, being at a stage greater than or equal to
// StageVisible means that the app is visible on the screen.
//
// A lifecycle event is a change from one stage to another, which crosses every
// intermediate stage. For example, changing from StageAlive to StageFocused
// implicitly crosses StageVisible.
//
// Crosses can be in a positive or negative direction. A positive crossing of
// StageFocused means that the app has gained the focus. A negative crossing
// means it has lost the focus.
//
// See the golang.org/x/mobile/app package for details on the event model.
package lifecycle // import "golang.org/x/mobile/event/lifecycle"

import (
	"fmt"
)

// Cross is whether a lifecycle stage was crossed.
type Cross uint32

func (c Cross) String() string {
	switch c {
	case CrossOn:
		return "on"
	case CrossOff:
		return "off"
	}
	return "none"
}

const (
	CrossNone Cross = 0
	CrossOn   Cross = 1
	CrossOff  Cross = 2
)

// Event is a lifecycle change from an old stage to a new stage.
type Event struct {
	From, To Stage

	// DrawContext is the state used for painting, if any is valid.
	//
	// For OpenGL apps, a non-nil DrawContext is a gl.Context.
	//
	// TODO: make this an App method if we move away from an event channel?
	DrawContext interface{}
}

// Crosses returns whether the transition from From to To crosses the stage s:
// 	- It returns CrossOn if it does, and the lifecycle change is positive.
// 	- It returns CrossOff if it does, and the lifecycle change is negative.
//	- Otherwise, it returns CrossNone.
// See the documentation for Stage for more discussion of positive and negative
// crosses.
func (e Event) Crosses(s Stage) Cross {
	switch {
	case e.From < s && e.To >= s:
		return CrossOn
	case e.From >= s && e.To < s:
		return CrossOff
	}
	return CrossNone
}

// Stage is a stage in the app's lifecycle. The values are ordered, so that a
// lifecycle change from stage From to stage To implicitly crosses every stage
// in the range (min, max], exclusive on the low end and inclusive on the high
// end, where min is the minimum of From and To, and max is the maximum.
//
// The documentation for individual stages talk about positive and negative
// crosses. A positive lifecycle change is one where its From stage is less
// than its To stage. Similarly, a negative lifecycle change is one where From
// is greater than To. Thus, a positive lifecycle change crosses every stage in
// the range (From, To] in increasing order, and a negative lifecycle change
// crosses every stage in the range (To, From] in decreasing order.
type Stage uint32

// TODO: how does iOS map to these stages? What do cross-platform mobile
// abstractions do?

const (
	// StageDead is the zero stage. No lifecycle change crosses this stage,
	// but:
	//	- A positive change from this stage is the very first lifecycle change.
	//	- A negative change to this stage is the very last lifecycle change.
	StageDead Stage = iota

	// StageAlive means that the app is alive.
	//	- A positive cross means that the app has been created.
	//	- A negative cross means that the app is being destroyed.
	// Each cross, either from or to StageDead, will occur only once.
	// On Android, these correspond to onCreate and onDestroy.
	StageAlive

	// StageVisible means that the app window is visible.
	//	- A positive cross means that the app window has become visible.
	//	- A negative cross means that the app window has become invisible.
	// On Android, these correspond to onStart and onStop.
	// On Desktop, an app window can become invisible if e.g. it is minimized,
	// unmapped, or not on a visible workspace.
	StageVisible

	// StageFocused means that the app window has the focus.
	//	- A positive cross means that the app window has gained the focus.
	//	- A negative cross means that the app window has lost the focus.
	// On Android, these correspond to onResume and onFreeze.
	StageFocused
)

func (s Stage) String() string {
	switch s {
	case StageDead:
		return "StageDead"
	case StageAlive:
		return "StageAlive"
	case StageVisible:
		return "StageVisible"
	case StageFocused:
		return "StageFocused"
	default:
		return fmt.Sprintf("lifecycle.Stage(%d)", s)
	}
}
<commit_msg>event/lifecycle: add an Event.String method.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package lifecycle defines an event for an app's lifecycle.
//
// The app lifecycle consists of moving back and forth between an ordered
// sequence of stages. For example, being at a stage greater than or equal to
// StageVisible means that the app is visible on the screen.
//
// A lifecycle event is a change from one stage to another, which crosses every
// intermediate stage. For example, changing from StageAlive to StageFocused
// implicitly crosses StageVisible.
//
// Crosses can be in a positive or negative direction. A positive crossing of
// StageFocused means that the app has gained the focus. A negative crossing
// means it has lost the focus.
//
// See the golang.org/x/mobile/app package for details on the event model.
package lifecycle // import "golang.org/x/mobile/event/lifecycle"

import (
	"fmt"
)

// Cross is whether a lifecycle stage was crossed.
type Cross uint32

func (c Cross) String() string {
	switch c {
	case CrossOn:
		return "on"
	case CrossOff:
		return "off"
	}
	return "none"
}

const (
	CrossNone Cross = 0
	CrossOn   Cross = 1
	CrossOff  Cross = 2
)

// Event is a lifecycle change from an old stage to a new stage.
type Event struct {
	From, To Stage

	// DrawContext is the state used for painting, if any is valid.
	//
	// For OpenGL apps, a non-nil DrawContext is a gl.Context.
	//
	// TODO: make this an App method if we move away from an event channel?
	DrawContext interface{}
}

func (e Event) String() string {
	return fmt.Sprintf("lifecycle.Event{From:%v, To:%v, DrawContext:%v}", e.From, e.To, e.DrawContext)
}

// Crosses returns whether the transition from From to To crosses the stage s:
// 	- It returns CrossOn if it does, and the lifecycle change is positive.
// 	- It returns CrossOff if it does, and the lifecycle change is negative.
//	- Otherwise, it returns CrossNone.
// See the documentation for Stage for more discussion of positive and negative
// crosses.
func (e Event) Crosses(s Stage) Cross {
	switch {
	case e.From < s && e.To >= s:
		return CrossOn
	case e.From >= s && e.To < s:
		return CrossOff
	}
	return CrossNone
}

// Stage is a stage in the app's lifecycle. The values are ordered, so that a
// lifecycle change from stage From to stage To implicitly crosses every stage
// in the range (min, max], exclusive on the low end and inclusive on the high
// end, where min is the minimum of From and To, and max is the maximum.
//
// The documentation for individual stages talk about positive and negative
// crosses. A positive lifecycle change is one where its From stage is less
// than its To stage. Similarly, a negative lifecycle change is one where From
// is greater than To. Thus, a positive lifecycle change crosses every stage in
// the range (From, To] in increasing order, and a negative lifecycle change
// crosses every stage in the range (To, From] in decreasing order.
type Stage uint32

// TODO: how does iOS map to these stages? What do cross-platform mobile
// abstractions do?

const (
	// StageDead is the zero stage. No lifecycle change crosses this stage,
	// but:
	//	- A positive change from this stage is the very first lifecycle change.
	//	- A negative change to this stage is the very last lifecycle change.
	StageDead Stage = iota

	// StageAlive means that the app is alive.
	//	- A positive cross means that the app has been created.
	//	- A negative cross means that the app is being destroyed.
	// Each cross, either from or to StageDead, will occur only once.
	// On Android, these correspond to onCreate and onDestroy.
	StageAlive

	// StageVisible means that the app window is visible.
	//	- A positive cross means that the app window has become visible.
	//	- A negative cross means that the app window has become invisible.
	// On Android, these correspond to onStart and onStop.
	// On Desktop, an app window can become invisible if e.g. it is minimized,
	// unmapped, or not on a visible workspace.
	StageVisible

	// StageFocused means that the app window has the focus.
	//	- A positive cross means that the app window has gained the focus.
	//	- A negative cross means that the app window has lost the focus.
	// On Android, these correspond to onResume and onFreeze.
	StageFocused
)

func (s Stage) String() string {
	switch s {
	case StageDead:
		return "StageDead"
	case StageAlive:
		return "StageAlive"
	case StageVisible:
		return "StageVisible"
	case StageFocused:
		return "StageFocused"
	default:
		return fmt.Sprintf("lifecycle.Stage(%d)", s)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package kbfsblock

import (
	"context"
	"errors"

	"github.com/keybase/backoff"
	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/kbfscodec"
	"github.com/keybase/kbfs/kbfscrypto"
	"github.com/keybase/kbfs/tlf"
)

func makeIDCombo(id ID, context Context) keybase1.BlockIdCombo {
	// ChargedTo is somewhat confusing when this BlockIdCombo is
	// used in a BlockReference -- it just refers to the original
	// creator of the block, i.e. the original user charged for
	// the block.
	//
	// This may all change once we implement groups.
	return keybase1.BlockIdCombo{
		BlockHash: id.String(),
		ChargedTo: context.GetCreator(),
		BlockType: context.GetBlockType(),
	}
}

func makeReference(id ID, context Context) keybase1.BlockReference {
	// Block references to MD blocks are allowed, because they can be
	// deleted in the case of an MD put failing.
	return keybase1.BlockReference{
		Bid: makeIDCombo(id, context),
		// The actual writer to modify quota for.
		ChargedTo: context.GetWriter(),
		Nonce:     keybase1.BlockRefNonce(context.GetRefNonce()),
	}
}

// MakeGetBlockArg builds a keybase1.GetBlockArg from the given params.
func MakeGetBlockArg(tlfID tlf.ID, id ID, context Context) keybase1.GetBlockArg {
	return keybase1.GetBlockArg{
		Bid:    makeIDCombo(id, context),
		Folder: tlfID.String(),
	}
}

// ParseGetBlockRes parses the given keybase1.GetBlockRes into its
// components.
func ParseGetBlockRes(res keybase1.GetBlockRes, resErr error) (
	buf []byte, serverHalf kbfscrypto.BlockCryptKeyServerHalf, err error) {
	if resErr != nil {
		return nil, kbfscrypto.BlockCryptKeyServerHalf{}, resErr
	}
	serverHalf, err = kbfscrypto.ParseBlockCryptKeyServerHalf(res.BlockKey)
	if err != nil {
		return nil, kbfscrypto.BlockCryptKeyServerHalf{}, err
	}
	return res.Buf, serverHalf, nil
}

// MakePutBlockArg builds a keybase1.PutBlockArg from the given params.
func MakePutBlockArg(tlfID tlf.ID, id ID,
	bContext Context, buf []byte,
	serverHalf kbfscrypto.BlockCryptKeyServerHalf) keybase1.PutBlockArg {
	return keybase1.PutBlockArg{
		Bid: makeIDCombo(id, bContext),
		// BlockKey is misnamed -- it contains just the server
		// half.
		BlockKey: serverHalf.String(),
		Folder:   tlfID.String(),
		Buf:      buf,
	}
}

// MakePutBlockAgainArg builds a keybase1.PutBlockAgainArg from the
// given params.
func MakePutBlockAgainArg(tlfID tlf.ID, id ID,
	bContext Context, buf []byte, serverHalf kbfscrypto.BlockCryptKeyServerHalf) keybase1.PutBlockAgainArg {
	return keybase1.PutBlockAgainArg{
		Ref: makeReference(id, bContext),
		// BlockKey is misnamed -- it contains just the server
		// half.
		BlockKey: serverHalf.String(),
		Folder:   tlfID.String(),
		Buf:      buf,
	}
}

// MakeAddReferenceArg builds a keybase1.AddReferenceArg from the
// given params.
func MakeAddReferenceArg(tlfID tlf.ID, id ID, context Context) keybase1.AddReferenceArg {
	return keybase1.AddReferenceArg{
		Ref:    makeReference(id, context),
		Folder: tlfID.String(),
	}
}

// getNotDone returns the set of block references in "all" that do not
// yet appear in "results"
func getNotDone(all ContextMap, doneRefs map[ID]map[RefNonce]int) (
	notDone []keybase1.BlockReference) {
	for id, idContexts := range all {
		for _, context := range idContexts {
			if _, ok := doneRefs[id]; ok {
				if _, ok1 := doneRefs[id][context.GetRefNonce()]; ok1 {
					continue
				}
			}
			ref := makeReference(id, context)
			notDone = append(notDone, ref)
		}
	}
	return notDone
}

// BatchDowngradeReferences archives or deletes a batch of references,
// handling all batching and throttles.
func BatchDowngradeReferences(ctx context.Context, log logger.Logger,
	tlfID tlf.ID, contexts ContextMap, archive bool,
	server keybase1.BlockInterface) (
	doneRefs map[ID]map[RefNonce]int, finalError error) {
	doneRefs = make(map[ID]map[RefNonce]int)
	notDone := getNotDone(contexts, doneRefs)

	throttleErr := backoff.Retry(func() error {
		var res keybase1.DowngradeReferenceRes
		var err error
		if archive {
			res, err = server.ArchiveReferenceWithCount(ctx,
				keybase1.ArchiveReferenceWithCountArg{
					Refs:   notDone,
					Folder: tlfID.String(),
				})
		} else {
			res, err = server.DelReferenceWithCount(ctx,
				keybase1.DelReferenceWithCountArg{
					Refs:   notDone,
					Folder: tlfID.String(),
				})
		}

		// log errors
		if err != nil {
			log.CWarningf(ctx, "batchDowngradeReferences archive=%t sent=%v done=%v failedRef=%v err=%v",
				archive, notDone, res.Completed, res.Failed, err)
		} else {
			log.CDebugf(ctx, "batchDowngradeReferences archive=%t notdone=%v all succeeded",
				archive, notDone)
		}

		// update the set of completed reference
		for _, ref := range res.Completed {
			bid, err := IDFromString(ref.Ref.Bid.BlockHash)
			if err != nil {
				continue
			}
			nonces, ok := doneRefs[bid]
			if !ok {
				nonces = make(map[RefNonce]int)
				doneRefs[bid] = nonces
			}
			nonces[RefNonce(ref.Ref.Nonce)] = ref.LiveCount
		}
		// update the list of references to downgrade
		notDone = getNotDone(contexts, doneRefs)

		//if context is cancelled, return immediately
		select {
		case <-ctx.Done():
			finalError = ctx.Err()
			return nil
		default:
		}

		// check whether to backoff and retry
		if err != nil {
			// if error is of type throttle, retry
			if IsThrottleError(err) {
				return err
			}
			// non-throttle error, do not retry here
			finalError = err
		}
		return nil
	}, backoff.NewExponentialBackOff())

	// if backoff has given up retrying, return error
	if throttleErr != nil {
		return doneRefs, throttleErr
	}

	if finalError == nil {
		if len(notDone) != 0 {
			log.CErrorf(ctx, "batchDowngradeReferences finished successfully with outstanding refs? all=%v done=%v notDone=%v\n", contexts, doneRefs, notDone)
			return doneRefs,
				errors.New("batchDowngradeReferences inconsistent result")
		}
	}
	return doneRefs, finalError
}

// GetLiveCounts computes the maximum live count for each ID over its
// RefNonces.
func GetLiveCounts(doneRefs map[ID]map[RefNonce]int) map[ID]int {
	liveCounts := make(map[ID]int)
	for id, nonces := range doneRefs {
		for _, count := range nonces {
			if existing, ok := liveCounts[id]; !ok || existing > count {
				liveCounts[id] = count
			}
		}
	}
	return liveCounts
}

// ParseGetQuotaInfoRes parses the given quota result into a
// *QuotaInfo.
func ParseGetQuotaInfoRes(codec kbfscodec.Codec, res []byte, resErr error) (
	info *QuotaInfo, err error) {
	if resErr != nil {
		return nil, resErr
	}
	return QuotaInfoDecode(res, codec)
}

// GetReferenceCount returns the number of live references (at least
// as "live" as `refStatus`) for each block ID.
func GetReferenceCount(
	ctx context.Context, tlfID tlf.ID, contexts ContextMap,
	refStatus keybase1.BlockStatus, server keybase1.BlockInterface) (
	liveCounts map[ID]int, err error) {
	arg := keybase1.GetReferenceCountArg{
		Ids:    make([]keybase1.BlockIdCombo, 0, len(contexts)),
		Folder: tlfID.String(),
		Status: refStatus,
	}
	for id, idContexts := range contexts {
		if len(idContexts) < 1 {
			return nil, errors.New("Each ID must have at least one context")
		}
		context := idContexts[0]
		arg.Ids = append(arg.Ids, makeIDCombo(id, context))
	}

	res, err := server.GetReferenceCount(ctx, arg)
	if err != nil {
		return nil, err
	}

	liveCounts = make(map[ID]int, len(res.Counts))
	for _, count := range res.Counts {
		id, err := IDFromString(count.Id.BlockHash)
		if err != nil {
			return nil, err
		}

		liveCounts[id] = count.LiveCount
	}
	return liveCounts, nil
}
<commit_msg>Revert "kbfsblock: protocol helper for getting ref counts"<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package kbfsblock

import (
	"context"
	"errors"

	"github.com/keybase/backoff"
	"github.com/keybase/client/go/logger"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/keybase/kbfs/kbfscodec"
	"github.com/keybase/kbfs/kbfscrypto"
	"github.com/keybase/kbfs/tlf"
)

func makeIDCombo(id ID, context Context) keybase1.BlockIdCombo {
	// ChargedTo is somewhat confusing when this BlockIdCombo is
	// used in a BlockReference -- it just refers to the original
	// creator of the block, i.e. the original user charged for
	// the block.
	//
	// This may all change once we implement groups.
	return keybase1.BlockIdCombo{
		BlockHash: id.String(),
		ChargedTo: context.GetCreator(),
		BlockType: context.GetBlockType(),
	}
}

func makeReference(id ID, context Context) keybase1.BlockReference {
	// Block references to MD blocks are allowed, because they can be
	// deleted in the case of an MD put failing.
	return keybase1.BlockReference{
		Bid: makeIDCombo(id, context),
		// The actual writer to modify quota for.
		ChargedTo: context.GetWriter(),
		Nonce:     keybase1.BlockRefNonce(context.GetRefNonce()),
	}
}

// MakeGetBlockArg builds a keybase1.GetBlockArg from the given params.
func MakeGetBlockArg(tlfID tlf.ID, id ID, context Context) keybase1.GetBlockArg {
	return keybase1.GetBlockArg{
		Bid:    makeIDCombo(id, context),
		Folder: tlfID.String(),
	}
}

// ParseGetBlockRes parses the given keybase1.GetBlockRes into its
// components.
func ParseGetBlockRes(res keybase1.GetBlockRes, resErr error) (
	buf []byte, serverHalf kbfscrypto.BlockCryptKeyServerHalf, err error) {
	if resErr != nil {
		return nil, kbfscrypto.BlockCryptKeyServerHalf{}, resErr
	}
	serverHalf, err = kbfscrypto.ParseBlockCryptKeyServerHalf(res.BlockKey)
	if err != nil {
		return nil, kbfscrypto.BlockCryptKeyServerHalf{}, err
	}
	return res.Buf, serverHalf, nil
}

// MakePutBlockArg builds a keybase1.PutBlockArg from the given params.
func MakePutBlockArg(tlfID tlf.ID, id ID,
	bContext Context, buf []byte,
	serverHalf kbfscrypto.BlockCryptKeyServerHalf) keybase1.PutBlockArg {
	return keybase1.PutBlockArg{
		Bid: makeIDCombo(id, bContext),
		// BlockKey is misnamed -- it contains just the server
		// half.
		BlockKey: serverHalf.String(),
		Folder:   tlfID.String(),
		Buf:      buf,
	}
}

// MakePutBlockAgainArg builds a keybase1.PutBlockAgainArg from the
// given params.
func MakePutBlockAgainArg(tlfID tlf.ID, id ID,
	bContext Context, buf []byte, serverHalf kbfscrypto.BlockCryptKeyServerHalf) keybase1.PutBlockAgainArg {
	return keybase1.PutBlockAgainArg{
		Ref: makeReference(id, bContext),
		// BlockKey is misnamed -- it contains just the server
		// half.
		BlockKey: serverHalf.String(),
		Folder:   tlfID.String(),
		Buf:      buf,
	}
}

// MakeAddReferenceArg builds a keybase1.AddReferenceArg from the
// given params.
func MakeAddReferenceArg(tlfID tlf.ID, id ID, context Context) keybase1.AddReferenceArg {
	return keybase1.AddReferenceArg{
		Ref:    makeReference(id, context),
		Folder: tlfID.String(),
	}
}

// getNotDone returns the set of block references in "all" that do not
// yet appear in "results"
func getNotDone(all ContextMap, doneRefs map[ID]map[RefNonce]int) (
	notDone []keybase1.BlockReference) {
	for id, idContexts := range all {
		for _, context := range idContexts {
			if _, ok := doneRefs[id]; ok {
				if _, ok1 := doneRefs[id][context.GetRefNonce()]; ok1 {
					continue
				}
			}
			ref := makeReference(id, context)
			notDone = append(notDone, ref)
		}
	}
	return notDone
}

// BatchDowngradeReferences archives or deletes a batch of references,
// handling all batching and throttles.
func BatchDowngradeReferences(ctx context.Context, log logger.Logger,
	tlfID tlf.ID, contexts ContextMap, archive bool,
	server keybase1.BlockInterface) (
	doneRefs map[ID]map[RefNonce]int, finalError error) {
	doneRefs = make(map[ID]map[RefNonce]int)
	notDone := getNotDone(contexts, doneRefs)

	throttleErr := backoff.Retry(func() error {
		var res keybase1.DowngradeReferenceRes
		var err error
		if archive {
			res, err = server.ArchiveReferenceWithCount(ctx,
				keybase1.ArchiveReferenceWithCountArg{
					Refs:   notDone,
					Folder: tlfID.String(),
				})
		} else {
			res, err = server.DelReferenceWithCount(ctx,
				keybase1.DelReferenceWithCountArg{
					Refs:   notDone,
					Folder: tlfID.String(),
				})
		}

		// log errors
		if err != nil {
			log.CWarningf(ctx, "batchDowngradeReferences archive=%t sent=%v done=%v failedRef=%v err=%v",
				archive, notDone, res.Completed, res.Failed, err)
		} else {
			log.CDebugf(ctx, "batchDowngradeReferences archive=%t notdone=%v all succeeded",
				archive, notDone)
		}

		// update the set of completed reference
		for _, ref := range res.Completed {
			bid, err := IDFromString(ref.Ref.Bid.BlockHash)
			if err != nil {
				continue
			}
			nonces, ok := doneRefs[bid]
			if !ok {
				nonces = make(map[RefNonce]int)
				doneRefs[bid] = nonces
			}
			nonces[RefNonce(ref.Ref.Nonce)] = ref.LiveCount
		}
		// update the list of references to downgrade
		notDone = getNotDone(contexts, doneRefs)

		//if context is cancelled, return immediately
		select {
		case <-ctx.Done():
			finalError = ctx.Err()
			return nil
		default:
		}

		// check whether to backoff and retry
		if err != nil {
			// if error is of type throttle, retry
			if IsThrottleError(err) {
				return err
			}
			// non-throttle error, do not retry here
			finalError = err
		}
		return nil
	}, backoff.NewExponentialBackOff())

	// if backoff has given up retrying, return error
	if throttleErr != nil {
		return doneRefs, throttleErr
	}

	if finalError == nil {
		if len(notDone) != 0 {
			log.CErrorf(ctx, "batchDowngradeReferences finished successfully with outstanding refs? all=%v done=%v notDone=%v\n", contexts, doneRefs, notDone)
			return doneRefs,
				errors.New("batchDowngradeReferences inconsistent result")
		}
	}
	return doneRefs, finalError
}

// GetLiveCounts computes the maximum live count for each ID over its
// RefNonces.
func GetLiveCounts(doneRefs map[ID]map[RefNonce]int) map[ID]int {
	liveCounts := make(map[ID]int)
	for id, nonces := range doneRefs {
		for _, count := range nonces {
			if existing, ok := liveCounts[id]; !ok || existing > count {
				liveCounts[id] = count
			}
		}
	}
	return liveCounts
}

// ParseGetQuotaInfoRes parses the given quota result into a
// *QuotaInfo.
func ParseGetQuotaInfoRes(codec kbfscodec.Codec, res []byte, resErr error) (
	info *QuotaInfo, err error) {
	if resErr != nil {
		return nil, resErr
	}
	return QuotaInfoDecode(res, codec)
}
<|endoftext|>


<commit_before>package gomposer

// TODO reanme
type PackageRepository struct {
	client *HttpClient
}

func (r *PackageRepository) Find(packageName string) (*Packages, error) {

	output := &Packages{}

	err := r.client.Request("GET", "/phpunit.json", output)

	return output, err
}

type Packages struct {
	Name        string       `json:"name"`
	Description string       `json:"description"`
	Time        string       `json:"time"`
	Maintainers []Maintainer `json:"maintainers"`
	Versions    []Version    `json:"versions"`
	Type        string       `json:"type"`
	Repository  string       `json:"repository"`
	Downloads   Downloads    `json:"downloads"`
	Favers      int          `json:"favers"`
}

type Maintainer struct {
	Name string `json:"name"`
}

type Author struct {
	Name  string `json:"name"`
	Email string `json:"email"`
	Role  string `json:"role"`
}

type Source struct {
	Type      string `json:"type"`
	Url       string `json:"url"`
	Reference string `json:"reference"`
	Shasum    string `json:"shasum"`
}

type Downloads struct {
	Total   int `json:"total"`
	Monthly int `json:"monthly"`
	Daily   int `json:daily`
}

type Autoload struct {
	ClassMap []string          `json:"classmap"`
	Files    []string          `json:"files"`
	Psr0     map[string]string `json:"psr-0"`
	Psr4     map[string]string `json:"psr-4"`
}

// TODO rename since this going to be the name composer.json
type Version struct {
	Name              string            `json:"name"`
	Description       string            `json:"description"`
	Keywords          []string          `json:"keywords"`
	Homepage          string            `json:"homepage"`
	Version           string            `json:"version"`
	VersionNormalized string            `json:version_normalized`
	License           []string          `json:"license"`
	Authors           Author            `json:"authors"`
	Source            Source            `json:"source"`
	Dist              Source            `json:"dist"`
	Type              string            `json:"string"`
	Time              string            `json:"time"`
	Autoload          Autoload          `json:"autoload"`
	Require           map[string]string `json:"require"`
	RequireDev        map[string]string `json:"require-dev"`
	Suggest           map[string]string `json:"suggest"`
}
<commit_msg>Refactor Packages to PackageInfo<commit_after>package gomposer

// TODO reanme
type PackageRepository struct {
	client *HttpClient
}

func (r *PackageRepository) Find(packageName string) (*PackageInfo, error) {

	output := &PackageInfo{}

	err := r.client.Request("GET", "/phpunit.json", output)

	return output, err
}

type PackageInfo struct {
	Name        string       `json:"name"`
	Description string       `json:"description"`
	Time        string       `json:"time"`
	Maintainers []Maintainer `json:"maintainers"`
	Versions    []Version    `json:"versions"`
	Type        string       `json:"type"`
	Repository  string       `json:"repository"`
	Downloads   Downloads    `json:"downloads"`
	Favers      int          `json:"favers"`
}

type Maintainer struct {
	Name string `json:"name"`
}

type Author struct {
	Name  string `json:"name"`
	Email string `json:"email"`
	Role  string `json:"role"`
}

type Source struct {
	Type      string `json:"type"`
	Url       string `json:"url"`
	Reference string `json:"reference"`
	Shasum    string `json:"shasum"`
}

type Downloads struct {
	Total   int `json:"total"`
	Monthly int `json:"monthly"`
	Daily   int `json:daily`
}

type Autoload struct {
	ClassMap []string          `json:"classmap"`
	Files    []string          `json:"files"`
	Psr0     map[string]string `json:"psr-0"`
	Psr4     map[string]string `json:"psr-4"`
}

// TODO rename since this going to be the name composer.json
type Version struct {
	Name              string            `json:"name"`
	Description       string            `json:"description"`
	Keywords          []string          `json:"keywords"`
	Homepage          string            `json:"homepage"`
	Version           string            `json:"version"`
	VersionNormalized string            `json:version_normalized`
	License           []string          `json:"license"`
	Authors           Author            `json:"authors"`
	Source            Source            `json:"source"`
	Dist              Source            `json:"dist"`
	Type              string            `json:"string"`
	Time              string            `json:"time"`
	Autoload          Autoload          `json:"autoload"`
	Require           map[string]string `json:"require"`
	RequireDev        map[string]string `json:"require-dev"`
	Suggest           map[string]string `json:"suggest"`
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"io"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/health"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/mesos"
	"github.com/mesosphere/kubernetes-mesos/executor"
)

const (
	MESOS_CFG_SOURCE = "mesos" // @see ConfigSourceAnnotationKey
	defaultRootDir   = "/var/lib/kubelet"
)

var (
	syncFrequency           = flag.Duration("sync_frequency", 10*time.Second, "Max period between synchronizing running containers and config")
	address                 = util.IP(net.ParseIP("0.0.0.0"))
	port                    = flag.Uint("port", ports.KubeletPort, "The port for the info server to serve on") // TODO(jdef): use kmmaster.KubeletExecutorPort
	hostnameOverride        = flag.String("hostname_override", "", "If non-empty, will use this string as identification instead of the actual hostname.")
	networkContainerImage   = flag.String("network_container_image", kubelet.NetworkContainerImage, "The image that network containers in each pod will use.")
	dockerEndpoint          = flag.String("docker_endpoint", "", "If non-empty, use this for the docker endpoint to communicate with")
	etcdServerList          util.StringList
	etcdConfigFile          = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers")
	rootDirectory           = flag.String("root_dir", defaultRootDir, "Directory path for managing kubelet files (volume mounts,etc).")
	allowPrivileged         = flag.Bool("allow_privileged", false, "If true, allow containers to request privileged mode. [default=false]")
	registryPullQPS         = flag.Float64("registry_qps", 0.0, "If > 0, limit registry pull QPS to this value.  If 0, unlimited. [default=0.0]")
	registryBurst           = flag.Int("registry_burst", 10, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry_qps.  Only used if --registry_qps > 0")
	enableDebuggingHandlers = flag.Bool("enable_debugging_handlers", true, "Enables server endpoints for log collection and local running of containers and commands. Default true.")
	minimumGCAge            = flag.Duration("minimum_container_ttl_duration", 0, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
	maxContainerCount       = flag.Int("maximum_dead_containers_per_container", 5, "Maximum number of old instances of a container to retain per container.  Each container takes up some disk space.  Default: 5.")
	authPath                = flag.String("auth_path", "", "Path to .kubernetes_auth file, specifying how to authenticate to API server.")
	apiServerList           util.StringList
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated")
	flag.Var(&address, "address", "The IP address for the info and proxy servers to serve on. Default to 0.0.0.0.")
	flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
}

func getDockerEndpoint() string {
	var endpoint string
	if len(*dockerEndpoint) > 0 {
		endpoint = *dockerEndpoint
	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
		endpoint = os.Getenv("DOCKER_HOST")
	} else {
		endpoint = "unix:///var/run/docker.sock"
	}
	log.Infof("Connecting to docker on %s", endpoint)

	return endpoint
}

func getHostname() string {
	hostname := []byte(*hostnameOverride)
	if string(hostname) == "" {
		// Note: We use exec here instead of os.Hostname() because we
		// want the FQDN, and this is the easiest way to get it.
		fqdn, err := exec.Command("hostname", "-f").Output()
		if err != nil {
			log.Fatalf("Couldn't determine hostname: %v", err)
		}
		hostname = fqdn
	}
	return strings.TrimSpace(string(hostname))
}

func getApiserverClient() (*client.Client, error) {
	authInfo, err := clientauth.LoadFromFile(*authPath)
	if err != nil {
		return nil, err
	}
	clientConfig, err := authInfo.MergeWithConfig(client.Config{})
	if err != nil {
		return nil, err
	}
	// TODO: adapt Kube client to support LB over several servers
	if len(apiServerList) > 1 {
		log.Infof("Mulitple api servers specified.  Picking first one")
	}
	clientConfig.Host = apiServerList[0]
	if c, err := client.New(&clientConfig); err != nil {
		return nil, err
	} else {
		return c, nil
	}
}

func main() {

	flag.Parse()
	util.InitLogs() // TODO(jdef) figure out where this actually sends logs by default
	defer util.FlushLogs()
	rand.Seed(time.Now().UTC().UnixNano())

	verflag.PrintAndExitIfRequested()

	etcd.SetLogger(util.NewLogger("etcd "))

	// Make an API client if possible.
	if len(apiServerList) < 1 {
		log.Info("No api servers specified.")
	} else {
		if apiClient, err := getApiserverClient(); err != nil {
			log.Errorf("Unable to make apiserver client: %v", err)
		} else {
			// Send events to APIserver if there is a client.
			log.Infof("Sending events to APIserver.")
			record.StartRecording(apiClient.Events(""), "kubelet")
		}
	}

	// Log the events locally too.
	record.StartLogging(log.Infof)

	capabilities.Initialize(capabilities.Capabilities{
		AllowPrivileged: *allowPrivileged,
	})

	dockerClient, err := docker.NewClient(getDockerEndpoint())
	if err != nil {
		log.Fatal("Couldn't connnect to docker.")
	}

	hostname := getHostname()

	if *rootDirectory == "" {
		log.Fatal("Invalid root directory path.")
	}
	*rootDirectory = path.Clean(*rootDirectory)
	if err := os.MkdirAll(*rootDirectory, 0750); err != nil {
		log.Warningf("Error creating root directory: %v", err)
	}

	// source of all configuration
	cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)

	// k8sm: no other pod configuration sources supported in this hybrid kubelet-executor

	// define etcd config source and initialize etcd client
	var etcdClient *etcd.Client
	if len(etcdServerList) > 0 {
		etcdClient = etcd.NewClient(etcdServerList)
	} else if *etcdConfigFile != "" {
		var err error
		etcdClient, err = etcd.NewClientFromFile(*etcdConfigFile)
		if err != nil {
			log.Fatalf("Error with etcd config file: %v", err)
		}
	}

	if etcdClient != nil {
		log.Infof("Connected to etcd at %v", etcdClient.GetCluster())
	}

	// TODO(???): Destroy existing k8s containers for now - we don't know how to reconcile yet.
	if containers, err := dockertools.GetKubeletDockerContainers(dockerClient, true); err == nil {
		for _, container := range containers {
			id := container.ID
			log.V(2).Infof("Removing container: %v", id)
			if err := dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: id, RemoveVolumes: true}); err != nil {
				log.Warning(err)
			}
		}
	} else {
		log.Warningf("Failed to list kubelet docker containers: %v", err)
	}

	// TODO(k8s): block until all sources have delivered at least one update to the channel, or break the sync loop
	// up into "per source" synchronizations

	kl := kubelet.NewMainKubelet(
		hostname,
		dockerClient,
		etcdClient,
		*rootDirectory,
		*networkContainerImage,
		*syncFrequency,
		float32(*registryPullQPS),
		*registryBurst,
		*minimumGCAge,
		*maxContainerCount)

	kl.BirthCry()

	go func() {
		util.Forever(func() {
			err := kl.GarbageCollectContainers()
			if err != nil {
				log.Errorf("Garbage collect failed: %v", err)
			}
		}, time.Minute*1)
	}()

	/* TODO(jdef): enable cadvisor integration
	go func() {
		defer util.HandleCrash()
		// TODO(k8s): Monitor this connection, reconnect if needed?
		log.V(1).Infof("Trying to create cadvisor client.")
		cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194")
		if err != nil {
			log.Errorf("Error on creating cadvisor client: %v", err)
			return
		}
		log.V(1).Infof("Successfully created cadvisor client.")
		kl.SetCadvisorClient(cadvisorClient)
	}()
	*/

	// TODO(k8s): These should probably become more plugin-ish: register a factory func
	// in each checker's init(), iterate those here.
	health.AddHealthChecker(health.NewExecHealthChecker(kl))
	health.AddHealthChecker(health.NewHTTPHealthChecker(&http.Client{}))
	health.AddHealthChecker(&health.TCPHealthChecker{})

	driver := new(mesos.MesosExecutorDriver)
	kubeletExecutor := executor.New(driver, kl, cfg.Channel(MESOS_CFG_SOURCE), MESOS_CFG_SOURCE)
	driver.Executor = kubeletExecutor

	log.V(2).Infof("Initialize executor driver...")
	driver.Init()
	defer driver.Destroy()

	log.V(2).Infof("Executor driver is running!")
	driver.Start()

	// TODO(who?) Recover running containers from check pointed pod list.

	// start the kubelet
	go util.Forever(func() { kl.Run(cfg.Updates()) }, 0)

	log.Infof("Starting kubelet server...")
	go util.Forever(func() {
		// TODO(nnielsen): Don't hardwire port, but use port from resource offer.
		log.Error(executor.ListenAndServeKubeletServer(kl, net.IP(address), *port, *enableDebuggingHandlers, MESOS_CFG_SOURCE))
	}, 0)

	go runProxyService()

	/*
		// TODO(nnielsen): Factor check-pointing into subsystem.
		dat, err := ioutil.ReadFile("/tmp/kubernetes-pods")
		if err == nil {
			var target []api.PodInfo
			err := json.Unmarshal(dat, &target)
			if err == nil {
				log.Infof("Checkpoint: '%v'", target)
			}
		}
	*/

	driver.Join()
}

// this function blocks as long as the proxy service is running; intended to be
// executed asynchronously.
func runProxyService() {
	// TODO(jdef): would be nice if we could run the proxy via an in-memory
	// kubelet config source (in case it crashes, kubelet would restart it);
	// not sure that k8s supports host-networking space for pods
	log.Infof("Starting proxy process...")

	args := []string{"-bind_address=" + address.String(), "-logtostderr=true", "-v=1"}
	if len(etcdServerList) > 0 {
		etcdServerArguments := strings.Join(etcdServerList, ",")
		args = append(args, "-etcd_servers="+etcdServerArguments)
	} else if *etcdConfigFile != "" {
		args = append(args, "-etcd_config="+*etcdConfigFile)
	}
	//TODO(jdef): don't hardcode name of the proxy executable here
	cmd := exec.Command("./kube-proxy", args...)
	_, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	proxylogs, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	logfile, err := os.Create("./proxy-log")
	if err != nil {
		log.Fatal(err)
	}
	defer logfile.Close()
	writer := bufio.NewWriter(logfile)
	defer writer.Flush()

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	defer func() {
		log.V(2).Infof("Cleaning up proxy process...")
		cmd.Process.Kill()
	}()
	io.Copy(writer, proxylogs)
}
<commit_msg>force container removal<commit_after>package main

import (
	"bufio"
	"flag"
	"io"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/capabilities"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client/record"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/health"
	_ "github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet"
	kconfig "github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/config"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/master/ports"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/version/verflag"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fsouza/go-dockerclient"
	log "github.com/golang/glog"
	"github.com/mesos/mesos-go/mesos"
	"github.com/mesosphere/kubernetes-mesos/executor"
)

const (
	MESOS_CFG_SOURCE = "mesos" // @see ConfigSourceAnnotationKey
	defaultRootDir   = "/var/lib/kubelet"
)

var (
	syncFrequency           = flag.Duration("sync_frequency", 10*time.Second, "Max period between synchronizing running containers and config")
	address                 = util.IP(net.ParseIP("0.0.0.0"))
	port                    = flag.Uint("port", ports.KubeletPort, "The port for the info server to serve on") // TODO(jdef): use kmmaster.KubeletExecutorPort
	hostnameOverride        = flag.String("hostname_override", "", "If non-empty, will use this string as identification instead of the actual hostname.")
	networkContainerImage   = flag.String("network_container_image", kubelet.NetworkContainerImage, "The image that network containers in each pod will use.")
	dockerEndpoint          = flag.String("docker_endpoint", "", "If non-empty, use this for the docker endpoint to communicate with")
	etcdServerList          util.StringList
	etcdConfigFile          = flag.String("etcd_config", "", "The config file for the etcd client. Mutually exclusive with -etcd_servers")
	rootDirectory           = flag.String("root_dir", defaultRootDir, "Directory path for managing kubelet files (volume mounts,etc).")
	allowPrivileged         = flag.Bool("allow_privileged", false, "If true, allow containers to request privileged mode. [default=false]")
	registryPullQPS         = flag.Float64("registry_qps", 0.0, "If > 0, limit registry pull QPS to this value.  If 0, unlimited. [default=0.0]")
	registryBurst           = flag.Int("registry_burst", 10, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry_qps.  Only used if --registry_qps > 0")
	enableDebuggingHandlers = flag.Bool("enable_debugging_handlers", true, "Enables server endpoints for log collection and local running of containers and commands. Default true.")
	minimumGCAge            = flag.Duration("minimum_container_ttl_duration", 0, "Minimum age for a finished container before it is garbage collected.  Examples: '300ms', '10s' or '2h45m'")
	maxContainerCount       = flag.Int("maximum_dead_containers_per_container", 5, "Maximum number of old instances of a container to retain per container.  Each container takes up some disk space.  Default: 5.")
	authPath                = flag.String("auth_path", "", "Path to .kubernetes_auth file, specifying how to authenticate to API server.")
	apiServerList           util.StringList
)

func init() {
	flag.Var(&etcdServerList, "etcd_servers", "List of etcd servers to watch (http://ip:port), comma separated")
	flag.Var(&address, "address", "The IP address for the info and proxy servers to serve on. Default to 0.0.0.0.")
	flag.Var(&apiServerList, "api_servers", "List of Kubernetes API servers to publish events to. (ip:port), comma separated.")
}

func getDockerEndpoint() string {
	var endpoint string
	if len(*dockerEndpoint) > 0 {
		endpoint = *dockerEndpoint
	} else if len(os.Getenv("DOCKER_HOST")) > 0 {
		endpoint = os.Getenv("DOCKER_HOST")
	} else {
		endpoint = "unix:///var/run/docker.sock"
	}
	log.Infof("Connecting to docker on %s", endpoint)

	return endpoint
}

func getHostname() string {
	hostname := []byte(*hostnameOverride)
	if string(hostname) == "" {
		// Note: We use exec here instead of os.Hostname() because we
		// want the FQDN, and this is the easiest way to get it.
		fqdn, err := exec.Command("hostname", "-f").Output()
		if err != nil {
			log.Fatalf("Couldn't determine hostname: %v", err)
		}
		hostname = fqdn
	}
	return strings.TrimSpace(string(hostname))
}

func getApiserverClient() (*client.Client, error) {
	authInfo, err := clientauth.LoadFromFile(*authPath)
	if err != nil {
		return nil, err
	}
	clientConfig, err := authInfo.MergeWithConfig(client.Config{})
	if err != nil {
		return nil, err
	}
	// TODO: adapt Kube client to support LB over several servers
	if len(apiServerList) > 1 {
		log.Infof("Mulitple api servers specified.  Picking first one")
	}
	clientConfig.Host = apiServerList[0]
	if c, err := client.New(&clientConfig); err != nil {
		return nil, err
	} else {
		return c, nil
	}
}

func main() {

	flag.Parse()
	util.InitLogs() // TODO(jdef) figure out where this actually sends logs by default
	defer util.FlushLogs()
	rand.Seed(time.Now().UTC().UnixNano())

	verflag.PrintAndExitIfRequested()

	etcd.SetLogger(util.NewLogger("etcd "))

	// Make an API client if possible.
	if len(apiServerList) < 1 {
		log.Info("No api servers specified.")
	} else {
		if apiClient, err := getApiserverClient(); err != nil {
			log.Errorf("Unable to make apiserver client: %v", err)
		} else {
			// Send events to APIserver if there is a client.
			log.Infof("Sending events to APIserver.")
			record.StartRecording(apiClient.Events(""), "kubelet")
		}
	}

	// Log the events locally too.
	record.StartLogging(log.Infof)

	capabilities.Initialize(capabilities.Capabilities{
		AllowPrivileged: *allowPrivileged,
	})

	dockerClient, err := docker.NewClient(getDockerEndpoint())
	if err != nil {
		log.Fatal("Couldn't connnect to docker.")
	}

	hostname := getHostname()

	if *rootDirectory == "" {
		log.Fatal("Invalid root directory path.")
	}
	*rootDirectory = path.Clean(*rootDirectory)
	if err := os.MkdirAll(*rootDirectory, 0750); err != nil {
		log.Warningf("Error creating root directory: %v", err)
	}

	// source of all configuration
	cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates)

	// k8sm: no other pod configuration sources supported in this hybrid kubelet-executor

	// define etcd config source and initialize etcd client
	var etcdClient *etcd.Client
	if len(etcdServerList) > 0 {
		etcdClient = etcd.NewClient(etcdServerList)
	} else if *etcdConfigFile != "" {
		var err error
		etcdClient, err = etcd.NewClientFromFile(*etcdConfigFile)
		if err != nil {
			log.Fatalf("Error with etcd config file: %v", err)
		}
	}

	if etcdClient != nil {
		log.Infof("Connected to etcd at %v", etcdClient.GetCluster())
	}

	// TODO(???): Destroy existing k8s containers for now - we don't know how to reconcile yet.
	if containers, err := dockertools.GetKubeletDockerContainers(dockerClient, true); err == nil {
		opts := docker.RemoveContainerOptions{
			RemoveVolumes: true,
			Force:         true,
		}
		for _, container := range containers {
			opts.ID = container.ID
			log.V(2).Infof("Removing container: %v", opts.ID)
			if err := dockerClient.RemoveContainer(opts); err != nil {
				log.Warning(err)
			}
		}
	} else {
		log.Warningf("Failed to list kubelet docker containers: %v", err)
	}

	// TODO(k8s): block until all sources have delivered at least one update to the channel, or break the sync loop
	// up into "per source" synchronizations

	kl := kubelet.NewMainKubelet(
		hostname,
		dockerClient,
		etcdClient,
		*rootDirectory,
		*networkContainerImage,
		*syncFrequency,
		float32(*registryPullQPS),
		*registryBurst,
		*minimumGCAge,
		*maxContainerCount)

	kl.BirthCry()

	go func() {
		util.Forever(func() {
			err := kl.GarbageCollectContainers()
			if err != nil {
				log.Errorf("Garbage collect failed: %v", err)
			}
		}, time.Minute*1)
	}()

	/* TODO(jdef): enable cadvisor integration
	go func() {
		defer util.HandleCrash()
		// TODO(k8s): Monitor this connection, reconnect if needed?
		log.V(1).Infof("Trying to create cadvisor client.")
		cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194")
		if err != nil {
			log.Errorf("Error on creating cadvisor client: %v", err)
			return
		}
		log.V(1).Infof("Successfully created cadvisor client.")
		kl.SetCadvisorClient(cadvisorClient)
	}()
	*/

	// TODO(k8s): These should probably become more plugin-ish: register a factory func
	// in each checker's init(), iterate those here.
	health.AddHealthChecker(health.NewExecHealthChecker(kl))
	health.AddHealthChecker(health.NewHTTPHealthChecker(&http.Client{}))
	health.AddHealthChecker(&health.TCPHealthChecker{})

	driver := new(mesos.MesosExecutorDriver)
	kubeletExecutor := executor.New(driver, kl, cfg.Channel(MESOS_CFG_SOURCE), MESOS_CFG_SOURCE)
	driver.Executor = kubeletExecutor

	log.V(2).Infof("Initialize executor driver...")
	driver.Init()
	defer driver.Destroy()

	log.V(2).Infof("Executor driver is running!")
	driver.Start()

	// TODO(who?) Recover running containers from check pointed pod list.

	// start the kubelet
	go util.Forever(func() { kl.Run(cfg.Updates()) }, 0)

	log.Infof("Starting kubelet server...")
	go util.Forever(func() {
		// TODO(nnielsen): Don't hardwire port, but use port from resource offer.
		log.Error(executor.ListenAndServeKubeletServer(kl, net.IP(address), *port, *enableDebuggingHandlers, MESOS_CFG_SOURCE))
	}, 0)

	go runProxyService()

	/*
		// TODO(nnielsen): Factor check-pointing into subsystem.
		dat, err := ioutil.ReadFile("/tmp/kubernetes-pods")
		if err == nil {
			var target []api.PodInfo
			err := json.Unmarshal(dat, &target)
			if err == nil {
				log.Infof("Checkpoint: '%v'", target)
			}
		}
	*/

	driver.Join()
}

// this function blocks as long as the proxy service is running; intended to be
// executed asynchronously.
func runProxyService() {
	// TODO(jdef): would be nice if we could run the proxy via an in-memory
	// kubelet config source (in case it crashes, kubelet would restart it);
	// not sure that k8s supports host-networking space for pods
	log.Infof("Starting proxy process...")

	args := []string{"-bind_address=" + address.String(), "-logtostderr=true", "-v=1"}
	if len(etcdServerList) > 0 {
		etcdServerArguments := strings.Join(etcdServerList, ",")
		args = append(args, "-etcd_servers="+etcdServerArguments)
	} else if *etcdConfigFile != "" {
		args = append(args, "-etcd_config="+*etcdConfigFile)
	}
	//TODO(jdef): don't hardcode name of the proxy executable here
	cmd := exec.Command("./kube-proxy", args...)
	_, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	proxylogs, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	logfile, err := os.Create("./proxy-log")
	if err != nil {
		log.Fatal(err)
	}
	defer logfile.Close()
	writer := bufio.NewWriter(logfile)
	defer writer.Flush()

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}
	defer func() {
		log.V(2).Infof("Cleaning up proxy process...")
		cmd.Process.Kill()
	}()
	io.Copy(writer, proxylogs)
}
<|endoftext|>