<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
)

var version = "head"

type wengerError struct {
	Error string
}

type wengerDownloadStatus struct {
	Percent int
}

const bufferSize = 1024 * 1024

func main() {
	if len(os.Args) < 2 {
		err("Missing command")
	}
	cmd := os.Args[1]

	switch cmd {
	case "version":
		fmt.Println(fmt.Sprintf("butler version %s", version))
	case "dl":
		dl()
	default:
		err("Invalid command")
	}
}

func send(v interface{}) {
	j, _ := json.Marshal(v)
	fmt.Println(string(j))
}

func err(msg string) {
	e := &wengerError{
		Error: msg}
	send(e)
	os.Exit(1)
}

func dl() {
	if len(os.Args) < 4 {
		err("Missing url or dest for dl command")
	}
	url := os.Args[2]
	dest := os.Args[3]

	out, _ := os.Create(dest)
	defer out.Close()

	resp, _ := http.Get(url)
	defer resp.Body.Close()

	bytesWritten := int64(0)

	for {
		n, _ := io.CopyN(out, resp.Body, bufferSize)
		bytesWritten += n

		status := &wengerDownloadStatus{
			Percent: int(bytesWritten * 100 / resp.ContentLength)}
		send(status)

		if n == 0 {
			break
		}
	}
}
<commit_msg>rename types<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
)

var version = "head"

type butlerError struct {
	Error string
}

type butlerDownloadStatus struct {
	Percent int
}

const bufferSize = 1024 * 1024

func main() {
	if len(os.Args) < 2 {
		err("Missing command")
	}
	cmd := os.Args[1]

	switch cmd {
	case "version":
		fmt.Println(fmt.Sprintf("butler version %s", version))
	case "dl":
		dl()
	default:
		err("Invalid command")
	}
}

func send(v interface{}) {
	j, _ := json.Marshal(v)
	fmt.Println(string(j))
}

func err(msg string) {
	e := &butlerError{Error: msg}
	send(e)
	os.Exit(1)
}

func dl() {
	if len(os.Args) < 4 {
		err("Missing url or dest for dl command")
	}
	url := os.Args[2]
	dest := os.Args[3]

	out, _ := os.Create(dest)
	defer out.Close()

	resp, _ := http.Get(url)
	defer resp.Body.Close()

	bytesWritten := int64(0)

	for {
		n, _ := io.CopyN(out, resp.Body, bufferSize)
		bytesWritten += n

		status := &butlerDownloadStatus{
			Percent: int(bytesWritten * 100 / resp.ContentLength)}
		send(status)

		if n == 0 {
			break
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

// Package cacher provides a uniform interface for different caching strategies.
package cacher

import "github.com/jelmersnoeck/cacher/memory"

// Cacher is the Caching interface that uniforms all the different strategies.
type Cacher interface {
	Add(key string, value []byte, ttl int64) bool
	CompareAndReplace(token, key string, value []byte, ttl int64) bool
	Set(key string, value []byte, ttl int64) bool
	SetMulti(keys map[string][]byte, ttl int64) map[string]bool
	Replace(key string, value []byte, ttl int64) bool
	Increment(key string, initial, offset, ttl int64) bool
	Decrement(key string, initial, offset, ttl int64) bool
	Delete(key string) bool
	DeleteMulti(keys []string) map[string]bool
	Get(key string) ([]byte, string, bool)
	GetMulti(keys []string) (map[string][]byte, map[string]string, map[string]bool)
	Flush() bool
	Touch(key string, ttl int64) bool
}

// The default cache which will be used for package level functions.
var DefaultCache Cacher = memory.New(500)

// Add adds a value to the cache under the specified key with a given TTL.
func Add(key string, value []byte, ttl int64) bool {
	return DefaultCache.Add(key, value, ttl)
}

// CompareAndReplace validates the token with the token in the store. If the
// tokens match, we will replace the value and return true. If it doesn't, we
// will not replace the value and return false.
func CompareAndReplace(token, key string, value []byte, ttl int64) bool {
	return DefaultCache.CompareAndReplace(token, key, value, ttl)
}

// Set sets the value of an item, regardless of wether or not the value is
// already cached.
//
// ttl defines the number of seconds the value should be cached. If ttl is 0,
// the item will be cached infinitely.
func Set(key string, value []byte, ttl int64) bool {
	return DefaultCache.Set(key, value, ttl)
}

// SetMulti sets multiple values for their respective keys. This is a shorthand
// to use `Set` multiple times.
func SetMulti(keys map[string][]byte, ttl int64) map[string]bool {
	return DefaultCache.SetMulty(keys, ttl)
}

// Replace will update and only update the value of a cache key. If the key is
// not previously used, we will return false.
func Replace(key string, value []byte, ttl int64) bool {
	return DefaultCache.Replace(key, value, ttl)
}

// Increment adds a value of offset to the initial value. If the initial value
// is already set, it will be added to the value currently stored in the cache.
func Increment(key string, initial, offset, ttl int64) bool {
	return DefaultCache.Increment(key, value, ttl)
}

// Decrement subtracts a value of offset to the initial value. If the initial
// value is already set, it will be added to the value currently stored in the
// cache.
func Decrement(key string, initial, offset, ttl int64) bool {
	return DefaultCache.Set(key, value, ttl)
}

// Delete will validate if the key actually is stored in the cache. If it is
// stored, it will remove the item from the cache. If it is not stored, it will
// return false.
func Delete(key string) bool {
	return DefaultCache.Set(key, value, ttl)
}

// DeleteMulti will delete multiple values at a time. It uses the `Delete`
// method internally to do so. It will return a map of results to see if the
// deletion is successful.
func DeleteMulti(keys []string) map[string]bool {
	return DefaultCache.Set(key, value, ttl)
}

// Get gets the value out of the map associated with the provided key.
func Get(key string) ([]byte, string, bool) {
	return DefaultCache.Set(key, value, ttl)
}

// GetMulti gets multiple values from the cache and returns them as a map. It
// uses `Get` internally to retrieve the data.
func GetMulti(keys []string) (map[string][]byte, map[string]string, map[string]bool) {
	return DefaultCache.Set(key, value, ttl)
}

// Flush will remove all the items from the storage.
func Flush() bool {
	return DefaultCache.Set(key, value, ttl)
}

// Touch will update the key's ttl to the given ttl value without altering the
// value.
func Touch(key string, ttl int64) bool {
	return DefaultCache.Set(key, value, ttl)
}
<commit_msg>Cacher: update methods with proper params.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

// Package cacher provides a uniform interface for different caching strategies.
package cacher

import "github.com/jelmersnoeck/cacher/memory"

// Cacher is the Caching interface that uniforms all the different strategies.
type Cacher interface {
	Add(key string, value []byte, ttl int64) bool
	CompareAndReplace(token, key string, value []byte, ttl int64) bool
	Set(key string, value []byte, ttl int64) bool
	SetMulti(keys map[string][]byte, ttl int64) map[string]bool
	Replace(key string, value []byte, ttl int64) bool
	Increment(key string, initial, offset, ttl int64) bool
	Decrement(key string, initial, offset, ttl int64) bool
	Delete(key string) bool
	DeleteMulti(keys []string) map[string]bool
	Get(key string) ([]byte, string, bool)
	GetMulti(keys []string) (map[string][]byte, map[string]string, map[string]bool)
	Flush() bool
	Touch(key string, ttl int64) bool
}

// The default cache which will be used for package level functions.
var DefaultCache Cacher = memory.New(500)

// Add adds a value to the cache under the specified key with a given TTL.
func Add(key string, value []byte, ttl int64) bool {
	return DefaultCache.Add(key, value, ttl)
}

// CompareAndReplace validates the token with the token in the store. If the
// tokens match, we will replace the value and return true. If it doesn't, we
// will not replace the value and return false.
func CompareAndReplace(token, key string, value []byte, ttl int64) bool {
	return DefaultCache.CompareAndReplace(token, key, value, ttl)
}

// Set sets the value of an item, regardless of wether or not the value is
// already cached.
//
// ttl defines the number of seconds the value should be cached. If ttl is 0,
// the item will be cached infinitely.
func Set(key string, value []byte, ttl int64) bool {
	return DefaultCache.Set(key, value, ttl)
}

// SetMulti sets multiple values for their respective keys. This is a shorthand
// to use `Set` multiple times.
func SetMulti(keys map[string][]byte, ttl int64) map[string]bool {
	return DefaultCache.SetMulti(keys, ttl)
}

// Replace will update and only update the value of a cache key. If the key is
// not previously used, we will return false.
func Replace(key string, value []byte, ttl int64) bool {
	return DefaultCache.Replace(key, value, ttl)
}

// Increment adds a value of offset to the initial value. If the initial value
// is already set, it will be added to the value currently stored in the cache.
func Increment(key string, initial, offset, ttl int64) bool {
	return DefaultCache.Increment(key, initial, offset, ttl)
}

// Decrement subtracts a value of offset to the initial value. If the initial
// value is already set, it will be added to the value currently stored in the
// cache.
func Decrement(key string, initial, offset, ttl int64) bool {
	return DefaultCache.Decrement(key, initial, offset, ttl)
}

// Delete will validate if the key actually is stored in the cache. If it is
// stored, it will remove the item from the cache. If it is not stored, it will
// return false.
func Delete(key string) bool {
	return DefaultCache.Delete(key)
}

// DeleteMulti will delete multiple values at a time. It uses the `Delete`
// method internally to do so. It will return a map of results to see if the
// deletion is successful.
func DeleteMulti(keys []string) map[string]bool {
	return DefaultCache.DeleteMulti(keys)
}

// Get gets the value out of the map associated with the provided key.
func Get(key string) ([]byte, string, bool) {
	return DefaultCache.Get(key)
}

// GetMulti gets multiple values from the cache and returns them as a map. It
// uses `Get` internally to retrieve the data.
func GetMulti(keys []string) (map[string][]byte, map[string]string, map[string]bool) {
	return DefaultCache.GetMulti(keys)
}

// Flush will remove all the items from the storage.
func Flush() bool {
	return DefaultCache.Flush()
}

// Touch will update the key's ttl to the given ttl value without altering the
// value.
func Touch(key string, ttl int64) bool {
	return DefaultCache.Touch(key, ttl)
}
<|endoftext|>


<commit_before>// Copyright 2016 Liam Stanley <me@liamstanley.io>. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

package girc

import (
	"errors"
	"fmt"
	"math/rand"
	"strings"
	"sync"
)

// RunCallbacks manually runs callbacks for a given event.
func (c *Client) RunCallbacks(event *Event) {
	// Log the event.
	c.log.Print("<-- " + event.Raw())

	// Regular wildcard callbacks.
	c.Callbacks.exec(ALLEVENTS, c, event)

	// Then regular non-threaded callbacks.
	c.Callbacks.exec(event.Command, c, event)
}

// Callback is lower level implementation of a callback. See
// Caller.AddHandler()
type Callback interface {
	Execute(*Client, Event)
}

// CallbackFunc is a type that represents the function necessary to
// implement Callback.
type CallbackFunc func(c *Client, e Event)

// Execute calls the CallbackFunc with the sender and irc message.
func (f CallbackFunc) Execute(c *Client, e Event) {
	f(c, e)
}

// Caller manages internal and external (user facing) callbacks.
//
// external/internal keys are of structure:
//   map[CALLBACK_TYPE][COMMAND][CUID]Callback
//
// Also of note: "COMMAND" should always be uppercase for normalization.
type Caller struct {
	// mu is the mutex that should be used when accessing callbacks.
	mu sync.RWMutex
	// external is a map of user facing callbacks.
	external map[string]map[string]map[string]Callback
	// internal is a map of internally used callbacks for the client.
	internal map[string]map[string]map[string]Callback
}

// newCaller creates and initializes a new callback handler.
func newCaller() *Caller {
	c := &Caller{}

	c.external = map[string]map[string]map[string]Callback{}
	c.external["routine"] = map[string]map[string]Callback{}
	c.external["std"] = map[string]map[string]Callback{}
	c.internal = map[string]map[string]map[string]Callback{}
	c.internal["routine"] = map[string]map[string]Callback{}
	c.internal["std"] = map[string]map[string]Callback{}

	return c
}

// Len returns the total amount of user-entered registered callbacks.
func (c *Caller) Len() int {
	var total int

	c.mu.RLock()
	for ctype := range c.external {
		for command := range c.external[ctype] {
			total += len(c.external[ctype][command])
		}
	}
	c.mu.RUnlock()

	return total
}

// Count is much like Caller.Len(), however it counts the number of
// registered callbacks for a given command.
func (c *Caller) Count(cmd string) int {
	var total int

	cmd = strings.ToUpper(cmd)

	c.mu.RLock()
	for ctype := range c.external {
		for command := range c.external[ctype] {
			if command == cmd {
				total += len(c.external[ctype][command])
			}
		}
	}
	c.mu.RUnlock()

	return total
}

func (c *Caller) String() string {
	var total int
	var ctypes []string

	c.mu.RLock()
	for ctype := range c.internal {
		ctypes = append(ctypes, ctype)
		for cmd := range c.internal[ctype] {
			total += len(c.internal[ctype][cmd])
		}
	}
	c.mu.RUnlock()

	return fmt.Sprintf(
		"<Caller() types[%d]:[%s] client:%d internal:%d>",
		len(c.external), strings.Join(ctypes, ","), c.Len(), len(c.internal),
	)
}

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

// cuid generates a unique UID string for each callback for ease of removal.
func (c *Caller) cuid(ctype, cmd string, n int) (cuid, uid string) {
	b := make([]byte, n)

	for i := range b {
		b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
	}

	return ctype + ":" + cmd + ":" + string(b), string(b)
}

// cuidToID allows easy mapping between a generated cuid and the caller``
// external/internal callback maps.
func (c *Caller) cuidToID(input string) (ctype, cmd, uid string) {
	// Ignore the errors because the strings will default to empty anyway.
	_, _ = fmt.Sscanf(input, "%s:%s:%s", &ctype, &cmd, &uid)
	return ctype, cmd, uid
}

// exec executes all callbacks pertaining to specified event. Internal first,
// then external.
//
// Please note that there is no specific order/priority for which the
// callback types themselves or the callbacks are executed.
func (c *Caller) exec(command string, client *Client, event *Event) {
	var execstack []Callback
	var routinestack []Callback

	c.mu.RLock()
	// Execute internal callbacks first.
	for callbackType := range c.internal {
		if _, ok := c.internal[callbackType][command]; !ok {
			continue
		}

		for cuid := range c.internal[callbackType][command] {
			switch callbackType {
			case "routine":
				routinestack = append(routinestack, c.internal[callbackType][command][cuid])
			default:
				execstack = append(execstack, c.internal[callbackType][command][cuid])
			}
		}
	}

	// Aaand then external callbacks.
	for callbackType := range c.external {
		if _, ok := c.external[callbackType][command]; !ok {
			continue
		}

		for cuid := range c.external[callbackType][command] {
			switch callbackType {
			case "routine":
				routinestack = append(routinestack, c.external[callbackType][command][cuid])
			default:
				execstack = append(execstack, c.external[callbackType][command][cuid])
			}
		}
	}
	c.mu.RUnlock()

	for i := 0; i < len(routinestack); i++ {
		routinestack[i].Execute(client, *event)
	}

	for i := 0; i < len(execstack); i++ {
		execstack[i].Execute(client, *event)
	}
}

// ClearAll clears all external callbacks currently setup within the client.
// This ignores internal callbacks.
func (c *Caller) ClearAll() {
	c.mu.Lock()
	c.external = map[string]map[string]map[string]Callback{}
	c.external["routine"] = map[string]map[string]Callback{}
	c.external["std"] = map[string]map[string]Callback{}
	c.mu.Unlock()
}

// Clear clears all of the callbacks for the given event.
// This ignores internal callbacks.
func (c *Caller) Clear(cmd string) {
	cmd = strings.ToUpper(cmd)

	c.mu.Lock()
	for ctype := range c.external {
		if _, ok := c.external[ctype][cmd]; ok {
			delete(c.external[ctype], cmd)
		}
	}
	c.mu.Unlock()
}

// Remove removes the callback with cuid from the callback stack. success
// indicates that it existed, and has been removed. If not success, it
// wasn't a registered callback.
func (c *Caller) Remove(cuid string) (success bool) {
	c.mu.Lock()
	success = c.remove(cuid)
	c.mu.Unlock()

	return success
}

// remove is much like Remove, however is NOT concurrency safe. Lock Caller.mu
// on your own.
func (c *Caller) remove(cuid string) (success bool) {
	ctype, cmd, uid := c.cuidToID(cuid)
	if len(ctype) == 0 || len(cmd) == 0 || len(uid) == 0 {
		return false
	}

	// Check if the callback type exists.
	if _, ok := c.external[ctype]; !ok {
		return false
	}

	// Check if the irc command/event has any callbacks on it.
	if _, ok := c.external[ctype][cmd]; !ok {
		return false
	}

	// Check to see if it's actually a registered callback.
	if _, ok := c.external[ctype][cmd][cuid]; !ok {
		return false
	}

	delete(c.external[ctype][cmd], uid)

	// Assume success.
	return true
}

// sregister is much like Caller.register(), except that it safely locks
// the Caller mutex.
func (c *Caller) sregister(internal bool, ctype, cmd string, callback Callback) (cuid string) {
	c.mu.Lock()
	cuid = c.register(internal, ctype, cmd, callback)
	c.mu.Unlock()

	return cuid
}

// register will register a callback in the internal tracker. Unsafe (you
// must lock c.mu yourself!)
func (c *Caller) register(internal bool, ctype, cmd string, callback Callback) (cuid string) {
	var uid string

	cmd = strings.ToUpper(cmd)

	if internal {
		if _, ok := c.internal[ctype]; !ok {
			panic(errors.New("callback type does not exist: " + ctype))
		}

		if _, ok := c.internal[ctype][cmd]; !ok {
			c.internal[ctype][cmd] = map[string]Callback{}
		}

		cuid, uid = c.cuid(ctype, cmd, 20)

		c.internal[ctype][cmd][uid] = callback
	} else {
		if _, ok := c.external[ctype]; !ok {
			panic(errors.New("callback type does not exist: " + ctype))
		}

		if _, ok := c.external[ctype][cmd]; !ok {
			c.external[ctype][cmd] = map[string]Callback{}
		}

		cuid, uid = c.cuid(ctype, cmd, 20)

		c.external[ctype][cmd][uid] = callback
	}

	return cuid
}

// AddHandler registers a callback (matching the Callback interface) for the
// given event. cuid is the callback uid which can be used to remove the
// callback with Caller.Remove().
func (c *Caller) AddHandler(cmd string, callback Callback) (cuid string) {
	return c.sregister(false, "std", cmd, callback)
}

// AddBgHandler registers a callback (matching the Callback interface) for
// the given event and executes it in a go-routine. cuid is the callback uid
// which can be used to remove the callback with Caller.Remove().
func (c *Caller) AddBgHandler(cmd string, callback Callback) (cuid string) {
	return c.sregister(false, "routine", cmd, callback)
}

// Add registers the callback function for the given event. cuid is the
// callback uid which can be used to remove the callback with Caller.Remove().
func (c *Caller) Add(cmd string, callback func(c *Client, e Event)) (cuid string) {
	return c.sregister(false, "std", cmd, CallbackFunc(callback))
}

// AddBg registers the callback function for the given event and executes it
// in a go-routine. cuid is the callback uid which can be used to remove the
// callback with Caller.Remove().
func (c *Caller) AddBg(cmd string, callback func(c *Client, e Event)) (cuid string) {
	return c.sregister(false, "routine", cmd, CallbackFunc(callback))
}
<commit_msg>fix callback goroutines<commit_after>// Copyright 2016 Liam Stanley <me@liamstanley.io>. All rights reserved.
// Use of this source code is governed by the MIT license that can be
// found in the LICENSE file.

package girc

import (
	"errors"
	"fmt"
	"math/rand"
	"strings"
	"sync"
)

// RunCallbacks manually runs callbacks for a given event.
func (c *Client) RunCallbacks(event *Event) {
	// Log the event.
	c.log.Print("<-- " + event.Raw())

	// Regular wildcard callbacks.
	c.Callbacks.exec(ALLEVENTS, c, event)

	// Then regular non-threaded callbacks.
	c.Callbacks.exec(event.Command, c, event)
}

// Callback is lower level implementation of a callback. See
// Caller.AddHandler()
type Callback interface {
	Execute(*Client, Event)
}

// CallbackFunc is a type that represents the function necessary to
// implement Callback.
type CallbackFunc func(c *Client, e Event)

// Execute calls the CallbackFunc with the sender and irc message.
func (f CallbackFunc) Execute(c *Client, e Event) {
	f(c, e)
}

// Caller manages internal and external (user facing) callbacks.
//
// external/internal keys are of structure:
//   map[CALLBACK_TYPE][COMMAND][CUID]Callback
//
// Also of note: "COMMAND" should always be uppercase for normalization.
type Caller struct {
	// mu is the mutex that should be used when accessing callbacks.
	mu sync.RWMutex
	// external is a map of user facing callbacks.
	external map[string]map[string]map[string]Callback
	// internal is a map of internally used callbacks for the client.
	internal map[string]map[string]map[string]Callback
}

// newCaller creates and initializes a new callback handler.
func newCaller() *Caller {
	c := &Caller{}

	c.external = map[string]map[string]map[string]Callback{}
	c.external["routine"] = map[string]map[string]Callback{}
	c.external["std"] = map[string]map[string]Callback{}
	c.internal = map[string]map[string]map[string]Callback{}
	c.internal["routine"] = map[string]map[string]Callback{}
	c.internal["std"] = map[string]map[string]Callback{}

	return c
}

// Len returns the total amount of user-entered registered callbacks.
func (c *Caller) Len() int {
	var total int

	c.mu.RLock()
	for ctype := range c.external {
		for command := range c.external[ctype] {
			total += len(c.external[ctype][command])
		}
	}
	c.mu.RUnlock()

	return total
}

// Count is much like Caller.Len(), however it counts the number of
// registered callbacks for a given command.
func (c *Caller) Count(cmd string) int {
	var total int

	cmd = strings.ToUpper(cmd)

	c.mu.RLock()
	for ctype := range c.external {
		for command := range c.external[ctype] {
			if command == cmd {
				total += len(c.external[ctype][command])
			}
		}
	}
	c.mu.RUnlock()

	return total
}

func (c *Caller) String() string {
	var total int
	var ctypes []string

	c.mu.RLock()
	for ctype := range c.internal {
		ctypes = append(ctypes, ctype)
		for cmd := range c.internal[ctype] {
			total += len(c.internal[ctype][cmd])
		}
	}
	c.mu.RUnlock()

	return fmt.Sprintf(
		"<Caller() types[%d]:[%s] client:%d internal:%d>",
		len(c.external), strings.Join(ctypes, ","), c.Len(), len(c.internal),
	)
}

const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"

// cuid generates a unique UID string for each callback for ease of removal.
func (c *Caller) cuid(ctype, cmd string, n int) (cuid, uid string) {
	b := make([]byte, n)

	for i := range b {
		b[i] = letterBytes[rand.Int63()%int64(len(letterBytes))]
	}

	return ctype + ":" + cmd + ":" + string(b), string(b)
}

// cuidToID allows easy mapping between a generated cuid and the caller``
// external/internal callback maps.
func (c *Caller) cuidToID(input string) (ctype, cmd, uid string) {
	// Ignore the errors because the strings will default to empty anyway.
	_, _ = fmt.Sscanf(input, "%s:%s:%s", &ctype, &cmd, &uid)
	return ctype, cmd, uid
}

// exec executes all callbacks pertaining to specified event. Internal first,
// then external.
//
// Please note that there is no specific order/priority for which the
// callback types themselves or the callbacks are executed.
func (c *Caller) exec(command string, client *Client, event *Event) {
	var execstack []Callback
	var routinestack []Callback

	c.mu.RLock()
	// Execute internal callbacks first.
	for callbackType := range c.internal {
		if _, ok := c.internal[callbackType][command]; !ok {
			continue
		}

		for cuid := range c.internal[callbackType][command] {
			switch callbackType {
			case "routine":
				routinestack = append(routinestack, c.internal[callbackType][command][cuid])
			default:
				execstack = append(execstack, c.internal[callbackType][command][cuid])
			}
		}
	}

	// Aaand then external callbacks.
	for callbackType := range c.external {
		if _, ok := c.external[callbackType][command]; !ok {
			continue
		}

		for cuid := range c.external[callbackType][command] {
			switch callbackType {
			case "routine":
				routinestack = append(routinestack, c.external[callbackType][command][cuid])
			default:
				execstack = append(execstack, c.external[callbackType][command][cuid])
			}
		}
	}
	c.mu.RUnlock()

	for i := 0; i < len(routinestack); i++ {
		go routinestack[i].Execute(client, *event)
	}

	for i := 0; i < len(execstack); i++ {
		execstack[i].Execute(client, *event)
	}
}

// ClearAll clears all external callbacks currently setup within the client.
// This ignores internal callbacks.
func (c *Caller) ClearAll() {
	c.mu.Lock()
	c.external = map[string]map[string]map[string]Callback{}
	c.external["routine"] = map[string]map[string]Callback{}
	c.external["std"] = map[string]map[string]Callback{}
	c.mu.Unlock()
}

// Clear clears all of the callbacks for the given event.
// This ignores internal callbacks.
func (c *Caller) Clear(cmd string) {
	cmd = strings.ToUpper(cmd)

	c.mu.Lock()
	for ctype := range c.external {
		if _, ok := c.external[ctype][cmd]; ok {
			delete(c.external[ctype], cmd)
		}
	}
	c.mu.Unlock()
}

// Remove removes the callback with cuid from the callback stack. success
// indicates that it existed, and has been removed. If not success, it
// wasn't a registered callback.
func (c *Caller) Remove(cuid string) (success bool) {
	c.mu.Lock()
	success = c.remove(cuid)
	c.mu.Unlock()

	return success
}

// remove is much like Remove, however is NOT concurrency safe. Lock Caller.mu
// on your own.
func (c *Caller) remove(cuid string) (success bool) {
	ctype, cmd, uid := c.cuidToID(cuid)
	if len(ctype) == 0 || len(cmd) == 0 || len(uid) == 0 {
		return false
	}

	// Check if the callback type exists.
	if _, ok := c.external[ctype]; !ok {
		return false
	}

	// Check if the irc command/event has any callbacks on it.
	if _, ok := c.external[ctype][cmd]; !ok {
		return false
	}

	// Check to see if it's actually a registered callback.
	if _, ok := c.external[ctype][cmd][cuid]; !ok {
		return false
	}

	delete(c.external[ctype][cmd], uid)

	// Assume success.
	return true
}

// sregister is much like Caller.register(), except that it safely locks
// the Caller mutex.
func (c *Caller) sregister(internal bool, ctype, cmd string, callback Callback) (cuid string) {
	c.mu.Lock()
	cuid = c.register(internal, ctype, cmd, callback)
	c.mu.Unlock()

	return cuid
}

// register will register a callback in the internal tracker. Unsafe (you
// must lock c.mu yourself!)
func (c *Caller) register(internal bool, ctype, cmd string, callback Callback) (cuid string) {
	var uid string

	cmd = strings.ToUpper(cmd)

	if internal {
		if _, ok := c.internal[ctype]; !ok {
			panic(errors.New("callback type does not exist: " + ctype))
		}

		if _, ok := c.internal[ctype][cmd]; !ok {
			c.internal[ctype][cmd] = map[string]Callback{}
		}

		cuid, uid = c.cuid(ctype, cmd, 20)

		c.internal[ctype][cmd][uid] = callback
	} else {
		if _, ok := c.external[ctype]; !ok {
			panic(errors.New("callback type does not exist: " + ctype))
		}

		if _, ok := c.external[ctype][cmd]; !ok {
			c.external[ctype][cmd] = map[string]Callback{}
		}

		cuid, uid = c.cuid(ctype, cmd, 20)

		c.external[ctype][cmd][uid] = callback
	}

	return cuid
}

// AddHandler registers a callback (matching the Callback interface) for the
// given event. cuid is the callback uid which can be used to remove the
// callback with Caller.Remove().
func (c *Caller) AddHandler(cmd string, callback Callback) (cuid string) {
	return c.sregister(false, "std", cmd, callback)
}

// AddBgHandler registers a callback (matching the Callback interface) for
// the given event and executes it in a go-routine. cuid is the callback uid
// which can be used to remove the callback with Caller.Remove().
func (c *Caller) AddBgHandler(cmd string, callback Callback) (cuid string) {
	return c.sregister(false, "routine", cmd, callback)
}

// Add registers the callback function for the given event. cuid is the
// callback uid which can be used to remove the callback with Caller.Remove().
func (c *Caller) Add(cmd string, callback func(c *Client, e Event)) (cuid string) {
	return c.sregister(false, "std", cmd, CallbackFunc(callback))
}

// AddBg registers the callback function for the given event and executes it
// in a go-routine. cuid is the callback uid which can be used to remove the
// callback with Caller.Remove().
func (c *Caller) AddBg(cmd string, callback func(c *Client, e Event)) (cuid string) {
	return c.sregister(false, "routine", cmd, CallbackFunc(callback))
}
<|endoftext|>


<commit_before>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"strings"
)

const (
	// fileDepMkTemplate holds a template for file-based
	// dependency tracking make file.
	fileDepMkTemplate = `# generated by !!!DEPS_GEN_APP_NAME!!!

define _DEPS_GEN_GET_ALL_FILES_
$(eval _DEPS_GEN_ALL_FILES_ := !!!DEPS_GEN_FILES_GENERATOR!!!)
$(strip $(_DEPS_GEN_ALL_FILES_)) \
$(eval _DEPS_GEN_ALL_FILES_ :=)
endef

_DEPS_GEN_ALL_FILES_STORED_ := !!!DEPS_GEN_ALL_FILES_STORED!!!

_DEPS_GEN_SPACE_ :=
_DEPS_GEN_SPACE_ +=

_DEPS_GEN_F1_ := $(sort $(strip $(_DEPS_GEN_ALL_FILES_STORED_)))
_DEPS_GEN_F2_ := $(sort $(strip $(call _DEPS_GEN_GET_ALL_FILES_)))

ifeq ($(_DEPS_GEN_F1_),$(_DEPS_GEN_F2_))

!!!DEPS_GEN_TARGET!!!: $(_DEPS_GEN_ALL_FILES_STORED_)

else

# invalidate the target
!!!DEPS_GEN_TARGET!!!: _DEPS_GEN_INVALIDATE_
.PHONY: _DEPS_GEN_INVALIDATE_

endif

$(call undefine-namespaces,_DEPS_GEN)
`

	// kvStoredPrefix is used to prefix the variable name we want
	// to store (so actual variable checked at runtime is FOO and
	// its stored value from previous run is $(kvStoredPrefix)FOO.
	kvStoredPrefix = "_DEPS_GEN_STORED_"
	// kvDepMkTemplate holds a template for key-value-based
	// dependency tracking make file.
	kvDepMkTemplate = `# generated by !!!DEPS_GEN_APP_NAME!!!
_DEPS_GEN_VARIABLES_ := !!!DEPS_GEN_VARIABLES!!!
!!!DEPS_GEN_STORED_VALUES_SET!!!
_DEPS_GEN_INVALIDATE_ := no

$(foreach v,$(_DEPS_GEN_VARIABLES_), \
        $(if $(call equal,$($v),$(!!!DEPS_GEN_STORED_PREFIX!!!$v)), \
                , \
                $(eval _DEPS_GEN_INVALIDATE_ := yes)))

ifeq ($(_DEPS_GEN_INVALIDATE_),yes)

# invalidate the target
!!!DEPS_GEN_TARGET!!!: _DEPS_GEN_INVALIDATE_
.PHONY: _DEPS_GEN_INVALIDATE_

endif

$(call undefine-namespaces,_DEPS_GEN)
`
)

// GenerateFileDeps returns contents of make file describing
// dependencies of given target on given files and checking if files
// weren't added or removed in directories where given files are.
func GenerateFileDeps(target, filesGenerator string, files []string) string {
	return replacePlaceholders(fileDepMkTemplate,
		"DEPS_GEN_APP_NAME", appName(),
		"DEPS_GEN_FILES_GENERATOR", filesGenerator,
		"DEPS_GEN_ALL_FILES_STORED", strings.Join(files, " "),
		"DEPS_GEN_TARGET", target,
	)
}

func GenerateKvDeps(target string, keysValues map[string]string) string {
	variables := make([]string, 0, len(keysValues))
	storedValuesSet := make([]string, 0, len(keysValues))
	for k, v := range keysValues {
		variables = append(variables, k)
		assign := kvStoredPrefix + k + " :="
		storedValuesSet = append(storedValuesSet, assign+" "+v)
	}

	return replacePlaceholders(kvDepMkTemplate,
		"DEPS_GEN_APP_NAME", appName(),
		"DEPS_GEN_VARIABLES", strings.Join(variables, " "),
		"DEPS_GEN_STORED_VALUES_SET", strings.Join(storedValuesSet, "\n"),
		"DEPS_GEN_STORED_PREFIX", kvStoredPrefix,
		"DEPS_GEN_TARGET", target,
	)
}
<commit_msg>build: Rename one make variable in generated deps.mk files<commit_after>// Copyright 2015 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"strings"
)

const (
	// fileDepMkTemplate holds a template for file-based
	// dependency tracking make file.
	fileDepMkTemplate = `# generated by !!!DEPS_GEN_APP_NAME!!!

define _DEPS_GEN_GET_ALL_FILES_
$(eval _DEPS_GEN_ALL_FILES_ := !!!DEPS_GEN_FILES_GENERATOR!!!)
$(strip $(_DEPS_GEN_ALL_FILES_)) \
$(eval _DEPS_GEN_ALL_FILES_ :=)
endef

_DEPS_GEN_ALL_FILES_STORED_ := !!!DEPS_GEN_ALL_FILES_STORED!!!

_DEPS_GEN_SPACE_ :=
_DEPS_GEN_SPACE_ +=

_DEPS_GEN_F1_ := $(sort $(strip $(_DEPS_GEN_ALL_FILES_STORED_)))
_DEPS_GEN_F2_ := $(sort $(strip $(call _DEPS_GEN_GET_ALL_FILES_)))

ifeq ($(_DEPS_GEN_F1_),$(_DEPS_GEN_F2_))

!!!DEPS_GEN_TARGET!!!: $(_DEPS_GEN_ALL_FILES_STORED_)

else

# invalidate the target
!!!DEPS_GEN_TARGET!!!: _DEPS_GEN_INVALIDATE_
.PHONY: _DEPS_GEN_INVALIDATE_

endif

$(call undefine-namespaces,_DEPS_GEN)
`

	// kvStoredPrefix is used to prefix the variable name we want
	// to store (so actual variable checked at runtime is FOO and
	// its stored value from previous run is $(kvStoredPrefix)FOO.
	kvStoredPrefix = "_DEPS_GEN_STORED_"
	// kvDepMkTemplate holds a template for key-value-based
	// dependency tracking make file.
	kvDepMkTemplate = `# generated by !!!DEPS_GEN_APP_NAME!!!
_DEPS_GEN_VARIABLES_ := !!!DEPS_GEN_VARIABLES!!!
!!!DEPS_GEN_STORED_VALUES_SET!!!
_DEPS_GEN_DO_INVALIDATE_ := no

$(foreach v,$(_DEPS_GEN_VARIABLES_), \
        $(if $(call equal,$($v),$(!!!DEPS_GEN_STORED_PREFIX!!!$v)), \
                , \
                $(eval _DEPS_GEN_DO_INVALIDATE_ := yes)))

ifeq ($(_DEPS_GEN_DO_INVALIDATE_),yes)

# invalidate the target
!!!DEPS_GEN_TARGET!!!: _DEPS_GEN_INVALIDATE_
.PHONY: _DEPS_GEN_INVALIDATE_

endif

$(call undefine-namespaces,_DEPS_GEN)
`
)

// GenerateFileDeps returns contents of make file describing
// dependencies of given target on given files and checking if files
// weren't added or removed in directories where given files are.
func GenerateFileDeps(target, filesGenerator string, files []string) string {
	return replacePlaceholders(fileDepMkTemplate,
		"DEPS_GEN_APP_NAME", appName(),
		"DEPS_GEN_FILES_GENERATOR", filesGenerator,
		"DEPS_GEN_ALL_FILES_STORED", strings.Join(files, " "),
		"DEPS_GEN_TARGET", target,
	)
}

func GenerateKvDeps(target string, keysValues map[string]string) string {
	variables := make([]string, 0, len(keysValues))
	storedValuesSet := make([]string, 0, len(keysValues))
	for k, v := range keysValues {
		variables = append(variables, k)
		assign := kvStoredPrefix + k + " :="
		storedValuesSet = append(storedValuesSet, assign+" "+v)
	}

	return replacePlaceholders(kvDepMkTemplate,
		"DEPS_GEN_APP_NAME", appName(),
		"DEPS_GEN_VARIABLES", strings.Join(variables, " "),
		"DEPS_GEN_STORED_VALUES_SET", strings.Join(storedValuesSet, "\n"),
		"DEPS_GEN_STORED_PREFIX", kvStoredPrefix,
		"DEPS_GEN_TARGET", target,
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Updated for Openshift CLI
*/

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/openshift/origin/pkg/cmd/admin"
	"github.com/openshift/origin/pkg/cmd/cli"
	"github.com/openshift/origin/pkg/cmd/openshift"
	mangen "github.com/openshift/origin/tools/genman/md2man"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"k8s.io/kubernetes/cmd/genutils"
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Root command not specified (oc | oadm | openshift).\n")
		os.Exit(1)
	}

	if strings.HasSuffix(os.Args[2], "oc") {
		genCmdMan("oc", cli.NewCommandCLI("oc", "oc", &bytes.Buffer{}, os.Stdout, ioutil.Discard))
	} else if strings.HasSuffix(os.Args[2], "openshift") {
		genCmdMan("openshift", openshift.NewCommandOpenShift("openshift"))
	} else if strings.HasSuffix(os.Args[2], "oadm") {
		genCmdMan("oadm", admin.NewCommandAdmin("oadm", "oadm", &bytes.Buffer{}, os.Stdout, ioutil.Discard))
	} else {
		fmt.Fprintf(os.Stderr, "Root command not specified (os | oadm | openshift).")
		os.Exit(1)
	}
}

func genCmdMan(cmdName string, cmd *cobra.Command) {
	// use os.Args instead of "flags" because "flags" will mess up the man pages!
	path := "docs/man/" + cmdName
	if len(os.Args) == 3 {
		path = os.Args[1]
	} else if len(os.Args) > 3 {
		fmt.Fprintf(os.Stderr, "usage: %s [output directory]\n", os.Args[0])
		os.Exit(1)
	}

	outDir, err := genutils.OutDir(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err)
		os.Exit(1)
	}

	// Set environment variables used by openshift so the output is consistent,
	// regardless of where we run.
	os.Setenv("HOME", "/home/username")
	// TODO os.Stdin should really be something like ioutil.Discard, but a Reader
	genMarkdown(cmd, "", outDir)
	for _, c := range cmd.Commands() {
		genMarkdown(c, cmdName, outDir)
	}
}

func preamble(out *bytes.Buffer, cmdName, name, short, long string) {
	out.WriteString(`% ` + strings.ToUpper(cmdName) + `(1) Openshift CLI User Manuals
% Openshift
% June 2016
# NAME
`)

	fmt.Fprintf(out, "%s \\- %s\n\n", name, short)
	fmt.Fprintf(out, "# SYNOPSIS\n")
	fmt.Fprintf(out, "**%s** [OPTIONS]\n\n", name)
	fmt.Fprintf(out, "# DESCRIPTION\n")
	fmt.Fprintf(out, "%s\n\n", long)
}

func printFlags(out *bytes.Buffer, flags *pflag.FlagSet) {
	flags.VisitAll(func(flag *pflag.Flag) {
		format := "**--%s**=%s\n\t%s\n\n"
		if flag.Value.Type() == "string" {
			// put quotes on the value
			format = "**--%s**=%q\n\t%s\n\n"
		}

		defValue := flag.DefValue
		if flag.Value.Type() == "duration" {
			defValue = "0"
		}

		if len(flag.Annotations["manpage-def-value"]) > 0 {
			defValue = flag.Annotations["manpage-def-value"][0]
		}

		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
			format = "**-%s**, " + format
			fmt.Fprintf(out, format, flag.Shorthand, flag.Name, defValue, flag.Usage)
		} else {
			fmt.Fprintf(out, format, flag.Name, defValue, flag.Usage)
		}
	})
}

func printOptions(out *bytes.Buffer, command *cobra.Command) {
	flags := command.NonInheritedFlags()
	if flags.HasFlags() {
		fmt.Fprintf(out, "# OPTIONS\n")
		printFlags(out, flags)
		fmt.Fprintf(out, "\n")
	}
	flags = command.InheritedFlags()
	if flags.HasFlags() {
		fmt.Fprintf(out, "# OPTIONS INHERITED FROM PARENT COMMANDS\n")
		printFlags(out, flags)
		fmt.Fprintf(out, "\n")
	}
}

func genMarkdown(command *cobra.Command, parent, docsDir string) {
	dparent := strings.Replace(parent, " ", "-", -1)
	name := command.Name()
	dname := name
	cmdName := name
	if len(parent) > 0 {
		dname = dparent + "-" + name
		name = parent + " " + name
		cmdName = parent
	}

	out := new(bytes.Buffer)
	short := command.Short
	long := command.Long
	if len(long) == 0 {
		long = short
	}

	preamble(out, cmdName, name, short, long)
	printOptions(out, command)

	if len(command.Example) > 0 {
		fmt.Fprintf(out, "# EXAMPLE\n")
		fmt.Fprintf(out, "```\n%s\n```\n", command.Example)
	}

	if len(command.Commands()) > 0 || len(parent) > 0 {
		fmt.Fprintf(out, "# SEE ALSO\n")
		if len(parent) > 0 {
			fmt.Fprintf(out, "**%s(1)**, ", dparent)
		}
		for _, c := range command.Commands() {
			fmt.Fprintf(out, "**%s-%s(1)**, ", dname, c.Name())
			genMarkdown(c, name, docsDir)
		}
		fmt.Fprintf(out, "\n")
	}

	out.WriteString(`
# HISTORY
June 2016, Ported from the Kubernetes man-doc generator
`)

	final := mangen.Render(out.Bytes())

	filename := docsDir + dname + ".1"
	outFile, err := os.Create(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer outFile.Close()
	_, err = outFile.Write(final)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

}
<commit_msg>Man page generation should skip hidden flags<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Updated for Openshift CLI
*/

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/openshift/origin/pkg/cmd/admin"
	"github.com/openshift/origin/pkg/cmd/cli"
	"github.com/openshift/origin/pkg/cmd/openshift"
	mangen "github.com/openshift/origin/tools/genman/md2man"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"k8s.io/kubernetes/cmd/genutils"
)

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Root command not specified (oc | oadm | openshift).\n")
		os.Exit(1)
	}

	if strings.HasSuffix(os.Args[2], "oc") {
		genCmdMan("oc", cli.NewCommandCLI("oc", "oc", &bytes.Buffer{}, os.Stdout, ioutil.Discard))
	} else if strings.HasSuffix(os.Args[2], "openshift") {
		genCmdMan("openshift", openshift.NewCommandOpenShift("openshift"))
	} else if strings.HasSuffix(os.Args[2], "oadm") {
		genCmdMan("oadm", admin.NewCommandAdmin("oadm", "oadm", &bytes.Buffer{}, os.Stdout, ioutil.Discard))
	} else {
		fmt.Fprintf(os.Stderr, "Root command not specified (os | oadm | openshift).")
		os.Exit(1)
	}
}

func genCmdMan(cmdName string, cmd *cobra.Command) {
	// use os.Args instead of "flags" because "flags" will mess up the man pages!
	path := "docs/man/" + cmdName
	if len(os.Args) == 3 {
		path = os.Args[1]
	} else if len(os.Args) > 3 {
		fmt.Fprintf(os.Stderr, "usage: %s [output directory]\n", os.Args[0])
		os.Exit(1)
	}

	outDir, err := genutils.OutDir(path)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to get output directory: %v\n", err)
		os.Exit(1)
	}

	// Set environment variables used by openshift so the output is consistent,
	// regardless of where we run.
	os.Setenv("HOME", "/home/username")
	// TODO os.Stdin should really be something like ioutil.Discard, but a Reader
	genMarkdown(cmd, "", outDir)
	for _, c := range cmd.Commands() {
		genMarkdown(c, cmdName, outDir)
	}
}

func preamble(out *bytes.Buffer, cmdName, name, short, long string) {
	out.WriteString(`% ` + strings.ToUpper(cmdName) + `(1) Openshift CLI User Manuals
% Openshift
% June 2016
# NAME
`)

	fmt.Fprintf(out, "%s \\- %s\n\n", name, short)
	fmt.Fprintf(out, "# SYNOPSIS\n")
	fmt.Fprintf(out, "**%s** [OPTIONS]\n\n", name)
	fmt.Fprintf(out, "# DESCRIPTION\n")
	fmt.Fprintf(out, "%s\n\n", long)
}

func printFlags(out *bytes.Buffer, flags *pflag.FlagSet) {
	flags.VisitAll(func(flag *pflag.Flag) {
		if flag.Hidden {
			return
		}
		format := "**--%s**=%s\n\t%s\n\n"
		if flag.Value.Type() == "string" {
			// put quotes on the value
			format = "**--%s**=%q\n\t%s\n\n"
		}

		defValue := flag.DefValue
		if flag.Value.Type() == "duration" {
			defValue = "0"
		}

		if len(flag.Annotations["manpage-def-value"]) > 0 {
			defValue = flag.Annotations["manpage-def-value"][0]
		}

		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
			format = "**-%s**, " + format
			fmt.Fprintf(out, format, flag.Shorthand, flag.Name, defValue, flag.Usage)
		} else {
			fmt.Fprintf(out, format, flag.Name, defValue, flag.Usage)
		}
	})
}

func printOptions(out *bytes.Buffer, command *cobra.Command) {
	flags := command.NonInheritedFlags()
	if flags.HasFlags() {
		fmt.Fprintf(out, "# OPTIONS\n")
		printFlags(out, flags)
		fmt.Fprintf(out, "\n")
	}
	flags = command.InheritedFlags()
	if flags.HasFlags() {
		fmt.Fprintf(out, "# OPTIONS INHERITED FROM PARENT COMMANDS\n")
		printFlags(out, flags)
		fmt.Fprintf(out, "\n")
	}
}

func genMarkdown(command *cobra.Command, parent, docsDir string) {
	dparent := strings.Replace(parent, " ", "-", -1)
	name := command.Name()
	dname := name
	cmdName := name
	if len(parent) > 0 {
		dname = dparent + "-" + name
		name = parent + " " + name
		cmdName = parent
	}

	out := new(bytes.Buffer)
	short := command.Short
	long := command.Long
	if len(long) == 0 {
		long = short
	}

	preamble(out, cmdName, name, short, long)
	printOptions(out, command)

	if len(command.Example) > 0 {
		fmt.Fprintf(out, "# EXAMPLE\n")
		fmt.Fprintf(out, "```\n%s\n```\n", command.Example)
	}

	if len(command.Commands()) > 0 || len(parent) > 0 {
		fmt.Fprintf(out, "# SEE ALSO\n")
		if len(parent) > 0 {
			fmt.Fprintf(out, "**%s(1)**, ", dparent)
		}
		for _, c := range command.Commands() {
			fmt.Fprintf(out, "**%s-%s(1)**, ", dname, c.Name())
			genMarkdown(c, name, docsDir)
		}
		fmt.Fprintf(out, "\n")
	}

	out.WriteString(`
# HISTORY
June 2016, Ported from the Kubernetes man-doc generator
`)

	final := mangen.Render(out.Bytes())

	filename := docsDir + dname + ".1"
	outFile, err := os.Create(filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer outFile.Close()
	_, err = outFile.Write(final)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

}
<|endoftext|>


<commit_before>package scheduler

import (
	"context"
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	pb "github.com/bleenco/abstruse/pb"
	"github.com/bleenco/abstruse/pkg/lib"
	"github.com/bleenco/abstruse/server/core"
	"github.com/bleenco/abstruse/server/ws"
	"go.uber.org/zap"
)

// New returns new scheduler.
func New(
	workers core.WorkerRegistry,
	jobStore core.JobStore,
	buildStore core.BuildStore,
	logger *zap.Logger,
	ws *ws.Server,
) core.Scheduler {
	s := &scheduler{
		ready:      make(chan struct{}, 1),
		interval:   time.Minute,
		workers:    workers,
		jobStore:   jobStore,
		buildStore: buildStore,
		logger:     logger.With(zap.String("type", "scheduler")).Sugar(),
		pending:    make(map[uint]*jobType),
		ws:         ws,
		ctx:        context.Background(),
	}
	go s.run()
	return s
}

type scheduler struct {
	mu         sync.Mutex
	ready      chan struct{}
	paused     bool
	interval   time.Duration
	workers    core.WorkerRegistry
	jobStore   core.JobStore
	buildStore core.BuildStore
	logger     *zap.SugaredLogger
	queued     []*core.Job
	pending    map[uint]*jobType
	ws         *ws.Server
	ctx        context.Context
}

type jobType struct {
	job *core.Job
	pb  *pb.Job
}

func (s *scheduler) Next(job *core.Job) error {
	s.logger.Infof("scheduling job %d from build %d", job.ID, job.BuildID)
	s.Stop(job.ID)
	s.mu.Lock()
	s.queued = append(s.queued, job)
	s.mu.Unlock()

	job.Status = "queued"
	job.StartTime = nil
	job.EndTime = nil
	if err := s.saveJob(job); err != nil {
		return err
	}

	s.next(s.ctx)

	return nil
}

func (s *scheduler) Stop(id uint) (bool, error) {
	if job, err := s.findJob(id); err == nil {
		s.removeJob(id)
		job.Status = "failing"
		job.EndTime = lib.TimeNow()
		if err := s.saveJob(job); err != nil {
			return true, nil
		}
		return false, nil
	}

	if job, ok := s.pending[id]; ok {
		worker, err := s.getWorker(job.pb.WorkerId)
		if err != nil {
			return false, err
		}

		res, err := worker.CLI.StopJob(context.Background(), job.pb)
		if err != nil {
			return false, err
		}

		job.job.Status = "failing"
		job.job.EndTime = lib.TimeNow()
		s.saveJob(job.job)

		return res.GetStopped(), nil
	}

	return false, nil
}

func (s *scheduler) RestartBuild(id uint) error {
	build, err := s.buildStore.Find(id)
	if err != nil {
		return err
	}
	build.StartTime = nil
	build.EndTime = nil
	if err := s.buildStore.Update(build); err != nil {
		return err
	}
	for _, job := range build.Jobs {
		job, _ := s.jobStore.Find(job.ID)
		s.Next(job)
	}

	return nil
}

func (s *scheduler) StopBuild(id uint) error {
	build, err := s.buildStore.Find(id)
	if err != nil {
		return err
	}
	var wg sync.WaitGroup
	wg.Add(len(build.Jobs))
	for _, job := range build.Jobs {
		job, _ := s.jobStore.Find(job.ID)
		go func(job *core.Job) {
			s.Stop(job.ID)
			wg.Done()
		}(job)
	}
	wg.Wait()
	build.EndTime = lib.TimeNow()
	return s.buildStore.Update(build)
}

func (s *scheduler) Pause() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.paused = true
	return nil
}

func (s *scheduler) Resume() error {
	s.mu.Lock()
	s.paused = false
	s.mu.Unlock()
	s.next(s.ctx)
	return nil
}

func (s *scheduler) JobLog(id uint) (string, error) {
	if job, ok := s.pending[id]; ok {
		return strings.Join(job.pb.GetLog(), ""), nil
	}
	return "", fmt.Errorf("job not running")
}

func (s *scheduler) process() error {
	worker, err := s.findWorker()
	if err != nil {
		return err
	}
	if worker == nil {
		return nil
	}
	job, err := s.enqueueJob()
	if err != nil || job == nil {
		return nil
	}

	s.logger.Infof("processing job %d, sending to worker %s...", job.ID, worker.ID)
	go s.startJob(job, worker)

	return nil
}

func (s *scheduler) startJob(job *core.Job, worker *core.Worker) error {
	s.removeJob(job.ID)

	job.Status = "running"
	job.Log = ""
	job.StartTime = lib.TimeNow()
	job.EndTime = nil
	s.saveJob(job)

	j := &pb.Job{
		Id:            uint64(job.ID),
		BuildId:       uint64(job.BuildID),
		Commands:      job.Commands,
		Image:         job.Image,
		Env:           job.Env,
		Url:           job.Build.Repository.URL,
		ProviderName:  job.Build.Repository.Provider.Name,
		ProviderURL:   job.Build.Repository.Provider.URL,
		ProviderToken: job.Build.Repository.Provider.AccessToken,
		Ref:           job.Build.Ref,
		CommitSHA:     job.Build.Commit,
		RepoName:      job.Build.Repository.FullName,
		Action:        pb.Job_JobStart,
		WorkerId:      worker.ID,
	}

	s.mu.Lock()
	s.pending[job.ID] = &jobType{job: job, pb: j}
	s.mu.Unlock()

	j, err := worker.StartJob(j)
	if err != nil {
		return err
	}

	job.Status = j.GetStatus()
	job.Log = strings.Join(j.GetLog(), "")
	job.EndTime = lib.TimeNow()
	s.saveJob(job)

	s.mu.Lock()
	delete(s.pending, job.ID)
	s.mu.Unlock()

	s.next(s.ctx)

	return nil
}

func (s *scheduler) next(ctx context.Context) {
	select {
	case s.ready <- struct{}{}:
	default:
	}
}

func (s *scheduler) enqueueJob() (*core.Job, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if len(s.queued) > 0 {
		job := s.queued[0]
		s.queued = s.queued[1:]
		return job, nil
	}
	return nil, fmt.Errorf("no jobs queued")
}

func (s *scheduler) findJob(id uint) (*core.Job, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	for _, job := range s.queued {
		if job.ID == id {
			return job, nil
		}
	}

	return nil, fmt.Errorf("job not found")
}

func (s *scheduler) removeJob(id uint) {
	s.mu.Lock()
	defer s.mu.Unlock()

	for i, job := range s.queued {
		if job.ID == id {
			s.queued = append(s.queued[:i], s.queued[i+1:]...)
			break
		}
	}
}

func (s *scheduler) findWorker() (*core.Worker, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	workers, err := s.workers.List()
	if err != nil {
		return nil, err
	}

	var worker *core.Worker
	var c int
	for _, w := range workers {
		w.Lock()
		diff := w.Max - w.Running
		if diff > c {
			worker, c = w, diff
		}
		w.Unlock()
	}

	return worker, nil
}

func (s *scheduler) getWorker(id string) (*core.Worker, error) {
	workers, err := s.workers.List()
	if err != nil {
		return nil, err
	}

	for _, w := range workers {
		if w.ID == id {
			return w, nil
		}
	}

	return nil, fmt.Errorf("worker not found")
}

func (s *scheduler) saveJob(job *core.Job) error {
	if err := s.jobStore.Update(job); err != nil {
		return err
	}
	go s.broadcastJobStatus(job)
	return s.updateBuildTime(job.BuildID)
}

func (s *scheduler) broadcastJobStatus(job *core.Job) {
	sub := path.Clean(path.Join("/subs", "jobs"))
	event := map[string]interface{}{
		"buildID": job.BuildID,
		"jobID":   job.ID,
		"status":  job.Status,
	}
	if job.StartTime != nil {
		event["startTime"] = job.StartTime
	}
	if job.EndTime != nil {
		event["endTime"] = job.EndTime
	}
	s.ws.App.Broadcast(sub, event)
}

func (s *scheduler) updateBuildTime(id uint) error {
	build, err := s.buildStore.Find(id)
	if err != nil {
		return err
	}
	if build.StartTime != nil && build.EndTime != nil {
		return nil
	}

	alldone := true
	var startTime *time.Time
	var endTime *time.Time
	for _, j := range build.Jobs {
		if j.EndTime == nil {
			alldone = false
			break
		} else {
			if endTime == nil || j.EndTime.After(*endTime) {
				endTime = j.EndTime
			}
		}
		if startTime == nil || (j.StartTime != nil && j.StartTime.Before(*startTime)) {
			startTime = j.StartTime
		}
	}
	if startTime != nil {
		build.StartTime = startTime
		if err := s.buildStore.Update(build); err != nil {
			return err
		}
	}
	if alldone && endTime != nil {
		build.EndTime = endTime
		if err := s.buildStore.Update(build); err != nil {
			return err
		}
	}
	return nil
}

func (s *scheduler) run() error {
	s.logger.Infof("starting scheduler loop")
	for {
		select {
		case <-s.ctx.Done():
			return s.ctx.Err()
		case <-s.ready:
			s.process()
		case <-time.After(s.interval):
			s.next(s.ctx)
		}
	}
}
<commit_msg>feat(scheduler): logging<commit_after>package scheduler

import (
	"context"
	"fmt"
	"path"
	"strings"
	"sync"
	"time"

	pb "github.com/bleenco/abstruse/pb"
	"github.com/bleenco/abstruse/pkg/lib"
	"github.com/bleenco/abstruse/server/core"
	"github.com/bleenco/abstruse/server/ws"
	"go.uber.org/zap"
)

// New returns new scheduler.
func New(
	workers core.WorkerRegistry,
	jobStore core.JobStore,
	buildStore core.BuildStore,
	logger *zap.Logger,
	ws *ws.Server,
) core.Scheduler {
	s := &scheduler{
		ready:      make(chan struct{}, 1),
		interval:   time.Minute,
		workers:    workers,
		jobStore:   jobStore,
		buildStore: buildStore,
		logger:     logger.With(zap.String("type", "scheduler")).Sugar(),
		pending:    make(map[uint]*jobType),
		ws:         ws,
		ctx:        context.Background(),
	}
	go s.run()
	return s
}

type scheduler struct {
	mu         sync.Mutex
	ready      chan struct{}
	paused     bool
	interval   time.Duration
	workers    core.WorkerRegistry
	jobStore   core.JobStore
	buildStore core.BuildStore
	logger     *zap.SugaredLogger
	queued     []*core.Job
	pending    map[uint]*jobType
	ws         *ws.Server
	ctx        context.Context
}

type jobType struct {
	job *core.Job
	pb  *pb.Job
}

func (s *scheduler) Next(job *core.Job) error {
	s.logger.Infof("scheduling job %d from build %d...", job.ID, job.BuildID)
	s.Stop(job.ID)
	s.mu.Lock()
	s.queued = append(s.queued, job)
	s.mu.Unlock()

	job.Status = "queued"
	job.StartTime = nil
	job.EndTime = nil
	if err := s.saveJob(job); err != nil {
		return err
	}
	s.logger.Infof("job %d scheduled", job.ID)

	s.next(s.ctx)

	return nil
}

func (s *scheduler) Stop(id uint) (bool, error) {
	if job, err := s.findJob(id); err == nil {
		s.removeJob(id)
		job.Status = "failing"
		job.EndTime = lib.TimeNow()
		s.logger.Infof("job %d removed from queue")
		if err := s.saveJob(job); err == nil {
			return true, nil
		}
		return false, nil
	}

	if job, ok := s.pending[id]; ok {
		worker, err := s.getWorker(job.pb.WorkerId)
		if err != nil {
			return false, err
		}

		res, err := worker.CLI.StopJob(context.Background(), job.pb)
		if err != nil {
			return false, err
		}

		s.logger.Infof("job %d stopped", id)
		job.job.Status = "failing"
		job.job.EndTime = lib.TimeNow()
		s.saveJob(job.job)

		return res.GetStopped(), nil
	}

	return false, nil
}

func (s *scheduler) RestartBuild(id uint) error {
	build, err := s.buildStore.Find(id)
	if err != nil {
		return err
	}
	build.StartTime = nil
	build.EndTime = nil
	if err := s.buildStore.Update(build); err != nil {
		return err
	}
	for _, job := range build.Jobs {
		job, _ := s.jobStore.Find(job.ID)
		s.Next(job)
	}

	return nil
}

func (s *scheduler) StopBuild(id uint) error {
	build, err := s.buildStore.Find(id)
	if err != nil {
		return err
	}
	var wg sync.WaitGroup
	wg.Add(len(build.Jobs))
	for _, job := range build.Jobs {
		job, _ := s.jobStore.Find(job.ID)
		go func(job *core.Job) {
			s.Stop(job.ID)
			wg.Done()
		}(job)
	}
	wg.Wait()
	return s.updateBuildTime(id)
}

func (s *scheduler) Pause() error {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.paused = true
	return nil
}

func (s *scheduler) Resume() error {
	s.mu.Lock()
	s.paused = false
	s.mu.Unlock()
	s.next(s.ctx)
	return nil
}

func (s *scheduler) JobLog(id uint) (string, error) {
	if job, ok := s.pending[id]; ok {
		return strings.Join(job.pb.GetLog(), ""), nil
	}
	return "", fmt.Errorf("job not running")
}

func (s *scheduler) process() error {
	worker, err := s.findWorker()
	if err != nil {
		return err
	}
	if worker == nil {
		return nil
	}
	job, err := s.enqueueJob()
	if err != nil || job == nil {
		return nil
	}

	s.logger.Infof("processing job %d, sending to worker %s...", job.ID, worker.ID)
	go s.startJob(job, worker)

	return nil
}

func (s *scheduler) startJob(job *core.Job, worker *core.Worker) error {
	s.removeJob(job.ID)

	job.Status = "running"
	job.Log = ""
	job.StartTime = lib.TimeNow()
	job.EndTime = nil
	s.saveJob(job)

	j := &pb.Job{
		Id:            uint64(job.ID),
		BuildId:       uint64(job.BuildID),
		Commands:      job.Commands,
		Image:         job.Image,
		Env:           job.Env,
		Url:           job.Build.Repository.URL,
		ProviderName:  job.Build.Repository.Provider.Name,
		ProviderURL:   job.Build.Repository.Provider.URL,
		ProviderToken: job.Build.Repository.Provider.AccessToken,
		Ref:           job.Build.Ref,
		CommitSHA:     job.Build.Commit,
		RepoName:      job.Build.Repository.FullName,
		Action:        pb.Job_JobStart,
		WorkerId:      worker.ID,
	}

	s.mu.Lock()
	s.pending[job.ID] = &jobType{job: job, pb: j}
	s.mu.Unlock()

	j, err := worker.StartJob(j)
	if err != nil {
		return err
	}

	job.Status = j.GetStatus()
	job.Log = strings.Join(j.GetLog(), "")
	job.EndTime = lib.TimeNow()
	s.saveJob(job)

	s.mu.Lock()
	delete(s.pending, job.ID)
	s.mu.Unlock()

	s.next(s.ctx)

	return nil
}

func (s *scheduler) next(ctx context.Context) {
	select {
	case s.ready <- struct{}{}:
	default:
	}
}

func (s *scheduler) enqueueJob() (*core.Job, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if len(s.queued) > 0 {
		job := s.queued[0]
		s.queued = s.queued[1:]
		return job, nil
	}
	return nil, fmt.Errorf("no jobs queued")
}

func (s *scheduler) findJob(id uint) (*core.Job, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	for _, job := range s.queued {
		if job.ID == id {
			return job, nil
		}
	}

	return nil, fmt.Errorf("job not found")
}

func (s *scheduler) removeJob(id uint) {
	s.mu.Lock()
	defer s.mu.Unlock()

	for i, job := range s.queued {
		if job.ID == id {
			s.queued = append(s.queued[:i], s.queued[i+1:]...)
			break
		}
	}
}

func (s *scheduler) findWorker() (*core.Worker, error) {
	s.mu.Lock()
	defer s.mu.Unlock()

	workers, err := s.workers.List()
	if err != nil {
		return nil, err
	}

	var worker *core.Worker
	var c int
	for _, w := range workers {
		w.Lock()
		diff := w.Max - w.Running
		if diff > c {
			worker, c = w, diff
		}
		w.Unlock()
	}

	return worker, nil
}

func (s *scheduler) getWorker(id string) (*core.Worker, error) {
	workers, err := s.workers.List()
	if err != nil {
		return nil, err
	}

	for _, w := range workers {
		if w.ID == id {
			return w, nil
		}
	}

	return nil, fmt.Errorf("worker not found")
}

func (s *scheduler) saveJob(job *core.Job) error {
	if err := s.jobStore.Update(job); err != nil {
		return err
	}
	go s.broadcastJobStatus(job)
	return s.updateBuildTime(job.BuildID)
}

func (s *scheduler) broadcastJobStatus(job *core.Job) {
	sub := path.Clean(path.Join("/subs", "jobs"))
	event := map[string]interface{}{
		"buildID": job.BuildID,
		"jobID":   job.ID,
		"status":  job.Status,
	}
	if job.StartTime != nil {
		event["startTime"] = job.StartTime
	}
	if job.EndTime != nil {
		event["endTime"] = job.EndTime
	}
	s.ws.App.Broadcast(sub, event)
}

func (s *scheduler) updateBuildTime(id uint) error {
	build, err := s.buildStore.Find(id)
	if err != nil {
		return err
	}
	if build.StartTime != nil && build.EndTime != nil {
		return nil
	}

	alldone := true
	var startTime *time.Time
	var endTime *time.Time
	for _, j := range build.Jobs {
		if j.EndTime == nil {
			alldone = false
			break
		} else {
			if endTime == nil || j.EndTime.After(*endTime) {
				endTime = j.EndTime
			}
		}
		if startTime == nil || (j.StartTime != nil && j.StartTime.Before(*startTime)) {
			startTime = j.StartTime
		}
	}
	if startTime != nil {
		build.StartTime = startTime
		if err := s.buildStore.Update(build); err != nil {
			return err
		}
	}
	if alldone && endTime != nil {
		build.EndTime = endTime
		if err := s.buildStore.Update(build); err != nil {
			return err
		}
	}
	return nil
}

func (s *scheduler) run() error {
	s.logger.Infof("starting scheduler loop")
	for {
		select {
		case <-s.ctx.Done():
			return s.ctx.Err()
		case <-s.ready:
			s.process()
		case <-time.After(s.interval):
			s.next(s.ctx)
		}
	}
}
<|endoftext|>


<commit_before>package replicate

import (
	crand "crypto/rand"
	"math/rand"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/drausin/libri/libri/common/ecid"
	cerrors "github.com/drausin/libri/libri/common/errors"
	"github.com/drausin/libri/libri/common/id"
	"github.com/drausin/libri/libri/common/storage"
	"github.com/drausin/libri/libri/librarian/api"
	"github.com/drausin/libri/libri/librarian/client"
	"github.com/drausin/libri/libri/librarian/server/peer"
	"github.com/drausin/libri/libri/librarian/server/routing"
	"github.com/drausin/libri/libri/librarian/server/search"
	"github.com/drausin/libri/libri/librarian/server/store"
	"github.com/drausin/libri/libri/librarian/server/verify"
	"github.com/golang/protobuf/proto"
	"github.com/pkg/errors"
	"go.uber.org/zap"
)

const (
	// DefaultVerifyInterval is the default amount of time between verify operations.
	DefaultVerifyInterval = 30 * time.Second

	// DefaultReplicateConcurrency is the default number of replicator routines.
	DefaultReplicateConcurrency = uint(1)

	// DefaultVerifyTimeout is the default timeout for the replicator's verify retries.
	DefaultVerifyTimeout = 10 * time.Second

	// DefaultMaxErrRate is the default maximum allowed error rate for verify & store requests
	// before a fatal error is thrown.
	DefaultMaxErrRate = 0.5

	// macKeySize is the size of the MAC key used for verify operations.
	macKeySize = 32

	// errQueueSize is the size of the error queue used to calculate the running error rate.
	errQueueSize = 100

	// underreplicatedQueueSize is the size of the queue of under-replicated documents to be
	// replicated
	underreplicatedQueueSize = 32

	// logger keys
	logVerify = "verify"
	logStore  = "store"
)

var (
	metricsKey              = []byte("replicationMetrics")
	errVerifyExhausted      = errors.New("verification failed because exhausted peers to query")
	errMissingStoredMetrics = errors.New("missing stored metrics")
)

// Parameters is the replicator parameters.
type Parameters struct {
	VerifyInterval       time.Duration
	ReplicateConcurrency uint
	VerifyTimeout        time.Duration
	MaxErrRate           float32
}

// NewDefaultParameters returns the default replicator parameters.
func NewDefaultParameters() *Parameters {
	return &Parameters{
		VerifyInterval:       DefaultVerifyInterval,
		ReplicateConcurrency: DefaultReplicateConcurrency,
		VerifyTimeout:        DefaultVerifyTimeout,
		MaxErrRate:           DefaultMaxErrRate,
	}
}

// NewVerifyDefaultParameters returns default verify parameters that exclude the replica stored by
// the replicator.
func NewVerifyDefaultParameters() *verify.Parameters {
	p := verify.NewDefaultParameters()
	p.NReplicas--
	p.NClosestResponses--
	p.ExcludeSelf = true
	return p
}

// Metrics contains (monotonically increasing) counters for different replication events.
type Metrics struct {
	NVerified        uint64
	NUnderreplicated uint64
	NReplicated      uint64
	LatestPass       int64
}

func (m *Metrics) clone() *Metrics {
	return &Metrics{
		NVerified:        m.NVerified,
		NReplicated:      m.NReplicated,
		NUnderreplicated: m.NUnderreplicated,
		LatestPass:       m.LatestPass,
	}
}

// Replicator is a long-running routine that iterates through stored documents and verified that
// they are fully replicated. When they are not, it issues Store requests to close peers to
// bring their replication up to the desired level.
type Replicator interface {
	// Start starts the replicator routines.
	Start() error

	// Stop gracefully stops the replicator routines.
	Stop()

	// Metrics returns a copy of the current metrics.
	Metrics() *Metrics
}

type replicator struct {
	selfID           ecid.ID
	rt               routing.Table
	metrics          *Metrics
	docS             storage.DocumentStorer
	metricsSL        metricsStorerLoader
	verifier         verify.Verifier
	storer           store.Storer
	replicatorParams *Parameters
	verifyParams     *verify.Parameters
	storeParams      *store.Parameters
	underreplicated  chan *verify.Verify
	stop             chan struct{}
	stopped          chan struct{}
	errs             chan error
	fatal            chan error
	logger           *zap.Logger
	rng              *rand.Rand
	mu               sync.Mutex
}

// NewReplicator returns a new Replicator.
func NewReplicator(
	selfID ecid.ID,
	rt routing.Table,
	docS storage.DocumentStorer,
	serverSL storage.StorerLoader,
	verifier verify.Verifier,
	storer store.Storer,
	replicatorParams *Parameters,
	verifyParams *verify.Parameters,
	storeParams *store.Parameters,
	rng *rand.Rand,
	logger *zap.Logger,
) Replicator {
	metricsSL := &metricsSLImpl{serverSL}

	// init metrics from stored value, if it exists
	metrics, err := metricsSL.Load()
	if err != nil {
		// if missing or other error, init new metrics
		metrics = &Metrics{}
	}
	return &replicator{
		selfID:           selfID,
		rt:               rt,
		metrics:          metrics,
		docS:             docS,
		metricsSL:        metricsSL,
		verifier:         verifier,
		storer:           storer,
		replicatorParams: replicatorParams,
		verifyParams:     verifyParams,
		storeParams:      storeParams,
		underreplicated:  make(chan *verify.Verify, underreplicatedQueueSize),
		errs:             make(chan error, errQueueSize),
		stop:             make(chan struct{}),
		stopped:          make(chan struct{}),
		fatal:            make(chan error, 1),
		rng:              rng,
		logger:           logger,
	}
}

func (r *replicator) Start() error {

	// listen for fatal error
	var err error
	go func() {
		err = <-r.fatal
		r.Stop()
	}()

	// monitor non-fatal errors, sending fatal error if too many
	maxErrRate := r.replicatorParams.MaxErrRate
	go cerrors.MonitorRunningErrors(r.errs, r.fatal, errQueueSize, maxErrRate, r.logger)

	// iterates through stored docs in a continuous loop
	go r.verify()

	// replicate under-replicated docs found during replication
	wg := new(sync.WaitGroup)
	for c := uint(0); c < r.replicatorParams.ReplicateConcurrency; c++ {
		wg.Add(1)
		go r.replicate(wg)
	}
	wg.Wait()

	return err
}

func (r *replicator) Stop() {
	r.logger.Info("ending replicator")
	safeClose(r.stop)
	r.wrapLock(func() {
		safeCloseErrChan(r.errs)
		safeCloseVerifyChan(r.underreplicated)
	})
	<-r.stopped
	r.logger.Debug("ended replicator")
}

func (r *replicator) Metrics() *Metrics {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.metrics.clone()
}

func (r *replicator) verify() {
	for {
		if r.docS.Metrics().NDocuments == 0 {
			pause := make(chan struct{})
			go func() {
				intervalMaxMs := int32(r.replicatorParams.VerifyInterval / time.Millisecond)
				waitMs := time.Duration(r.rng.Int31n(intervalMaxMs))
				time.Sleep(waitMs * time.Millisecond)
				close(pause)
			}()
			select {
			case <-r.stop:
			case <-pause:
			}
		} else {
			if err := r.docS.Iterate(r.stop, r.verifyValue); err != nil {
				r.fatal <- err
			}
			r.wrapLock(func() {
				r.metrics.LatestPass = time.Now().Unix()
				if err := r.metricsSL.Store(r.metrics.clone()); err != nil {
					r.fatal <- err
				}
			})
		}
		select {
		case <-r.stop: // exit if we've received stop signal
			close(r.stopped)
			return
		default:
			// otherwise, do another pass
		}
	}
}

func (r *replicator) verifyValue(key id.ID, value []byte) {
	pause := make(chan struct{})
	go func() {
		time.Sleep(r.replicatorParams.VerifyInterval)
		close(pause)
	}()
	macKey := make([]byte, macKeySize)
	_, err := crand.Read(macKey)
	cerrors.MaybePanic(err) // should never happen

	v := verify.NewVerify(r.selfID, key, value, macKey, r.verifyParams)
	seeds := r.rt.Peak(key, r.verifyParams.NClosestResponses)

	operation := func() error {
		v.Result = verify.NewInitialResult(key, r.verifyParams)
		return r.verifier.Verify(v, seeds)
	}
	err = backoff.Retry(operation, client.NewExpBackoff(r.replicatorParams.VerifyTimeout))

	if err != nil { // implies v.Errored()
		r.logger.Debug("document verification errored", zap.Object(logVerify, v))
		r.wrapLock(func() { maybeSendErrChan(r.errs, err) })
		return
	}
	if v.Exhausted() {
		r.logger.Debug("verify exhausted peers", zap.Object(logVerify, v))
		r.wrapLock(func() { maybeSendErrChan(r.errs, errVerifyExhausted) })
		return
	}

	if v.FullyReplicated() {
		r.logger.Debug("document fully-replicated", zap.Object(logVerify, v))
	} else if v.UnderReplicated() {
		r.wrapLock(func() {
			r.metrics.NUnderreplicated++
			maybeSendVerifyChan(r.underreplicated, v)
		})
		r.logger.Info("document under-replicated", zap.Object(logVerify, v))
	}
	r.wrapLock(func() {
		r.metrics.NVerified++
		maybeSendErrChan(r.errs, nil)
	})
	for _, p := range v.Result.Closest.Peers() {
		r.rt.Push(p)
	}
	select {
	case <-r.stop:
	case <-pause:
	}
}

func (r *replicator) replicate(wg *sync.WaitGroup) {
	defer wg.Done()
	for v := range r.underreplicated {
		s := newStore(r.selfID, v, *r.storeParams)
		// empty seeds b/c verification has already, in effect, replaced the search component of
		// the store operation
		if err := r.storer.Store(s, []peer.Peer{}); err != nil {
			maybeSendErrChan(r.errs, err)
			continue
		}
		if s.Stored() {
			r.wrapLock(func() { r.metrics.NReplicated++ })
			r.logger.Info("stored additional document replicas", zap.Object(logStore, s))
		}
		// for all other non-Stored outcomes for the store, we basically give up and hope to
		// replicate on next pass
		r.wrapLock(func() { r.errs <- nil })
	}
}

func (r *replicator) wrapLock(operation func()) {
	r.mu.Lock()
	defer r.mu.Unlock()
	operation()
}

func newStore(selfID ecid.ID, v *verify.Verify, storeParams store.Parameters) *store.Store {
	value := &api.Document{}
	cerrors.MaybePanic(proto.Unmarshal(v.Value, value)) // should never happen
	searchParams := &search.Parameters{
		NClosestResponses: uint(v.Result.Closest.Len()),
	}
	s := store.NewStore(selfID, v.Key, value, searchParams, &storeParams)

	// update number of replicas to store to be just enough to get back to full replication
	storeParams.NReplicas = v.Params.NReplicas - uint(len(v.Result.Replicas))
	s.Search.Params.NMaxErrors = v.Params.NMaxErrors
	s.Search.Params.NClosestResponses = storeParams.NReplicas + s.Search.Params.NMaxErrors

	// construct minimal search result from verify
	s.Search.Params.NMaxErrors = v.Params.NMaxErrors
	s.Search.Result = &search.Result{
		Closest:   v.Result.Closest,
		Unqueried: v.Result.Unqueried,
		Responded: v.Result.Responded,
		Errored:   v.Result.Errored,
	}
	// s.Search.FoundClosestPeers() == true

	return s
}

type metricsStorerLoader interface {
	Store(m *Metrics) error
	Load() (*Metrics, error)
}

type metricsSLImpl struct {
	inner storage.StorerLoader
}

func (sl *metricsSLImpl) Store(m *Metrics) error {
	storageMetrics := &storage.ReplicationMetrics{
		NVerified:        m.NVerified,
		NReplicated:      m.NReplicated,
		NUnderreplicated: m.NUnderreplicated,
		LatestPass:       m.LatestPass,
	}
	metricsBytes, err := proto.Marshal(storageMetrics)
	cerrors.MaybePanic(err) // should never happen
	return sl.inner.Store(metricsKey, metricsBytes)
}

func (sl *metricsSLImpl) Load() (*Metrics, error) {
	storageMetricsBytes, err := sl.inner.Load(metricsKey)
	if err != nil {
		return nil, err
	}
	if storageMetricsBytes == nil {
		return nil, errMissingStoredMetrics
	}
	stored := &storage.ReplicationMetrics{}
	cerrors.MaybePanic(proto.Unmarshal(storageMetricsBytes, stored)) // should never happen
	return &Metrics{
		NVerified:        stored.NVerified,
		NReplicated:      stored.NReplicated,
		NUnderreplicated: stored.NUnderreplicated,
		LatestPass:       stored.LatestPass,
	}, nil
}

func safeClose(ch chan struct{}) {
	select {
	case <-ch: // already closed
	default:
		close(ch)
	}
}

func safeCloseErrChan(ch chan error) {
	select {
	case <-ch: // already closed
	default:
		close(ch)
	}
}

func maybeSendErrChan(ch chan error, err error) {
	select {
	case <-ch: // already closed
	default:
		ch <- err
	}
}

func maybeSendVerifyChan(ch chan *verify.Verify, v *verify.Verify) {
	select {
	case <-ch: // already closed
	default:
		ch <- v
	}
}

func safeCloseVerifyChan(ch chan *verify.Verify) {
	select {
	case <-ch: // already closed
	default:
		close(ch)
	}
}
<commit_msg>fix another channel data race (#131)<commit_after>package replicate

import (
	crand "crypto/rand"
	"math/rand"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
	"github.com/drausin/libri/libri/common/ecid"
	cerrors "github.com/drausin/libri/libri/common/errors"
	"github.com/drausin/libri/libri/common/id"
	"github.com/drausin/libri/libri/common/storage"
	"github.com/drausin/libri/libri/librarian/api"
	"github.com/drausin/libri/libri/librarian/client"
	"github.com/drausin/libri/libri/librarian/server/peer"
	"github.com/drausin/libri/libri/librarian/server/routing"
	"github.com/drausin/libri/libri/librarian/server/search"
	"github.com/drausin/libri/libri/librarian/server/store"
	"github.com/drausin/libri/libri/librarian/server/verify"
	"github.com/golang/protobuf/proto"
	"github.com/pkg/errors"
	"go.uber.org/zap"
)

const (
	// DefaultVerifyInterval is the default amount of time between verify operations.
	DefaultVerifyInterval = 30 * time.Second

	// DefaultReplicateConcurrency is the default number of replicator routines.
	DefaultReplicateConcurrency = uint(1)

	// DefaultVerifyTimeout is the default timeout for the replicator's verify retries.
	DefaultVerifyTimeout = 10 * time.Second

	// DefaultMaxErrRate is the default maximum allowed error rate for verify & store requests
	// before a fatal error is thrown.
	DefaultMaxErrRate = 0.5

	// macKeySize is the size of the MAC key used for verify operations.
	macKeySize = 32

	// errQueueSize is the size of the error queue used to calculate the running error rate.
	errQueueSize = 100

	// underreplicatedQueueSize is the size of the queue of under-replicated documents to be
	// replicated
	underreplicatedQueueSize = 32

	// logger keys
	logVerify = "verify"
	logStore  = "store"
)

var (
	metricsKey              = []byte("replicationMetrics")
	errVerifyExhausted      = errors.New("verification failed because exhausted peers to query")
	errMissingStoredMetrics = errors.New("missing stored metrics")
)

// Parameters is the replicator parameters.
type Parameters struct {
	VerifyInterval       time.Duration
	ReplicateConcurrency uint
	VerifyTimeout        time.Duration
	MaxErrRate           float32
}

// NewDefaultParameters returns the default replicator parameters.
func NewDefaultParameters() *Parameters {
	return &Parameters{
		VerifyInterval:       DefaultVerifyInterval,
		ReplicateConcurrency: DefaultReplicateConcurrency,
		VerifyTimeout:        DefaultVerifyTimeout,
		MaxErrRate:           DefaultMaxErrRate,
	}
}

// NewVerifyDefaultParameters returns default verify parameters that exclude the replica stored by
// the replicator.
func NewVerifyDefaultParameters() *verify.Parameters {
	p := verify.NewDefaultParameters()
	p.NReplicas--
	p.NClosestResponses--
	p.ExcludeSelf = true
	return p
}

// Metrics contains (monotonically increasing) counters for different replication events.
type Metrics struct {
	NVerified        uint64
	NUnderreplicated uint64
	NReplicated      uint64
	LatestPass       int64
}

func (m *Metrics) clone() *Metrics {
	return &Metrics{
		NVerified:        m.NVerified,
		NReplicated:      m.NReplicated,
		NUnderreplicated: m.NUnderreplicated,
		LatestPass:       m.LatestPass,
	}
}

// Replicator is a long-running routine that iterates through stored documents and verified that
// they are fully replicated. When they are not, it issues Store requests to close peers to
// bring their replication up to the desired level.
type Replicator interface {
	// Start starts the replicator routines.
	Start() error

	// Stop gracefully stops the replicator routines.
	Stop()

	// Metrics returns a copy of the current metrics.
	Metrics() *Metrics
}

type replicator struct {
	selfID           ecid.ID
	rt               routing.Table
	metrics          *Metrics
	docS             storage.DocumentStorer
	metricsSL        metricsStorerLoader
	verifier         verify.Verifier
	storer           store.Storer
	replicatorParams *Parameters
	verifyParams     *verify.Parameters
	storeParams      *store.Parameters
	underreplicated  chan *verify.Verify
	stop             chan struct{}
	stopped          chan struct{}
	errs             chan error
	fatal            chan error
	logger           *zap.Logger
	rng              *rand.Rand
	mu               sync.Mutex
}

// NewReplicator returns a new Replicator.
func NewReplicator(
	selfID ecid.ID,
	rt routing.Table,
	docS storage.DocumentStorer,
	serverSL storage.StorerLoader,
	verifier verify.Verifier,
	storer store.Storer,
	replicatorParams *Parameters,
	verifyParams *verify.Parameters,
	storeParams *store.Parameters,
	rng *rand.Rand,
	logger *zap.Logger,
) Replicator {
	metricsSL := &metricsSLImpl{serverSL}

	// init metrics from stored value, if it exists
	metrics, err := metricsSL.Load()
	if err != nil {
		// if missing or other error, init new metrics
		metrics = &Metrics{}
	}
	return &replicator{
		selfID:           selfID,
		rt:               rt,
		metrics:          metrics,
		docS:             docS,
		metricsSL:        metricsSL,
		verifier:         verifier,
		storer:           storer,
		replicatorParams: replicatorParams,
		verifyParams:     verifyParams,
		storeParams:      storeParams,
		underreplicated:  make(chan *verify.Verify, underreplicatedQueueSize),
		errs:             make(chan error, errQueueSize),
		stop:             make(chan struct{}),
		stopped:          make(chan struct{}),
		fatal:            make(chan error, 1),
		rng:              rng,
		logger:           logger,
	}
}

func (r *replicator) Start() error {

	// listen for fatal error
	var err error
	go func() {
		err = <-r.fatal
		r.Stop()
	}()

	// monitor non-fatal errors, sending fatal error if too many
	maxErrRate := r.replicatorParams.MaxErrRate
	go cerrors.MonitorRunningErrors(r.errs, r.fatal, errQueueSize, maxErrRate, r.logger)

	// iterates through stored docs in a continuous loop
	go r.verify()

	// replicate under-replicated docs found during replication
	wg := new(sync.WaitGroup)
	for c := uint(0); c < r.replicatorParams.ReplicateConcurrency; c++ {
		wg.Add(1)
		go r.replicate(wg)
	}
	wg.Wait()

	return err
}

func (r *replicator) Stop() {
	r.logger.Info("ending replicator")
	safeClose(r.stop)
	r.wrapLock(func() {
		safeCloseErrChan(r.errs)
		safeCloseVerifyChan(r.underreplicated)
	})
	<-r.stopped
	r.logger.Debug("ended replicator")
}

func (r *replicator) Metrics() *Metrics {
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.metrics.clone()
}

func (r *replicator) verify() {
	for {
		if r.docS.Metrics().NDocuments == 0 {
			pause := make(chan struct{})
			go func() {
				intervalMaxMs := int32(r.replicatorParams.VerifyInterval / time.Millisecond)
				waitMs := time.Duration(r.rng.Int31n(intervalMaxMs))
				time.Sleep(waitMs * time.Millisecond)
				close(pause)
			}()
			select {
			case <-r.stop:
			case <-pause:
			}
		} else {
			if err := r.docS.Iterate(r.stop, r.verifyValue); err != nil {
				r.fatal <- err
			}
			r.wrapLock(func() {
				r.metrics.LatestPass = time.Now().Unix()
				if err := r.metricsSL.Store(r.metrics.clone()); err != nil {
					r.fatal <- err
				}
			})
		}
		select {
		case <-r.stop: // exit if we've received stop signal
			close(r.stopped)
			return
		default:
			// otherwise, do another pass
		}
	}
}

func (r *replicator) verifyValue(key id.ID, value []byte) {
	pause := make(chan struct{})
	go func() {
		time.Sleep(r.replicatorParams.VerifyInterval)
		close(pause)
	}()
	macKey := make([]byte, macKeySize)
	_, err := crand.Read(macKey)
	cerrors.MaybePanic(err) // should never happen

	v := verify.NewVerify(r.selfID, key, value, macKey, r.verifyParams)
	seeds := r.rt.Peak(key, r.verifyParams.NClosestResponses)

	operation := func() error {
		v.Result = verify.NewInitialResult(key, r.verifyParams)
		return r.verifier.Verify(v, seeds)
	}
	err = backoff.Retry(operation, client.NewExpBackoff(r.replicatorParams.VerifyTimeout))

	if err != nil { // implies v.Errored()
		r.logger.Debug("document verification errored", zap.Object(logVerify, v))
		r.wrapLock(func() { maybeSendErrChan(r.errs, err) })
		return
	}
	if v.Exhausted() {
		r.logger.Debug("verify exhausted peers", zap.Object(logVerify, v))
		r.wrapLock(func() { maybeSendErrChan(r.errs, errVerifyExhausted) })
		return
	}

	if v.FullyReplicated() {
		r.logger.Debug("document fully-replicated", zap.Object(logVerify, v))
	} else if v.UnderReplicated() {
		r.wrapLock(func() {
			r.metrics.NUnderreplicated++
			maybeSendVerifyChan(r.underreplicated, v)
		})
		r.logger.Info("document under-replicated", zap.Object(logVerify, v))
	}
	r.wrapLock(func() {
		r.metrics.NVerified++
		maybeSendErrChan(r.errs, nil)
	})
	for _, p := range v.Result.Closest.Peers() {
		r.rt.Push(p)
	}
	select {
	case <-r.stop:
	case <-pause:
	}
}

func (r *replicator) replicate(wg *sync.WaitGroup) {
	defer wg.Done()
	for v := range r.underreplicated {
		s := newStore(r.selfID, v, *r.storeParams)
		// empty seeds b/c verification has already, in effect, replaced the search component of
		// the store operation
		if err := r.storer.Store(s, []peer.Peer{}); err != nil {
			maybeSendErrChan(r.errs, err)
			continue
		}
		if s.Stored() {
			r.wrapLock(func() { r.metrics.NReplicated++ })
			r.logger.Info("stored additional document replicas", zap.Object(logStore, s))
		}
		// for all other non-Stored outcomes for the store, we basically give up and hope to
		// replicate on next pass
		r.wrapLock(func() { maybeSendErrChan(r.errs, nil) })
	}
}

func (r *replicator) wrapLock(operation func()) {
	r.mu.Lock()
	defer r.mu.Unlock()
	operation()
}

func newStore(selfID ecid.ID, v *verify.Verify, storeParams store.Parameters) *store.Store {
	value := &api.Document{}
	cerrors.MaybePanic(proto.Unmarshal(v.Value, value)) // should never happen
	searchParams := &search.Parameters{
		NClosestResponses: uint(v.Result.Closest.Len()),
	}
	s := store.NewStore(selfID, v.Key, value, searchParams, &storeParams)

	// update number of replicas to store to be just enough to get back to full replication
	storeParams.NReplicas = v.Params.NReplicas - uint(len(v.Result.Replicas))
	s.Search.Params.NMaxErrors = v.Params.NMaxErrors
	s.Search.Params.NClosestResponses = storeParams.NReplicas + s.Search.Params.NMaxErrors

	// construct minimal search result from verify
	s.Search.Params.NMaxErrors = v.Params.NMaxErrors
	s.Search.Result = &search.Result{
		Closest:   v.Result.Closest,
		Unqueried: v.Result.Unqueried,
		Responded: v.Result.Responded,
		Errored:   v.Result.Errored,
	}
	// s.Search.FoundClosestPeers() == true

	return s
}

type metricsStorerLoader interface {
	Store(m *Metrics) error
	Load() (*Metrics, error)
}

type metricsSLImpl struct {
	inner storage.StorerLoader
}

func (sl *metricsSLImpl) Store(m *Metrics) error {
	storageMetrics := &storage.ReplicationMetrics{
		NVerified:        m.NVerified,
		NReplicated:      m.NReplicated,
		NUnderreplicated: m.NUnderreplicated,
		LatestPass:       m.LatestPass,
	}
	metricsBytes, err := proto.Marshal(storageMetrics)
	cerrors.MaybePanic(err) // should never happen
	return sl.inner.Store(metricsKey, metricsBytes)
}

func (sl *metricsSLImpl) Load() (*Metrics, error) {
	storageMetricsBytes, err := sl.inner.Load(metricsKey)
	if err != nil {
		return nil, err
	}
	if storageMetricsBytes == nil {
		return nil, errMissingStoredMetrics
	}
	stored := &storage.ReplicationMetrics{}
	cerrors.MaybePanic(proto.Unmarshal(storageMetricsBytes, stored)) // should never happen
	return &Metrics{
		NVerified:        stored.NVerified,
		NReplicated:      stored.NReplicated,
		NUnderreplicated: stored.NUnderreplicated,
		LatestPass:       stored.LatestPass,
	}, nil
}

func safeClose(ch chan struct{}) {
	select {
	case <-ch: // already closed
	default:
		close(ch)
	}
}

func safeCloseErrChan(ch chan error) {
	select {
	case <-ch: // already closed
	default:
		close(ch)
	}
}

func maybeSendErrChan(ch chan error, err error) {
	select {
	case <-ch: // already closed
	default:
		ch <- err
	}
}

func maybeSendVerifyChan(ch chan *verify.Verify, v *verify.Verify) {
	select {
	case <-ch: // already closed
	default:
		ch <- v
	}
}

func safeCloseVerifyChan(ch chan *verify.Verify) {
	select {
	case <-ch: // already closed
	default:
		close(ch)
	}
}
<|endoftext|>