<commit_before>package jman

import (
	"bytes"
	"errors"
	"io/ioutil"
	"strconv"
	"strings"
	"sync"
)

var (
	// ErrSpecificNode is for when retrieving a node does not return a specific key/value, but perhaps a map
	ErrSpecificNode = errors.New("Could not find a specific node that matched the given path")
)

// JFile represents a JSON file and contains the filename and root node
type JFile struct {
	filename string
	rootnode *Node
	rw       *sync.RWMutex
}

// NewFile will read the given filename and return a JFile struct
func NewFile(filename string) (*JFile, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	js, err := New(data)
	if err != nil {
		return nil, err
	}
	rw := &sync.RWMutex{}
	return &JFile{filename, js, rw}, nil
}

// Add two byte slices together
func badd(a, b []byte) []byte {
	var buf bytes.Buffer
	buf.Write(a)
	buf.Write(b)
	return buf.Bytes()
}

// Recursively look up a given JSON path
func jsonpath(js *Node, JSONpath string) (*Node, string, error) {
	if JSONpath == "" {
		// Could not find end node
		return js, JSONpath, ErrSpecificNode
	}
	firstpart := JSONpath
	secondpart := ""
	if strings.Contains(JSONpath, ".") {
		fields := strings.SplitN(JSONpath, ".", 2)
		firstpart = fields[0]
		secondpart = fields[1]
	}
	if firstpart == "x" {
		return jsonpath(js, secondpart)
	} else if strings.Contains(firstpart, "[") && strings.Contains(firstpart, "]") {
		fields := strings.SplitN(firstpart, "[", 2)
		name := fields[0]
		if name != "x" {
			js = js.Get(name)
		}
		fields = strings.SplitN(fields[1], "]", 2)
		index, err := strconv.Atoi(fields[0])
		if err != nil {
			return js, JSONpath, errors.New("Invalid index: " + fields[0] + " (" + err.Error() + ")")
		}
		node, ok := js.GetIndex(index)
		if !ok {
			return js, JSONpath, errors.New("Could not find index: " + fields[0])
		}
		return jsonpath(node, secondpart)
	}
	name := firstpart
	if secondpart != "" {
		return js, JSONpath, errors.New("JSON path left unparsed: " + secondpart)
	}
	return js.Get(name), "", nil
}

// GetNode will find the JSON node that corresponds to the given JSON path
func (jf *JFile) GetNode(JSONpath string) (*Node, error) {
	foundnode, leftoverpath, err := jsonpath(jf.rootnode, JSONpath)
	if err != nil {
		return nil, err
	}
	if leftoverpath != "" {
		return nil, errors.New("JSON path left unparsed: " + leftoverpath)
	}
	if foundnode == nil {
		return nil, errors.New("Could not lookup: " + JSONpath)
	}
	return foundnode, nil
}

// GetString will find the string that corresponds to the given JSON path
func (jf *JFile) GetString(JSONpath string) (string, error) {
	node, err := jf.GetNode(JSONpath)
	if err != nil {
		return "", err
	}
	return node.String(), nil
}

// SetString will change the value of the key that the given JSON path points to
func (jf *JFile) SetString(JSONpath, value string) error {
	firstpart := ""
	lastpart := JSONpath
	if strings.Contains(JSONpath, ".") {
		pos := strings.LastIndex(JSONpath, ".")
		firstpart = JSONpath[:pos]
		lastpart = JSONpath[pos+1:]
	}

	node, _, err := jsonpath(jf.rootnode, firstpart)
	if (err != nil) && (err != ErrSpecificNode) {
		return err
	}

	_, hasNode := node.CheckGet(lastpart)
	if !hasNode {
		return errors.New("Index out of range? Could not set value.")
	}

	// It's weird that simplejson Set does not return an error value
	node.Set(lastpart, value)

	newdata, err := jf.rootnode.EncodePretty()
	if err != nil {
		return err
	}

	return jf.Write(newdata)
}

// Write writes the current JSON data to the file
func (jf *JFile) Write(data []byte) error {
	jf.rw.Lock()
	defer jf.rw.Unlock()
	// TODO: Add newline as well?
	return ioutil.WriteFile(jf.filename, data, 0666)
}

// AddJSON adds JSON data at the given JSON path
func (jf *JFile) AddJSON(JSONpath, JSONdata string) error {
	firstpart := ""
	lastpart := JSONpath
	if strings.Contains(JSONpath, ".") {
		pos := strings.LastIndex(JSONpath, ".")
		firstpart = JSONpath[:pos]
		lastpart = JSONpath[pos+1:]
	}

	node, _, err := jsonpath(jf.rootnode, firstpart)
	if (err != nil) && (err != ErrSpecificNode) {
		return err
	}

	_, hasNode := node.CheckGet(lastpart)
	if hasNode {
		return errors.New("The JSON path should not point to a single key when adding JSON data.")
	}

	listJSON, err := node.Encode()
	if err != nil {
		return err
	}

	fullJSON, err := jf.rootnode.Encode()
	if err != nil {
		return err
	}

	// TODO: Implement a more efficient way of adding data.
	newFullJSON := bytes.Replace(fullJSON, listJSON, badd(listJSON[:len(listJSON)-1], []byte(","+JSONdata+"]")), 1)

	js, err := New(newFullJSON)
	if err != nil {
		return err
	}

	newFullJSON, err = js.EncodePretty()
	if err != nil {
		return err
	}

	return jf.Write(newFullJSON)
}

// SetString sets a value to the given JSON file at the given JSON path
func SetString(filename, JSONpath, value string) error {
	jf, err := NewFile(filename)
	if err != nil {
		return err
	}
	return jf.SetString(JSONpath, value)
}

// AddJSON adds JSON data to the given JSON file at the given JSON path
func AddJSON(filename, JSONpath, JSONdata string) error {
	jf, err := NewFile(filename)
	if err != nil {
		return err
	}
	return jf.AddJSON(JSONpath, JSONdata)
}

// GetString will find the string that corresponds to the given JSON Path,
// given a filename and a simple JSON path expression.
func GetString(filename, JSONpath string) (string, error) {
	jf, err := NewFile(filename)
	if err != nil {
		return "", err
	}
	return jf.GetString(JSONpath)
}
<commit_msg>Added GetAll method<commit_after>package jman

import (
	"bytes"
	"errors"
	"io/ioutil"
	"strconv"
	"strings"
	"sync"
)

var (
	// ErrSpecificNode is for when retrieving a node does not return a specific key/value, but perhaps a map
	ErrSpecificNode = errors.New("Could not find a specific node that matched the given path")
)

// JFile represents a JSON file and contains the filename and root node
type JFile struct {
	filename string
	rootnode *Node
	rw       *sync.RWMutex
}

// NewFile will read the given filename and return a JFile struct
func NewFile(filename string) (*JFile, error) {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}
	js, err := New(data)
	if err != nil {
		return nil, err
	}
	rw := &sync.RWMutex{}
	return &JFile{filename, js, rw}, nil
}

// Add two byte slices together
func badd(a, b []byte) []byte {
	var buf bytes.Buffer
	buf.Write(a)
	buf.Write(b)
	return buf.Bytes()
}

// Recursively look up a given JSON path
func jsonpath(js *Node, JSONpath string) (*Node, string, error) {
	if JSONpath == "" {
		// Could not find end node
		return js, JSONpath, ErrSpecificNode
	}
	firstpart := JSONpath
	secondpart := ""
	if strings.Contains(JSONpath, ".") {
		fields := strings.SplitN(JSONpath, ".", 2)
		firstpart = fields[0]
		secondpart = fields[1]
	}
	if firstpart == "x" {
		return jsonpath(js, secondpart)
	} else if strings.Contains(firstpart, "[") && strings.Contains(firstpart, "]") {
		fields := strings.SplitN(firstpart, "[", 2)
		name := fields[0]
		if name != "x" {
			js = js.Get(name)
		}
		fields = strings.SplitN(fields[1], "]", 2)
		index, err := strconv.Atoi(fields[0])
		if err != nil {
			return js, JSONpath, errors.New("Invalid index: " + fields[0] + " (" + err.Error() + ")")
		}
		node, ok := js.GetIndex(index)
		if !ok {
			return js, JSONpath, errors.New("Could not find index: " + fields[0])
		}
		return jsonpath(node, secondpart)
	}
	name := firstpart
	if secondpart != "" {
		return js, JSONpath, errors.New("JSON path left unparsed: " + secondpart)
	}
	return js.Get(name), "", nil
}

// GetNode will find the JSON node that corresponds to the given JSON path
func (jf *JFile) GetNode(JSONpath string) (*Node, error) {
	foundnode, leftoverpath, err := jsonpath(jf.rootnode, JSONpath)
	if err != nil {
		return nil, err
	}
	if leftoverpath != "" {
		return nil, errors.New("JSON path left unparsed: " + leftoverpath)
	}
	if foundnode == nil {
		return nil, errors.New("Could not lookup: " + JSONpath)
	}
	return foundnode, nil
}

// GetString will find the string that corresponds to the given JSON path
func (jf *JFile) GetString(JSONpath string) (string, error) {
	node, err := jf.GetNode(JSONpath)
	if err != nil {
		return "", err
	}
	return node.String(), nil
}

// SetString will change the value of the key that the given JSON path points to
func (jf *JFile) SetString(JSONpath, value string) error {
	firstpart := ""
	lastpart := JSONpath
	if strings.Contains(JSONpath, ".") {
		pos := strings.LastIndex(JSONpath, ".")
		firstpart = JSONpath[:pos]
		lastpart = JSONpath[pos+1:]
	}

	node, _, err := jsonpath(jf.rootnode, firstpart)
	if (err != nil) && (err != ErrSpecificNode) {
		return err
	}

	_, hasNode := node.CheckGet(lastpart)
	if !hasNode {
		return errors.New("Index out of range? Could not set value.")
	}

	// It's weird that simplejson Set does not return an error value
	node.Set(lastpart, value)

	newdata, err := jf.rootnode.EncodePretty()
	if err != nil {
		return err
	}

	return jf.Write(newdata)
}

// Write writes the current JSON data to the file
func (jf *JFile) Write(data []byte) error {
	jf.rw.Lock()
	defer jf.rw.Unlock()
	// TODO: Add newline as well?
	return ioutil.WriteFile(jf.filename, data, 0666)
}

// AddJSON adds JSON data at the given JSON path
func (jf *JFile) AddJSON(JSONpath, JSONdata string) error {
	firstpart := ""
	lastpart := JSONpath
	if strings.Contains(JSONpath, ".") {
		pos := strings.LastIndex(JSONpath, ".")
		firstpart = JSONpath[:pos]
		lastpart = JSONpath[pos+1:]
	}

	node, _, err := jsonpath(jf.rootnode, firstpart)
	if (err != nil) && (err != ErrSpecificNode) {
		return err
	}

	_, hasNode := node.CheckGet(lastpart)
	if hasNode {
		return errors.New("The JSON path should not point to a single key when adding JSON data.")
	}

	listJSON, err := node.Encode()
	if err != nil {
		return err
	}

	fullJSON, err := jf.rootnode.Encode()
	if err != nil {
		return err
	}

	// TODO: Implement a more efficient way of adding data.
	newFullJSON := bytes.Replace(fullJSON, listJSON, badd(listJSON[:len(listJSON)-1], []byte(","+JSONdata+"]")), 1)

	js, err := New(newFullJSON)
	if err != nil {
		return err
	}

	newFullJSON, err = js.EncodePretty()
	if err != nil {
		return err
	}

	return jf.Write(newFullJSON)
}

// GetAll returns the current JSON data
func (jf *JFile) GetAll() ([]byte, error) {
	return jf.rootnode.EncodePretty()
}

// SetString sets a value to the given JSON file at the given JSON path
func SetString(filename, JSONpath, value string) error {
	jf, err := NewFile(filename)
	if err != nil {
		return err
	}
	return jf.SetString(JSONpath, value)
}

// AddJSON adds JSON data to the given JSON file at the given JSON path
func AddJSON(filename, JSONpath, JSONdata string) error {
	jf, err := NewFile(filename)
	if err != nil {
		return err
	}
	return jf.AddJSON(JSONpath, JSONdata)
}

// GetString will find the string that corresponds to the given JSON Path,
// given a filename and a simple JSON path expression.
func GetString(filename, JSONpath string) (string, error) {
	jf, err := NewFile(filename)
	if err != nil {
		return "", err
	}
	return jf.GetString(JSONpath)
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/caarlos0/env"
)

type app struct {
	Environtment string `env:"APP_ENV"    envDefault:"development"`
	Name         string `env:"APP_NAME"   envDefault:"Haste"`
	URL          string `env:"APP_URL"    envDefault:"http://127.0.0.1"`
	Port         int    `env:"APP_PORT"   envDefault:"3000"`
	SecretKey    string `env:"SECRET_KEY" envDefault:"somesupersecretkey"`
}

func App() *app {
	cfg := &app{}
	env.Parse(cfg)
	return cfg
}
<commit_msg>fix typo in configuration key name<commit_after>package config

import (
	"github.com/caarlos0/env"
)

type app struct {
	Environment string `env:"APP_ENV"    envDefault:"development"`
	Name        string `env:"APP_NAME"   envDefault:"Haste"`
	URL         string `env:"APP_URL"    envDefault:"http://127.0.0.1"`
	Port        int    `env:"APP_PORT"   envDefault:"3000"`
	SecretKey   string `env:"SECRET_KEY" envDefault:"somesupersecretkey"`
}

func App() *app {
	cfg := &app{}
	env.Parse(cfg)
	return cfg
}
<|endoftext|>


<commit_before>package conn

import (
	"github.com/garyburd/redigo/redis"
)

type Memo struct {
	Time    string `redis:"time"`
	Content string `redis:"content"`
}

//All redis actions

func SetMasterId(id int) {
	c := Pool.Get()
	defer c.Close()
	c.Do("SET", "evolsnowChatId", id)
}

func GetMasterId() int {
	c := Pool.Get()
	defer c.Close()
	id, _ := redis.Int(c.Do("GET", "evolsnowChatId"))
	return id
}

func SetUserChatId(user string, id int) {
	c := Pool.Get()
	defer c.Close()
	key := user + "ChatId"
	c.Do("SET", key, id)
}

func GetUserChatId(user string) int {
	c := Pool.Get()
	defer c.Close()
	key := user + "ChatId"
	id, _ := redis.Int(c.Do("GET", key))
	return id
}

func HSetMemo(user, time, memo string) {
	c := Pool.Get()
	defer c.Close()
	var setMemoLua = `
	local id = redis.call("INCR", "memoIncrId")
	redis.call("RPUSH", KEYS[1]..":memos", id)
	redis.call("HMSET", "memo:"..id, "time", KEYS[2], "content", KEYS[3])
	`
	script := redis.NewScript(3, setMemoLua)
	script.Do(c, user, time, memo)
}

func HGetAllMemos(user string) []Memo {
	c := Pool.Get()
	defer c.Close()
	var multiGetMemoLua = `
	local data = redis.call("LRANGE", KEYS[1]..":memos", "0", "-1")
	local ret = {}
  	for idx=1, #data do
  		ret[idx] = redis.call("HGETALL", "memo:"..data[idx])
  	end
  	return ret
   `
	var memos *[]Memo
	script := redis.NewScript(1, multiGetMemoLua)
	values, _ := redis.Values(script.Do(c, user))
	//	for i := range values {
	//		m := new(Memo)
	//		redis.ScanStruct(values[i], &m)
	//		memos = append(memos, m)
	//	}
	redis.ScanStruct(values, memos)
	return *memos
}

//
//var multiGetScript = redis.NewScript(0, multiGetMemoLua)
<commit_msg>log redis error<commit_after>package conn

import (
	"github.com/garyburd/redigo/redis"
	"log"
)

type Memo struct {
	Time    string `redis:"time"`
	Content string `redis:"content"`
}

//All redis actions

func SetMasterId(id int) {
	c := Pool.Get()
	defer c.Close()
	c.Do("SET", "evolsnowChatId", id)
}

func GetMasterId() int {
	c := Pool.Get()
	defer c.Close()
	id, _ := redis.Int(c.Do("GET", "evolsnowChatId"))
	return id
}

func SetUserChatId(user string, id int) {
	c := Pool.Get()
	defer c.Close()
	key := user + "ChatId"
	c.Do("SET", key, id)
}

func GetUserChatId(user string) int {
	c := Pool.Get()
	defer c.Close()
	key := user + "ChatId"
	id, _ := redis.Int(c.Do("GET", key))
	return id
}

func HSetMemo(user, time, memo string) {
	c := Pool.Get()
	defer c.Close()
	var setMemoLua = `
	local id = redis.call("INCR", "memoIncrId")
	redis.call("RPUSH", KEYS[1]..":memos", id)
	redis.call("HMSET", "memo:"..id, "time", KEYS[2], "content", KEYS[3])
	`
	script := redis.NewScript(3, setMemoLua)
	script.Do(c, user, time, memo)
}

func HGetAllMemos(user string) []Memo {
	c := Pool.Get()
	defer c.Close()
	var multiGetMemoLua = `
	local data = redis.call("LRANGE", KEYS[1]..":memos", "0", "-1")
	local ret = {}
  	for idx=1, #data do
  		ret[idx] = redis.call("HGETALL", "memo:"..data[idx])
  	end
  	return ret
   `
	var memos *[]Memo
	script := redis.NewScript(1, multiGetMemoLua)
	values, err := redis.Values(script.Do(c, user))
	if err != nil {
		log.Println(err)
	}
	//	for i := range values {
	//		m := new(Memo)
	//		redis.ScanStruct(values[i], &m)
	//		memos = append(memos, m)
	//	}
	if err = redis.ScanStruct(values, memos); err != nil {
		log.Println(err)
	}
	return *memos
}

//
//var multiGetScript = redis.NewScript(0, multiGetMemoLua)
<|endoftext|>


<commit_before>package hostdb

import (
	"math"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// updateHostDBEntry updates a HostDBEntries's historic interactions if more
// than one block passed since the last update. This should be called every time
// before the recent interactions are updated.  if passedTime is e.g. 10, this
// means that the recent interactions were updated 10 blocks ago but never
// since. So we need to apply the decay of 1 block before we append the recent
// interactions from 10 blocks ago and then apply the decay of 9 more blocks in
// which the recent interactions have been 0
func updateHostHistoricInteractions(host *modules.HostDBEntry, bh types.BlockHeight) {
	passedTime := bh - host.LastHistoricUpdate
	if passedTime == 0 {
		// no time passed. nothing to do.
		return
	}

	// tmp float64 values for more accurate decay
	hsi := float64(host.HistoricSuccessfulInteractions)
	hfi := float64(host.HistoricFailedInteractions)

	// Apply the decay of a single block, but only if there are more than
	// historicInteractionDecalyLimit datapoints total.
	if hsi+hfi > historicInteractionDecayLimit {
		decay := historicInteractionDecay
		hsi *= decay
		hfi *= decay
	}

	// Apply the recent interactions of that single block. Recent interactions
	// cannot represent more than recentInteractionWeightLimit of historic
	// interactions, unless there are less than historicInteractionDecayLimit
	// total interactions, and then the recent interactions cannot count for
	// more than recentInteractionWeightLimit of the decay limit.
	rsi := float64(host.RecentSuccessfulInteractions)
	rfi := float64(host.RecentFailedInteractions)
	if hsi+hfi > historicInteractionDecayLimit {
		if rsi+rfi > recentInteractionWeightLimit*(hsi+hfi) {
			adjustment := recentInteractionWeightLimit * (hsi + hfi) / (rsi + rfi)
			rsi *= adjustment
			rfi *= adjustment
		}
	} else {
		if rsi+rfi > recentInteractionWeightLimit*historicInteractionDecayLimit {
			adjustment := recentInteractionWeightLimit * historicInteractionDecayLimit / (rsi + rfi)
			rsi *= adjustment
			rfi *= adjustment
		}
	}
	hsi += rsi
	hfi += rfi

	// Apply the decay of the rest of the blocks
	if passedTime > 1 && hsi+hfi > historicInteractionDecayLimit {
		decay := math.Pow(historicInteractionDecay, float64(passedTime-1))
		hsi *= decay
		hfi *= decay
	}

	// Set new values
	host.HistoricSuccessfulInteractions = uint64(hsi)
	host.HistoricFailedInteractions = uint64(hfi)
	host.RecentSuccessfulInteractions = 0
	host.RecentFailedInteractions = 0

	// Update the time of the last update
	host.LastHistoricUpdate = bh
}

// IncrementSuccessfulInteractions increments the number of successful
// interactions with a host for a given key
func (hdb *HostDB) IncrementSuccessfulInteractions(key types.SiaPublicKey) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// Fetch the host.
	host, haveHost := hdb.hostTree.Select(key)
	if !haveHost {
		return
	}

	// Update historic values if necessary
	updateHostHistoricInteractions(&host, hdb.blockHeight)

	// Increment the successful interactions
	host.RecentSuccessfulInteractions++
	hdb.hostTree.Modify(host)
}

// IncrementFailedInteractions increments the number of failed interactions with
// a host for a given key
func (hdb *HostDB) IncrementFailedInteractions(key types.SiaPublicKey) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// Fetch the host.
	host, haveHost := hdb.hostTree.Select(key)
	if !haveHost || !hdb.online {
		// If we are offline it probably wasn't the host's fault
		return
	}

	// Update historic values if necessary
	updateHostHistoricInteractions(&host, hdb.blockHeight)

	// Increment the failed interactions
	host.RecentFailedInteractions++
	hdb.hostTree.Modify(host)
}
<commit_msg>add sanity check for historic interaction updates<commit_after>package hostdb

import (
	"math"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// updateHostDBEntry updates a HostDBEntries's historic interactions if more
// than one block passed since the last update. This should be called every time
// before the recent interactions are updated.  if passedTime is e.g. 10, this
// means that the recent interactions were updated 10 blocks ago but never
// since. So we need to apply the decay of 1 block before we append the recent
// interactions from 10 blocks ago and then apply the decay of 9 more blocks in
// which the recent interactions have been 0
func updateHostHistoricInteractions(host *modules.HostDBEntry, bh types.BlockHeight) {
	// Check that the last historic update was not in the future.
	if host.LastHistoricUpdate >= bh {
		// The hostdb may be performing a rescan, or maybe no time has passed
		// since the last update, so there is nothing to do.
		return
	}
	passedTime := bh - host.LastHistoricUpdate

	// tmp float64 values for more accurate decay
	hsi := float64(host.HistoricSuccessfulInteractions)
	hfi := float64(host.HistoricFailedInteractions)

	// Apply the decay of a single block, but only if there are more than
	// historicInteractionDecalyLimit datapoints total.
	if hsi+hfi > historicInteractionDecayLimit {
		decay := historicInteractionDecay
		hsi *= decay
		hfi *= decay
	}

	// Apply the recent interactions of that single block. Recent interactions
	// cannot represent more than recentInteractionWeightLimit of historic
	// interactions, unless there are less than historicInteractionDecayLimit
	// total interactions, and then the recent interactions cannot count for
	// more than recentInteractionWeightLimit of the decay limit.
	rsi := float64(host.RecentSuccessfulInteractions)
	rfi := float64(host.RecentFailedInteractions)
	if hsi+hfi > historicInteractionDecayLimit {
		if rsi+rfi > recentInteractionWeightLimit*(hsi+hfi) {
			adjustment := recentInteractionWeightLimit * (hsi + hfi) / (rsi + rfi)
			rsi *= adjustment
			rfi *= adjustment
		}
	} else {
		if rsi+rfi > recentInteractionWeightLimit*historicInteractionDecayLimit {
			adjustment := recentInteractionWeightLimit * historicInteractionDecayLimit / (rsi + rfi)
			rsi *= adjustment
			rfi *= adjustment
		}
	}
	hsi += rsi
	hfi += rfi

	// Apply the decay of the rest of the blocks
	if passedTime > 1 && hsi+hfi > historicInteractionDecayLimit {
		decay := math.Pow(historicInteractionDecay, float64(passedTime-1))
		hsi *= decay
		hfi *= decay
	}

	// Set new values
	host.HistoricSuccessfulInteractions = uint64(hsi)
	host.HistoricFailedInteractions = uint64(hfi)
	host.RecentSuccessfulInteractions = 0
	host.RecentFailedInteractions = 0

	// Update the time of the last update
	host.LastHistoricUpdate = bh
}

// IncrementSuccessfulInteractions increments the number of successful
// interactions with a host for a given key
func (hdb *HostDB) IncrementSuccessfulInteractions(key types.SiaPublicKey) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// Fetch the host.
	host, haveHost := hdb.hostTree.Select(key)
	if !haveHost {
		return
	}

	// Update historic values if necessary
	updateHostHistoricInteractions(&host, hdb.blockHeight)

	// Increment the successful interactions
	host.RecentSuccessfulInteractions++
	hdb.hostTree.Modify(host)
}

// IncrementFailedInteractions increments the number of failed interactions with
// a host for a given key
func (hdb *HostDB) IncrementFailedInteractions(key types.SiaPublicKey) {
	hdb.mu.Lock()
	defer hdb.mu.Unlock()

	// Fetch the host.
	host, haveHost := hdb.hostTree.Select(key)
	if !haveHost || !hdb.online {
		// If we are offline it probably wasn't the host's fault
		return
	}

	// Update historic values if necessary
	updateHostHistoricInteractions(&host, hdb.blockHeight)

	// Increment the failed interactions
	host.RecentFailedInteractions++
	hdb.hostTree.Modify(host)
}
<|endoftext|>


<commit_before>package contractmanager

import (
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
)

// findUnfinishedStorageFolderAdditions will scroll through a set of state
// changes and figure out which of the unfinished storage folder additions are
// still unfinished.
func findUnfinishedStorageFolderAdditions(scs []stateChange) []savedStorageFolder {
	// Use a map to figure out what unfinished storage folders exist and use it
	// to remove the ones that have terminated.
	usfMap := make(map[uint16]savedStorageFolder)
	for _, sc := range scs {
		for _, sf := range sc.UnfinishedStorageFolderAdditions {
			usfMap[sf.Index] = sf
		}
		for _, sf := range sc.StorageFolderAdditions {
			delete(usfMap, sf.Index)
		}
		for _, index := range sc.ErroredStorageFolderAdditions {
			delete(usfMap, index)
		}
		for _, sfr := range sc.StorageFolderRemovals {
			delete(usfMap, sfr.Index)
		}
	}

	// Return the active unifinished storage folders as a slice.
	var sfs []savedStorageFolder
	for _, sf := range usfMap {
		sfs = append(sfs, sf)
	}
	return sfs
}

// cleanupUnfinishedStorageFolderAdditions will purge any unfinished storage
// folder additions from the previous run.
func (wal *writeAheadLog) cleanupUnfinishedStorageFolderAdditions(scs []stateChange) {
	usfs := findUnfinishedStorageFolderAdditions(scs)
	for _, usf := range usfs {
		sf, exists := wal.cm.storageFolders[usf.Index]
		if exists {
			// Close the storage folder file handles.
			err := sf.metadataFile.Close()
			if err != nil {
				wal.cm.log.Println("Unable to close metadata file for storage folder", sf.path)
			}
			err = sf.sectorFile.Close()
			if err != nil {
				wal.cm.log.Println("Unable to close sector file for storage folder", sf.path)
			}

			// Delete the storage folder from the storage folders map.
			delete(wal.cm.storageFolders, sf.index)
		}

		// Remove any leftover files.
		sectorLookupName := filepath.Join(usf.Path, metadataFile)
		sectorHousingName := filepath.Join(usf.Path, sectorFile)
		err := wal.cm.dependencies.removeFile(sectorLookupName)
		if err != nil {
			wal.cm.log.Println("Unable to remove documented sector metadata lookup:", sectorLookupName, err)
		}
		err = wal.cm.dependencies.removeFile(sectorHousingName)
		if err != nil {
			wal.cm.log.Println("Unable to remove documented sector housing:", sectorHousingName, err)
		}

		// Append an error call to the changeset, indicating that the storage
		// folder add was not completed successfully.
		wal.appendChange(stateChange{
			ErroredStorageFolderAdditions: []uint16{usf.Index},
		})
	}
}

// managedAddStorageFolder will add a storage folder to the contract manager.
// The parent fucntion, contractmanager.AddStorageFolder, has already performed
// any error checking that can be performed without accessing the contract
// manager state.
//
// managedAddStorageFolder can take a long time, as it writes a giant, zeroed
// out file to disk covering the entire range of the storage folder, and
// failure can occur late in the operation. The WAL is notified that a long
// running operation is in progress, so that any changes to disk can be
// reverted in the event of unclean shutdown.
func (wal *writeAheadLog) managedAddStorageFolder(sf *storageFolder) error {
	// Lock the storage folder for the duration of the function.
	sf.mu.Lock()
	defer sf.mu.Unlock()

	numSectors := uint64(len(sf.usage)) * 64
	sectorLookupSize := numSectors * sectorMetadataDiskSize
	sectorHousingSize := numSectors * modules.SectorSize
	totalSize := sectorLookupSize + sectorHousingSize
	sectorLookupName := filepath.Join(sf.path, metadataFile)
	sectorHousingName := filepath.Join(sf.path, sectorFile)

	// Update the uncommitted state to include the storage folder, returning an
	// error if any checks fail.
	var syncChan chan struct{}
	err := func() error {
		wal.mu.Lock()
		defer wal.mu.Unlock()

		// Check that the storage folder is not a duplicate. That requires
		// first checking the contract manager and then checking the WAL. The
		// number of storage folders are also counted, to make sure that the
		// maximum number of storage folders allowed is not exceeded.
		for _, csf := range wal.cm.storageFolders {
			// The conflicting storage folder may e in the process of being
			// removed, however we refuse to add a replacement storage folder
			// until the existing one has been removed entirely.
			if sf.path == csf.path {
				return ErrRepeatFolder
			}
		}

		// Check that there is room for another storage folder.
		if uint64(len(wal.cm.storageFolders)) > maximumStorageFolders {
			return errMaxStorageFolders
		}

		// Determine the index of the storage folder by scanning for an empty
		// spot in the folderLocations map. A random starting place is chosen
		// to keep good average and worst-case runtime.
		var iterator int
		var index uint16
		rand, err := crypto.RandIntn(65536)
		if err != nil {
			wal.cm.log.Critical("no entropy for random iteration when adding a storage folder")
		}
		index = uint16(rand)
		for iterator = 0; iterator < 65536; iterator++ {
			// check the list of unique folders we created earlier.
			_, exists := wal.cm.storageFolders[index]
			if !exists {
				break
			}
			index++
		}
		if iterator == 65536 {
			wal.cm.log.Critical("Previous check indicated that there was room to add another storage folder, but folderLocations set is full.")
			return errMaxStorageFolders
		}
		// Assign the empty index to the storage folder.
		sf.index = index

		// Create the file that is used with the storage folder.
		sf.metadataFile, err = wal.cm.dependencies.createFile(sectorLookupName)
		if err != nil {
			return build.ExtendErr("could not create storage folder file", err)
		}
		sf.sectorFile, err = wal.cm.dependencies.createFile(sectorHousingName)
		if err != nil {
			err = build.ComposeErrors(err, sf.metadataFile.Close())
			err = build.ComposeErrors(err, wal.cm.dependencies.removeFile(sectorLookupName))
			return build.ExtendErr("could not create storage folder file", err)
		}
		// Establish the progress fields for the add operation in the storage
		// folder.
		atomic.StoreUint64(&sf.atomicProgressDenominator, totalSize)

		// Add the storage folder to the list of storage folders.
		wal.cm.storageFolders[index] = sf

		// Add the storage folder to the list of unfinished storage folder
		// additions. There should be no chance of error between this append
		// operation and the completed commitment to the unfinished storage
		// folder addition (signaled by `<-syncChan` a few lines down).
		wal.appendChange(stateChange{
			UnfinishedStorageFolderAdditions: []savedStorageFolder{sf.savedStorageFolder()},
		})
		// Grab the sync channel so we know when the unfinished storage folder
		// addition has been committed to on disk.
		syncChan = wal.syncChan
		return nil
	}()
	if err != nil {
		return err
	}
	// Block until the commitment to the unfinished storage folder addition is
	// complete.
	<-syncChan

	// Simulate a disk failure at this point.
	if wal.cm.dependencies.disrupt("storageFolderAddFinish") {
		return nil
	}

	// If there's an error in the rest of the function, the storage folder
	// needs to be removed from the list of unfinished storage folder
	// additions. Because the WAL is append-only, a stateChange needs to be
	// appended which indicates that the storage folder was unable to be added
	// successfully.
	defer func(sf *storageFolder) {
		if err != nil {
			wal.mu.Lock()
			defer wal.mu.Unlock()

			// Delete the storage folder from the storage folders map.
			delete(wal.cm.storageFolders, sf.index)

			// Remove the leftover files from the failed operation.
			err = build.ComposeErrors(err, sf.sectorFile.Close())
			err = build.ComposeErrors(err, sf.metadataFile.Close())
			err = build.ComposeErrors(err, wal.cm.dependencies.removeFile(sectorLookupName))
			err = build.ComposeErrors(err, wal.cm.dependencies.removeFile(sectorHousingName))

			// Signal in the WAL that the unfinished storage folder addition
			// has failed.
			wal.appendChange(stateChange{
				ErroredStorageFolderAdditions: []uint16{sf.index},
			})
		}
	}(sf)

	// Allocate the files on disk for the storage folder.
	writeSize := uint64(64e3)
	writeData := make([]byte, writeSize)
	writeLocation := sectorHousingSize - writeSize
	_, err = sf.sectorFile.WriteAt(writeData, int64(writeLocation))
	if err != nil {
		return build.ExtendErr("could not allocate sector data file", err)
	}

	// Write the metadata file.
	writeData = make([]byte, sectorLookupSize)
	_, err = sf.metadataFile.WriteAt(writeData, 0)
	if err != nil {
		return build.ExtendErr("could not allocate sector metadata file", err)
	}

	// The file creation process is essentially complete at this point, report
	// complete progress.
	atomic.StoreUint64(&sf.atomicProgressNumerator, totalSize)

	// Sync the files.
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		err := sf.metadataFile.Sync()
		if err != nil {
			wal.cm.log.Println("could not synchronize allocated sector metadata file:", err)
		}
	}()
	go func() {
		defer wg.Done()
		err := sf.sectorFile.Sync()
		if err != nil {
			wal.cm.log.Println("could not synchronize allocated sector data file:", err)
		}
	}()
	wg.Wait()

	// Simulate power failure at this point for some testing scenarios.
	if wal.cm.dependencies.disrupt("incompleteAddStorageFolder") {
		return nil
	}

	// Storage folder addition has completed successfully, commit the addition
	// through the WAL.
	wal.mu.Lock()
	wal.cm.storageFolders[sf.index] = sf
	wal.appendChange(stateChange{
		StorageFolderAdditions: []savedStorageFolder{sf.savedStorageFolder()},
	})
	syncChan = wal.syncChan
	wal.mu.Unlock()

	// Wait to confirm the storage folder addition has completed until the WAL
	// entry has synced.
	<-syncChan

	// Set the progress back to '0'.
	atomic.StoreUint64(&sf.atomicProgressNumerator, 0)
	atomic.StoreUint64(&sf.atomicProgressDenominator, 0)
	return nil
}

// commitAddStorageFolder integrates a pending AddStorageFolder call into the
// state. commitAddStorageFolder should only be called during WAL recovery.
func (wal *writeAheadLog) commitAddStorageFolder(ssf savedStorageFolder) {
	sf, exists := wal.cm.storageFolders[ssf.Index]
	if exists {
		if sf.metadataFile != nil {
			sf.metadataFile.Close()
		}
		if sf.sectorFile != nil {
			sf.sectorFile.Close()
		}
	}

	sf = &storageFolder{
		index: ssf.Index,
		path:  ssf.Path,
		usage: ssf.Usage,

		availableSectors: make(map[sectorID]uint32),
	}

	var err error
	sf.metadataFile, err = wal.cm.dependencies.openFile(filepath.Join(sf.path, metadataFile), os.O_RDWR, 0700)
	if err != nil {
		wal.cm.log.Println("Difficulties opening sector file for ", sf.path, ":", err)
		return
	}
	sf.sectorFile, err = wal.cm.dependencies.openFile(filepath.Join(sf.path, sectorFile), os.O_RDWR, 0700)
	if err != nil {
		wal.cm.log.Println("Difficulties opening sector metadata file for", sf.path, ":", err)
		sf.metadataFile.Close()
		return
	}
	wal.cm.storageFolders[sf.index] = sf
}

// AddStorageFolder adds a storage folder to the contract manager.
func (cm *ContractManager) AddStorageFolder(path string, size uint64) error {
	err := cm.tg.Add()
	if err != nil {
		return err
	}
	defer cm.tg.Done()

	// Check that the storage folder being added meets the size requirements.
	sectors := size / modules.SectorSize
	if sectors > MaximumSectorsPerStorageFolder {
		return ErrLargeStorageFolder
	}
	if sectors < MinimumSectorsPerStorageFolder {
		return ErrSmallStorageFolder
	}
	if (size/modules.SectorSize)%storageFolderGranularity != 0 {
		return errStorageFolderGranularity
	}
	// Check that the path is an absolute path.
	if !filepath.IsAbs(path) {
		return errRelativePath
	}

	// Check that the folder being linked to both exists and is a folder.
	pathInfo, err := os.Stat(path)
	if err != nil {
		return err
	}
	if !pathInfo.Mode().IsDir() {
		return errStorageFolderNotFolder
	}

	// Create a storage folder object and add it to the WAL.
	newSF := &storageFolder{
		path:  path,
		usage: make([]uint64, size/modules.SectorSize/64),

		availableSectors: make(map[sectorID]uint32),
	}
	err = cm.wal.managedAddStorageFolder(newSF)
	if err != nil {
		cm.log.Println("Call to AddStorageFolder has failed:", err)
		return err
	}
	return nil
}
<commit_msg>remove progress indicator from AddStorageFolder<commit_after>package contractmanager

import (
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
)

// findUnfinishedStorageFolderAdditions will scroll through a set of state
// changes and figure out which of the unfinished storage folder additions are
// still unfinished.
func findUnfinishedStorageFolderAdditions(scs []stateChange) []savedStorageFolder {
	// Use a map to figure out what unfinished storage folders exist and use it
	// to remove the ones that have terminated.
	usfMap := make(map[uint16]savedStorageFolder)
	for _, sc := range scs {
		for _, sf := range sc.UnfinishedStorageFolderAdditions {
			usfMap[sf.Index] = sf
		}
		for _, sf := range sc.StorageFolderAdditions {
			delete(usfMap, sf.Index)
		}
		for _, index := range sc.ErroredStorageFolderAdditions {
			delete(usfMap, index)
		}
		for _, sfr := range sc.StorageFolderRemovals {
			delete(usfMap, sfr.Index)
		}
	}

	// Return the active unifinished storage folders as a slice.
	var sfs []savedStorageFolder
	for _, sf := range usfMap {
		sfs = append(sfs, sf)
	}
	return sfs
}

// cleanupUnfinishedStorageFolderAdditions will purge any unfinished storage
// folder additions from the previous run.
func (wal *writeAheadLog) cleanupUnfinishedStorageFolderAdditions(scs []stateChange) {
	usfs := findUnfinishedStorageFolderAdditions(scs)
	for _, usf := range usfs {
		sf, exists := wal.cm.storageFolders[usf.Index]
		if exists {
			// Close the storage folder file handles.
			err := sf.metadataFile.Close()
			if err != nil {
				wal.cm.log.Println("Unable to close metadata file for storage folder", sf.path)
			}
			err = sf.sectorFile.Close()
			if err != nil {
				wal.cm.log.Println("Unable to close sector file for storage folder", sf.path)
			}

			// Delete the storage folder from the storage folders map.
			delete(wal.cm.storageFolders, sf.index)
		}

		// Remove any leftover files.
		sectorLookupName := filepath.Join(usf.Path, metadataFile)
		sectorHousingName := filepath.Join(usf.Path, sectorFile)
		err := wal.cm.dependencies.removeFile(sectorLookupName)
		if err != nil {
			wal.cm.log.Println("Unable to remove documented sector metadata lookup:", sectorLookupName, err)
		}
		err = wal.cm.dependencies.removeFile(sectorHousingName)
		if err != nil {
			wal.cm.log.Println("Unable to remove documented sector housing:", sectorHousingName, err)
		}

		// Append an error call to the changeset, indicating that the storage
		// folder add was not completed successfully.
		wal.appendChange(stateChange{
			ErroredStorageFolderAdditions: []uint16{usf.Index},
		})
	}
}

// managedAddStorageFolder will add a storage folder to the contract manager.
// The parent fucntion, contractmanager.AddStorageFolder, has already performed
// any error checking that can be performed without accessing the contract
// manager state.
func (wal *writeAheadLog) managedAddStorageFolder(sf *storageFolder) error {
	// Lock the storage folder for the duration of the function.
	sf.mu.Lock()
	defer sf.mu.Unlock()

	numSectors := uint64(len(sf.usage)) * 64
	sectorLookupSize := numSectors * sectorMetadataDiskSize
	sectorHousingSize := numSectors * modules.SectorSize
	totalSize := sectorLookupSize + sectorHousingSize
	sectorLookupName := filepath.Join(sf.path, metadataFile)
	sectorHousingName := filepath.Join(sf.path, sectorFile)

	// Update the uncommitted state to include the storage folder, returning an
	// error if any checks fail.
	var syncChan chan struct{}
	err := func() error {
		wal.mu.Lock()
		defer wal.mu.Unlock()

		// Check that the storage folder is not a duplicate. That requires
		// first checking the contract manager and then checking the WAL. The
		// number of storage folders are also counted, to make sure that the
		// maximum number of storage folders allowed is not exceeded.
		for _, csf := range wal.cm.storageFolders {
			// The conflicting storage folder may e in the process of being
			// removed, however we refuse to add a replacement storage folder
			// until the existing one has been removed entirely.
			if sf.path == csf.path {
				return ErrRepeatFolder
			}
		}

		// Check that there is room for another storage folder.
		if uint64(len(wal.cm.storageFolders)) > maximumStorageFolders {
			return errMaxStorageFolders
		}

		// Determine the index of the storage folder by scanning for an empty
		// spot in the folderLocations map. A random starting place is chosen
		// to keep good average and worst-case runtime.
		var iterator int
		var index uint16
		rand, err := crypto.RandIntn(65536)
		if err != nil {
			wal.cm.log.Critical("no entropy for random iteration when adding a storage folder")
		}
		index = uint16(rand)
		for iterator = 0; iterator < 65536; iterator++ {
			// check the list of unique folders we created earlier.
			_, exists := wal.cm.storageFolders[index]
			if !exists {
				break
			}
			index++
		}
		if iterator == 65536 {
			wal.cm.log.Critical("Previous check indicated that there was room to add another storage folder, but folderLocations set is full.")
			return errMaxStorageFolders
		}
		// Assign the empty index to the storage folder.
		sf.index = index

		// Create the files that get used with the storage folder.
		sf.metadataFile, err = wal.cm.dependencies.createFile(sectorLookupName)
		if err != nil {
			return build.ExtendErr("could not create storage folder file", err)
		}
		sf.sectorFile, err = wal.cm.dependencies.createFile(sectorHousingName)
		if err != nil {
			err = build.ComposeErrors(err, sf.metadataFile.Close())
			err = build.ComposeErrors(err, wal.cm.dependencies.removeFile(sectorLookupName))
			return build.ExtendErr("could not create storage folder file", err)
		}

		// Add the storage folder to the list of storage folders.
		wal.cm.storageFolders[index] = sf

		// Add the storage folder to the list of unfinished storage folder
		// additions. There should be no chance of error between this append
		// operation and the completed commitment to the unfinished storage
		// folder addition (signaled by `<-syncChan` a few lines down).
		wal.appendChange(stateChange{
			UnfinishedStorageFolderAdditions: []savedStorageFolder{sf.savedStorageFolder()},
		})
		// Grab the sync channel so we know when the unfinished storage folder
		// addition has been committed to on disk.
		syncChan = wal.syncChan
		return nil
	}()
	if err != nil {
		return err
	}
	// Block until the commitment to the unfinished storage folder addition is
	// complete.
	<-syncChan

	// Simulate a disk failure at this point.
	if wal.cm.dependencies.disrupt("storageFolderAddFinish") {
		return nil
	}

	// If there's an error in the rest of the function, the storage folder
	// needs to be removed from the list of unfinished storage folder
	// additions. Because the WAL is append-only, a stateChange needs to be
	// appended which indicates that the storage folder was unable to be added
	// successfully.
	defer func(sf *storageFolder) {
		if err != nil {
			wal.mu.Lock()
			defer wal.mu.Unlock()

			// Delete the storage folder from the storage folders map.
			delete(wal.cm.storageFolders, sf.index)

			// Remove the leftover files from the failed operation.
			err = build.ComposeErrors(err, sf.sectorFile.Close())
			err = build.ComposeErrors(err, sf.metadataFile.Close())
			err = build.ComposeErrors(err, wal.cm.dependencies.removeFile(sectorLookupName))
			err = build.ComposeErrors(err, wal.cm.dependencies.removeFile(sectorHousingName))

			// Signal in the WAL that the unfinished storage folder addition
			// has failed.
			wal.appendChange(stateChange{
				ErroredStorageFolderAdditions: []uint16{sf.index},
			})
		}
	}(sf)

	// Allocate the files on disk for the storage folder.
	writeSize := uint64(64e3)
	writeData := make([]byte, writeSize)
	writeLocation := sectorHousingSize - writeSize
	_, err = sf.sectorFile.WriteAt(writeData, int64(writeLocation))
	if err != nil {
		return build.ExtendErr("could not allocate sector data file", err)
	}

	// Write the metadata file.
	writeData = make([]byte, sectorLookupSize)
	_, err = sf.metadataFile.WriteAt(writeData, 0)
	if err != nil {
		return build.ExtendErr("could not allocate sector metadata file", err)
	}

	// Sync the files.
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		err := sf.metadataFile.Sync()
		if err != nil {
			wal.cm.log.Println("could not synchronize allocated sector metadata file:", err)
		}
	}()
	go func() {
		defer wg.Done()
		err := sf.sectorFile.Sync()
		if err != nil {
			wal.cm.log.Println("could not synchronize allocated sector data file:", err)
		}
	}()
	wg.Wait()

	// Simulate power failure at this point for some testing scenarios.
	if wal.cm.dependencies.disrupt("incompleteAddStorageFolder") {
		return nil
	}

	// Storage folder addition has completed successfully, commit the addition
	// through the WAL.
	wal.mu.Lock()
	wal.cm.storageFolders[sf.index] = sf
	wal.appendChange(stateChange{
		StorageFolderAdditions: []savedStorageFolder{sf.savedStorageFolder()},
	})
	syncChan = wal.syncChan
	wal.mu.Unlock()

	// Wait to confirm the storage folder addition has completed until the WAL
	// entry has synced.
	<-syncChan
	return nil
}

// commitAddStorageFolder integrates a pending AddStorageFolder call into the
// state. commitAddStorageFolder should only be called during WAL recovery.
func (wal *writeAheadLog) commitAddStorageFolder(ssf savedStorageFolder) {
	sf, exists := wal.cm.storageFolders[ssf.Index]
	if exists {
		if sf.metadataFile != nil {
			sf.metadataFile.Close()
		}
		if sf.sectorFile != nil {
			sf.sectorFile.Close()
		}
	}

	sf = &storageFolder{
		index: ssf.Index,
		path:  ssf.Path,
		usage: ssf.Usage,

		availableSectors: make(map[sectorID]uint32),
	}

	var err error
	sf.metadataFile, err = wal.cm.dependencies.openFile(filepath.Join(sf.path, metadataFile), os.O_RDWR, 0700)
	if err != nil {
		wal.cm.log.Println("Difficulties opening sector file for ", sf.path, ":", err)
		return
	}
	sf.sectorFile, err = wal.cm.dependencies.openFile(filepath.Join(sf.path, sectorFile), os.O_RDWR, 0700)
	if err != nil {
		wal.cm.log.Println("Difficulties opening sector metadata file for", sf.path, ":", err)
		sf.metadataFile.Close()
		return
	}
	wal.cm.storageFolders[sf.index] = sf
}

// AddStorageFolder adds a storage folder to the contract manager.
func (cm *ContractManager) AddStorageFolder(path string, size uint64) error {
	err := cm.tg.Add()
	if err != nil {
		return err
	}
	defer cm.tg.Done()

	// Check that the storage folder being added meets the size requirements.
	sectors := size / modules.SectorSize
	if sectors > MaximumSectorsPerStorageFolder {
		return ErrLargeStorageFolder
	}
	if sectors < MinimumSectorsPerStorageFolder {
		return ErrSmallStorageFolder
	}
	if (size/modules.SectorSize)%storageFolderGranularity != 0 {
		return errStorageFolderGranularity
	}
	// Check that the path is an absolute path.
	if !filepath.IsAbs(path) {
		return errRelativePath
	}

	// Check that the folder being linked to both exists and is a folder.
	pathInfo, err := os.Stat(path)
	if err != nil {
		return err
	}
	if !pathInfo.Mode().IsDir() {
		return errStorageFolderNotFolder
	}

	// Create a storage folder object and add it to the WAL.
	newSF := &storageFolder{
		path:  path,
		usage: make([]uint64, size/modules.SectorSize/64),

		availableSectors: make(map[sectorID]uint32),
	}
	err = cm.wal.managedAddStorageFolder(newSF)
	if err != nil {
		cm.log.Println("Call to AddStorageFolder has failed:", err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>//
// Copyright © 2011-2013 Guy M. Allard
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package stompngo

import (
	"log"
	"time"
)

// Exported Connection methods

/*
	Connected returns the current connection status.
*/
func (c *Connection) Connected() bool {
	return c.connected
}

/*
	Session returns the broker assigned session id.
*/
func (c *Connection) Session() string {
	return c.session
}

/*
	Protocol returns the current connection protocol level.
*/
func (c *Connection) Protocol() string {
	return c.protocol
}

/*
	SetLogger enables a client defined logger for this connection.

	Set to "nil" to disable logging.

	Example:
		// Start logging
		l := log.New(os.Stdout, "", log.Ldate|log.Lmicroseconds)
		c.SetLogger(l)
*/
func (c *Connection) SetLogger(l *log.Logger) {
	c.logger = l
}

/*
	SendTickerInterval returns any heartbeat send ticker interval in ms.  A return
	value of zero means	no heartbeats are being sent.
*/
func (c *Connection) SendTickerInterval() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.sti / 1000000
}

/*
	ReceiveTickerInterval returns any heartbeat receive ticker interval in ms.
	A return value of zero means no heartbeats are being received.
*/
func (c *Connection) ReceiveTickerInterval() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.rti / 1000000
}

/*
	SendTickerCount returns any heartbeat send ticker count.  A return value of
	zero usually indicates no send heartbeats are enabled.
*/
func (c *Connection) SendTickerCount() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.sc
}

/*
	ReceiveTickerCount returns any heartbeat receive ticker count. A return
	value of zero usually indicates no read heartbeats are enabled.
*/
func (c *Connection) ReceiveTickerCount() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.rc
}

// Package exported functions

/*
	Supported checks if a particular STOMP version is supported in the current
	implementation.
*/
func Supported(v string) bool {
	return hasValue(supported, v)
}

/*
	Protocols returns a slice of client supported protocol levels.
*/
func Protocols() []string {
	return supported
}

/*
	Frames Read Count.
*/
func (c *Connection) FramesRead() int64 {
	return c.mets.tfr
}

/*
	Bytes Read Count.
*/
func (c *Connection) BytesRead() int64 {
	return c.mets.tbr
}

/*
	Frames Written Count.
*/
func (c *Connection) FramesWritten() int64 {
	return c.mets.tfw
}

/*
	Bytes Written Count.
*/
func (c *Connection) BytesWritten() int64 {
	return c.mets.tbw
}

/*
	Duration since client start.
*/
func (c *Connection) Running() time.Duration {
	return time.Since(c.mets.st)
}

/*
	Subscribe channel capacity.
*/
func (c *Connection) SubChanCap() int {
	return c.scc
}

/*
	Set Subscribe channel capacity.
*/
func (c *Connection) SetSubChanCap(nc int) {
	c.scc = nc
	return
}

// Unexported Connection methods

/*
	Log data if possible.
*/
func (c *Connection) log(v ...interface{}) {
	if c.logger == nil {
		return
	}
	c.logger.Print(c.session, v)
	return
}

/*
	Shutdown logic.
*/
func (c *Connection) shutdown() {
	// Shutdown heartbeats if necessary
	if c.hbd != nil {
		if c.hbd.hbs {
			c.hbd.ssd <- true
		}
		if c.hbd.hbr {
			c.hbd.rsd <- true
		}
	}
	// Stop writer go routine
	c.wsd <- true
	// We are not connected
	c.connected = false
	return
}

/*
	Read error handler.
*/
func (c *Connection) handleReadError(md MessageData) {
	// Notify any general subscriber of error
	c.input <- md
	// Notify all individual subscribers of error
	c.subsLock.Lock()
	for key := range c.subs {
		c.subs[key] <- md
	}
	c.subsLock.Unlock()
	// Let further shutdown logic proceed normally.
	return
}
<commit_msg>Use more idiomatic method comments.<commit_after>//
// Copyright © 2011-2013 Guy M. Allard
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package stompngo

import (
	"log"
	"time"
)

// Exported Connection methods

/*
	Connected returns the current connection status.
*/
func (c *Connection) Connected() bool {
	return c.connected
}

/*
	Session returns the broker assigned session id.
*/
func (c *Connection) Session() string {
	return c.session
}

/*
	Protocol returns the current connection protocol level.
*/
func (c *Connection) Protocol() string {
	return c.protocol
}

/*
	SetLogger enables a client defined logger for this connection.

	Set to "nil" to disable logging.

	Example:
		// Start logging
		l := log.New(os.Stdout, "", log.Ldate|log.Lmicroseconds)
		c.SetLogger(l)
*/
func (c *Connection) SetLogger(l *log.Logger) {
	c.logger = l
}

/*
	SendTickerInterval returns any heartbeat send ticker interval in ms.  A return
	value of zero means	no heartbeats are being sent.
*/
func (c *Connection) SendTickerInterval() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.sti / 1000000
}

/*
	ReceiveTickerInterval returns any heartbeat receive ticker interval in ms.
	A return value of zero means no heartbeats are being received.
*/
func (c *Connection) ReceiveTickerInterval() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.rti / 1000000
}

/*
	SendTickerCount returns any heartbeat send ticker count.  A return value of
	zero usually indicates no send heartbeats are enabled.
*/
func (c *Connection) SendTickerCount() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.sc
}

/*
	ReceiveTickerCount returns any heartbeat receive ticker count. A return
	value of zero usually indicates no read heartbeats are enabled.
*/
func (c *Connection) ReceiveTickerCount() int64 {
	if c.hbd == nil {
		return 0
	}
	return c.hbd.rc
}

// Package exported functions

/*
	Supported checks if a particular STOMP version is supported in the current
	implementation.
*/
func Supported(v string) bool {
	return hasValue(supported, v)
}

/*
	Protocols returns a slice of client supported protocol levels.
*/
func Protocols() []string {
	return supported
}

/*
	FramesRead returns a count of the number of frames read on the connection.
*/
func (c *Connection) FramesRead() int64 {
	return c.mets.tfr
}

/*
	BytesRead returns a count of the number of bytes read on the connection.
*/
func (c *Connection) BytesRead() int64 {
	return c.mets.tbr
}

/*
	FramesWritten returns a count of the number of frames written on the connection.
*/
func (c *Connection) FramesWritten() int64 {
	return c.mets.tfw
}

/*
	BytesWritten returns a count of the number of bytes written on the connection.
*/
func (c *Connection) BytesWritten() int64 {
	return c.mets.tbw
}

/*
	Running returns a time duration since connection start.
*/
func (c *Connection) Running() time.Duration {
	return time.Since(c.mets.st)
}

/*
	SubChanCap returns the current scribe channel capacity.
*/
func (c *Connection) SubChanCap() int {
	return c.scc
}

/*
	SetSubChanCap sets a new subscribe channel capacity, to be used during future
	SUBSCRIBE operations.
*/
func (c *Connection) SetSubChanCap(nc int) {
	c.scc = nc
	return
}

// Unexported Connection methods

/*
	Log data if possible.
*/
func (c *Connection) log(v ...interface{}) {
	if c.logger == nil {
		return
	}
	c.logger.Print(c.session, v)
	return
}

/*
	Shutdown logic.
*/
func (c *Connection) shutdown() {
	// Shutdown heartbeats if necessary
	if c.hbd != nil {
		if c.hbd.hbs {
			c.hbd.ssd <- true
		}
		if c.hbd.hbr {
			c.hbd.rsd <- true
		}
	}
	// Stop writer go routine
	c.wsd <- true
	// We are not connected
	c.connected = false
	return
}

/*
	Read error handler.
*/
func (c *Connection) handleReadError(md MessageData) {
	// Notify any general subscriber of error
	c.input <- md
	// Notify all individual subscribers of error
	c.subsLock.Lock()
	for key := range c.subs {
		c.subs[key] <- md
	}
	c.subsLock.Unlock()
	// Let further shutdown logic proceed normally.
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"log"
	"sync"

	"github.com/jacobsa/bazilfuse"
	"github.com/jacobsa/fuse/fuseops"
)

// A connection to the fuse kernel process.
type Connection struct {
	logger      *log.Logger
	wrapped     *bazilfuse.Conn
	opsInFlight sync.WaitGroup
}

// Responsibility for closing the wrapped connection is transferred to the
// result. You must call c.close() eventually.
func newConnection(
	logger *log.Logger,
	wrapped *bazilfuse.Conn) (c *Connection, err error) {
	c = &Connection{
		logger:  logger,
		wrapped: wrapped,
	}

	return
}

// Read the next op from the kernel process. Return io.EOF if the kernel has
// closed the connection.
//
// This function delivers ops in exactly the order they are received from
// /dev/fuse. Be wary of naively calling it concurrently: you probably want
// ordering guarantees between e.g. write ops and flush ops. For example,
// close(2) causes WriteFileOps to be issued before a FlushFileOp, but doesn't
// wait for their response before issuing the latter (cf.
// https://github.com/jacobsa/fuse/issues/3).
func (c *Connection) ReadOp() (op fuseops.Op, err error) {
	var bfReq bazilfuse.Request

	// Keep going until we find a request we know how to convert.
	for {
		// Read a bazilfuse request.
		bfReq, err = c.wrapped.ReadRequest()
		if err != nil {
			return
		}

		c.logger.Printf("Received: %v", bfReq)

		// Special case: responding to this is required to make mounting work on OS
		// X. We don't currently expose the capability for the file system to
		// intercept this.
		if statfsReq, ok := bfReq.(*bazilfuse.StatfsRequest); ok {
			c.logger.Println("Responding OK to Statfs.")
			statfsReq.Respond(&bazilfuse.StatfsResponse{})
			continue
		}

		// Convert it, if possible.
		if op = fuseops.Convert(bfReq, c.logger, &c.opsInFlight); op == nil {
			c.logger.Printf("Returning ENOSYS for unknown bazilfuse request: %v", bfReq)
			bfReq.RespondError(ENOSYS)
			continue
		}

		c.opsInFlight.Add(1)
		return
	}
}

func (c *Connection) waitForReady() (err error) {
	<-c.wrapped.Ready
	err = c.wrapped.MountError
	return
}

// Close the connection and wait for in-flight ops.
func (c *Connection) close() (err error) {
	err = c.wrapped.Close()
	c.opsInFlight.Wait()
	return
}
<commit_msg>Updated an out of date comment.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuse

import (
	"log"
	"sync"

	"github.com/jacobsa/bazilfuse"
	"github.com/jacobsa/fuse/fuseops"
)

// A connection to the fuse kernel process.
type Connection struct {
	logger      *log.Logger
	wrapped     *bazilfuse.Conn
	opsInFlight sync.WaitGroup
}

// Responsibility for closing the wrapped connection is transferred to the
// result. You must call c.close() eventually.
func newConnection(
	logger *log.Logger,
	wrapped *bazilfuse.Conn) (c *Connection, err error) {
	c = &Connection{
		logger:  logger,
		wrapped: wrapped,
	}

	return
}

// Read the next op from the kernel process. Return io.EOF if the kernel has
// closed the connection.
//
// This function delivers ops in exactly the order they are received from
// /dev/fuse. It must not be called multiple times concurrently.
func (c *Connection) ReadOp() (op fuseops.Op, err error) {
	var bfReq bazilfuse.Request

	// Keep going until we find a request we know how to convert.
	for {
		// Read a bazilfuse request.
		bfReq, err = c.wrapped.ReadRequest()
		if err != nil {
			return
		}

		c.logger.Printf("Received: %v", bfReq)

		// Special case: responding to this is required to make mounting work on OS
		// X. We don't currently expose the capability for the file system to
		// intercept this.
		if statfsReq, ok := bfReq.(*bazilfuse.StatfsRequest); ok {
			c.logger.Println("Responding OK to Statfs.")
			statfsReq.Respond(&bazilfuse.StatfsResponse{})
			continue
		}

		// Convert it, if possible.
		if op = fuseops.Convert(bfReq, c.logger, &c.opsInFlight); op == nil {
			c.logger.Printf("Returning ENOSYS for unknown bazilfuse request: %v", bfReq)
			bfReq.RespondError(ENOSYS)
			continue
		}

		c.opsInFlight.Add(1)
		return
	}
}

func (c *Connection) waitForReady() (err error) {
	<-c.wrapped.Ready
	err = c.wrapped.MountError
	return
}

// Close the connection and wait for in-flight ops.
func (c *Connection) close() (err error) {
	err = c.wrapped.Close()
	c.opsInFlight.Wait()
	return
}
<|endoftext|>


<commit_before>package kafka

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"net"
	"sync"
	"time"

	"github.com/optiopay/kafka/proto"
)

// ErrClosed is returned as result of any request made using closed connection.
var ErrClosed = errors.New("closed")

// Low level abstraction over connection to Kafka.
type connection struct {
	rw     io.ReadWriteCloser
	stop   chan struct{}
	nextID chan int32
	logger Logger

	mu      sync.Mutex
	respc   map[int32]chan []byte
	stopErr error
}

// newConnection returns new, initialized connection or error
func newTCPConnection(address string, timeout time.Duration) (*connection, error) {
	conn, err := net.DialTimeout("tcp", address, timeout)
	if err != nil {
		return nil, err
	}
	c := &connection{
		stop:   make(chan struct{}),
		nextID: make(chan int32),
		rw:     conn,
		respc:  make(map[int32]chan []byte),
		logger: &nullLogger{},
	}
	go c.nextIDLoop()
	go c.readRespLoop()
	return c, nil
}

// nextIDLoop generates correlation IDs, making sure they are always in order
// and within the scope of request-response mapping array.
func (c *connection) nextIDLoop() {
	var id int32 = 1
	for {
		select {
		case <-c.stop:
			close(c.nextID)
			return
		case c.nextID <- id:
			id++
			if id == math.MaxInt32 {
				id = 1
			}
		}
	}
}

// readRespLoop constantly reading response messages from the socket and after
// partial parsing, sends byte representation of the whole message to request
// sending process.
func (c *connection) readRespLoop() {
	defer func() {
		c.mu.Lock()
		for _, cc := range c.respc {
			close(cc)
		}
		c.respc = make(map[int32]chan []byte)
		c.mu.Unlock()
	}()

	rd := bufio.NewReader(c.rw)
	for {
		correlationID, b, err := proto.ReadResp(rd)
		if err != nil {
			c.mu.Lock()
			if c.stopErr == nil {
				c.stopErr = err
				close(c.stop)
			}
			c.mu.Unlock()
			return
		}

		c.mu.Lock()
		rc, ok := c.respc[correlationID]
		delete(c.respc, correlationID)
		c.mu.Unlock()
		if !ok {
			c.logger.Warn(
				"msg", "response to unknown request",
				"correlationID", correlationID)
			continue
		}

		select {
		case <-c.stop:
			c.mu.Lock()
			if c.stopErr == nil {
				c.stopErr = ErrClosed
			}
			c.mu.Unlock()
		case rc <- b:
		}
		close(rc)
	}
}

// respWaiter register listener to response message with given correlationID
// and return channel that single response message will be pushed to once it
// will arrive.
// After pushing response message, channel is closed.
//
// Upon connection close, all unconsumed channels are closed.
func (c *connection) respWaiter(correlationID int32) (respc chan []byte, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.stopErr != nil {
		return nil, c.stopErr
	}
	if _, ok := c.respc[correlationID]; ok {
		c.logger.Error("msg", "correlation conflict", "correlationID", correlationID)
		return nil, fmt.Errorf("correlation conflict: %d", correlationID)
	}
	respc = make(chan []byte)
	c.respc[correlationID] = respc
	return respc, nil
}

// releaseWaiter removes response channel from waiters pool and close it.
// Calling this method for unknown correlationID has no effect.
func (c *connection) releaseWaiter(correlationID int32) {
	c.mu.Lock()
	rc, ok := c.respc[correlationID]
	if ok {
		delete(c.respc, correlationID)
		close(rc)
	}
	c.mu.Unlock()
}

// Close close underlying transport connection and cancel all pending response
// waiters.
func (c *connection) Close() error {
	c.mu.Lock()
	if c.stopErr == nil {
		c.stopErr = ErrClosed
		close(c.stop)
	}
	c.mu.Unlock()
	return c.rw.Close()
}

// Metadata sends given metadata request to kafka node and returns related
// metadata response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Metadata(req *proto.MetadataReq) (*proto.MetadataResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadMetadataResp(bytes.NewReader(b))
}

// Produce sends given produce request to kafka node and returns related
// response. Sending request with no ACKs flag will result with returning nil
// right after sending request, without waiting for response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Produce(req *proto.ProduceReq) (*proto.ProduceResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	if req.RequiredAcks == proto.RequiredAcksNone {
		_, err := req.WriteTo(c.rw)
		return nil, err
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadProduceResp(bytes.NewReader(b))
}

// Fetch sends given fetch request to kafka node and returns related response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Fetch(req *proto.FetchReq) (*proto.FetchResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	resp, err := proto.ReadFetchResp(bytes.NewReader(b))
	if err != nil {
		return nil, err
	}

	// Compressed messages are returned in full batches for efficiency
	// (the broker doesn't need to decompress).
	// This means that it's possible to get some leading messages
	// with a smaller offset than requested. Trim those.
	for ti := range resp.Topics {
		topic := &resp.Topics[ti]
		reqTopic := &req.Topics[ti]
		for pi := range topic.Partitions {
			partition := &topic.Partitions[pi]
			reqPartition := &reqTopic.Partitions[pi]
			i := 0
			for _, msg := range partition.Messages {
				if msg.Offset >= reqPartition.FetchOffset {
					break
				}
				i++
			}
			partition.Messages = partition.Messages[i:]
		}
	}
	return resp, nil
}

// Offset sends given offset request to kafka node and returns related response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Offset(req *proto.OffsetReq) (*proto.OffsetResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	// TODO(husio) documentation is not mentioning this directly, but I assume
	// -1 is for non node clients
	req.ReplicaID = -1
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadOffsetResp(bytes.NewReader(b))
}

func (c *connection) ConsumerMetadata(req *proto.ConsumerMetadataReq) (*proto.ConsumerMetadataResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}
	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadConsumerMetadataResp(bytes.NewReader(b))
}

func (c *connection) OffsetCommit(req *proto.OffsetCommitReq) (*proto.OffsetCommitResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}
	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadOffsetCommitResp(bytes.NewReader(b))
}

func (c *connection) OffsetFetch(req *proto.OffsetFetchReq) (*proto.OffsetFetchResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}
	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadOffsetFetchResp(bytes.NewReader(b))
}
<commit_msg>Use a 30s TCP keep-alive on all connections<commit_after>package kafka

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"math"
	"net"
	"sync"
	"time"

	"github.com/optiopay/kafka/proto"
)

// ErrClosed is returned as result of any request made using closed connection.
var ErrClosed = errors.New("closed")

// Low level abstraction over connection to Kafka.
type connection struct {
	rw     io.ReadWriteCloser
	stop   chan struct{}
	nextID chan int32
	logger Logger

	mu      sync.Mutex
	respc   map[int32]chan []byte
	stopErr error
}

// newConnection returns new, initialized connection or error
func newTCPConnection(address string, timeout time.Duration) (*connection, error) {
	dialer := net.Dialer{
		Timeout:   timeout,
		KeepAlive: 30 * time.Second,
	}
	conn, err := dialer.Dial("tcp", address)
	if err != nil {
		return nil, err
	}
	c := &connection{
		stop:   make(chan struct{}),
		nextID: make(chan int32),
		rw:     conn,
		respc:  make(map[int32]chan []byte),
		logger: &nullLogger{},
	}
	go c.nextIDLoop()
	go c.readRespLoop()
	return c, nil
}

// nextIDLoop generates correlation IDs, making sure they are always in order
// and within the scope of request-response mapping array.
func (c *connection) nextIDLoop() {
	var id int32 = 1
	for {
		select {
		case <-c.stop:
			close(c.nextID)
			return
		case c.nextID <- id:
			id++
			if id == math.MaxInt32 {
				id = 1
			}
		}
	}
}

// readRespLoop constantly reading response messages from the socket and after
// partial parsing, sends byte representation of the whole message to request
// sending process.
func (c *connection) readRespLoop() {
	defer func() {
		c.mu.Lock()
		for _, cc := range c.respc {
			close(cc)
		}
		c.respc = make(map[int32]chan []byte)
		c.mu.Unlock()
	}()

	rd := bufio.NewReader(c.rw)
	for {
		correlationID, b, err := proto.ReadResp(rd)
		if err != nil {
			c.mu.Lock()
			if c.stopErr == nil {
				c.stopErr = err
				close(c.stop)
			}
			c.mu.Unlock()
			return
		}

		c.mu.Lock()
		rc, ok := c.respc[correlationID]
		delete(c.respc, correlationID)
		c.mu.Unlock()
		if !ok {
			c.logger.Warn(
				"msg", "response to unknown request",
				"correlationID", correlationID)
			continue
		}

		select {
		case <-c.stop:
			c.mu.Lock()
			if c.stopErr == nil {
				c.stopErr = ErrClosed
			}
			c.mu.Unlock()
		case rc <- b:
		}
		close(rc)
	}
}

// respWaiter register listener to response message with given correlationID
// and return channel that single response message will be pushed to once it
// will arrive.
// After pushing response message, channel is closed.
//
// Upon connection close, all unconsumed channels are closed.
func (c *connection) respWaiter(correlationID int32) (respc chan []byte, err error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	if c.stopErr != nil {
		return nil, c.stopErr
	}
	if _, ok := c.respc[correlationID]; ok {
		c.logger.Error("msg", "correlation conflict", "correlationID", correlationID)
		return nil, fmt.Errorf("correlation conflict: %d", correlationID)
	}
	respc = make(chan []byte)
	c.respc[correlationID] = respc
	return respc, nil
}

// releaseWaiter removes response channel from waiters pool and close it.
// Calling this method for unknown correlationID has no effect.
func (c *connection) releaseWaiter(correlationID int32) {
	c.mu.Lock()
	rc, ok := c.respc[correlationID]
	if ok {
		delete(c.respc, correlationID)
		close(rc)
	}
	c.mu.Unlock()
}

// Close close underlying transport connection and cancel all pending response
// waiters.
func (c *connection) Close() error {
	c.mu.Lock()
	if c.stopErr == nil {
		c.stopErr = ErrClosed
		close(c.stop)
	}
	c.mu.Unlock()
	return c.rw.Close()
}

// Metadata sends given metadata request to kafka node and returns related
// metadata response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Metadata(req *proto.MetadataReq) (*proto.MetadataResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadMetadataResp(bytes.NewReader(b))
}

// Produce sends given produce request to kafka node and returns related
// response. Sending request with no ACKs flag will result with returning nil
// right after sending request, without waiting for response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Produce(req *proto.ProduceReq) (*proto.ProduceResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	if req.RequiredAcks == proto.RequiredAcksNone {
		_, err := req.WriteTo(c.rw)
		return nil, err
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadProduceResp(bytes.NewReader(b))
}

// Fetch sends given fetch request to kafka node and returns related response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Fetch(req *proto.FetchReq) (*proto.FetchResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	resp, err := proto.ReadFetchResp(bytes.NewReader(b))
	if err != nil {
		return nil, err
	}

	// Compressed messages are returned in full batches for efficiency
	// (the broker doesn't need to decompress).
	// This means that it's possible to get some leading messages
	// with a smaller offset than requested. Trim those.
	for ti := range resp.Topics {
		topic := &resp.Topics[ti]
		reqTopic := &req.Topics[ti]
		for pi := range topic.Partitions {
			partition := &topic.Partitions[pi]
			reqPartition := &reqTopic.Partitions[pi]
			i := 0
			for _, msg := range partition.Messages {
				if msg.Offset >= reqPartition.FetchOffset {
					break
				}
				i++
			}
			partition.Messages = partition.Messages[i:]
		}
	}
	return resp, nil
}

// Offset sends given offset request to kafka node and returns related response.
// Calling this method on closed connection will always return ErrClosed.
func (c *connection) Offset(req *proto.OffsetReq) (*proto.OffsetResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}

	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}

	// TODO(husio) documentation is not mentioning this directly, but I assume
	// -1 is for non node clients
	req.ReplicaID = -1
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadOffsetResp(bytes.NewReader(b))
}

func (c *connection) ConsumerMetadata(req *proto.ConsumerMetadataReq) (*proto.ConsumerMetadataResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}
	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadConsumerMetadataResp(bytes.NewReader(b))
}

func (c *connection) OffsetCommit(req *proto.OffsetCommitReq) (*proto.OffsetCommitResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}
	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadOffsetCommitResp(bytes.NewReader(b))
}

func (c *connection) OffsetFetch(req *proto.OffsetFetchReq) (*proto.OffsetFetchResp, error) {
	var ok bool
	if req.CorrelationID, ok = <-c.nextID; !ok {
		return nil, c.stopErr
	}
	respc, err := c.respWaiter(req.CorrelationID)
	if err != nil {
		c.logger.Error("msg", "failed waiting for response", "error", err)
		return nil, fmt.Errorf("wait for response: %s", err)
	}
	if _, err := req.WriteTo(c.rw); err != nil {
		c.logger.Error("msg", "cannot write", "error", err)
		c.releaseWaiter(req.CorrelationID)
		return nil, err
	}
	b, ok := <-respc
	if !ok {
		return nil, c.stopErr
	}
	return proto.ReadOffsetFetchResp(bytes.NewReader(b))
}
<|endoftext|>


<commit_before>// The meek-client-torbrowser program starts a copy of Tor Browser running
// meek-http-helper in a special profile, and then starts meek-client set up to
// use the browser helper.
//
// Arguments to this program are passed unmodified to meek-client, with the
// addition of a --helper option pointing to the browser helper.
package main

import (
	"bufio"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"regexp"
	"syscall"
)

var helperAddrPattern *regexp.Regexp

// Log a call to os.Process.Kill.
func logKill(p *os.Process) error {
	log.Printf("killing PID %d", p.Pid)
	err := p.Kill()
	if err != nil {
		log.Print(err)
	}
	return err
}

// Log a call to os.Process.Signal.
func logSignal(p *os.Process, sig os.Signal) error {
	log.Printf("sending signal %s to PID %d", sig, p.Pid)
	err := p.Signal(sig)
	if err != nil {
		log.Print(err)
	}
	return err
}

// Run firefox and return its exec.Cmd and stdout pipe.
func runFirefox() (cmd *exec.Cmd, stdout io.Reader, err error) {
	var profilePath string
	// Mac OS X needs an absolute profile path.
	profilePath, err = filepath.Abs(firefoxProfilePath)
	if err != nil {
		return
	}
	cmd = exec.Command(firefoxPath, "-no-remote", "-profile", profilePath)
	cmd.Stderr = os.Stderr
	stdout, err = cmd.StdoutPipe()
	if err != nil {
		return
	}
	log.Printf("running firefox command %q", cmd.Args)
	err = cmd.Start()
	if err != nil {
		return
	}
	log.Printf("firefox started with pid %d", cmd.Process.Pid)
	return cmd, stdout, nil
}

// Look for the magic meek-http-helper address string in the Reader, and return
// the address it contains. Start a goroutine to continue reading and discarding
// output of the Reader before returning.
func grepHelperAddr(r io.Reader) (string, error) {
	var helperAddr string
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if m := helperAddrPattern.FindStringSubmatch(line); m != nil {
			helperAddr = m[1]
			break
		}
	}
	err := scanner.Err()
	if err != nil {
		return "", err
	}
	// Ran out of input before finding the pattern.
	if helperAddr == "" {
		return "", io.EOF
	}
	// Keep reading from the browser to avoid its output buffer filling.
	go io.Copy(ioutil.Discard, r)
	return helperAddr, nil
}

// Run meek-client and return its exec.Cmd.
func runMeekClient(helperAddr string) (cmd *exec.Cmd, err error) {
	args := os.Args[1:]
	args = append(args, []string{"--helper", helperAddr}...)
	cmd = exec.Command(meekClientPath, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	log.Printf("running meek-client command %q", cmd.Args)
	err = cmd.Start()
	if err != nil {
		return
	}
	log.Printf("meek-client started with pid %d", cmd.Process.Pid)
	return cmd, nil
}

func main() {
	var err error

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// This magic string is emitted by meek-http-helper.
	helperAddrPattern, err = regexp.Compile(`^meek-http-helper: listen (127\.0\.0\.1:\d+)$`)
	if err != nil {
		log.Fatal(err)
	}

	// Start firefox.
	firefoxCmd, stdout, err := runFirefox()
	if err != nil {
		log.Print(err)
		return
	}
	defer logKill(firefoxCmd.Process)

	// Find out the helper's listening address.
	helperAddr, err := grepHelperAddr(stdout)
	if err != nil {
		log.Print(err)
		return
	}

	// Start meek-client with the helper address.
	meekClientCmd, err := runMeekClient(helperAddr)
	if err != nil {
		log.Print(err)
		return
	}
	defer logKill(meekClientCmd.Process)

	sig := <-sigChan
	log.Printf("sig %s", sig)
	err = logSignal(meekClientCmd.Process, sig)
	if err != nil {
		log.Print(err)
	}

	// If SIGINT, wait for a second SIGINT.
	if sig == syscall.SIGINT {
		sig := <-sigChan
		log.Printf("sig %s", sig)
		err = logSignal(meekClientCmd.Process, sig)
		if err != nil {
			log.Print(err)
		}
	}
}
<commit_msg>Synthesize a SIGTERM on stdin close.<commit_after>// The meek-client-torbrowser program starts a copy of Tor Browser running
// meek-http-helper in a special profile, and then starts meek-client set up to
// use the browser helper.
//
// Arguments to this program are passed unmodified to meek-client, with the
// addition of a --helper option pointing to the browser helper.
package main

import (
	"bufio"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"regexp"
	"syscall"
)

var helperAddrPattern *regexp.Regexp

// Log a call to os.Process.Kill.
func logKill(p *os.Process) error {
	log.Printf("killing PID %d", p.Pid)
	err := p.Kill()
	if err != nil {
		log.Print(err)
	}
	return err
}

// Log a call to os.Process.Signal.
func logSignal(p *os.Process, sig os.Signal) error {
	log.Printf("sending signal %s to PID %d", sig, p.Pid)
	err := p.Signal(sig)
	if err != nil {
		log.Print(err)
	}
	return err
}

// Run firefox and return its exec.Cmd and stdout pipe.
func runFirefox() (cmd *exec.Cmd, stdout io.Reader, err error) {
	var profilePath string
	// Mac OS X needs an absolute profile path.
	profilePath, err = filepath.Abs(firefoxProfilePath)
	if err != nil {
		return
	}
	cmd = exec.Command(firefoxPath, "-no-remote", "-profile", profilePath)
	cmd.Stderr = os.Stderr
	stdout, err = cmd.StdoutPipe()
	if err != nil {
		return
	}
	log.Printf("running firefox command %q", cmd.Args)
	err = cmd.Start()
	if err != nil {
		return
	}
	log.Printf("firefox started with pid %d", cmd.Process.Pid)
	return cmd, stdout, nil
}

// Look for the magic meek-http-helper address string in the Reader, and return
// the address it contains. Start a goroutine to continue reading and discarding
// output of the Reader before returning.
func grepHelperAddr(r io.Reader) (string, error) {
	var helperAddr string
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		line := scanner.Text()
		if m := helperAddrPattern.FindStringSubmatch(line); m != nil {
			helperAddr = m[1]
			break
		}
	}
	err := scanner.Err()
	if err != nil {
		return "", err
	}
	// Ran out of input before finding the pattern.
	if helperAddr == "" {
		return "", io.EOF
	}
	// Keep reading from the browser to avoid its output buffer filling.
	go io.Copy(ioutil.Discard, r)
	return helperAddr, nil
}

// Run meek-client and return its exec.Cmd.
func runMeekClient(helperAddr string) (cmd *exec.Cmd, err error) {
	args := os.Args[1:]
	args = append(args, []string{"--helper", helperAddr}...)
	cmd = exec.Command(meekClientPath, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	log.Printf("running meek-client command %q", cmd.Args)
	err = cmd.Start()
	if err != nil {
		return
	}
	log.Printf("meek-client started with pid %d", cmd.Process.Pid)
	return cmd, nil
}

func main() {
	var err error

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// This magic string is emitted by meek-http-helper.
	helperAddrPattern, err = regexp.Compile(`^meek-http-helper: listen (127\.0\.0\.1:\d+)$`)
	if err != nil {
		log.Fatal(err)
	}

	// Start firefox.
	firefoxCmd, stdout, err := runFirefox()
	if err != nil {
		log.Print(err)
		return
	}
	defer logKill(firefoxCmd.Process)

	// Find out the helper's listening address.
	helperAddr, err := grepHelperAddr(stdout)
	if err != nil {
		log.Print(err)
		return
	}

	// Start meek-client with the helper address.
	meekClientCmd, err := runMeekClient(helperAddr)
	if err != nil {
		log.Print(err)
		return
	}
	defer logKill(meekClientCmd.Process)

	// On Windows, we don't get a SIGINT or SIGTERM, rather we are killed
	// without a chance to clean up our subprocesses. When run inside
	// processterminate-buffer, it is instead processterminate-buffer that
	// is killed, and we can detect that event by that our stdout gets
	// closed.
	// https://trac.torproject.org/projects/tor/ticket/9330
	go func() {
		io.Copy(ioutil.Discard, os.Stdin)
		log.Printf("synthesizing SIGTERM because of stdin close")
		sigChan <-syscall.SIGTERM
	}()

	sig := <-sigChan
	log.Printf("sig %s", sig)
	err = logSignal(meekClientCmd.Process, sig)
	if err != nil {
		log.Print(err)
	}

	// If SIGINT, wait for a second SIGINT.
	if sig == syscall.SIGINT {
		sig := <-sigChan
		log.Printf("sig %s", sig)
		err = logSignal(meekClientCmd.Process, sig)
		if err != nil {
			log.Print(err)
		}
	}
}
<|endoftext|>