<commit_before>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3git

import (
	"encoding/hex"
	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"
	"io/ioutil"
	"os"
)

// Pull updates for the repository
func (repo Repository) Pull(progress func(maxTicks int64)) error {

	return pull(progress)
}

func pull(progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToFetch := []string{}

	// Iterate over prefixes and find out if they are already available locally
	for prefix, _ := range prefixesInBackend {
		key, _ := hex.DecodeString(prefix)
		_, _, err := kv.GetLevel1(key)
		if err != nil && !lmdb.IsNotFound(err) {
			return err
		}

		if lmdb.IsNotFound(err) {
			// Prefix is not available locally  --> add to get fetched
			prefixesToFetch = append(prefixesToFetch, prefix)
		}
	}

	if len(prefixesToFetch) == 0 {
		return nil
	}

	progress(int64(len(prefixesToFetch)))

	// TODO: [perf] Speed up by running parallel in multiple go routines
	for _, prefix := range prefixesToFetch {

		// Fetch Prefix object and all objects directly and indirectly referenced by it
		err = fetchPrefix(prefix, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToFetch)))
	}

	return nil
}

// Fetch Prefix object and all objects directly and indirectly referenced by it
func fetchPrefix(prefix string, client backend.Backend) error {

	// Make sure that we do the storage of the object in the cas in
	// reverse order, that is store prefix object in the cas as the
	// *last* action, just before that the commit object, etc.
	// This ensures that, if the pull action is prematurely terminated
	// (either by user/application, or some sort of error/crash) that
	// a subsequent pull will still miss the prefix object and pull it
	// again

	// First fetch blob for prefix object
	prefixName, prefixBytes, err := fetchBlobTempFileAndContents(prefix, client)
	if err != nil {
		return err
	}
	defer os.Remove(prefixName)

	po, err := core.GetPrefixObjectFromString(string(prefixBytes))
	if err != nil {
		return err
	}

	{
		// Now pull down commit object
		commitName, commitBytes, err := fetchBlobTempFileAndContents(po.S3gitFollowMe, client)
		if err != nil {
			return err
		}
		defer os.Remove(commitName)

		co, err := core.GetCommitObjectFromString(string(commitBytes))
		if err != nil {
			return err
		}

		{
			// Now pull down tree object
			treeName, treeBytes, err := fetchBlobTempFileAndContents(co.S3gitTree, client)
			if err != nil {
				return err
			}
			defer os.Remove(treeName)

			to, err := core.GetTreeObjectFromString(string(treeBytes))
			if err != nil {
				return err
			}

			// Cache root keys for all added blobs in this commit ...
			err = cacheKeysForBlobs(to.S3gitAdded)
			if err != nil {
				return err
			}

			// Add tree object to cas
			_, err = cas.StoreBlobInCache(treeName, kv.TREE)
			if err != nil {
				return err
			}

			// Delete the chunks for the tree object since we are unlikely the need it again
			err = cas.DeleteLeavesForBlob(co.S3gitTree)
			if err != nil {
				return err
			}
		}

		// Add commit object to cas
		_, err = cas.StoreBlobInCache(commitName, kv.COMMIT)
		if err != nil {
			return err
		}

		// Mark warm and cold parents as parents
		err = co.MarkWarmAndColdParents()
		if err != nil {
			return err
		}
	}

	// Add prefix object to cas
	_, err = cas.StoreBlobInCache(prefixName, kv.PREFIX)
	if err != nil {
		return err
	}

	return nil
}

// Fetch blob down to temp file along with contents
func fetchBlobTempFileAndContents(prefix string, client backend.Backend) (tempFile string, contents []byte, err error) {

	name, err := cas.FetchBlobToTempFile(prefix, client)
	if err != nil {
		return "", nil, err
	}

	contents, err = ioutil.ReadFile(name)
	if err != nil {
		return "", nil, err
	}

	return name, contents, nil
}

const treeBatchSize = 0x4000

// Just cache the key for BLOBs, content will be pulled down later when needed
// For performance do not write per key to the KV but write in larger batches
func cacheKeysForBlobs(added []string) error {

	// Create arrays to be able to batch the operation
	keys := make([][]byte, 0, treeBatchSize)
	values := make([][]byte, 0, treeBatchSize)

	count := 0
	for _, add := range added {
		key, _ := hex.DecodeString(add)

		keys = append(keys, key)
		values = append(values, nil)

		if len(keys) == cap(keys) {
			err := kv.AddMultiToLevel1(keys, values, kv.BLOB)
			if err != nil {
				return err
			}

			keys = keys[:0]
			values = values[:0]
		}
		count++
	}

	if len(keys) > 0 {
		err := kv.AddMultiToLevel1(keys, values, kv.BLOB)
		if err != nil {
			return err
		}
		count++
	}

	return nil
}
<commit_msg>Small typo<commit_after>/*
 * Copyright 2016 Frank Wessels <fwessels@xs4all.nl>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package s3git

import (
	"encoding/hex"
	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/s3git/s3git-go/internal/backend"
	"github.com/s3git/s3git-go/internal/cas"
	"github.com/s3git/s3git-go/internal/core"
	"github.com/s3git/s3git-go/internal/kv"
	"io/ioutil"
	"os"
)

// Pull updates for the repository
func (repo Repository) Pull(progress func(maxTicks int64)) error {

	return pull(progress)
}

func pull(progress func(maxTicks int64)) error {

	client, err := backend.GetDefaultClient()
	if err != nil {
		return err
	}

	// Get map of prefixes already in store
	prefixesInBackend, err := listPrefixes(client)
	if err != nil {
		return err
	}

	prefixesToFetch := []string{}

	// Iterate over prefixes and find out if they are already available locally
	for prefix, _ := range prefixesInBackend {
		key, _ := hex.DecodeString(prefix)
		_, _, err := kv.GetLevel1(key)
		if err != nil && !lmdb.IsNotFound(err) {
			return err
		}

		if lmdb.IsNotFound(err) {
			// Prefix is not available locally  --> add to get fetched
			prefixesToFetch = append(prefixesToFetch, prefix)
		}
	}

	if len(prefixesToFetch) == 0 {
		return nil
	}

	progress(int64(len(prefixesToFetch)))

	// TODO: [perf] Speed up by running parallel in multiple go routines
	for _, prefix := range prefixesToFetch {

		// Fetch Prefix object and all objects directly and indirectly referenced by it
		err = fetchPrefix(prefix, client)
		if err != nil {
			return err
		}

		progress(int64(len(prefixesToFetch)))
	}

	return nil
}

// Fetch Prefix object and all objects directly and indirectly referenced by it
func fetchPrefix(prefix string, client backend.Backend) error {

	// Make sure that we do the storage of the object in the cas in
	// reverse order, that is store prefix object in the cas as the
	// *last* action, just before that the commit object, etc.
	// This ensures that, if the pull action is prematurely terminated
	// (either by user/application, or some sort of error/crash) that
	// a subsequent pull will still miss the prefix object and pull it
	// again

	// First fetch blob for prefix object
	prefixName, prefixBytes, err := fetchBlobTempFileAndContents(prefix, client)
	if err != nil {
		return err
	}
	defer os.Remove(prefixName)

	po, err := core.GetPrefixObjectFromString(string(prefixBytes))
	if err != nil {
		return err
	}

	{
		// Now pull down commit object
		commitName, commitBytes, err := fetchBlobTempFileAndContents(po.S3gitFollowMe, client)
		if err != nil {
			return err
		}
		defer os.Remove(commitName)

		co, err := core.GetCommitObjectFromString(string(commitBytes))
		if err != nil {
			return err
		}

		{
			// Now pull down tree object
			treeName, treeBytes, err := fetchBlobTempFileAndContents(co.S3gitTree, client)
			if err != nil {
				return err
			}
			defer os.Remove(treeName)

			to, err := core.GetTreeObjectFromString(string(treeBytes))
			if err != nil {
				return err
			}

			// Cache root keys for all added blobs in this commit ...
			err = cacheKeysForBlobs(to.S3gitAdded)
			if err != nil {
				return err
			}

			// Add tree object to cas
			_, err = cas.StoreBlobInCache(treeName, kv.TREE)
			if err != nil {
				return err
			}

			// Delete the chunks for the tree object since we are unlikely to need it again
			err = cas.DeleteLeavesForBlob(co.S3gitTree)
			if err != nil {
				return err
			}
		}

		// Add commit object to cas
		_, err = cas.StoreBlobInCache(commitName, kv.COMMIT)
		if err != nil {
			return err
		}

		// Mark warm and cold parents as parents
		err = co.MarkWarmAndColdParents()
		if err != nil {
			return err
		}
	}

	// Add prefix object to cas
	_, err = cas.StoreBlobInCache(prefixName, kv.PREFIX)
	if err != nil {
		return err
	}

	return nil
}

// Fetch blob down to temp file along with contents
func fetchBlobTempFileAndContents(prefix string, client backend.Backend) (tempFile string, contents []byte, err error) {

	name, err := cas.FetchBlobToTempFile(prefix, client)
	if err != nil {
		return "", nil, err
	}

	contents, err = ioutil.ReadFile(name)
	if err != nil {
		return "", nil, err
	}

	return name, contents, nil
}

const treeBatchSize = 0x4000

// Just cache the key for BLOBs, content will be pulled down later when needed
// For performance do not write per key to the KV but write in larger batches
func cacheKeysForBlobs(added []string) error {

	// Create arrays to be able to batch the operation
	keys := make([][]byte, 0, treeBatchSize)
	values := make([][]byte, 0, treeBatchSize)

	count := 0
	for _, add := range added {
		key, _ := hex.DecodeString(add)

		keys = append(keys, key)
		values = append(values, nil)

		if len(keys) == cap(keys) {
			err := kv.AddMultiToLevel1(keys, values, kv.BLOB)
			if err != nil {
				return err
			}

			keys = keys[:0]
			values = values[:0]
		}
		count++
	}

	if len(keys) > 0 {
		err := kv.AddMultiToLevel1(keys, values, kv.BLOB)
		if err != nil {
			return err
		}
		count++
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"./mysql";
	"fmt";
)

func main() {
	dbh, err := mysql.Connect("127.0.0.1:3306", "test", "test", "test");
	if err != nil {
		fmt.Printf("%s\n", err);
	}
	fmt.Printf("Connected to %s\n", dbh.ServerVersion);
	err = dbh.Query("SHOW PROCESSLIST");
	if err != nil {
		fmt.Printf("%s\n", err);
	}
}
<commit_msg>Fix call<commit_after>package main

import (
	"./mysql";
	"fmt";
)

func main() {
	dbh, err := mysql.Connect("127.0.0.1:3306", "test", "test", "test");
	if err != nil {
		fmt.Printf("%s\n", err);
	}
	fmt.Printf("Connected to %s\n", dbh.ServerVersion);
	//var res * mysql.MySQLResponse;
	_, err = dbh.Query("SHOW PROCESSLIST");
	if err != nil {
		fmt.Printf("%s\n", err);
	}
}
<|endoftext|>


<commit_before>package warserver

import (
    "github.com/gorilla/websocket"
    "io"
    "net"
    "net/http"
    "warserver/logger"
)

const (
    RECV_BUF_LEN = 1024
    SERVER_IP = "localhost"
    SERVER_PORT = "5269"
)

type websocketHandler interface {
    handleWebsocket(message []byte, cconn *clientConnection)
}

type clientInfo struct {
    Id int
}

type clientConnection struct {
    ws *websocket.Conn
    currentHandler websocketHandler
    handlers chan websocketHandler
    toClient chan []byte
    info clientInfo
}

type pipe struct {
    wsRecv chan []byte
    sockRecv chan []byte
}

type proxy struct {
    proxyConns []*clientConnection
}

func (p *proxy) slotClientConnection(slot int, cconn *clientConnection) {
    p.proxyConns[slot] = cconn;
}

func (p *proxy) removeClientConnection(pos int) {
    j := pos + 1
    copy(p.proxyConns[pos:], p.proxyConns[j:])
    for k, n := len(p.proxyConns)- j + pos, len(p.proxyConns); k < n; k ++ {
        p.proxyConns[k] = nil // or the zero value of T
    } // for k
    p.proxyConns = p.proxyConns[:len(p.proxyConns) - j + pos]
}

func (p *proxy) handleWebsocket(message []byte, cconn *clientConnection) {
    logger.Infof("Proxying message: %s", message)
}

func (pc *clientConnection) wsReadPump() {
    defer func() {
        pc.ws.Close()
    }()
    pc.ws.SetReadLimit(RECV_BUF_LEN)
    for {
        _, msg, err := pc.ws.ReadMessage()
        if err != nil {
            if err == io.EOF {
                // the client ID here is redundant...
                killcconn := fmt.Sprintf("killClient:{\"Id\": %d}", pc.info.Id)
                pc.currentHandler.handleWebsocket([]byte(killcconn), pc)
            } else {
                logger.Errorf("Error while reading from websocket: %s", err)
            }
            break
        }
        logger.Debugf("Received %s from websocket", msg)
        select {
        case newHandler := <-pc.handlers:
            pc.currentHandler = newHandler
        default:
        }
        pc.currentHandler.handleWebsocket(msg, pc)
    }
}

func (pc *clientConnection) wsWritePump() {
    for msg := range pc.toClient {
        logger.Debugf("Writing %s to websocket", msg)
        err := pc.ws.WriteMessage(websocket.TextMessage, msg)
        if err != nil {
            logger.Errorf("Error while writing to websocket: %s", err)
            break
        }
    }
}

func connectToServer() (net.Conn, error) {
    return net.Dial("tcp", SERVER_IP + ":" + SERVER_PORT)
}

func serveWs(w http.ResponseWriter, r *http.Request) {
    if r.Method != "GET" {
        http.Error(w, "Method not allowed", 405)
        return
    }
    if r.Header.Get("Origin") != "http://"+r.Host {
        http.Error(w, "Origin not allowed", 403)
        return
    }
    ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
    if _, ok := err.(websocket.HandshakeError); ok {
        http.Error(w, "Not a websocket handshake", 400)
        return
    } else if err != nil {
        logger.Errorf("Websocket upgrade error: %s", err)
        return
    }
    gamehub.wsRegister<- ws
}
<commit_msg>Extremely anal whitespace<commit_after>package warserver

import (
    "github.com/gorilla/websocket"
    "io"
    "net"
    "net/http"
    "warserver/logger"
)

const (
    RECV_BUF_LEN = 1024
    SERVER_IP = "localhost"
    SERVER_PORT = "5269"
)

type websocketHandler interface {
    handleWebsocket(message []byte, cconn *clientConnection)
}

type clientInfo struct {
    Id int
}

type clientConnection struct {
    ws *websocket.Conn
    currentHandler websocketHandler
    handlers chan websocketHandler
    toClient chan []byte
    info clientInfo
}

type pipe struct {
    wsRecv chan []byte
    sockRecv chan []byte
}

type proxy struct {
    proxyConns []*clientConnection
}

func (p *proxy) slotClientConnection(slot int, cconn *clientConnection) {
    p.proxyConns[slot] = cconn;
}

func (p *proxy) removeClientConnection(pos int) {
    j := pos + 1
    copy(p.proxyConns[pos:], p.proxyConns[j:])
    for k, n := len(p.proxyConns) - j + pos, len(p.proxyConns); k < n; k ++ {
        p.proxyConns[k] = nil // or the zero value of T
    } // for k
    p.proxyConns = p.proxyConns[:len(p.proxyConns) - j + pos]
}

func (p *proxy) handleWebsocket(message []byte, cconn *clientConnection) {
    logger.Infof("Proxying message: %s", message)
}

func (pc *clientConnection) wsReadPump() {
    defer func() {
        pc.ws.Close()
    }()
    pc.ws.SetReadLimit(RECV_BUF_LEN)
    for {
        _, msg, err := pc.ws.ReadMessage()
        if err != nil {
            if err == io.EOF {
                // the client ID here is redundant...
                killcconn := fmt.Sprintf("killClient:{\"Id\": %d}", pc.info.Id)
                pc.currentHandler.handleWebsocket([]byte(killcconn), pc)
            } else {
                logger.Errorf("Error while reading from websocket: %s", err)
            }
            break
        }
        logger.Debugf("Received %s from websocket", msg)
        select {
        case newHandler := <-pc.handlers:
            pc.currentHandler = newHandler
        default:
        }
        pc.currentHandler.handleWebsocket(msg, pc)
    }
}

func (pc *clientConnection) wsWritePump() {
    for msg := range pc.toClient {
        logger.Debugf("Writing %s to websocket", msg)
        err := pc.ws.WriteMessage(websocket.TextMessage, msg)
        if err != nil {
            logger.Errorf("Error while writing to websocket: %s", err)
            break
        }
    }
}

func connectToServer() (net.Conn, error) {
    return net.Dial("tcp", SERVER_IP + ":" + SERVER_PORT)
}

func serveWs(w http.ResponseWriter, r *http.Request) {
    if r.Method != "GET" {
        http.Error(w, "Method not allowed", 405)
        return
    }
    if r.Header.Get("Origin") != "http://"+r.Host {
        http.Error(w, "Origin not allowed", 403)
        return
    }
    ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
    if _, ok := err.(websocket.HandshakeError); ok {
        http.Error(w, "Not a websocket handshake", 400)
        return
    } else if err != nil {
        logger.Errorf("Websocket upgrade error: %s", err)
        return
    }
    gamehub.wsRegister<- ws
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"strings"

	imap "github.com/emersion/imap/common"
	"github.com/emersion/imap/commands"
	"github.com/emersion/imap/responses"
)

// Requests a checkpoint of the currently selected mailbox. A checkpoint refers
// to any implementation-dependent housekeeping associated with the mailbox that
// is not normally executed as part of each command.
func (c *Client) Check() (err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	cmd := &commands.Check{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Permanently removes all messages that have the \Deleted flag set from the
// currently selected mailbox, and returns to the authenticated state from the
// selected state.
func (c *Client) Close() (err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	cmd := &commands.Close{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	if err != nil {
		return
	}

	c.State = imap.AuthenticatedState
	c.Mailbox = nil
	return
}

// Permanently removes all messages that have the \Deleted flag set from the
// currently selected mailbox.
// If ch is not nil, sends sequence IDs of each deleted message to this channel.
func (c *Client) Expunge(ch chan<- uint32) (err error) {
	defer close(ch)

	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	cmd := &commands.Expunge{}

	var res *responses.Expunge
	if ch != nil {
		res = &responses.Expunge{SeqIds: ch}
	}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

func (c *Client) search(uid bool, criteria []interface{}) (ids []uint32, err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	var cmd imap.Commander
	cmd = &commands.Search{
		Charset: "UTF-8",
		Criteria: criteria,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	res := &responses.Search{}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	ids = res.Ids
	return
}

// Searches the mailbox for messages that match the given searching criteria.
// Searching criteria consist of one or more search keys. The response contains
// a list of message sequence IDs corresponding to those messages that match the
// searching criteria.
// When multiple keys are specified, the result is the intersection (AND
// function) of all the messages that match those keys.
// Criteria must be UTF-8 encoded.
// See RFC 3501 section 6.4.4 for a list of searching criteria.
func (c *Client) Search(criteria []interface{}) (seqids []uint32, err error) {
	return c.search(false, criteria)
}

// Identical to Search, but unique identifiers are returned instead of message
// sequence numbers.
func (c *Client) UidSearch(criteria []interface{}) (uids []uint32, err error) {
	return c.search(true, criteria)
}

func (c *Client) fetch(uid bool, seqset *imap.SeqSet, items []string, ch chan<- *imap.Message) (err error) {
	defer close(ch)

	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	var cmd imap.Commander
	cmd = &commands.Fetch{
		SeqSet: seqset,
		Items: items,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	res := &responses.Fetch{Messages: ch}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Retrieves data associated with a message in the mailbox.
// See RFC 3501 section 6.4.5 for a list of items that can be requested.
func (c *Client) Fetch(seqset *imap.SeqSet, items []string, ch chan<- *imap.Message) (err error) {
	return c.fetch(false, seqset, items, ch)
}

// Identical to Fetch, but seqset is interpreted as containing unique
// identifiers instead of message sequence numbers.
func (c *Client) UidFetch(seqset *imap.SeqSet, items []string, ch chan<- *imap.Message) (err error) {
	return c.fetch(true, seqset, items, ch)
}

func (c *Client) store(uid bool, seqset *imap.SeqSet, item string, value interface{}, ch chan<- *imap.Message) (err error) {
	defer (func () {
		if ch != nil {
			close(ch)
		}
	})()

	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	// If ch is nil, the updated values are data which will be lost, so don't
	// retrieve it.
	if ch == nil && !strings.HasSuffix(item, ".SILENT") {
		item += ".SILENT"
	}

	var cmd imap.Commander
	cmd = &commands.Store{
		SeqSet: seqset,
		Item: item,
		Value: value,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	var res *responses.Fetch
	if ch != nil {
		res = &responses.Fetch{Messages: ch}
	}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Alters data associated with a message in the mailbox. If ch is not nil, the
// updated value of the data will be sent to this channel.
// See RFC 3501 section 6.4.6 for a list of items that can be updated.
func (c *Client) Store(seqset *imap.SeqSet, item string, value interface{}, ch chan<- *imap.Message) (err error) {
	return c.store(false, seqset, item, value, ch)
}

// Identical to Store, but seqset is interpreted as containing unique
// identifiers instead of message sequence numbers.
func (c *Client) UidStore(seqset *imap.SeqSet, item string, value interface{}, ch chan<- *imap.Message) (err error) {
	return c.store(true, seqset, item, value, ch)
}

func (c *Client) copy(uid bool, seqset *imap.SeqSet, dest string) (err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	var cmd imap.Commander
	cmd = &commands.Copy{
		SeqSet: seqset,
		Mailbox: dest,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Copies the specified message(s) to the end of the specified destination
// mailbox.
func (c *Client) Copy(seqset *imap.SeqSet, dest string) (err error) {
	return c.copy(false, seqset, dest)
}

// Identical to Copy, but seqset is interpreted as containing unique
// identifiers instead of message sequence numbers.
func (c *Client) UidCopy(seqset *imap.SeqSet, dest string) (err error) {
	return c.copy(true, seqset, dest)
}
<commit_msg>Fixes build error<commit_after>package client

import (
	"errors"
	"strings"

	imap "github.com/emersion/imap/common"
	"github.com/emersion/imap/commands"
	"github.com/emersion/imap/responses"
)

// Requests a checkpoint of the currently selected mailbox. A checkpoint refers
// to any implementation-dependent housekeeping associated with the mailbox that
// is not normally executed as part of each command.
func (c *Client) Check() (err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	cmd := &commands.Check{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Permanently removes all messages that have the \Deleted flag set from the
// currently selected mailbox, and returns to the authenticated state from the
// selected state.
func (c *Client) Close() (err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	cmd := &commands.Close{}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	if err != nil {
		return
	}

	c.State = imap.AuthenticatedState
	c.Mailbox = nil
	return
}

// Permanently removes all messages that have the \Deleted flag set from the
// currently selected mailbox.
// If ch is not nil, sends sequence IDs of each deleted message to this channel.
func (c *Client) Expunge(ch chan<- uint32) (err error) {
	defer close(ch)

	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	cmd := &commands.Expunge{}

	var res *responses.Expunge
	if ch != nil {
		res = &responses.Expunge{SeqIds: ch}
	}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

func (c *Client) search(uid bool, criteria []interface{}) (ids []uint32, err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	var cmd imap.Commander
	cmd = &commands.Search{
		Charset: "UTF-8",
		Criteria: criteria,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	res := &responses.Search{}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	ids = res.Ids
	return
}

// Searches the mailbox for messages that match the given searching criteria.
// Searching criteria consist of one or more search keys. The response contains
// a list of message sequence IDs corresponding to those messages that match the
// searching criteria.
// When multiple keys are specified, the result is the intersection (AND
// function) of all the messages that match those keys.
// Criteria must be UTF-8 encoded.
// See RFC 3501 section 6.4.4 for a list of searching criteria.
func (c *Client) Search(criteria []interface{}) (seqids []uint32, err error) {
	return c.search(false, criteria)
}

// Identical to Search, but unique identifiers are returned instead of message
// sequence numbers.
func (c *Client) UidSearch(criteria []interface{}) (uids []uint32, err error) {
	return c.search(true, criteria)
}

func (c *Client) fetch(uid bool, seqset *imap.SeqSet, items []string, ch chan *imap.Message) (err error) {
	defer close(ch)

	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	var cmd imap.Commander
	cmd = &commands.Fetch{
		SeqSet: seqset,
		Items: items,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	res := &responses.Fetch{Messages: ch}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Retrieves data associated with a message in the mailbox.
// See RFC 3501 section 6.4.5 for a list of items that can be requested.
func (c *Client) Fetch(seqset *imap.SeqSet, items []string, ch chan *imap.Message) (err error) {
	return c.fetch(false, seqset, items, ch)
}

// Identical to Fetch, but seqset is interpreted as containing unique
// identifiers instead of message sequence numbers.
func (c *Client) UidFetch(seqset *imap.SeqSet, items []string, ch chan *imap.Message) (err error) {
	return c.fetch(true, seqset, items, ch)
}

func (c *Client) store(uid bool, seqset *imap.SeqSet, item string, value interface{}, ch chan *imap.Message) (err error) {
	defer (func () {
		if ch != nil {
			close(ch)
		}
	})()

	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	// If ch is nil, the updated values are data which will be lost, so don't
	// retrieve it.
	if ch == nil && !strings.HasSuffix(item, ".SILENT") {
		item += ".SILENT"
	}

	var cmd imap.Commander
	cmd = &commands.Store{
		SeqSet: seqset,
		Item: item,
		Value: value,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	var res *responses.Fetch
	if ch != nil {
		res = &responses.Fetch{Messages: ch}
	}

	status, err := c.execute(cmd, res)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Alters data associated with a message in the mailbox. If ch is not nil, the
// updated value of the data will be sent to this channel.
// See RFC 3501 section 6.4.6 for a list of items that can be updated.
func (c *Client) Store(seqset *imap.SeqSet, item string, value interface{}, ch chan *imap.Message) (err error) {
	return c.store(false, seqset, item, value, ch)
}

// Identical to Store, but seqset is interpreted as containing unique
// identifiers instead of message sequence numbers.
func (c *Client) UidStore(seqset *imap.SeqSet, item string, value interface{}, ch chan *imap.Message) (err error) {
	return c.store(true, seqset, item, value, ch)
}

func (c *Client) copy(uid bool, seqset *imap.SeqSet, dest string) (err error) {
	if c.State != imap.SelectedState {
		err = errors.New("No mailbox selected")
		return
	}

	var cmd imap.Commander
	cmd = &commands.Copy{
		SeqSet: seqset,
		Mailbox: dest,
	}
	if uid {
		cmd = &commands.Uid{Cmd: cmd}
	}

	status, err := c.execute(cmd, nil)
	if err != nil {
		return
	}

	err = status.Err()
	return
}

// Copies the specified message(s) to the end of the specified destination
// mailbox.
func (c *Client) Copy(seqset *imap.SeqSet, dest string) (err error) {
	return c.copy(false, seqset, dest)
}

// Identical to Copy, but seqset is interpreted as containing unique
// identifiers instead of message sequence numbers.
func (c *Client) UidCopy(seqset *imap.SeqSet, dest string) (err error) {
	return c.copy(true, seqset, dest)
}
<|endoftext|>


<commit_before>/*
 *   Copyright 2015 Benoit LETONDOR
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

package main

import (
	"fmt"
	"math/rand"
	"strings"

	"github.com/ChimeraCoder/anaconda"
	"github.com/jsgoecke/go-wit"
)

const (
	INTENT_HI           = "hi"
	INTENT_NICE_ARTICLE = "nice_article"
	INTENT_THANK_FOLLOW = "thank_follow"
)

func buildReply(tweet anaconda.Tweet) (string, error) {
	message := cleanTweetMessage(tweet.Text)
	if message == "" {
		return "", nil
	}

	// Process a text message
	request := &wit.MessageRequest{}
	request.Query = message

	result, err := witclient.Message(request)
	if err != nil {
		return "", err
	}

	intent := result.Outcome.Intent
	if result.Outcome.Confidence < 0.5 {
		fmt.Println("Not enough confidence for intent : " + intent)
		return "", nil
	}

	if intent == INTENT_HI {
		return buildHiIntentResponse(tweet), nil
	} else if intent == INTENT_NICE_ARTICLE {
		return buildNiceArticleIntentResponse(tweet), nil
	} else if intent == INTENT_THANK_FOLLOW {
		return buildThanksFollowIntentResponse(tweet), nil
	}

	return "", nil
}

func buildHiIntentResponse(tweet anaconda.Tweet) string {
	greetings := []string{"hello!", "hey", "yo"}

	return buildMention(tweet.User, greetings[rand.Intn(len(greetings))])
}

func buildNiceArticleIntentResponse(tweet anaconda.Tweet) string {
	greetings := []string{"hello!", "hey", "hi", "well,", ""}
	thanks := []string{"thanks", "thank you", "many thanks", "thx"}
	messages := []string{"reading", "your tweet", "your message"}

	greet := greetings[rand.Intn(len(greetings))]
	thank := thanks[rand.Intn(len(thanks))]
	message := messages[rand.Intn(len(messages))]

	return buildMention(tweet.User, greet+" "+thank+" for "+message)
}

func buildThanksFollowIntentResponse(tweet anaconda.Tweet) string {
	greetings := []string{"hello!", "hey", "hi", "well,", ""}
	thanks := []string{"thanks", "thank you", "many thanks", "thx"}
	follows := []string{"following me", "the follow"}
	messages := []string{"your message", "your tweet", "your mention"}
	reciprocals := []string{"too", "as well", ""}

	following, err := isUserFollowing(tweet.User.ScreenName)
	if following && err == nil {
		greet := greetings[rand.Intn(len(greetings))]
		thank := thanks[rand.Intn(len(thanks))]
		follow := follows[rand.Intn(len(follows))]
		reciprocal := reciprocals[rand.Intn(len(reciprocals))]

		return buildMention(tweet.User, greet+" "+thank+" for "+follow+" "+reciprocal)
	} else {
		greet := greetings[rand.Intn(len(greetings))]
		thank := thanks[rand.Intn(len(thanks))]
		message := messages[rand.Intn(len(messages))]

		return buildMention(tweet.User, greet+" "+thank+" for "+message)
	}
}

func buildMention(user anaconda.User, text string) string {
	return "@" + user.ScreenName + " " + text
}

func cleanTweetMessage(message string) string {
	cleaned := ""

	words := strings.Split(message, " ")
	for _, word := range words {
		if strings.HasPrefix(word, "@") || strings.HasPrefix(word, "http") {
			continue
		} else if strings.HasPrefix(word, "#") {
			cleaned += strings.TrimPrefix(word, "#") + " "
		}

		cleaned += word + " "
	}

	return cleaned
}
<commit_msg>Fix ia.go<commit_after>/*
 *   Copyright 2015 Benoit LETONDOR
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */

package main

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"strings"

	"github.com/ChimeraCoder/anaconda"
	"github.com/jsgoecke/go-wit"
)

const (
	INTENT_HI           = "hi"
	INTENT_NICE_ARTICLE = "nice_article"
	INTENT_THANK_FOLLOW = "thank_follow"
)

func buildReply(tweet anaconda.Tweet) (string, error) {
	message := cleanTweetMessage(tweet.Text)
	if message == "" {
		return "", nil
	}

	// Process a text message
	request := &wit.MessageRequest{}
	request.Query = message

	result, err := witclient.Message(request)
	if err != nil {
		log.Println(err)
		os.Exit(-1)
	}

	intent := result.Outcomes[0].Intent
	if result.Outcomes[0].Confidence < 0.5 {
		fmt.Println("Not enough confidence for intent : " + intent)
		return "", nil
	}

	if intent == INTENT_HI {
		return buildHiIntentResponse(tweet), nil
	} else if intent == INTENT_NICE_ARTICLE {
		return buildNiceArticleIntentResponse(tweet), nil
	} else if intent == INTENT_THANK_FOLLOW {
		return buildThanksFollowIntentResponse(tweet), nil
	}

	return "", nil
}

func buildHiIntentResponse(tweet anaconda.Tweet) string {
	greetings := []string{"hello!", "hey", "yo"}

	return buildMention(tweet.User, greetings[rand.Intn(len(greetings))])
}

func buildNiceArticleIntentResponse(tweet anaconda.Tweet) string {
	greetings := []string{"hello!", "hey", "hi", "well,", ""}
	thanks := []string{"thanks", "thank you", "many thanks", "thx"}
	messages := []string{"reading", "your tweet", "your message"}

	greet := greetings[rand.Intn(len(greetings))]
	thank := thanks[rand.Intn(len(thanks))]
	message := messages[rand.Intn(len(messages))]

	return buildMention(tweet.User, greet+" "+thank+" for "+message)
}

func buildThanksFollowIntentResponse(tweet anaconda.Tweet) string {
	greetings := []string{"hello!", "hey", "hi", "well,", ""}
	thanks := []string{"thanks", "thank you", "many thanks", "thx"}
	follows := []string{"following me", "the follow"}
	messages := []string{"your message", "your tweet", "your mention"}
	reciprocals := []string{"too", "as well", ""}

	following, err := isUserFollowing(tweet.User.ScreenName)
	if following && err == nil {
		greet := greetings[rand.Intn(len(greetings))]
		thank := thanks[rand.Intn(len(thanks))]
		follow := follows[rand.Intn(len(follows))]
		reciprocal := reciprocals[rand.Intn(len(reciprocals))]

		return buildMention(tweet.User, greet+" "+thank+" for "+follow+" "+reciprocal)
	} else {
		greet := greetings[rand.Intn(len(greetings))]
		thank := thanks[rand.Intn(len(thanks))]
		message := messages[rand.Intn(len(messages))]

		return buildMention(tweet.User, greet+" "+thank+" for "+message)
	}
}

func buildMention(user anaconda.User, text string) string {
	return "@" + user.ScreenName + " " + text
}

func cleanTweetMessage(message string) string {
	cleaned := ""

	words := strings.Split(message, " ")
	for _, word := range words {
		if strings.HasPrefix(word, "@") || strings.HasPrefix(word, "http") {
			continue
		} else if strings.HasPrefix(word, "#") {
			cleaned += strings.TrimPrefix(word, "#") + " "
		}

		cleaned += word + " "
	}

	return cleaned
}
<|endoftext|>


<commit_before>// Package xid is a globally unique id generator suited for web scale
//
// Xid is using Mongo Object ID algorithm to generate globally unique ids:
// https://docs.mongodb.org/manual/reference/object-id/
//
//   - 4-byte value representing the seconds since the Unix epoch,
//   - 3-byte machine identifier,
//   - 2-byte process id, and
//   - 3-byte counter, starting with a random value.
//
// The binary representation of the id is compatible with Mongo 12 bytes Object IDs.
// The string representation is using base32 hex (w/o padding) for better space efficiency
// when stored in that form (20 bytes). The hex variant of base32 is used to retain the
// sortable property of the id.
//
// Xid doesn't use base64 because case sensitivity and the 2 non alphanum chars may be an
// issue when transported as a string between various systems. Base36 wasn't retained either
// because 1/ it's not standard 2/ the resulting size is not predictable (not bit aligned)
// and 3/ it would not remain sortable. To validate a base32 `xid`, expect a 20 chars long,
// all lowercase sequence of `a` to `v` letters and `0` to `9` numbers (`[0-9a-v]{20}`).
//
// UUID is 16 bytes (128 bits), snowflake is 8 bytes (64 bits), xid stands in between
// with 12 bytes with a more compact string representation ready for the web and no
// required configuration or central generation server.
//
// Features:
//
//   - Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake
//   - Base32 hex encoded by default (16 bytes storage when transported as printable string)
//   - Non configured, you don't need set a unique machine and/or data center id
//   - K-ordered
//   - Embedded time with 1 second precision
//   - Unicity guaranteed for 16,777,216 (24 bits) unique ids per second and per host/process
//
// Best used with xlog's RequestIDHandler (https://godoc.org/github.com/rs/xlog#RequestIDHandler).
//
// References:
//
//   - http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems
//   - https://en.wikipedia.org/wiki/Universally_unique_identifier
//   - https://blog.twitter.com/2010/announcing-snowflake
package xid

import (
	"crypto/md5"
	"crypto/rand"
	"database/sql/driver"
	"encoding/binary"
	"errors"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"os"
	"sync/atomic"
	"time"
)

// Code inspired from mgo/bson ObjectId

// ID represents a unique request id
type ID [rawLen]byte

const (
	encodedLen = 20 // string encoded len
	rawLen     = 12 // binary raw len

	// encoding stores a custom version of the base32 encoding with lower case
	// letters.
	encoding = "0123456789abcdefghijklmnopqrstuv"
)

var (
	// ErrInvalidID is returned when trying to unmarshal an invalid ID
	ErrInvalidID = errors.New("xid: invalid ID")

	// objectIDCounter is atomically incremented when generating a new ObjectId
	// using NewObjectId() function. It's used as a counter part of an id.
	// This id is initialized with a random value.
	objectIDCounter = randInt()

	// machineId stores machine id generated once and used in subsequent calls
	// to NewObjectId function.
	machineID = readMachineID()

	// pid stores the current process id
	pid = os.Getpid()

	nilID ID

	// dec is the decoding map for base32 encoding
	dec [256]byte
)

func init() {
	for i := 0; i < len(dec); i++ {
		dec[i] = 0xFF
	}
	for i := 0; i < len(encoding); i++ {
		dec[encoding[i]] = byte(i)
	}

	// If PID is 1 and /proc/1/cpuset exists and is not /, we can assume that we
	// are in a form of container and use the content of /proc/1/cpuset instead
	// of the PID.
	if pid == 1 {
		b, err := ioutil.ReadFile("/proc/1/cpuset")
		if err == nil && len(b) > 1 {
			pid = int(crc32.ChecksumIEEE(b))
		}
	}
}

// readMachineId generates machine id and puts it into the machineId global
// variable. If this function fails to get the hostname, it will cause
// a runtime error.
func readMachineID() []byte {
	id := make([]byte, 3)
	if hostname, err := os.Hostname(); err == nil {
		hw := md5.New()
		if _, err = hw.Write([]byte(hostname)); err != nil {
			panic(fmt.Errorf("xid: cannot write hostname hash: %v;", err))
		}
		copy(id, hw.Sum(nil))
	} else {
		// Fallback to rand number if machine id can't be gathered
		if _, randErr := rand.Reader.Read(id); randErr != nil {
			panic(fmt.Errorf("xid: cannot get hostname nor generate a random number: %v; %v", err, randErr))
		}
	}
	return id
}

// randInt generates a random uint32
func randInt() uint32 {
	b := make([]byte, 3)
	if _, err := rand.Reader.Read(b); err != nil {
		panic(fmt.Errorf("xid: cannot generate random number: %v;", err))
	}
	return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])
}

// New generates a globally unique ID
func New() ID {
	var id ID
	// Timestamp, 4 bytes, big endian
	binary.BigEndian.PutUint32(id[:], uint32(time.Now().Unix()))
	// Machine, first 3 bytes of md5(hostname)
	id[4] = machineID[0]
	id[5] = machineID[1]
	id[6] = machineID[2]
	// Pid, 2 bytes, specs don't specify endianness, but we use big endian.
	id[7] = byte(pid >> 8)
	id[8] = byte(pid)
	// Increment, 3 bytes, big endian
	i := atomic.AddUint32(&objectIDCounter, 1)
	id[9] = byte(i >> 16)
	id[10] = byte(i >> 8)
	id[11] = byte(i)
	return id
}

// FromString reads an ID from its string representation
func FromString(id string) (ID, error) {
	i := &ID{}
	err := i.UnmarshalText([]byte(id))
	return *i, err
}

// String returns a base32 hex lowercased with no padding representation of the id (char set is 0-9, a-v).
func (id ID) String() string {
	text := make([]byte, encodedLen)
	encode(text, id[:])
	return string(text)
}

// MarshalText implements encoding/text TextMarshaler interface
func (id ID) MarshalText() ([]byte, error) {
	text := make([]byte, encodedLen)
	encode(text, id[:])
	return text, nil
}

// MarshalJSON implements encoding/json Marshaler interface
func (id ID) MarshalJSON() ([]byte, error) {
	if id.IsNil() {
		return []byte("null"), nil
	}
	text, err := id.MarshalText()
	return []byte(`"` + string(text) + `"`), err
}

// encode by unrolling the stdlib base32 algorithm + removing all safe checks
func encode(dst, id []byte) {
	dst[0] = encoding[id[0]>>3]
	dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F]
	dst[2] = encoding[(id[1]>>1)&0x1F]
	dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F]
	dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F]
	dst[5] = encoding[(id[3]>>2)&0x1F]
	dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F]
	dst[7] = encoding[id[4]&0x1F]
	dst[8] = encoding[id[5]>>3]
	dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F]
	dst[10] = encoding[(id[6]>>1)&0x1F]
	dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F]
	dst[12] = encoding[id[8]>>7|(id[7]<<1)&0x1F]
	dst[13] = encoding[(id[8]>>2)&0x1F]
	dst[14] = encoding[(id[9]>>5)|(id[8]<<3)&0x1F]
	dst[15] = encoding[id[9]&0x1F]
	dst[16] = encoding[id[10]>>3]
	dst[17] = encoding[(id[11]>>6)&0x1F|(id[10]<<2)&0x1F]
	dst[18] = encoding[(id[11]>>1)&0x1F]
	dst[19] = encoding[(id[11]<<4)&0x1F]
}

// UnmarshalText implements encoding/text TextUnmarshaler interface
func (id *ID) UnmarshalText(text []byte) error {
	if len(text) != encodedLen {
		return ErrInvalidID
	}
	for _, c := range text {
		if dec[c] == 0xFF {
			return ErrInvalidID
		}
	}
	decode(id, text)
	return nil
}

// UnmarshalJSON implements encoding/json Unmarshaler interface
func (id *ID) UnmarshalJSON(b []byte) error {
	s := string(b)
	if s == "null" {
		*id = nilID
		return nil
	}
	return id.UnmarshalText(b[1 : len(b)-1])
}

// decode by unrolling the stdlib base32 algorithm + removing all safe checks
func decode(id *ID, src []byte) {
	id[0] = dec[src[0]]<<3 | dec[src[1]]>>2
	id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4
	id[2] = dec[src[3]]<<4 | dec[src[4]]>>1
	id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3
	id[4] = dec[src[6]]<<5 | dec[src[7]]
	id[5] = dec[src[8]]<<3 | dec[src[9]]>>2
	id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4
	id[7] = dec[src[11]]<<4 | dec[src[12]]>>1
	id[8] = dec[src[12]]<<7 | dec[src[13]]<<2 | dec[src[14]]>>3
	id[9] = dec[src[14]]<<5 | dec[src[15]]
	id[10] = dec[src[16]]<<3 | dec[src[17]]>>2
	id[11] = dec[src[17]]<<6 | dec[src[18]]<<1 | dec[src[19]]>>4
}

// Time returns the timestamp part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Time() time.Time {
	// First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch.
	secs := int64(binary.BigEndian.Uint32(id[0:4]))
	return time.Unix(secs, 0)
}

// Machine returns the 3-byte machine id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Machine() []byte {
	return id[4:7]
}

// Pid returns the process id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Pid() uint16 {
	return binary.BigEndian.Uint16(id[7:9])
}

// Counter returns the incrementing value part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Counter() int32 {
	b := id[9:12]
	// Counter is stored as big-endian 3-byte value
	return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]))
}

// Value implements the driver.Valuer interface.
func (id ID) Value() (driver.Value, error) {
	if id.IsNil() {
		return nil, nil
	}
	b, err := id.MarshalText()
	return string(b), err
}

// Scan implements the sql.Scanner interface.
func (id *ID) Scan(value interface{}) (err error) {
	switch val := value.(type) {
	case string:
		return id.UnmarshalText([]byte(val))
	case []byte:
		return id.UnmarshalText(val)
	case nil:
		*id = nilID
		return nil
	default:
		return fmt.Errorf("xid: scanning unsupported type: %T", value)
	}
}

// IsNil Returns true if this is a "nil" ID
func (id ID) IsNil() bool {
	return id == nilID
}
<commit_msg>Handle containerized process with non 1 pid (fix #31)<commit_after>// Package xid is a globally unique id generator suited for web scale
//
// Xid is using Mongo Object ID algorithm to generate globally unique ids:
// https://docs.mongodb.org/manual/reference/object-id/
//
//   - 4-byte value representing the seconds since the Unix epoch,
//   - 3-byte machine identifier,
//   - 2-byte process id, and
//   - 3-byte counter, starting with a random value.
//
// The binary representation of the id is compatible with Mongo 12 bytes Object IDs.
// The string representation is using base32 hex (w/o padding) for better space efficiency
// when stored in that form (20 bytes). The hex variant of base32 is used to retain the
// sortable property of the id.
//
// Xid doesn't use base64 because case sensitivity and the 2 non alphanum chars may be an
// issue when transported as a string between various systems. Base36 wasn't retained either
// because 1/ it's not standard 2/ the resulting size is not predictable (not bit aligned)
// and 3/ it would not remain sortable. To validate a base32 `xid`, expect a 20 chars long,
// all lowercase sequence of `a` to `v` letters and `0` to `9` numbers (`[0-9a-v]{20}`).
//
// UUID is 16 bytes (128 bits), snowflake is 8 bytes (64 bits), xid stands in between
// with 12 bytes with a more compact string representation ready for the web and no
// required configuration or central generation server.
//
// Features:
//
//   - Size: 12 bytes (96 bits), smaller than UUID, larger than snowflake
//   - Base32 hex encoded by default (16 bytes storage when transported as printable string)
//   - Non configured, you don't need set a unique machine and/or data center id
//   - K-ordered
//   - Embedded time with 1 second precision
//   - Unicity guaranteed for 16,777,216 (24 bits) unique ids per second and per host/process
//
// Best used with xlog's RequestIDHandler (https://godoc.org/github.com/rs/xlog#RequestIDHandler).
//
// References:
//
//   - http://www.slideshare.net/davegardnerisme/unique-id-generation-in-distributed-systems
//   - https://en.wikipedia.org/wiki/Universally_unique_identifier
//   - https://blog.twitter.com/2010/announcing-snowflake
package xid

import (
	"crypto/md5"
	"crypto/rand"
	"database/sql/driver"
	"encoding/binary"
	"errors"
	"fmt"
	"hash/crc32"
	"io/ioutil"
	"os"
	"sync/atomic"
	"time"
)

// Code inspired from mgo/bson ObjectId

// ID represents a unique request id
type ID [rawLen]byte

const (
	encodedLen = 20 // string encoded len
	rawLen     = 12 // binary raw len

	// encoding stores a custom version of the base32 encoding with lower case
	// letters.
	encoding = "0123456789abcdefghijklmnopqrstuv"
)

var (
	// ErrInvalidID is returned when trying to unmarshal an invalid ID
	ErrInvalidID = errors.New("xid: invalid ID")

	// objectIDCounter is atomically incremented when generating a new ObjectId
	// using NewObjectId() function. It's used as a counter part of an id.
	// This id is initialized with a random value.
	objectIDCounter = randInt()

	// machineId stores machine id generated once and used in subsequent calls
	// to NewObjectId function.
	machineID = readMachineID()

	// pid stores the current process id
	pid = os.Getpid()

	nilID ID

	// dec is the decoding map for base32 encoding
	dec [256]byte
)

func init() {
	for i := 0; i < len(dec); i++ {
		dec[i] = 0xFF
	}
	for i := 0; i < len(encoding); i++ {
		dec[encoding[i]] = byte(i)
	}

	// If /proc/self/cpuset exists and is not /, we can assume that we are in a
	// form of container and use the content of cpuset xor-ed with the PID in
	// order get a reasonable machine global unique PID.
	b, err := ioutil.ReadFile("/proc/self/cpuset")
	if err == nil && len(b) > 1 {
		pid ^= int(crc32.ChecksumIEEE(b))
	}
}

// readMachineId generates machine id and puts it into the machineId global
// variable. If this function fails to get the hostname, it will cause
// a runtime error.
func readMachineID() []byte {
	id := make([]byte, 3)
	if hostname, err := os.Hostname(); err == nil {
		hw := md5.New()
		if _, err = hw.Write([]byte(hostname)); err != nil {
			panic(fmt.Errorf("xid: cannot write hostname hash: %v;", err))
		}
		copy(id, hw.Sum(nil))
	} else {
		// Fallback to rand number if machine id can't be gathered
		if _, randErr := rand.Reader.Read(id); randErr != nil {
			panic(fmt.Errorf("xid: cannot get hostname nor generate a random number: %v; %v", err, randErr))
		}
	}
	return id
}

// randInt generates a random uint32
func randInt() uint32 {
	b := make([]byte, 3)
	if _, err := rand.Reader.Read(b); err != nil {
		panic(fmt.Errorf("xid: cannot generate random number: %v;", err))
	}
	return uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])
}

// New generates a globally unique ID
func New() ID {
	var id ID
	// Timestamp, 4 bytes, big endian
	binary.BigEndian.PutUint32(id[:], uint32(time.Now().Unix()))
	// Machine, first 3 bytes of md5(hostname)
	id[4] = machineID[0]
	id[5] = machineID[1]
	id[6] = machineID[2]
	// Pid, 2 bytes, specs don't specify endianness, but we use big endian.
	id[7] = byte(pid >> 8)
	id[8] = byte(pid)
	// Increment, 3 bytes, big endian
	i := atomic.AddUint32(&objectIDCounter, 1)
	id[9] = byte(i >> 16)
	id[10] = byte(i >> 8)
	id[11] = byte(i)
	return id
}

// FromString reads an ID from its string representation
func FromString(id string) (ID, error) {
	i := &ID{}
	err := i.UnmarshalText([]byte(id))
	return *i, err
}

// String returns a base32 hex lowercased with no padding representation of the id (char set is 0-9, a-v).
func (id ID) String() string {
	text := make([]byte, encodedLen)
	encode(text, id[:])
	return string(text)
}

// MarshalText implements encoding/text TextMarshaler interface
func (id ID) MarshalText() ([]byte, error) {
	text := make([]byte, encodedLen)
	encode(text, id[:])
	return text, nil
}

// MarshalJSON implements encoding/json Marshaler interface
func (id ID) MarshalJSON() ([]byte, error) {
	if id.IsNil() {
		return []byte("null"), nil
	}
	text, err := id.MarshalText()
	return []byte(`"` + string(text) + `"`), err
}

// encode by unrolling the stdlib base32 algorithm + removing all safe checks
func encode(dst, id []byte) {
	dst[0] = encoding[id[0]>>3]
	dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F]
	dst[2] = encoding[(id[1]>>1)&0x1F]
	dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F]
	dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F]
	dst[5] = encoding[(id[3]>>2)&0x1F]
	dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F]
	dst[7] = encoding[id[4]&0x1F]
	dst[8] = encoding[id[5]>>3]
	dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F]
	dst[10] = encoding[(id[6]>>1)&0x1F]
	dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F]
	dst[12] = encoding[id[8]>>7|(id[7]<<1)&0x1F]
	dst[13] = encoding[(id[8]>>2)&0x1F]
	dst[14] = encoding[(id[9]>>5)|(id[8]<<3)&0x1F]
	dst[15] = encoding[id[9]&0x1F]
	dst[16] = encoding[id[10]>>3]
	dst[17] = encoding[(id[11]>>6)&0x1F|(id[10]<<2)&0x1F]
	dst[18] = encoding[(id[11]>>1)&0x1F]
	dst[19] = encoding[(id[11]<<4)&0x1F]
}

// UnmarshalText implements encoding/text TextUnmarshaler interface
func (id *ID) UnmarshalText(text []byte) error {
	if len(text) != encodedLen {
		return ErrInvalidID
	}
	for _, c := range text {
		if dec[c] == 0xFF {
			return ErrInvalidID
		}
	}
	decode(id, text)
	return nil
}

// UnmarshalJSON implements encoding/json Unmarshaler interface
func (id *ID) UnmarshalJSON(b []byte) error {
	s := string(b)
	if s == "null" {
		*id = nilID
		return nil
	}
	return id.UnmarshalText(b[1 : len(b)-1])
}

// decode by unrolling the stdlib base32 algorithm + removing all safe checks
func decode(id *ID, src []byte) {
	id[0] = dec[src[0]]<<3 | dec[src[1]]>>2
	id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4
	id[2] = dec[src[3]]<<4 | dec[src[4]]>>1
	id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3
	id[4] = dec[src[6]]<<5 | dec[src[7]]
	id[5] = dec[src[8]]<<3 | dec[src[9]]>>2
	id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4
	id[7] = dec[src[11]]<<4 | dec[src[12]]>>1
	id[8] = dec[src[12]]<<7 | dec[src[13]]<<2 | dec[src[14]]>>3
	id[9] = dec[src[14]]<<5 | dec[src[15]]
	id[10] = dec[src[16]]<<3 | dec[src[17]]>>2
	id[11] = dec[src[17]]<<6 | dec[src[18]]<<1 | dec[src[19]]>>4
}

// Time returns the timestamp part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Time() time.Time {
	// First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch.
	secs := int64(binary.BigEndian.Uint32(id[0:4]))
	return time.Unix(secs, 0)
}

// Machine returns the 3-byte machine id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Machine() []byte {
	return id[4:7]
}

// Pid returns the process id part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Pid() uint16 {
	return binary.BigEndian.Uint16(id[7:9])
}

// Counter returns the incrementing value part of the id.
// It's a runtime error to call this method with an invalid id.
func (id ID) Counter() int32 {
	b := id[9:12]
	// Counter is stored as big-endian 3-byte value
	return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2]))
}

// Value implements the driver.Valuer interface.
func (id ID) Value() (driver.Value, error) {
	if id.IsNil() {
		return nil, nil
	}
	b, err := id.MarshalText()
	return string(b), err
}

// Scan implements the sql.Scanner interface.
func (id *ID) Scan(value interface{}) (err error) {
	switch val := value.(type) {
	case string:
		return id.UnmarshalText([]byte(val))
	case []byte:
		return id.UnmarshalText(val)
	case nil:
		*id = nilID
		return nil
	default:
		return fmt.Errorf("xid: scanning unsupported type: %T", value)
	}
}

// IsNil Returns true if this is a "nil" ID
func (id ID) IsNil() bool {
	return id == nilID
}
<|endoftext|>


<commit_before>package p2p

import (
	"bytes"
	"fmt"
	"math/rand"
	"reflect"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
	wire "github.com/tendermint/go-wire"
	cmn "github.com/tendermint/tmlibs/common"

	"github.com/bytom/errors"
)

const (
	// PexChannel is a channel for PEX messages
	PexChannel = byte(0x00)

	// period to ensure peers connected
	defaultEnsurePeersPeriod = 120 * time.Second
	minNumOutboundPeers      = 10
	maxPexMessageSize        = 1048576 // 1MB

	// maximum messages one peer can send to us during `msgCountByPeerFlushInterval`
	defaultMaxMsgCountByPeer    = 1000
	msgCountByPeerFlushInterval = 1 * time.Hour
)

var ErrSendPexFail = errors.New("Send pex message fail")

// PEXReactor handles PEX (peer exchange) and ensures that an
// adequate number of peers are connected to the switch.
//
// It uses `AddrBook` (address book) to store `NetAddress`es of the peers.
//
// ## Preventing abuse
//
// For now, it just limits the number of messages from one peer to
// `defaultMaxMsgCountByPeer` messages per `msgCountByPeerFlushInterval` (1000
// msg/hour).
//
// NOTE [2017-01-17]:
//   Limiting is fine for now. Maybe down the road we want to keep track of the
//   quality of peer messages so if peerA keeps telling us about peers we can't
//   connect to then maybe we should care less about peerA. But I don't think
//   that kind of complexity is priority right now.
type PEXReactor struct {
	BaseReactor

	sw                *Switch
	book              *AddrBook
	ensurePeersPeriod time.Duration

	// tracks message count by peer, so we can prevent abuse
	msgCountByPeer    *cmn.CMap
	maxMsgCountByPeer uint16
}

// NewPEXReactor creates new PEX reactor.
func NewPEXReactor(b *AddrBook, sw *Switch) *PEXReactor {
	r := &PEXReactor{
		sw:                sw,
		book:              b,
		ensurePeersPeriod: defaultEnsurePeersPeriod,
		msgCountByPeer:    cmn.NewCMap(),
		maxMsgCountByPeer: defaultMaxMsgCountByPeer,
	}
	r.BaseReactor = *NewBaseReactor("PEXReactor", r)
	return r
}

// OnStart implements BaseService
func (r *PEXReactor) OnStart() error {
	r.BaseReactor.OnStart()
	r.book.Start()
	go r.ensurePeersRoutine()
	go r.flushMsgCountByPeer()
	return nil
}

// OnStop implements BaseService
func (r *PEXReactor) OnStop() {
	r.BaseReactor.OnStop()
	r.book.Stop()
}

// GetChannels implements Reactor
func (r *PEXReactor) GetChannels() []*ChannelDescriptor {
	return []*ChannelDescriptor{
		&ChannelDescriptor{
			ID:                PexChannel,
			Priority:          1,
			SendQueueCapacity: 10,
		},
	}
}

// AddPeer implements Reactor by adding peer to the address book (if inbound)
// or by requesting more addresses (if outbound).
func (r *PEXReactor) AddPeer(p *Peer) error {
	if p.IsOutbound() {
		// For outbound peers, the address is already in the books.
		// Either it was added in DialSeeds or when we
		// received the peer's address in r.Receive
		if r.book.NeedMoreAddrs() {
			if ok := r.RequestPEX(p); !ok {
				return ErrSendPexFail
			}
		}
	} else { // For inbound connections, the peer is its own source
		addr, err := NewNetAddressString(p.ListenAddr)
		if err != nil {
			// this should never happen
			log.WithFields(log.Fields{
				"addr":  p.ListenAddr,
				"error": err,
			}).Error("Error in AddPeer: Invalid peer address")
			return errors.New("Error in AddPeer: Invalid peer address")
		}
		r.book.AddAddress(addr, addr)
	}
	if r.sw.peers.Size() >= r.sw.config.MaxNumPeers {
		if ok := r.SendAddrs(p, r.book.GetSelection()); ok {
			r.sw.StopPeerGracefully(p)
		}
		return errors.New("Error in AddPeer: reach the max peer, exchange then close")
	}
	return nil
}

// RemovePeer implements Reactor.
func (r *PEXReactor) RemovePeer(p *Peer, reason interface{}) {
	// If we aren't keeping track of local temp data for each peer here, then we
	// don't have to do anything.
}

// Receive implements Reactor by handling incoming PEX messages.
func (r *PEXReactor) Receive(chID byte, src *Peer, msgBytes []byte) {
	srcAddr := src.Connection().RemoteAddress
	srcAddrStr := srcAddr.String()

	r.IncrementMsgCountForPeer(srcAddrStr)
	if r.ReachedMaxMsgCountForPeer(srcAddrStr) {
		log.WithField("peer", srcAddrStr).Error("Maximum number of messages reached for peer")
		// TODO remove src from peers?
		return
	}

	_, msg, err := DecodeMessage(msgBytes)
	if err != nil {
		log.WithField("error", err).Error("Error decoding message")
		return
	}
	log.WithField("msg", msg).Info("Reveived message")

	switch msg := msg.(type) {
	case *pexRequestMessage:
		// src requested some peers.
		if ok := r.SendAddrs(src, r.book.GetSelection()); !ok {
			log.Info("Send address message failed. Stop peer.")
		}
	case *pexAddrsMessage:
		// We received some peer addresses from src.
		// (We don't want to get spammed with bad peers)
		for _, addr := range msg.Addrs {
			if addr != nil {
				r.book.AddAddress(addr, srcAddr)
			}
		}
	default:
		log.WithField("type", reflect.TypeOf(msg)).Error("Unknown message type")
	}
}

// RequestPEX asks peer for more addresses.
func (r *PEXReactor) RequestPEX(p *Peer) bool {
	ok := p.TrySend(PexChannel, struct{ PexMessage }{&pexRequestMessage{}})
	if !ok {
		r.sw.StopPeerGracefully(p)
	}
	return ok
}

// SendAddrs sends addrs to the peer.
func (r *PEXReactor) SendAddrs(p *Peer, addrs []*NetAddress) bool {
	ok := p.TrySend(PexChannel, struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}})
	if !ok {
		r.sw.StopPeerGracefully(p)
	}
	return ok
}

// SetEnsurePeersPeriod sets period to ensure peers connected.
func (r *PEXReactor) SetEnsurePeersPeriod(d time.Duration) {
	r.ensurePeersPeriod = d
}

// SetMaxMsgCountByPeer sets maximum messages one peer can send to us during 'msgCountByPeerFlushInterval'.
func (r *PEXReactor) SetMaxMsgCountByPeer(v uint16) {
	r.maxMsgCountByPeer = v
}

// ReachedMaxMsgCountForPeer returns true if we received too many
// messages from peer with address `addr`.
// NOTE: assumes the value in the CMap is non-nil
func (r *PEXReactor) ReachedMaxMsgCountForPeer(addr string) bool {
	return r.msgCountByPeer.Get(addr).(uint16) >= r.maxMsgCountByPeer
}

// Increment or initialize the msg count for the peer in the CMap
func (r *PEXReactor) IncrementMsgCountForPeer(addr string) {
	var count uint16
	countI := r.msgCountByPeer.Get(addr)
	if countI != nil {
		count = countI.(uint16)
	}
	count++
	r.msgCountByPeer.Set(addr, count)
}

// Ensures that sufficient peers are connected. (continuous)
func (r *PEXReactor) ensurePeersRoutine() {
	// Randomize when routine starts
	ensurePeersPeriodMs := int64(10000)
	time.Sleep(time.Duration(rand.Int63n(ensurePeersPeriodMs)) * time.Millisecond)

	// fire once immediately.
	r.ensurePeers()

	// fire periodically
	ticker := time.NewTicker(r.ensurePeersPeriod)

	for {
		select {
		case <-ticker.C:
			r.ensurePeers()
		case <-r.Quit:
			ticker.Stop()
			return
		}
	}
}

// ensurePeers ensures that sufficient peers are connected. (once)
//
// Old bucket / New bucket are arbitrary categories to denote whether an
// address is vetted or not, and this needs to be determined over time via a
// heuristic that we haven't perfected yet, or, perhaps is manually edited by
// the node operator. It should not be used to compute what addresses are
// already connected or not.
//
// TODO Basically, we need to work harder on our good-peer/bad-peer marking.
// What we're currently doing in terms of marking good/bad peers is just a
// placeholder. It should not be the case that an address becomes old/vetted
// upon a single successful connection.
func (r *PEXReactor) ensurePeers() {
	numOutPeers, _, numDialing := r.Switch.NumPeers()
	numToDial := minNumOutboundPeers - (numOutPeers + numDialing)
	log.WithFields(log.Fields{
		"numOutPeers": numOutPeers,
		"numDialing":  numDialing,
		"numToDial":   numToDial,
	}).Info("Ensure peers")
	if numToDial <= 0 {
		return
	}

	newBias := cmn.MinInt(numOutPeers, 8)*10 + 10
	toDial := make(map[string]*NetAddress)

	// Try to pick numToDial addresses to dial.
	for i := 0; i < numToDial; i++ {
		// The purpose of newBias is to first prioritize old (more vetted) peers
		// when we have few connections, but to allow for new (less vetted) peers
		// if we already have many connections. This algorithm isn't perfect, but
		// it somewhat ensures that we prioritize connecting to more-vetted
		// peers.

		var picked *NetAddress
		// Try to fetch a new peer 3 times.
		// This caps the maximum number of tries to 3 * numToDial.
		for j := 0; j < 3; j++ {
			try := r.book.PickAddress(newBias)
			if try == nil {
				break
			}
			ka := r.book.addrLookup[try.String()]
			if ka != nil {
				if ka.isBad() {
					continue
				}
			}
			_, alreadySelected := toDial[try.IP.String()]
			alreadyDialing := r.Switch.IsDialing(try)
			var alreadyConnected bool

			for _, v := range r.Switch.Peers().list {
				if strings.Compare(v.mconn.RemoteAddress.IP.String(), try.IP.String()) == 0 {
					alreadyConnected = true
					break
				}
			}
			if alreadySelected || alreadyDialing || alreadyConnected {
				continue
			} else {
				log.WithField("addr", try).Info("Will dial address")
				picked = try
				break
			}
		}
		if picked == nil {
			continue
		}
		toDial[picked.IP.String()] = picked
	}

	// Dial picked addresses
	for _, item := range toDial {
		if _, err := r.Switch.DialPeerWithAddress(item, false); err != nil {
			r.book.MarkAttempt(item)
		} else {
			r.book.MarkGood(item)
		}
	}

	// If we need more addresses, pick a random peer and ask for more.
	if r.book.NeedMoreAddrs() {
		if peers := r.Switch.Peers().List(); len(peers) > 0 {
			i := rand.Int() % len(peers)
			peer := peers[i]
			log.WithField("peer", peer).Info("No addresses to dial. Sending pexRequest to random peer")
			if ok := r.RequestPEX(peer); !ok {
				log.Info("Send request address message failed. Stop peer.")
			}
		}
	}
}

func (r *PEXReactor) flushMsgCountByPeer() {
	ticker := time.NewTicker(msgCountByPeerFlushInterval)

	for {
		select {
		case <-ticker.C:
			r.msgCountByPeer.Clear()
		case <-r.Quit:
			ticker.Stop()
			return
		}
	}
}

//-----------------------------------------------------------------------------
// Messages

const (
	msgTypeRequest = byte(0x01)
	msgTypeAddrs   = byte(0x02)
)

// PexMessage is a primary type for PEX messages. Underneath, it could contain
// either pexRequestMessage, or pexAddrsMessage messages.
type PexMessage interface{}

var _ = wire.RegisterInterface(
	struct{ PexMessage }{},
	wire.ConcreteType{&pexRequestMessage{}, msgTypeRequest},
	wire.ConcreteType{&pexAddrsMessage{}, msgTypeAddrs},
)

// DecodeMessage implements interface registered above.
func DecodeMessage(bz []byte) (msgType byte, msg PexMessage, err error) {
	msgType = bz[0]
	n := new(int)
	r := bytes.NewReader(bz)
	msg = wire.ReadBinary(struct{ PexMessage }{}, r, maxPexMessageSize, n, &err).(struct{ PexMessage }).PexMessage
	return
}

/*
A pexRequestMessage requests additional peer addresses.
*/
type pexRequestMessage struct {
}

func (m *pexRequestMessage) String() string {
	return "[pexRequest]"
}

/*
A message with announced peer addresses.
*/
type pexAddrsMessage struct {
	Addrs []*NetAddress
}

func (m *pexAddrsMessage) String() string {
	return fmt.Sprintf("[pexAddrs %v]", m.Addrs)
}
<commit_msg>edit for code review<commit_after>package p2p

import (
	"bytes"
	"fmt"
	"math/rand"
	"reflect"
	"strings"
	"time"

	log "github.com/sirupsen/logrus"
	wire "github.com/tendermint/go-wire"
	cmn "github.com/tendermint/tmlibs/common"

	"github.com/bytom/errors"
)

const (
	// PexChannel is a channel for PEX messages
	PexChannel = byte(0x00)

	// period to ensure peers connected
	defaultEnsurePeersPeriod = 120 * time.Second
	minNumOutboundPeers      = 10
	maxPexMessageSize        = 1048576 // 1MB

	// maximum messages one peer can send to us during `msgCountByPeerFlushInterval`
	defaultMaxMsgCountByPeer    = 1000
	msgCountByPeerFlushInterval = 1 * time.Hour
)

var ErrSendPexFail = errors.New("Send pex message fail")

// PEXReactor handles PEX (peer exchange) and ensures that an
// adequate number of peers are connected to the switch.
//
// It uses `AddrBook` (address book) to store `NetAddress`es of the peers.
//
// ## Preventing abuse
//
// For now, it just limits the number of messages from one peer to
// `defaultMaxMsgCountByPeer` messages per `msgCountByPeerFlushInterval` (1000
// msg/hour).
//
// NOTE [2017-01-17]:
//   Limiting is fine for now. Maybe down the road we want to keep track of the
//   quality of peer messages so if peerA keeps telling us about peers we can't
//   connect to then maybe we should care less about peerA. But I don't think
//   that kind of complexity is priority right now.
type PEXReactor struct {
	BaseReactor

	sw                *Switch
	book              *AddrBook
	ensurePeersPeriod time.Duration

	// tracks message count by peer, so we can prevent abuse
	msgCountByPeer    *cmn.CMap
	maxMsgCountByPeer uint16
}

// NewPEXReactor creates new PEX reactor.
func NewPEXReactor(b *AddrBook, sw *Switch) *PEXReactor {
	r := &PEXReactor{
		sw:                sw,
		book:              b,
		ensurePeersPeriod: defaultEnsurePeersPeriod,
		msgCountByPeer:    cmn.NewCMap(),
		maxMsgCountByPeer: defaultMaxMsgCountByPeer,
	}
	r.BaseReactor = *NewBaseReactor("PEXReactor", r)
	return r
}

// OnStart implements BaseService
func (r *PEXReactor) OnStart() error {
	r.BaseReactor.OnStart()
	r.book.Start()
	go r.ensurePeersRoutine()
	go r.flushMsgCountByPeer()
	return nil
}

// OnStop implements BaseService
func (r *PEXReactor) OnStop() {
	r.BaseReactor.OnStop()
	r.book.Stop()
}

// GetChannels implements Reactor
func (r *PEXReactor) GetChannels() []*ChannelDescriptor {
	return []*ChannelDescriptor{
		&ChannelDescriptor{
			ID:                PexChannel,
			Priority:          1,
			SendQueueCapacity: 10,
		},
	}
}

// AddPeer implements Reactor by adding peer to the address book (if inbound)
// or by requesting more addresses (if outbound).
func (r *PEXReactor) AddPeer(p *Peer) error {
	if p.IsOutbound() {
		// For outbound peers, the address is already in the books.
		// Either it was added in DialSeeds or when we
		// received the peer's address in r.Receive
		if r.book.NeedMoreAddrs() {
			if ok := r.RequestPEX(p); !ok {
				return ErrSendPexFail
			}
		}
		return nil
	}

	// For inbound connections, the peer is its own source
	addr, err := NewNetAddressString(p.ListenAddr)
	if err != nil {
		// this should never happen
		log.WithFields(log.Fields{
			"addr":  p.ListenAddr,
			"error": err,
		}).Error("Error in AddPeer: Invalid peer address")
		return errors.New("Error in AddPeer: Invalid peer address")
	}
	r.book.AddAddress(addr, addr)

	// close the connect if connect is big than max limit
	if r.sw.peers.Size() >= r.sw.config.MaxNumPeers {
		if ok := r.SendAddrs(p, r.book.GetSelection()); ok {
			r.sw.StopPeerGracefully(p)
		}
		return errors.New("Error in AddPeer: reach the max peer, exchange then close")
	}
	return nil
}

// RemovePeer implements Reactor.
func (r *PEXReactor) RemovePeer(p *Peer, reason interface{}) {
	// If we aren't keeping track of local temp data for each peer here, then we
	// don't have to do anything.
}

// Receive implements Reactor by handling incoming PEX messages.
func (r *PEXReactor) Receive(chID byte, src *Peer, msgBytes []byte) {
	srcAddr := src.Connection().RemoteAddress
	srcAddrStr := srcAddr.String()

	r.IncrementMsgCountForPeer(srcAddrStr)
	if r.ReachedMaxMsgCountForPeer(srcAddrStr) {
		log.WithField("peer", srcAddrStr).Error("Maximum number of messages reached for peer")
		// TODO remove src from peers?
		return
	}

	_, msg, err := DecodeMessage(msgBytes)
	if err != nil {
		log.WithField("error", err).Error("Error decoding message")
		return
	}
	log.WithField("msg", msg).Info("Reveived message")

	switch msg := msg.(type) {
	case *pexRequestMessage:
		// src requested some peers.
		if ok := r.SendAddrs(src, r.book.GetSelection()); !ok {
			log.Info("Send address message failed. Stop peer.")
		}
	case *pexAddrsMessage:
		// We received some peer addresses from src.
		// (We don't want to get spammed with bad peers)
		for _, addr := range msg.Addrs {
			if addr != nil {
				r.book.AddAddress(addr, srcAddr)
			}
		}
	default:
		log.WithField("type", reflect.TypeOf(msg)).Error("Unknown message type")
	}
}

// RequestPEX asks peer for more addresses.
func (r *PEXReactor) RequestPEX(p *Peer) bool {
	ok := p.TrySend(PexChannel, struct{ PexMessage }{&pexRequestMessage{}})
	if !ok {
		r.sw.StopPeerGracefully(p)
	}
	return ok
}

// SendAddrs sends addrs to the peer.
func (r *PEXReactor) SendAddrs(p *Peer, addrs []*NetAddress) bool {
	ok := p.TrySend(PexChannel, struct{ PexMessage }{&pexAddrsMessage{Addrs: addrs}})
	if !ok {
		r.sw.StopPeerGracefully(p)
	}
	return ok
}

// SetEnsurePeersPeriod sets period to ensure peers connected.
func (r *PEXReactor) SetEnsurePeersPeriod(d time.Duration) {
	r.ensurePeersPeriod = d
}

// SetMaxMsgCountByPeer sets maximum messages one peer can send to us during 'msgCountByPeerFlushInterval'.
func (r *PEXReactor) SetMaxMsgCountByPeer(v uint16) {
	r.maxMsgCountByPeer = v
}

// ReachedMaxMsgCountForPeer returns true if we received too many
// messages from peer with address `addr`.
// NOTE: assumes the value in the CMap is non-nil
func (r *PEXReactor) ReachedMaxMsgCountForPeer(addr string) bool {
	return r.msgCountByPeer.Get(addr).(uint16) >= r.maxMsgCountByPeer
}

// Increment or initialize the msg count for the peer in the CMap
func (r *PEXReactor) IncrementMsgCountForPeer(addr string) {
	var count uint16
	countI := r.msgCountByPeer.Get(addr)
	if countI != nil {
		count = countI.(uint16)
	}
	count++
	r.msgCountByPeer.Set(addr, count)
}

// Ensures that sufficient peers are connected. (continuous)
func (r *PEXReactor) ensurePeersRoutine() {
	// Randomize when routine starts
	ensurePeersPeriodMs := int64(10000)
	time.Sleep(time.Duration(rand.Int63n(ensurePeersPeriodMs)) * time.Millisecond)

	// fire once immediately.
	r.ensurePeers()

	// fire periodically
	ticker := time.NewTicker(r.ensurePeersPeriod)

	for {
		select {
		case <-ticker.C:
			r.ensurePeers()
		case <-r.Quit:
			ticker.Stop()
			return
		}
	}
}

// ensurePeers ensures that sufficient peers are connected. (once)
//
// Old bucket / New bucket are arbitrary categories to denote whether an
// address is vetted or not, and this needs to be determined over time via a
// heuristic that we haven't perfected yet, or, perhaps is manually edited by
// the node operator. It should not be used to compute what addresses are
// already connected or not.
//
// TODO Basically, we need to work harder on our good-peer/bad-peer marking.
// What we're currently doing in terms of marking good/bad peers is just a
// placeholder. It should not be the case that an address becomes old/vetted
// upon a single successful connection.
func (r *PEXReactor) ensurePeers() {
	numOutPeers, _, numDialing := r.Switch.NumPeers()
	numToDial := minNumOutboundPeers - (numOutPeers + numDialing)
	log.WithFields(log.Fields{
		"numOutPeers": numOutPeers,
		"numDialing":  numDialing,
		"numToDial":   numToDial,
	}).Info("Ensure peers")
	if numToDial <= 0 {
		return
	}

	newBias := cmn.MinInt(numOutPeers, 8)*10 + 10
	toDial := make(map[string]*NetAddress)

	// Try to pick numToDial addresses to dial.
	for i := 0; i < numToDial; i++ {
		// The purpose of newBias is to first prioritize old (more vetted) peers
		// when we have few connections, but to allow for new (less vetted) peers
		// if we already have many connections. This algorithm isn't perfect, but
		// it somewhat ensures that we prioritize connecting to more-vetted
		// peers.

		var picked *NetAddress
		// Try to fetch a new peer 3 times.
		// This caps the maximum number of tries to 3 * numToDial.
		for j := 0; j < 3; j++ {
			try := r.book.PickAddress(newBias)
			if try == nil {
				break
			}
			ka := r.book.addrLookup[try.String()]
			if ka != nil {
				if ka.isBad() {
					continue
				}
			}
			_, alreadySelected := toDial[try.IP.String()]
			alreadyDialing := r.Switch.IsDialing(try)
			var alreadyConnected bool

			for _, v := range r.Switch.Peers().list {
				if strings.Compare(v.mconn.RemoteAddress.IP.String(), try.IP.String()) == 0 {
					alreadyConnected = true
					break
				}
			}
			if alreadySelected || alreadyDialing || alreadyConnected {
				continue
			} else {
				log.WithField("addr", try).Info("Will dial address")
				picked = try
				break
			}
		}
		if picked == nil {
			continue
		}
		toDial[picked.IP.String()] = picked
	}

	// Dial picked addresses
	for _, item := range toDial {
		if _, err := r.Switch.DialPeerWithAddress(item, false); err != nil {
			r.book.MarkAttempt(item)
		} else {
			r.book.MarkGood(item)
		}
	}

	// If we need more addresses, pick a random peer and ask for more.
	if r.book.NeedMoreAddrs() {
		if peers := r.Switch.Peers().List(); len(peers) > 0 {
			i := rand.Int() % len(peers)
			peer := peers[i]
			log.WithField("peer", peer).Info("No addresses to dial. Sending pexRequest to random peer")
			if ok := r.RequestPEX(peer); !ok {
				log.Info("Send request address message failed. Stop peer.")
			}
		}
	}
}

func (r *PEXReactor) flushMsgCountByPeer() {
	ticker := time.NewTicker(msgCountByPeerFlushInterval)

	for {
		select {
		case <-ticker.C:
			r.msgCountByPeer.Clear()
		case <-r.Quit:
			ticker.Stop()
			return
		}
	}
}

//-----------------------------------------------------------------------------
// Messages

const (
	msgTypeRequest = byte(0x01)
	msgTypeAddrs   = byte(0x02)
)

// PexMessage is a primary type for PEX messages. Underneath, it could contain
// either pexRequestMessage, or pexAddrsMessage messages.
type PexMessage interface{}

var _ = wire.RegisterInterface(
	struct{ PexMessage }{},
	wire.ConcreteType{&pexRequestMessage{}, msgTypeRequest},
	wire.ConcreteType{&pexAddrsMessage{}, msgTypeAddrs},
)

// DecodeMessage implements interface registered above.
func DecodeMessage(bz []byte) (msgType byte, msg PexMessage, err error) {
	msgType = bz[0]
	n := new(int)
	r := bytes.NewReader(bz)
	msg = wire.ReadBinary(struct{ PexMessage }{}, r, maxPexMessageSize, n, &err).(struct{ PexMessage }).PexMessage
	return
}

/*
A pexRequestMessage requests additional peer addresses.
*/
type pexRequestMessage struct {
}

func (m *pexRequestMessage) String() string {
	return "[pexRequest]"
}

/*
A message with announced peer addresses.
*/
type pexAddrsMessage struct {
	Addrs []*NetAddress
}

func (m *pexAddrsMessage) String() string {
	return fmt.Sprintf("[pexAddrs %v]", m.Addrs)
}
<|endoftext|>


<commit_before>package coreutils

import (
	"errors"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

// AbsPath get the absolute directory path, cleaning out any file names, home directory references, etc.
func AbsPath(path string) string {
	if !filepath.IsAbs(path) { // If the path provided isn't already absolute
		user, userGetErr := user.Current()

		if userGetErr == nil { // If we didn't fail getting the current user
			path = strings.Replace(path, "~", user.HomeDir+Separator, -1) // Replace any home directory reference
		}

		path, _ = filepath.Abs(path) // Get the absolute path of path

		var stripLastElement bool

		if file, openErr := os.Open(path); openErr == nil { // Attempt to open the path, to validate if it is a file or directory
			stat, statErr := file.Stat()
			stripLastElement = (statErr == nil) && !stat.IsDir() // Sets stripLastElement to true if stat.IsDir is not true
		} else { // If we failed to open the directory or file
			lastElement := filepath.Base(path)
			stripLastElement = filepath.Ext(lastElement) != "" // If lastElement is either a dotfile or has an extension, assume it is a file
		}

		if stripLastElement {
			path = filepath.Dir(path) + Separator // Strip out the last element and add the separator
		}
	}

	return path
}

// CopyDirectory will copy a directory, sub-directories, and files
func CopyDirectory(sourceDirectory, destinationDirectory string) error {
	var copyError error

	if IsDir(sourceDirectory) { // If sourceDirectory is a valid directory
		os.MkdirAll(destinationDirectory, NonGlobalFileMode)                     // Make all the needed directories to destinationDirectory
		sourceDirectoryFile, _ := os.Open(sourceDirectory)                       // Get the source directory "file" struct
		directoryContents, directoryReadError := sourceDirectoryFile.Readdir(-1) // Read the directory contents

		if directoryReadError == nil { /// If there was no read error on the directory
			if len(directoryContents) != 0 { // If there is content
				for _, contentItemFileInfo := range directoryContents { // For each FileInfo struct in directoryContents
					contentItemName := contentItemFileInfo.Name() // Get the name of the item
					sourceItemPath := sourceDirectory + "/" + contentItemName
					destinationItemPath := destinationDirectory + "/" + contentItemName

					if contentItemFileInfo.IsDir() { // If this is a directory
						copyError = CopyDirectory(sourceItemPath, destinationItemPath) // Copy this sub-directory and its contents
					} else { // If this is a file
						copyError = CopyFile(sourceItemPath, destinationItemPath) // Copy the directory
					}
				}
			}
		} else { // If there was a read error on the directory
			copyError = errors.New("Unable to read: " + sourceDirectory)
		}
	} else { // If sourceDirectory is not a valid directory
		copyError = errors.New(sourceDirectory + " is not a valid directory.")
	}

	return copyError
}

// CopyFile will copy a file and its relevant permissions
func CopyFile(sourceFile, destinationFile string) error {
	var copyError error

	sourceFileStruct, sourceFileError := os.Open(sourceFile) // Attempt to open the sourceFile

	if sourceFileError == nil { // If there was not an error opening the source file
		sourceFileStats, _ := sourceFileStruct.Stat() // Get the stats of the file

		if sourceFileStats.IsDir() { // If this is actually a directory
			copyError = errors.New(sourceFile + " is a directory. Please use CopyDirectory instead.")
		} else { // If it is indeed a file
			var fileContent []byte
			sourceFileMode := sourceFileStats.Mode() // Get the FileMode of this file
			sourceFileStruct.Close()                 // Close the file

			fileContent, copyError = ioutil.ReadFile(sourceFile) // Read the source file
			copyError = WriteOrUpdateFile(destinationFile, fileContent, sourceFileMode)
		}
	} else { // If the file does not exist
		copyError = errors.New(sourceFile + " does not exist.")
	}

	return copyError
}

// GetFiles will get all the files from a directory.
func GetFiles(path string, recursive bool) ([]string, error) {
	var files []string      // Define files as a []string
	var getFilesError error // Define getFilesError as an error

	if directory, openErr := os.Open(path); openErr == nil {
		directoryContents, directoryReadError := directory.Readdir(-1)

		if directoryReadError == nil { // If there was no issue reading the directory contents
			for _, fileInfoStruct := range directoryContents { // For each FileInfo struct in directoryContents
				name := fileInfoStruct.Name()

				if recursive && fileInfoStruct.IsDir() { // If the FileInfo indicates the object is a directory and we're doing recursive file fetching
					additionalFiles, _ := GetFiles(path + Separator + name, true)
					files = append(files, additionalFiles...)
				} else if !fileInfoStruct.IsDir() { // FileInfo is not a directory
					files = append(files, path+ Separator + name) // Add to files the file's name
				}
			}
		} else { // If there was ano issue reading the directory content
			getFilesError = errors.New("Cannot read the contents of " + path)
		}
	} else { // If path is not a directory
		getFilesError = errors.New(path + " is not a directory.")
	}

	return files, getFilesError
}

// GetFilesContains will return any files from a directory containing a particular string
func GetFilesContains(path, substring string) ([]string, error) {
	var files []string                // Define files as the parsed files
	var getFilesError error           // Define getFilesError as an error
	var allDirectoryContents []string // Define allDirectoryContents as the contents returned (if any) from GetFiles

	allDirectoryContents, getFilesError = GetFiles(path, false) // Get all the files from the path

	if getFilesError == nil { // If there was no issue getting the directory contents
		for _, fileName := range allDirectoryContents { // For each file name in directory contents
			if strings.Contains(filepath.Base(fileName), substring) { // If the file name contains our substring
				files = append(files, fileName) // Append to files
			}
		}
	}

	return files, getFilesError
}

// IsDir checks if the path provided is a directory or not
func IsDir(path string) bool {
	var isDir bool
	fileObject, fileOpenError := os.Open(path) // Open currentDirectory + path

	if fileOpenError == nil { // If there was no error opening the file object
		stat, filePathError := fileObject.Stat() // Get any stats

		if filePathError == nil { // If we got the statistics properly
			isDir = stat.IsDir() // Set isDir to result from stat
		}
	}

	return isDir
}

// WriteOrUpdateFile writes or updates the file contents of the passed file under the leading filepath with the specified sourceFileMode
func WriteOrUpdateFile(file string, fileContent []byte, sourceFileMode os.FileMode) error {
	var writeDirectory string // Directory to write file

	currentDirectory, _ := os.Getwd()            // Get the working directory
	currentDirectory = AbsPath(currentDirectory) // Get the absolute path of the current working directory
	fileName := filepath.Base(file)

	if file == fileName { // If we did not specify a directory to write to
		writeDirectory = currentDirectory // Set to the current directory
	} else {
		writeDirectory = AbsPath(file)
	}

	if currentDirectory != writeDirectory { // If the currentDirectory is not the same directory as the writeDirectory
		if createDirsErr := os.MkdirAll(writeDirectory, sourceFileMode); createDirsErr != nil { // If we failed to make all the directories needed
			return errors.New("Failed to create the path leading up to " + fileName + ": " + writeDirectory)
		}
	}

	writeErr := ioutil.WriteFile(writeDirectory+Separator+fileName, fileContent, sourceFileMode)

	if writeErr != nil {
		writeErr = errors.New("Failed to write " + fileName + " in directory " + writeDirectory)
	}

	return writeErr
}
<commit_msg>WriteOrUpdateFile: Ensure we not treating file as the full path. Instead, get the directory path of file for writeDirectory.<commit_after>package coreutils

import (
	"errors"
	"io/ioutil"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

// AbsPath get the absolute directory path, cleaning out any file names, home directory references, etc.
func AbsPath(path string) string {
	if !filepath.IsAbs(path) { // If the path provided isn't already absolute
		user, userGetErr := user.Current()

		if userGetErr == nil { // If we didn't fail getting the current user
			path = strings.Replace(path, "~", user.HomeDir+Separator, -1) // Replace any home directory reference
		}

		path, _ = filepath.Abs(path) // Get the absolute path of path

		var stripLastElement bool

		if file, openErr := os.Open(path); openErr == nil { // Attempt to open the path, to validate if it is a file or directory
			stat, statErr := file.Stat()
			stripLastElement = (statErr == nil) && !stat.IsDir() // Sets stripLastElement to true if stat.IsDir is not true
		} else { // If we failed to open the directory or file
			lastElement := filepath.Base(path)
			stripLastElement = filepath.Ext(lastElement) != "" // If lastElement is either a dotfile or has an extension, assume it is a file
		}

		if stripLastElement {
			path = filepath.Dir(path) + Separator // Strip out the last element and add the separator
		}
	}

	return path
}

// CopyDirectory will copy a directory, sub-directories, and files
func CopyDirectory(sourceDirectory, destinationDirectory string) error {
	var copyError error

	if IsDir(sourceDirectory) { // If sourceDirectory is a valid directory
		os.MkdirAll(destinationDirectory, NonGlobalFileMode)                     // Make all the needed directories to destinationDirectory
		sourceDirectoryFile, _ := os.Open(sourceDirectory)                       // Get the source directory "file" struct
		directoryContents, directoryReadError := sourceDirectoryFile.Readdir(-1) // Read the directory contents

		if directoryReadError == nil { /// If there was no read error on the directory
			if len(directoryContents) != 0 { // If there is content
				for _, contentItemFileInfo := range directoryContents { // For each FileInfo struct in directoryContents
					contentItemName := contentItemFileInfo.Name() // Get the name of the item
					sourceItemPath := sourceDirectory + "/" + contentItemName
					destinationItemPath := destinationDirectory + "/" + contentItemName

					if contentItemFileInfo.IsDir() { // If this is a directory
						copyError = CopyDirectory(sourceItemPath, destinationItemPath) // Copy this sub-directory and its contents
					} else { // If this is a file
						copyError = CopyFile(sourceItemPath, destinationItemPath) // Copy the directory
					}
				}
			}
		} else { // If there was a read error on the directory
			copyError = errors.New("Unable to read: " + sourceDirectory)
		}
	} else { // If sourceDirectory is not a valid directory
		copyError = errors.New(sourceDirectory + " is not a valid directory.")
	}

	return copyError
}

// CopyFile will copy a file and its relevant permissions
func CopyFile(sourceFile, destinationFile string) error {
	var copyError error

	sourceFileStruct, sourceFileError := os.Open(sourceFile) // Attempt to open the sourceFile

	if sourceFileError == nil { // If there was not an error opening the source file
		sourceFileStats, _ := sourceFileStruct.Stat() // Get the stats of the file

		if sourceFileStats.IsDir() { // If this is actually a directory
			copyError = errors.New(sourceFile + " is a directory. Please use CopyDirectory instead.")
		} else { // If it is indeed a file
			var fileContent []byte
			sourceFileMode := sourceFileStats.Mode() // Get the FileMode of this file
			sourceFileStruct.Close()                 // Close the file

			fileContent, copyError = ioutil.ReadFile(sourceFile) // Read the source file
			copyError = WriteOrUpdateFile(destinationFile, fileContent, sourceFileMode)
		}
	} else { // If the file does not exist
		copyError = errors.New(sourceFile + " does not exist.")
	}

	return copyError
}

// GetFiles will get all the files from a directory.
func GetFiles(path string, recursive bool) ([]string, error) {
	var files []string      // Define files as a []string
	var getFilesError error // Define getFilesError as an error

	if directory, openErr := os.Open(path); openErr == nil {
		directoryContents, directoryReadError := directory.Readdir(-1)

		if directoryReadError == nil { // If there was no issue reading the directory contents
			for _, fileInfoStruct := range directoryContents { // For each FileInfo struct in directoryContents
				name := fileInfoStruct.Name()

				if recursive && fileInfoStruct.IsDir() { // If the FileInfo indicates the object is a directory and we're doing recursive file fetching
					additionalFiles, _ := GetFiles(path + Separator + name, true)
					files = append(files, additionalFiles...)
				} else if !fileInfoStruct.IsDir() { // FileInfo is not a directory
					files = append(files, path+ Separator + name) // Add to files the file's name
				}
			}
		} else { // If there was ano issue reading the directory content
			getFilesError = errors.New("Cannot read the contents of " + path)
		}
	} else { // If path is not a directory
		getFilesError = errors.New(path + " is not a directory.")
	}

	return files, getFilesError
}

// GetFilesContains will return any files from a directory containing a particular string
func GetFilesContains(path, substring string) ([]string, error) {
	var files []string                // Define files as the parsed files
	var getFilesError error           // Define getFilesError as an error
	var allDirectoryContents []string // Define allDirectoryContents as the contents returned (if any) from GetFiles

	allDirectoryContents, getFilesError = GetFiles(path, false) // Get all the files from the path

	if getFilesError == nil { // If there was no issue getting the directory contents
		for _, fileName := range allDirectoryContents { // For each file name in directory contents
			if strings.Contains(filepath.Base(fileName), substring) { // If the file name contains our substring
				files = append(files, fileName) // Append to files
			}
		}
	}

	return files, getFilesError
}

// IsDir checks if the path provided is a directory or not
func IsDir(path string) bool {
	var isDir bool
	fileObject, fileOpenError := os.Open(path) // Open currentDirectory + path

	if fileOpenError == nil { // If there was no error opening the file object
		stat, filePathError := fileObject.Stat() // Get any stats

		if filePathError == nil { // If we got the statistics properly
			isDir = stat.IsDir() // Set isDir to result from stat
		}
	}

	return isDir
}

// WriteOrUpdateFile writes or updates the file contents of the passed file under the leading filepath with the specified sourceFileMode
func WriteOrUpdateFile(file string, fileContent []byte, sourceFileMode os.FileMode) error {
	var writeDirectory string // Directory to write file

	currentDirectory, _ := os.Getwd()            // Get the working directory
	currentDirectory = AbsPath(currentDirectory) // Get the absolute path of the current working directory
	fileName := filepath.Base(file)

	if file == fileName { // If we did not specify a directory to write to
		writeDirectory = currentDirectory // Set to the current directory
	} else {
		writeDirectory = AbsPath(filepath.Dir(file))
	}

	if currentDirectory != writeDirectory { // If the currentDirectory is not the same directory as the writeDirectory
		if createDirsErr := os.MkdirAll(writeDirectory, sourceFileMode); createDirsErr != nil { // If we failed to make all the directories needed
			return errors.New("Failed to create the path leading up to " + fileName + ": " + writeDirectory)
		}
	}

	writeErr := ioutil.WriteFile(writeDirectory+Separator+fileName, fileContent, sourceFileMode)

	if writeErr != nil {
		writeErr = errors.New("Failed to write " + fileName + " in directory " + writeDirectory)
	}

	return writeErr
}
<|endoftext|>