<commit_before>// Package buffstreams provides a simple interface for creating and storing
// sockets that have a pre-defined set of behaviors, making them simple to use
// for consuming streaming protocol buffer messages over TCP
package buffstreams

import ()

import (
	"encoding/binary"
	"errors"
	"io"
	"log"
	"net"
	"sync"
)

// BuffManager represents the object used to govern interactions between tcp endpoints.
// You can use it to read from and write to streaming or non-streaming TCP connections
// and have it handle packaging data with a header describing the size of the data payload.
// This is to make it easy to work with wire formats like ProtocolBuffers, which require
// a custom-delimeter situation to be sent in a streaming fashion.
type BuffManager struct {
	dialedConnections map[string]*net.TCPConn
	listeningSockets  map[string]*net.TCPListener
	// TODO find a way to sanely provide this to a Dialer or a Receiver on a per-connection basis
	// TODO I could control access to the maps better if I centralized how they got accessed - less locking code littered around
	headerByteSize int
	maxMessageSize int
	enableLogging  bool
	sync.RWMutex
}

// BuffManagerConfig represents a set of options that the person building systems ontop of
type BuffManagerConfig struct {
	// Controls how large the largest Message may be. The server will reject any messages whose clients
	// header size does not match this configuration
	MaxMessageSize int
	// Controls the ability to enable logging errors occuring in the library
	EnableLogging bool
}

// New creates a new *BuffManager based on the provided BuffManagerConfig
func New(cfg BuffManagerConfig) *BuffManager {
	bm := &BuffManager{
		dialedConnections: make(map[string]*net.TCPConn),
		listeningSockets:  make(map[string]*net.TCPListener),
		enableLogging:     cfg.EnableLogging,
	}
	maxMessageSize := 4096
	// 0 is the default, and the message must be atleast 1 byte large
	if cfg.MaxMessageSize != 0 {
		maxMessageSize = cfg.MaxMessageSize
	}
	bm.maxMessageSize = maxMessageSize
	bm.headerByteSize = messageSizeToBitLength(maxMessageSize)
	return bm
}

// ListenCallback is a function type that calling code will need to implement in order
// to receive arrays of bytes from the socket. Each slice of bytes will be stripped of the
// size header, meaning you can directly serialize the raw slice. You would then perform your
// custom logic for interpretting the message, before returning. You can optionally
// return an error, which in turn will be logged if EnableLogging is set to true.
type ListenCallback func([]byte) error

// FormatAddress is to cover the event that you want/need a programmtically correct way
// to format an address/port to use with StartListening or WriteTo
func FormatAddress(address string, port string) string {
	return address + ":" + port
}

// StartListening is an asyncrhonous, non-blocking method. It begins listening on the given
// port, and fire off a goroutine for every client connection it receives. That goroutine will
// read the fixed header, then the message payload, and then invoke the povided ListenCallbacl.
// In the event of an transport error, it will disconnect the client. It is the clients responsibility
// to re-connect if needed.
func (bm *BuffManager) StartListening(port string, cb ListenCallback) error {
	address := FormatAddress("", port)
	tcpAddr, err := net.ResolveTCPAddr("tcp", address)
	if err != nil {
		return err
	}
	receiveSocket, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return err
	}
	bm.startListening(address, receiveSocket, cb)
	return nil
}

func (bm *BuffManager) startListening(address string, socket *net.TCPListener, cb ListenCallback) {
	bm.Lock()
	bm.listeningSockets[address] = socket
	bm.Unlock()

	go func(address string, headerByteSize int, maxMessageSize int, enableLogging bool, listener *net.TCPListener) {
		for {
			// Wait for someone to connect
			conn, err := listener.AcceptTCP()
			if err != nil {
				if enableLogging {
					log.Printf("Error attempting to accept connection: %s", err)
				}
			} else {
				// Hand this off and immediately listen for more
				go handleListenedConn(address, conn, headerByteSize, maxMessageSize, enableLogging, cb)
			}
		}
	}(address, bm.headerByteSize, bm.maxMessageSize, bm.enableLogging, socket)
}

func handleListenedConn(address string, conn *net.TCPConn, headerByteSize int, maxMessageSize int, enableLogging bool, cb ListenCallback) {
	// If there is any error, close the connection officially and break out of the listen-loop.
	// We don't store these connections anywhere else, and if we can't recover from an error on the socket
	// we want to kill the connection, exit the goroutine, and let the client handle re-connecting if need be.
	// Handle getting the data header

	// We can cheat a tiny bit here, and only allocate this buffer one time. It will be overwritten on each call
	// to read, and we always pass in a slice the size of the total bytes read so far, so there should
	// never be any resultant cross-contamination from earlier runs of the loop.
	headerBuffer := make([]byte, headerByteSize)
	dataBuffer := make([]byte, maxMessageSize)
	for {
		var headerReadError error
		var totalHeaderBytesRead = 0
		var bytesRead = 0
		// First, read the number of bytes required to determine the message length
		for totalHeaderBytesRead < headerByteSize && headerReadError == nil {
			// While we haven't read enough yet, pass in the slice that represents where we are in the buffer
			bytesRead, headerReadError = readFromConnection(conn, headerBuffer[totalHeaderBytesRead:])
			totalHeaderBytesRead += bytesRead
		}
		if headerReadError != nil {
			if enableLogging {
				if headerReadError != io.EOF {
					// Log the error we got from the call to read
					log.Printf("Error when trying to read from address %s. Tried to read %d, actually read %d. Underlying error: %s", address, headerByteSize, totalHeaderBytesRead, headerReadError)
				} else {
					// Client closed the conn
					log.Printf("Address %s: Client closed connection during header read. Underlying error: %s", address, headerReadError)
				}
			}
			conn.Close()
			return
		}
		// Now turn that buffer of bytes into an integer - represnts size of message body
		msgLength, bytesParsed := binary.Uvarint(headerBuffer)
		iMsgLength := int(msgLength)
		// Not sure what the correct way to handle these errors are. For now, bomb out
		if bytesParsed == 0 {
			// "Buffer too small"
			if enableLogging {
				log.Printf("Address %s: 0 Bytes parsed from header. Underlying error: %s", address, headerReadError)
			}
			conn.Close()
			return
		} else if bytesParsed < 0 {
			// "Buffer overflow"
			if enableLogging {
				log.Printf("Address %s: Buffer Less than zero bytes parsed from header. Underlying error: %s", address, headerReadError)
			}
			conn.Close()
			return
		}

		var dataReadError error
		var totalDataBytesRead = 0
		bytesRead = 0
		for totalDataBytesRead < iMsgLength && dataReadError == nil {
			// While we haven't read enough yet, pass in the slice that represents where we are in the buffer
			bytesRead, dataReadError = readFromConnection(conn, dataBuffer[totalDataBytesRead:iMsgLength])
			totalDataBytesRead += bytesRead
		}

		if dataReadError != nil {
			if enableLogging {
				if dataReadError != io.EOF {
					// log the error from the call to read
					log.Printf("Address %s: Failure to read from connection. Was told to read %d by the header, actually read %d. Underlying error: %s", address, msgLength, totalDataBytesRead, dataReadError)
				} else {
					// The client wrote the header but closed the connection
					log.Printf("Address %s: Client closed connection during data read. Underlying error: %s", address, dataReadError)
				}
			}
			conn.Close()
			return
		}

		// If we read bytes, there wasn't an error, or if there was it was only EOF
		// And readbytes + EOF is normal, just as readbytes + no err, next read 0 bytes EOF
		// So... we take action on the actual message data
		if totalDataBytesRead == 0 && (dataReadError == nil || (dataReadError != nil && dataReadError == io.EOF)) {
			err := cb(dataBuffer[:iMsgLength])
			if err != nil && enableLogging {
				log.Printf("Error in Callback: %s", err)
				// TODO if it's a protobuffs error, it means we likely had an issue and can't
				// deserialize data? Should we kill the connection and have the client start over?
				// At this point, there isn't a reliable recovery mechanic for the server
			}
		}
	}
}

func readFromConnection(reader *net.TCPConn, buffer []byte) (int, error) {
	// This fills the buffer
	bytesLen, err := reader.Read(buffer)
	// Output the content of the bytes to the queue
	if bytesLen == 0 {
		if err != nil && err == io.EOF {
			// "End of individual transmission"
			// We're just done reading from that conn
			return bytesLen, err
		}
	}

	if err != nil {
		//"Underlying network failure?"
		// Not sure what this error would be, but it could exist and i've seen it handled
		// as a general case in other networking code. Following in the footsteps of (greatness|madness)
		return bytesLen, err
	}
	// Read some bytes, return the length
	return bytesLen, nil
}

func (bm *BuffManager) dialOut(address string) (*net.TCPConn, error) {
	// We want to hold this lock for both the lookup as well as for setting it in the map
	// Skipping defer for now, need to profile it's performance
	bm.Lock()
	if _, ok := bm.dialedConnections[address]; ok {
		bm.Unlock()
		return nil, errors.New("You have a connection to this ip and port open already")
	}
	tcpAddr, err := net.ResolveTCPAddr("tcp", address)
	if err != nil {
		bm.Unlock()
		return nil, err
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		bm.Unlock()
		return nil, err
	}

	// Store the connection, it's valid
	bm.dialedConnections[address] = conn
	bm.Unlock()
	return conn, nil
}

func (bm *BuffManager) closeDialer(address string) error {
	bm.Lock()
	// We want to hold this lock for both the lookup as well as the delete
	// Either way, we can't release until we're ready to return.
	//Skipping defer for now, need to profile it's performance
	if conn, ok := bm.dialedConnections[address]; ok {
		err := conn.Close()
		// Grab lock to delete from the map
		delete(bm.dialedConnections, address)
		// Release immediately
		bm.Unlock()
		return err
	}
	bm.Unlock()
	return nil
}

// WriteTo allows you to dial to a remote or local TCP endpoint, and send either 1 or a stream
// of bytes as messages. Each array of bytes you pass in will be pre-pended with it's size
// within the size of the pre-defined maximum message size. If the connection isn't open yet,
// WriteTo will open it, and cache it. If for anyreason the connection breaks, it will be disposed
// and upon the next write, a new one will dial out.
func (bm *BuffManager) WriteTo(address string, data []byte, persist bool) (int, error) {
	var conn *net.TCPConn
	var err error
	var ok bool

	// Get the connection if it's cached, or open a new one
	bm.RLock()
	conn, ok = bm.dialedConnections[address]
	bm.RUnlock()
	if !ok {
		conn, err = bm.dialOut(address)
		if err != nil {
			// Error dialing out, cannot write
			// bail
			return 0, err
		}
	}
	// Calculate how big the message is, using a consistent header size.
	msgLenHeader := uInt16ToByteArray(uint16(len(data)), bm.headerByteSize)
	// Append the size to the message, so now it has a header
	toWrite := append(msgLenHeader, data...)

	toWriteLen := len(toWrite)

	// Three conditions could have occured:
	// 1. There was an error
	// 2. Not all bytes were written
	// 3. Both 1 and 2

	// If there was an error, that should take handling precedence. If the connection
	// was closed, or is otherwise in a bad state, we have to abort and re-open the connection
	// to try again, as we can't realistically finish the write. We have to retry it, or return
	// and error to the user?

	// TODO configurable message retries

	// If there was not an error, and we simply didn't finish the write, we should enter
	// a write-until-complete loop, where we continue to write the data until the server accepts
	// all of it.

	// If both issues occurred, we'll need to find a way to determine if the error
	// is recoverable (is the connection in a bad state) or not

	var writeError error
	var totalBytesWritten = 0
	var bytesWritten = 0
	// First, read the number of bytes required to determine the message length
	for totalBytesWritten < toWriteLen && writeError == nil {
		// While we haven't read enough yet
		// If there are remainder bytes, adjust the contents of toWrite
		// totalBytesWritten will be the index of the nextByte waiting to be read
		bytesWritten, writeError = conn.Write(toWrite[totalBytesWritten:])
		totalBytesWritten += bytesWritten
	}

	if writeError != nil || !persist {
		if writeError != nil && bm.enableLogging {
			log.Printf("Error while writing data to %s. Expected to write %d, actually wrote %d. Underlying error: %s", address, len(toWrite), totalBytesWritten, writeError)
		}
		writeError = bm.closeDialer(address)
		if writeError != nil {
			// TODO ponder the following:
			// What if some bytes written, then failure, then also the close throws an error
			// []error is a better return type, but not sure if thats a thing you're supposed to do...
			// Possibilities for error not as complicated as i'm thinking?
			if bm.enableLogging {
				// The error will get returned up the stack, no need to log it here?
				log.Print("There was a subsequent error cleaning up the connection to %s")
			}
			return totalBytesWritten, writeError
		}
	}

	// Return the bytes written, any error
	return totalBytesWritten, writeError
}
<commit_msg>Removing bogus import statement that somehow snuck in...<commit_after>// Package buffstreams provides a simple interface for creating and storing
// sockets that have a pre-defined set of behaviors, making them simple to use
// for consuming streaming protocol buffer messages over TCP
package buffstreams

import (
	"encoding/binary"
	"errors"
	"io"
	"log"
	"net"
	"sync"
)

// BuffManager represents the object used to govern interactions between tcp endpoints.
// You can use it to read from and write to streaming or non-streaming TCP connections
// and have it handle packaging data with a header describing the size of the data payload.
// This is to make it easy to work with wire formats like ProtocolBuffers, which require
// a custom-delimeter situation to be sent in a streaming fashion.
type BuffManager struct {
	dialedConnections map[string]*net.TCPConn
	listeningSockets  map[string]*net.TCPListener
	// TODO find a way to sanely provide this to a Dialer or a Receiver on a per-connection basis
	// TODO I could control access to the maps better if I centralized how they got accessed - less locking code littered around
	headerByteSize int
	maxMessageSize int
	enableLogging  bool
	sync.RWMutex
}

// BuffManagerConfig represents a set of options that the person building systems ontop of
type BuffManagerConfig struct {
	// Controls how large the largest Message may be. The server will reject any messages whose clients
	// header size does not match this configuration
	MaxMessageSize int
	// Controls the ability to enable logging errors occuring in the library
	EnableLogging bool
}

// New creates a new *BuffManager based on the provided BuffManagerConfig
func New(cfg BuffManagerConfig) *BuffManager {
	bm := &BuffManager{
		dialedConnections: make(map[string]*net.TCPConn),
		listeningSockets:  make(map[string]*net.TCPListener),
		enableLogging:     cfg.EnableLogging,
	}
	maxMessageSize := 4096
	// 0 is the default, and the message must be atleast 1 byte large
	if cfg.MaxMessageSize != 0 {
		maxMessageSize = cfg.MaxMessageSize
	}
	bm.maxMessageSize = maxMessageSize
	bm.headerByteSize = messageSizeToBitLength(maxMessageSize)
	return bm
}

// ListenCallback is a function type that calling code will need to implement in order
// to receive arrays of bytes from the socket. Each slice of bytes will be stripped of the
// size header, meaning you can directly serialize the raw slice. You would then perform your
// custom logic for interpretting the message, before returning. You can optionally
// return an error, which in turn will be logged if EnableLogging is set to true.
type ListenCallback func([]byte) error

// FormatAddress is to cover the event that you want/need a programmtically correct way
// to format an address/port to use with StartListening or WriteTo
func FormatAddress(address string, port string) string {
	return address + ":" + port
}

// StartListening is an asyncrhonous, non-blocking method. It begins listening on the given
// port, and fire off a goroutine for every client connection it receives. That goroutine will
// read the fixed header, then the message payload, and then invoke the povided ListenCallbacl.
// In the event of an transport error, it will disconnect the client. It is the clients responsibility
// to re-connect if needed.
func (bm *BuffManager) StartListening(port string, cb ListenCallback) error {
	address := FormatAddress("", port)
	tcpAddr, err := net.ResolveTCPAddr("tcp", address)
	if err != nil {
		return err
	}
	receiveSocket, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		return err
	}
	bm.startListening(address, receiveSocket, cb)
	return nil
}

func (bm *BuffManager) startListening(address string, socket *net.TCPListener, cb ListenCallback) {
	bm.Lock()
	bm.listeningSockets[address] = socket
	bm.Unlock()

	go func(address string, headerByteSize int, maxMessageSize int, enableLogging bool, listener *net.TCPListener) {
		for {
			// Wait for someone to connect
			conn, err := listener.AcceptTCP()
			if err != nil {
				if enableLogging {
					log.Printf("Error attempting to accept connection: %s", err)
				}
			} else {
				// Hand this off and immediately listen for more
				go handleListenedConn(address, conn, headerByteSize, maxMessageSize, enableLogging, cb)
			}
		}
	}(address, bm.headerByteSize, bm.maxMessageSize, bm.enableLogging, socket)
}

func handleListenedConn(address string, conn *net.TCPConn, headerByteSize int, maxMessageSize int, enableLogging bool, cb ListenCallback) {
	// If there is any error, close the connection officially and break out of the listen-loop.
	// We don't store these connections anywhere else, and if we can't recover from an error on the socket
	// we want to kill the connection, exit the goroutine, and let the client handle re-connecting if need be.
	// Handle getting the data header

	// We can cheat a tiny bit here, and only allocate this buffer one time. It will be overwritten on each call
	// to read, and we always pass in a slice the size of the total bytes read so far, so there should
	// never be any resultant cross-contamination from earlier runs of the loop.
	headerBuffer := make([]byte, headerByteSize)
	dataBuffer := make([]byte, maxMessageSize)
	for {
		var headerReadError error
		var totalHeaderBytesRead = 0
		var bytesRead = 0
		// First, read the number of bytes required to determine the message length
		for totalHeaderBytesRead < headerByteSize && headerReadError == nil {
			// While we haven't read enough yet, pass in the slice that represents where we are in the buffer
			bytesRead, headerReadError = readFromConnection(conn, headerBuffer[totalHeaderBytesRead:])
			totalHeaderBytesRead += bytesRead
		}
		if headerReadError != nil {
			if enableLogging {
				if headerReadError != io.EOF {
					// Log the error we got from the call to read
					log.Printf("Error when trying to read from address %s. Tried to read %d, actually read %d. Underlying error: %s", address, headerByteSize, totalHeaderBytesRead, headerReadError)
				} else {
					// Client closed the conn
					log.Printf("Address %s: Client closed connection during header read. Underlying error: %s", address, headerReadError)
				}
			}
			conn.Close()
			return
		}
		// Now turn that buffer of bytes into an integer - represnts size of message body
		msgLength, bytesParsed := binary.Uvarint(headerBuffer)
		iMsgLength := int(msgLength)
		// Not sure what the correct way to handle these errors are. For now, bomb out
		if bytesParsed == 0 {
			// "Buffer too small"
			if enableLogging {
				log.Printf("Address %s: 0 Bytes parsed from header. Underlying error: %s", address, headerReadError)
			}
			conn.Close()
			return
		} else if bytesParsed < 0 {
			// "Buffer overflow"
			if enableLogging {
				log.Printf("Address %s: Buffer Less than zero bytes parsed from header. Underlying error: %s", address, headerReadError)
			}
			conn.Close()
			return
		}

		var dataReadError error
		var totalDataBytesRead = 0
		bytesRead = 0
		for totalDataBytesRead < iMsgLength && dataReadError == nil {
			// While we haven't read enough yet, pass in the slice that represents where we are in the buffer
			bytesRead, dataReadError = readFromConnection(conn, dataBuffer[totalDataBytesRead:iMsgLength])
			totalDataBytesRead += bytesRead
		}

		if dataReadError != nil {
			if enableLogging {
				if dataReadError != io.EOF {
					// log the error from the call to read
					log.Printf("Address %s: Failure to read from connection. Was told to read %d by the header, actually read %d. Underlying error: %s", address, msgLength, totalDataBytesRead, dataReadError)
				} else {
					// The client wrote the header but closed the connection
					log.Printf("Address %s: Client closed connection during data read. Underlying error: %s", address, dataReadError)
				}
			}
			conn.Close()
			return
		}

		// If we read bytes, there wasn't an error, or if there was it was only EOF
		// And readbytes + EOF is normal, just as readbytes + no err, next read 0 bytes EOF
		// So... we take action on the actual message data
		if totalDataBytesRead == 0 && (dataReadError == nil || (dataReadError != nil && dataReadError == io.EOF)) {
			err := cb(dataBuffer[:iMsgLength])
			if err != nil && enableLogging {
				log.Printf("Error in Callback: %s", err)
				// TODO if it's a protobuffs error, it means we likely had an issue and can't
				// deserialize data? Should we kill the connection and have the client start over?
				// At this point, there isn't a reliable recovery mechanic for the server
			}
		}
	}
}

func readFromConnection(reader *net.TCPConn, buffer []byte) (int, error) {
	// This fills the buffer
	bytesLen, err := reader.Read(buffer)
	// Output the content of the bytes to the queue
	if bytesLen == 0 {
		if err != nil && err == io.EOF {
			// "End of individual transmission"
			// We're just done reading from that conn
			return bytesLen, err
		}
	}

	if err != nil {
		//"Underlying network failure?"
		// Not sure what this error would be, but it could exist and i've seen it handled
		// as a general case in other networking code. Following in the footsteps of (greatness|madness)
		return bytesLen, err
	}
	// Read some bytes, return the length
	return bytesLen, nil
}

func (bm *BuffManager) dialOut(address string) (*net.TCPConn, error) {
	// We want to hold this lock for both the lookup as well as for setting it in the map
	// Skipping defer for now, need to profile it's performance
	bm.Lock()
	if _, ok := bm.dialedConnections[address]; ok {
		bm.Unlock()
		return nil, errors.New("You have a connection to this ip and port open already")
	}
	tcpAddr, err := net.ResolveTCPAddr("tcp", address)
	if err != nil {
		bm.Unlock()
		return nil, err
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		bm.Unlock()
		return nil, err
	}

	// Store the connection, it's valid
	bm.dialedConnections[address] = conn
	bm.Unlock()
	return conn, nil
}

func (bm *BuffManager) closeDialer(address string) error {
	bm.Lock()
	// We want to hold this lock for both the lookup as well as the delete
	// Either way, we can't release until we're ready to return.
	//Skipping defer for now, need to profile it's performance
	if conn, ok := bm.dialedConnections[address]; ok {
		err := conn.Close()
		// Grab lock to delete from the map
		delete(bm.dialedConnections, address)
		// Release immediately
		bm.Unlock()
		return err
	}
	bm.Unlock()
	return nil
}

// WriteTo allows you to dial to a remote or local TCP endpoint, and send either 1 or a stream
// of bytes as messages. Each array of bytes you pass in will be pre-pended with it's size
// within the size of the pre-defined maximum message size. If the connection isn't open yet,
// WriteTo will open it, and cache it. If for anyreason the connection breaks, it will be disposed
// and upon the next write, a new one will dial out.
func (bm *BuffManager) WriteTo(address string, data []byte, persist bool) (int, error) {
	var conn *net.TCPConn
	var err error
	var ok bool

	// Get the connection if it's cached, or open a new one
	bm.RLock()
	conn, ok = bm.dialedConnections[address]
	bm.RUnlock()
	if !ok {
		conn, err = bm.dialOut(address)
		if err != nil {
			// Error dialing out, cannot write
			// bail
			return 0, err
		}
	}
	// Calculate how big the message is, using a consistent header size.
	msgLenHeader := uInt16ToByteArray(uint16(len(data)), bm.headerByteSize)
	// Append the size to the message, so now it has a header
	toWrite := append(msgLenHeader, data...)

	toWriteLen := len(toWrite)

	// Three conditions could have occured:
	// 1. There was an error
	// 2. Not all bytes were written
	// 3. Both 1 and 2

	// If there was an error, that should take handling precedence. If the connection
	// was closed, or is otherwise in a bad state, we have to abort and re-open the connection
	// to try again, as we can't realistically finish the write. We have to retry it, or return
	// and error to the user?

	// TODO configurable message retries

	// If there was not an error, and we simply didn't finish the write, we should enter
	// a write-until-complete loop, where we continue to write the data until the server accepts
	// all of it.

	// If both issues occurred, we'll need to find a way to determine if the error
	// is recoverable (is the connection in a bad state) or not

	var writeError error
	var totalBytesWritten = 0
	var bytesWritten = 0
	// First, read the number of bytes required to determine the message length
	for totalBytesWritten < toWriteLen && writeError == nil {
		// While we haven't read enough yet
		// If there are remainder bytes, adjust the contents of toWrite
		// totalBytesWritten will be the index of the nextByte waiting to be read
		bytesWritten, writeError = conn.Write(toWrite[totalBytesWritten:])
		totalBytesWritten += bytesWritten
	}

	if writeError != nil || !persist {
		if writeError != nil && bm.enableLogging {
			log.Printf("Error while writing data to %s. Expected to write %d, actually wrote %d. Underlying error: %s", address, len(toWrite), totalBytesWritten, writeError)
		}
		writeError = bm.closeDialer(address)
		if writeError != nil {
			// TODO ponder the following:
			// What if some bytes written, then failure, then also the close throws an error
			// []error is a better return type, but not sure if thats a thing you're supposed to do...
			// Possibilities for error not as complicated as i'm thinking?
			if bm.enableLogging {
				// The error will get returned up the stack, no need to log it here?
				log.Print("There was a subsequent error cleaning up the connection to %s")
			}
			return totalBytesWritten, writeError
		}
	}

	// Return the bytes written, any error
	return totalBytesWritten, writeError
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/daviddengcn/go-colortext"
	"github.com/phrase/phraseapp-go/phraseapp"
	"regexp"
)

var Debug bool

type LocaleFiles []*LocaleFile
type LocaleFile struct {
	Path, Name, ID, RFC, Tag, FileFormat string
	ExistsRemote                         bool
}

var placeholderRegexp = regexp.MustCompile("<(locale_name|tag|locale_code)>")

func ValidPath(file string) error {
	if strings.TrimSpace(file) == "" {
		return fmt.Errorf(
			"File patterns of a source may not be empty! Please use a valid file pattern: %s",
			"http://docs.phraseapp.com/developers/cli/configuration/",
		)
	}

	extension := filepath.Ext(file)
	if strings.TrimSpace(extension) == "" {
		abs, err := filepath.Abs(file)
		if err != nil {
			return err
		}
		return fmt.Errorf(
			"'%s' does not have a valid extension. Please use a valid extension: %s",
			abs, "http://docs.phraseapp.com/guides/formats/",
		)
	}

	return nil
}

func (localeFile *LocaleFile) RelPath() string {
	callerPath, _ := os.Getwd()
	relativePath, _ := filepath.Rel(callerPath, localeFile.Path)
	return relativePath
}

// Locale to Path mapping
func (localeFile *LocaleFile) Message() string {
	str := ""
	if Debug {
		if localeFile.Name != "" {
			str = fmt.Sprintf("%s Name: %s", str, localeFile.Name)
		}
		if localeFile.ID != "" {
			str = fmt.Sprintf("%s Id: %s", str, localeFile.ID)
		}
		if localeFile.RFC != "" {
			str = fmt.Sprintf("%s RFC5646: %s", str, localeFile.RFC)
		}
		if localeFile.Tag != "" {
			str = fmt.Sprintf("%s Tag: %s", str, localeFile.Tag)
		}
		if localeFile.FileFormat != "" {
			str = fmt.Sprintf("%s Format: %s", str, localeFile.FileFormat)
		}
	} else {
		str = fmt.Sprintf("%s", localeFile.Name)
	}
	return strings.TrimSpace(str)
}

func printErr(err error) {
	ct.Foreground(ct.Red, true)
	fmt.Fprintf(os.Stderr, "\nERROR: %s\n", err)
	ct.ResetColor()
}

func sharedMessage(method string, localeFile *LocaleFile) {
	local := localeFile.RelPath()

	if method == "pull" {
		remote := localeFile.Message()
		fmt.Print("Downloaded ")
		ct.Foreground(ct.Green, true)
		fmt.Print(remote)
		ct.ResetColor()
		fmt.Print(" to ")
		ct.Foreground(ct.Green, true)
		fmt.Print(local, "\n")
		ct.ResetColor()
	} else {
		fmt.Print("Uploaded ")
		ct.Foreground(ct.Green, true)
		fmt.Print(local)
		ct.ResetColor()
		fmt.Println(" successfully.")
	}
}

func RemoteLocales(client *phraseapp.Client, projectId string) ([]*phraseapp.Locale, error) {
	page := 1
	locales, err := client.LocalesList(projectId, page, 25)
	if err != nil {
		return nil, err
	}
	result := locales
	for len(locales) == 25 {
		page = page + 1
		locales, err = client.LocalesList(projectId, page, 25)
		if err != nil {
			return nil, err
		}
		result = append(result, locales...)
	}
	return result, nil
}

func Contains(seq []string, str string) bool {
	for _, elem := range seq {
		if str == elem {
			return true
		}
	}
	return false
}

func TakeWhile(seq []string, predicate func(string) bool) []string {
	take := []string{}
	for _, elem := range seq {
		if !predicate(elem) {
			break
		}
		take = append(take, elem)
	}
	return take
}

func Exists(absPath string) error {
	if _, err := os.Stat(absPath); os.IsNotExist(err) {
		return fmt.Errorf("no such file or directory:", absPath)
	}
	return nil
}
<commit_msg>add ReportError func<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"bytes"
	"path"
	"runtime"
	"path/filepath"
	"strings"
	"github.com/daviddengcn/go-colortext"
	"github.com/phrase/phraseapp-go/phraseapp"
	"regexp"
)

var Debug bool

type LocaleFiles []*LocaleFile
type LocaleFile struct {
	Path, Name, ID, RFC, Tag, FileFormat string
	ExistsRemote                         bool
}

var placeholderRegexp = regexp.MustCompile("<(locale_name|tag|locale_code)>")

func ValidPath(file string) error {
	if strings.TrimSpace(file) == "" {
		return fmt.Errorf(
			"File patterns of a source may not be empty! Please use a valid file pattern: %s",
			"http://docs.phraseapp.com/developers/cli/configuration/",
		)
	}

	extension := filepath.Ext(file)
	if strings.TrimSpace(extension) == "" {
		abs, err := filepath.Abs(file)
		if err != nil {
			return err
		}
		return fmt.Errorf(
			"'%s' does not have a valid extension. Please use a valid extension: %s",
			abs, "http://docs.phraseapp.com/guides/formats/",
		)
	}

	return nil
}

func (localeFile *LocaleFile) RelPath() string {
	callerPath, _ := os.Getwd()
	relativePath, _ := filepath.Rel(callerPath, localeFile.Path)
	return relativePath
}

// Locale to Path mapping
func (localeFile *LocaleFile) Message() string {
	str := ""
	if Debug {
		if localeFile.Name != "" {
			str = fmt.Sprintf("%s Name: %s", str, localeFile.Name)
		}
		if localeFile.ID != "" {
			str = fmt.Sprintf("%s Id: %s", str, localeFile.ID)
		}
		if localeFile.RFC != "" {
			str = fmt.Sprintf("%s RFC5646: %s", str, localeFile.RFC)
		}
		if localeFile.Tag != "" {
			str = fmt.Sprintf("%s Tag: %s", str, localeFile.Tag)
		}
		if localeFile.FileFormat != "" {
			str = fmt.Sprintf("%s Format: %s", str, localeFile.FileFormat)
		}
	} else {
		str = fmt.Sprintf("%s", localeFile.Name)
	}
	return strings.TrimSpace(str)
}

func printErr(err error) {
	ct.Foreground(ct.Red, true)
	fmt.Fprintf(os.Stderr, "\nERROR: %s\n", err)
	ct.ResetColor()
}

func sharedMessage(method string, localeFile *LocaleFile) {
	local := localeFile.RelPath()

	if method == "pull" {
		remote := localeFile.Message()
		fmt.Print("Downloaded ")
		ct.Foreground(ct.Green, true)
		fmt.Print(remote)
		ct.ResetColor()
		fmt.Print(" to ")
		ct.Foreground(ct.Green, true)
		fmt.Print(local, "\n")
		ct.ResetColor()
	} else {
		fmt.Print("Uploaded ")
		ct.Foreground(ct.Green, true)
		fmt.Print(local)
		ct.ResetColor()
		fmt.Println(" successfully.")
	}
}

func RemoteLocales(client *phraseapp.Client, projectId string) ([]*phraseapp.Locale, error) {
	page := 1
	locales, err := client.LocalesList(projectId, page, 25)
	if err != nil {
		return nil, err
	}
	result := locales
	for len(locales) == 25 {
		page = page + 1
		locales, err = client.LocalesList(projectId, page, 25)
		if err != nil {
			return nil, err
		}
		result = append(result, locales...)
	}
	return result, nil
}

func Contains(seq []string, str string) bool {
	for _, elem := range seq {
		if str == elem {
			return true
		}
	}
	return false
}

func TakeWhile(seq []string, predicate func(string) bool) []string {
	take := []string{}
	for _, elem := range seq {
		if !predicate(elem) {
			break
		}
		take = append(take, elem)
	}
	return take
}

func Exists(absPath string) error {
	if _, err := os.Stat(absPath); os.IsNotExist(err) {
		return fmt.Errorf("no such file or directory:", absPath)
	}
	return nil
}

func ReportError(name string, message string){
	body := []byte(fmt.Sprintf("{"+
    "\"name\":\"%s\"," +
    "\"message\":\"%s\"," +
    "\"data\": {" +
			"\"version\":\"%s\"," +
			"\"os\":\"%s\"," +
			"\"arch\":\"%s\"" +
		"}}",
    name, message, PHRASEAPP_CLIENT_VERSION, runtime.GOOS, runtime.GOARCH))

	req, err := http.NewRequest("POST", "https://phraseapp.com/errors", bytes.NewBuffer(body))
  req.Header.Set("Content-Type", "application/json")

	client := http.Client{}
  resp, err := client.Do(req)
	if err != nil {
		panic(err)
	}
	resp.Body.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"gopkg.in/Iwark/spreadsheet.v2"
)

// Represents the board members of a Toastmasters meeting.
type board struct {
	president string
	vpe       string
	vpm       string
	vppr      string
	secretary string
	treasurer string
	saa       string
}

// Factory method using a spreadsheet to fill in board members.
func (board) new(sheet *spreadsheet.Sheet) board {
	board := board{}
	board.president = sheet.Columns[1][0].Value
	board.vpe = sheet.Columns[1][1].Value
	board.vpm = sheet.Columns[1][2].Value
	board.vppr = sheet.Columns[1][3].Value
	board.secretary = sheet.Columns[1][4].Value
	board.treasurer = sheet.Columns[1][5].Value
	board.saa = sheet.Columns[1][6].Value

	return board
}

// Represents the editable fields on a Toastmasters agenda.
type agendaRoles struct {
	toastmaster       string
	ge                string
	timer             string
	ahCounter         string
	grammarian        string
	tableTopicsMaster string
	jokeMaster        string
	speakers          []speaker
	boardMembers      board
	futureWeeks       [][]string
}

// Factory method to create agenda roles from a google doc based on the date of the meeting.
func (agendaRoles) new(agendaDate string) (agendaRoles, error) {
	spreadsheets, err := getSheet()
	if err != nil {
		return agendaRoles{}, err
	}
	boardMembers := board{}.new(spreadsheets.boardSheet)

	agendaRoles := agendaRoles{}
	agendaRoles.boardMembers = boardMembers

	const speakerCellStart = 7
	const speakerCellEnd = 13
	rolesSheet := spreadsheets.meetingRoles
	for i := range rolesSheet.Columns {
		if rolesSheet.Columns[i][0].Value == agendaDate {
			agendaRoles.toastmaster = rolesSheet.Columns[i][1].Value
			agendaRoles.jokeMaster = rolesSheet.Columns[i][2].Value
			agendaRoles.ge = rolesSheet.Columns[i][3].Value
			agendaRoles.timer = rolesSheet.Columns[i][4].Value
			agendaRoles.ahCounter = rolesSheet.Columns[i][5].Value
			agendaRoles.grammarian = rolesSheet.Columns[i][6].Value

			for j := speakerCellStart; j <= speakerCellEnd; j += 2 {
				agendaRoles.speakers = append(agendaRoles.speakers, NewSpeaker(rolesSheet.Columns[i][j].Value,
					rolesSheet.Columns[i][j+1].Value))
			}

			agendaRoles.tableTopicsMaster = rolesSheet.Columns[i][16].Value
			agendaRoles.futureWeeks = getFutureWeeks(rolesSheet, i)
			break
		}
	}
	return agendaRoles, nil
}

//  Represents a speaker in a Toastmasters meeting.
type speaker struct {
	name string
	Speech
	evaluator string
}

// Factory method to create a speaker based on the spreadsheet speaker and evaluator.
func NewSpeaker(s string, eval string) speaker {
	name, manual, number := parseManualAndNumber(s)
	info := NewSpeech(manual, number)

	speaker := speaker{}
	speaker.name = name
	speaker.evaluator = eval
	speaker.Speech = info

	return speaker
}

// Helper method that returns the first name of a speaker.
func (s speaker) firstName() string {
	return strings.Split(s.name, " ")[0]
}

// Represents the spreadsheet tabs.
type googleDocsSheet struct {
	boardSheet   *spreadsheet.Sheet
	meetingRoles *spreadsheet.Sheet
}

//  GetSheet reads a Google Docs spreadsheet and returns a sheet with roles and another sheet with the board members.
func getSheet() (googleDocsSheet, error) {
	data, err := ioutil.ReadFile("client_secret.json")
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot read client_secret.json")
	}

	conf, err := google.JWTConfigFromJSON(data, spreadsheet.Scope)
	if err != nil {
		return googleDocsSheet{}, errors.New("problem with google.JWTConfigFromJSON(data, s.Scope)")
	}

	client := conf.Client(context.TODO())

	service := spreadsheet.NewServiceWithClient(client)
	s, err := service.FetchSpreadsheet("1CBlORqCzL6YvyAUZTk8jezvhyuDzjjumghwGKk5VIK8")
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot fetch spread sheet: ")
	}

	roles, err := s.SheetByIndex(0)
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot read s by index 0")
	}

	board, err := s.SheetByIndex(1)
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot read s by index 1")
	}

	return googleDocsSheet{boardSheet: board, meetingRoles: roles}, nil
}

// Find the speaker name, manual and number from a string that looks like "Ann Addicks\nCC #9".
func parseManualAndNumber(speaker string) (string, string, int) {
	re := regexp.MustCompile(`([a-zA-Z]+ [a-zA-Z]+)\n([a-zA-Z]+) #(\d{1,2})`)
	result := re.FindStringSubmatch(speaker)
	name := speaker
	manual := ""
	speechNum := 0

	if len(result) > 0 {
		name = result[1]
		manual = result[2]
		speechNum, _ = strconv.Atoi(result[3])
	}
	return name, manual, speechNum
}

// The number of weeks in the future to capture.
const futureWeeks = 4
const numberOfRoles = 17

// GetFutureWeeks finds the next several weeks after the current week based on the constant futureWeeks.
func getFutureWeeks(sheet *spreadsheet.Sheet, thisWeek int) [][]string {
	week := 0
	var nextSchedule = make([][]string, 0, futureWeeks)

	for i := thisWeek + 1; i < len(sheet.Columns) && week <= futureWeeks; i++ {
		nextWeek := make([]string, numberOfRoles)

		for j := 0; j < numberOfRoles; j++ {
			nextWeek[j] = sheet.Columns[i][j].Value
		}
		nextSchedule = append(nextSchedule, nextWeek)
		week++

	}
	return nextSchedule
}
<commit_msg>conslidate new speaker func.<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"gopkg.in/Iwark/spreadsheet.v2"
)

// Represents the board members of a Toastmasters meeting.
type board struct {
	president string
	vpe       string
	vpm       string
	vppr      string
	secretary string
	treasurer string
	saa       string
}

// Factory method using a spreadsheet to fill in board members.
func (board) new(sheet *spreadsheet.Sheet) board {
	board := board{}
	board.president = sheet.Columns[1][0].Value
	board.vpe = sheet.Columns[1][1].Value
	board.vpm = sheet.Columns[1][2].Value
	board.vppr = sheet.Columns[1][3].Value
	board.secretary = sheet.Columns[1][4].Value
	board.treasurer = sheet.Columns[1][5].Value
	board.saa = sheet.Columns[1][6].Value

	return board
}

// Represents the editable fields on a Toastmasters agenda.
type agendaRoles struct {
	toastmaster       string
	ge                string
	timer             string
	ahCounter         string
	grammarian        string
	tableTopicsMaster string
	jokeMaster        string
	speakers          []speaker
	boardMembers      board
	futureWeeks       [][]string
}

// Factory method to create agenda roles from a google doc based on the date of the meeting.
func (agendaRoles) new(agendaDate string) (agendaRoles, error) {
	spreadsheets, err := getSheet()
	if err != nil {
		return agendaRoles{}, err
	}
	boardMembers := board{}.new(spreadsheets.boardSheet)

	agendaRoles := agendaRoles{}
	agendaRoles.boardMembers = boardMembers

	const speakerCellStart = 7
	const speakerCellEnd = 13
	rolesSheet := spreadsheets.meetingRoles
	for i := range rolesSheet.Columns {
		if rolesSheet.Columns[i][0].Value == agendaDate {
			agendaRoles.toastmaster = rolesSheet.Columns[i][1].Value
			agendaRoles.jokeMaster = rolesSheet.Columns[i][2].Value
			agendaRoles.ge = rolesSheet.Columns[i][3].Value
			agendaRoles.timer = rolesSheet.Columns[i][4].Value
			agendaRoles.ahCounter = rolesSheet.Columns[i][5].Value
			agendaRoles.grammarian = rolesSheet.Columns[i][6].Value

			for j := speakerCellStart; j <= speakerCellEnd; j += 2 {
				agendaRoles.speakers = append(agendaRoles.speakers, NewSpeaker(rolesSheet.Columns[i][j].Value,
					rolesSheet.Columns[i][j+1].Value))
			}

			agendaRoles.tableTopicsMaster = rolesSheet.Columns[i][16].Value
			agendaRoles.futureWeeks = getFutureWeeks(rolesSheet, i)
			break
		}
	}
	return agendaRoles, nil
}

//  Represents a speaker in a Toastmasters meeting.
type speaker struct {
	name string
	Speech
	evaluator string
}

// Factory method to create a speaker based on the spreadsheet speaker and evaluator.
func NewSpeaker(s string, eval string) speaker {
	name, manual, number := parseManualAndNumber(s)

	return speaker{
		name:      name,
		evaluator: eval,
		Speech:    NewSpeech(manual, number),
	}
}

// Helper method that returns the first name of a speaker.
func (s speaker) firstName() string {
	return strings.Split(s.name, " ")[0]
}

// Represents the spreadsheet tabs.
type googleDocsSheet struct {
	boardSheet   *spreadsheet.Sheet
	meetingRoles *spreadsheet.Sheet
}

//  GetSheet reads a Google Docs spreadsheet and returns a sheet with roles and another sheet with the board members.
func getSheet() (googleDocsSheet, error) {
	data, err := ioutil.ReadFile("client_secret.json")
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot read client_secret.json")
	}

	conf, err := google.JWTConfigFromJSON(data, spreadsheet.Scope)
	if err != nil {
		return googleDocsSheet{}, errors.New("problem with google.JWTConfigFromJSON(data, s.Scope)")
	}

	client := conf.Client(context.TODO())

	service := spreadsheet.NewServiceWithClient(client)
	s, err := service.FetchSpreadsheet("1CBlORqCzL6YvyAUZTk8jezvhyuDzjjumghwGKk5VIK8")
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot fetch spread sheet: ")
	}

	roles, err := s.SheetByIndex(0)
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot read s by index 0")
	}

	board, err := s.SheetByIndex(1)
	if err != nil {
		return googleDocsSheet{}, errors.New("cannot read s by index 1")
	}

	return googleDocsSheet{boardSheet: board, meetingRoles: roles}, nil
}

// Find the speaker name, manual and number from a string that looks like "Ann Addicks\nCC #9".
func parseManualAndNumber(speaker string) (string, string, int) {
	re := regexp.MustCompile(`([a-zA-Z]+ [a-zA-Z]+)\n([a-zA-Z]+) #(\d{1,2})`)
	result := re.FindStringSubmatch(speaker)
	name := speaker
	manual := ""
	speechNum := 0

	if len(result) > 0 {
		name = result[1]
		manual = result[2]
		speechNum, _ = strconv.Atoi(result[3])
	}
	return name, manual, speechNum
}

// The number of weeks in the future to capture.
const futureWeeks = 4
const numberOfRoles = 17

// GetFutureWeeks finds the next several weeks after the current week based on the constant futureWeeks.
func getFutureWeeks(sheet *spreadsheet.Sheet, thisWeek int) [][]string {
	week := 0
	var nextSchedule = make([][]string, 0, futureWeeks)

	for i := thisWeek + 1; i < len(sheet.Columns) && week <= futureWeeks; i++ {
		nextWeek := make([]string, numberOfRoles)

		for j := 0; j < numberOfRoles; j++ {
			nextWeek[j] = sheet.Columns[i][j].Value
		}
		nextSchedule = append(nextSchedule, nextWeek)
		week++

	}
	return nextSchedule
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/jlaffaye/ftp"
)

// Loggerer ...
type Loggerer struct {
	logger ILogger
	err    error
}

// SetLogger sets objects logger
func (l *Loggerer) SetLogger(logger ILogger) {
	l.logger = logger
}

// Log logs the input text
func (l *Loggerer) Log(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(text...)
	}
}

// Error ...
func (l *Loggerer) Error(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log("ERROR: " + fmt.Sprint(text...))
	}
	if l.err == nil {
		l.err = errors.New(fmt.Sprint(text...))
	}
}

// ResetError ...
func (l *Loggerer) ResetError() {
	l.err = nil
}

// GetError ...
func (l *Loggerer) GetError() error {
	return l.err
}

type ftpConn struct {
	Loggerer
	conn      *ftp.ServerConn
	connected bool
}

func newFtpConn() *ftpConn {
	return &ftpConn{}
}

func (f *ftpConn) dial(addr string) {
	f.ResetError()
	conn, err := ftp.Dial(addr)
	if err != nil {
		f.Error(err)
		return
	}
	f.conn = conn
	f.connected = true
	f.Log("Connected to " + addr)
}

func (f *ftpConn) quit() {
	if !f.connected {
		return
	}
	f.conn.Quit()
	f.connected = false
	f.Log("Connection closed correctly")
}

func (f *ftpConn) login(user string, pass string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.Login(user, pass)
	if err != nil {
		f.Error(err)
		return
	}
	f.Log("Logged in as " + user)
}

func (f *ftpConn) cwd() string {
	if f.GetError() != nil {
		return ""
	}
	cwd, err := f.conn.CurrentDir()
	if err != nil {
		f.Error(err)
		return ""
	}
	return cwd
}

func (f *ftpConn) cd(path string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.ChangeDir(path)
	if err != nil {
		f.Error(err)
		return
	}
}

func (f *ftpConn) cdup() {
	if f.GetError() != nil {
		return
	}
	f.conn.ChangeDirToParent()
}

func (f *ftpConn) ls(path string) (entries []*ftp.Entry) {
	if f.GetError() != nil {
		return
	}
	entries, err := f.conn.List(path)
	if err != nil {
		f.Error(err)
		return
	}
	return entries
}

func (f *ftpConn) walk(fl map[string]fileEntry) {
	if f.GetError() != nil {
		return
	}
	entries := f.ls("")
	cwd := f.cwd()
	newLine := pad(cwd, len(lastLine))
	fmt.Print(newLine + "\r")
	lastLine = cwd
	for _, element := range entries {
		switch element.Type {
		case ftp.EntryTypeFile:
			if acceptFileName(element.Name) {
				key := cwd + "/" + element.Name
				entry, fileExists := fl[key]
				if fileExists {
					// Old file with new date
					if !entry.Time.Equal(element.Time) {
						f.Log("~ " + truncPad(key, 40, 'l') + " datetime changed")
						fl[key] = newFileEntry(element)
					} else if entry.Size != element.Size {
						f.Log("~ " + truncPad(key, 40, 'l') + " size changed")
						fl[key] = newFileEntry(element)
					} else {
						entry.Found = true
						fl[key] = entry
					}
				} else {
					// New file
					f.Log("+ " + truncPad(key, 40, 'l') + " new file")
					fl[key] = newFileEntry(element)
				}
			}
		case ftp.EntryTypeFolder:
			f.cd(element.Name)
			f.walk(fl)
			f.cdup()
		}
	}
}

func pad(s string, n int) string {
	if n > len(s) {
		return s + strings.Repeat(" ", n-len(s))
	}
	return s
}

func truncPad(s string, n int, side byte) string {
	if len(s) > n {
		if n >= 3 {
			return "..." + s[len(s)-n+3:len(s)]
		}
		return s[len(s)-n : len(s)]
	}
	if side == 'r' {
		return strings.Repeat(" ", n-len(s)) + s
	}
	return s + strings.Repeat(" ", n-len(s))
}

func acceptFileName(fileName string) bool {
	if fileMask.MatchString(fileName) {
		return true
	}
	return false
}

func newFileEntry(entry *ftp.Entry) fileEntry {
	file := fileEntry{}
	file.Name = entry.Name
	file.Size = entry.Size
	file.Time = entry.Time
	file.Found = true
	return file
}

type fileEntry struct {
	Name  string
	Size  uint64
	Time  time.Time
	Found bool
}

func (fe *fileEntry) pack() string {
	return fe.Name + "?|" + fmt.Sprintf("%v", fe.Size) + "?|" + fe.Time.String()
}

type tFileList struct {
	Loggerer
	file map[string]fileEntry
}

func newFileList() *tFileList {
	return &tFileList{file: map[string]fileEntry{}}
}

func (fl *tFileList) pack() string {
	output := []string{}
	for key, value := range fl.file {
		output = append(output, "?{"+key+"?}"+value.pack()+"\n")
	}
	sort.Strings(output)
	return strings.Join(output, "")
}

func (fl *tFileList) clean() {
	for key, value := range fl.file {
		if !value.Found {
			delete(fl.file, key)
			fl.Log("- " + truncPad(key, 40, 'l') + " deleted")
		} else {
			value.Found = false
		}
	}
}

func (fl tFileList) String() string {
	return fl.pack()
}

func (fl *tFileList) save(filepath string) {
	file, err := os.Create(filepath)
	if err != nil {
		log.Panicln(err)
	}
	defer file.Close()

	_, err = io.Copy(file, strings.NewReader(fl.pack()))
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) load(filepath string) {
	fl.Log("Loading \"" + filepath + "\"...")
	file, err := os.Open(filepath)
	if err != nil {
		fl.Log("\"" + filepath + "\" not found")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		key, entry := fl.parseLine(scanner.Text())
		if key != "" {
			fl.file[key] = entry
		}
	}

	err = scanner.Err()
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) parseLine(line string) (string, fileEntry) {
	// "?{/AMEDIATEKA/ANIMALS_2/SER_05620.mxf?}SER_05620.mxf?|13114515508?|2017-03-17 14:39:39 +0000 UTC"
	if !regExpLine.MatchString(line) {
		fl.Log("ERROR: Wrong input in file list (" + line + ")")
		return "", fileEntry{}
	}
	matches := regExpLine.FindStringSubmatch(line)
	key := matches[1]
	entry := fileEntry{}
	entry.Name = matches[2]
	entrySize, err := strconv.Atoi(matches[3])
	entry.Size = uint64(entrySize)
	entry.Time, err = time.Parse("2006-01-02 15:04:05 +0000 UTC", matches[4])
	if err != nil {
		fl.Log(err)
		return "", fileEntry{}
	}
	return key, entry
}

// ILogger ...
type ILogger interface {
	Log(text ...interface{})
}

type logger struct {
	file   *os.File
	logger *log.Logger
}

func newLogger() *logger {
	l := &logger{}
	file, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
	if err != nil {
		log.Panicln(err)
	}
	l.file = file
	l.logger = log.New(file, "", log.LstdFlags)
	return l
}

func (l *logger) close() {
	l.file.Close()
}

func (l *logger) Log(text ...interface{}) {
	log.Println(text...)
	l.logger.Print(text...)
}

// Global variables are set in private file.
// Ftp server address with port.
// var addr = ""
// var user = ""
// var pass = ""

var regExpLine = regexp.MustCompile(`\?\{(.*)\?\}(.*)\?\|(\d+)\?\|(.*)$`)
var logFilePath = "shuher.log"
var fileListPath = "shuherFileList.txt"
var watcherRootPath = "/AMEDIATEKA"
var fileMask = regexp.MustCompile(`^.*\.mxf$`)
var lastLine string
var longSleepTime = 15 * time.Minute
var shortSleepTime = 1 * time.Minute

func main() {
	// Create objects.
	logger := newLogger()
	defer logger.close()
	ftpConn := newFtpConn()
	ftpConn.SetLogger(logger)
	fileList := newFileList()
	fileList.SetLogger(logger)
	// Load file list.
	fileList.load(fileListPath)
	// Properly close the connection on exit.
	defer ftpConn.quit()

	for {
		// Initialize the connection to the specified ftp server address.
		ftpConn.dial(addr)
		// Authenticate the client with specified user and password.
		ftpConn.login(user, pass)
		// Change directory to watcherRootPath.
		ftpConn.cd(watcherRootPath)
		// Walk the directory tree.
		if ftpConn.GetError() == nil {
			logger.Log("Looking for new files...")
			ftpConn.walk(fileList.file)
		}
		// Terminate the FTP connection.
		ftpConn.quit()
		// Remove deleted files from the fileList.
		fileList.clean()
		// Save new fileList.
		if ftpConn.GetError() == nil {
			fileList.save(fileListPath)
			// Wait for sleepTime before checking again.
			time.Sleep(longSleepTime)
		} else {
			// Wait for sleepTime before checking again.
			time.Sleep(shortSleepTime)
		}
	}
}
<commit_msg>Add logger<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/jlaffaye/ftp"
)

// Loggerer ...
type Loggerer struct {
	logger ILogger
	err    error
}

// SetLogger sets objects logger
func (l *Loggerer) SetLogger(logger ILogger) {
	l.logger = logger
}

// Log logs the input text
func (l *Loggerer) Log(loglevel int, text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(loglevel, text...)
	}
}

// Error ...
func (l *Loggerer) Error(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(Error, "ERROR: "+fmt.Sprint(text...))
	}
	if l.err == nil {
		l.err = errors.New(fmt.Sprint(text...))
	}
}

// ResetError ...
func (l *Loggerer) ResetError() {
	l.err = nil
}

// GetError ...
func (l *Loggerer) GetError() error {
	return l.err
}

type ftpConn struct {
	Loggerer
	conn      *ftp.ServerConn
	connected bool
}

func newFtpConn() *ftpConn {
	return &ftpConn{}
}

func (f *ftpConn) dial(addr string) {
	f.ResetError()
	conn, err := ftp.Dial(addr)
	if err != nil {
		f.Error(err)
		return
	}
	f.conn = conn
	f.connected = true
	f.Log(Debug, "Connected to "+addr)
}

func (f *ftpConn) quit() {
	if !f.connected {
		return
	}
	f.conn.Quit()
	f.connected = false
	f.Log(Debug, "Connection closed correctly")
}

func (f *ftpConn) login(user string, pass string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.Login(user, pass)
	if err != nil {
		f.Error(err)
		return
	}
	f.Log(Debug, "Logged in as "+user)
}

func (f *ftpConn) cwd() string {
	if f.GetError() != nil {
		return ""
	}
	cwd, err := f.conn.CurrentDir()
	if err != nil {
		f.Error(err)
		return ""
	}
	return cwd
}

func (f *ftpConn) cd(path string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.ChangeDir(path)
	if err != nil {
		f.Error(err)
		return
	}
}

func (f *ftpConn) cdup() {
	if f.GetError() != nil {
		return
	}
	f.conn.ChangeDirToParent()
}

func (f *ftpConn) ls(path string) (entries []*ftp.Entry) {
	if f.GetError() != nil {
		return
	}
	entries, err := f.conn.List(path)
	if err != nil {
		f.Error(err)
		return
	}
	return entries
}

func (f *ftpConn) walk(fl map[string]fileEntry) {
	if f.GetError() != nil {
		return
	}
	entries := f.ls("")
	cwd := f.cwd()
	newLine := pad(cwd, len(lastLine))
	fmt.Print(newLine + "\r")
	lastLine = cwd
	for _, element := range entries {
		switch element.Type {
		case ftp.EntryTypeFile:
			if acceptFileName(element.Name) {
				key := cwd + "/" + element.Name
				entry, fileExists := fl[key]
				if fileExists {
					// Old file with new date
					if !entry.Time.Equal(element.Time) {
						f.Log(Notice, "~ "+truncPad(key, 40, 'l')+" datetime changed")
						fl[key] = newFileEntry(element)
					} else if entry.Size != element.Size {
						f.Log(Notice, "~ "+truncPad(key, 40, 'l')+" size changed")
						fl[key] = newFileEntry(element)
					} else {
						entry.Found = true
						fl[key] = entry
					}
				} else {
					// New file
					f.Log(Notice, "+ "+truncPad(key, 40, 'l')+" new file")
					fl[key] = newFileEntry(element)
				}
			}
		case ftp.EntryTypeFolder:
			f.cd(element.Name)
			f.walk(fl)
			f.cdup()
		}
	}
}

func pad(s string, n int) string {
	if n > len(s) {
		return s + strings.Repeat(" ", n-len(s))
	}
	return s
}

func truncPad(s string, n int, side byte) string {
	if len(s) > n {
		if n >= 3 {
			return "..." + s[len(s)-n+3:len(s)]
		}
		return s[len(s)-n : len(s)]
	}
	if side == 'r' {
		return strings.Repeat(" ", n-len(s)) + s
	}
	return s + strings.Repeat(" ", n-len(s))
}

func acceptFileName(fileName string) bool {
	if fileMask.MatchString(fileName) {
		return true
	}
	return false
}

func newFileEntry(entry *ftp.Entry) fileEntry {
	file := fileEntry{}
	file.Name = entry.Name
	file.Size = entry.Size
	file.Time = entry.Time
	file.Found = true
	return file
}

type fileEntry struct {
	Name  string
	Size  uint64
	Time  time.Time
	Found bool
}

func (fe *fileEntry) pack() string {
	return fe.Name + "?|" + fmt.Sprintf("%v", fe.Size) + "?|" + fe.Time.String()
}

type tFileList struct {
	Loggerer
	file map[string]fileEntry
}

func newFileList() *tFileList {
	return &tFileList{file: map[string]fileEntry{}}
}

func (fl *tFileList) pack() string {
	output := []string{}
	for key, value := range fl.file {
		output = append(output, "?{"+key+"?}"+value.pack()+"\n")
	}
	sort.Strings(output)
	return strings.Join(output, "")
}

func (fl *tFileList) clean() {
	for key, value := range fl.file {
		if !value.Found {
			delete(fl.file, key)
			fl.Log(Notice, "- "+truncPad(key, 40, 'l')+" deleted")
		} else {
			value.Found = false
		}
	}
}

func (fl tFileList) String() string {
	return fl.pack()
}

func (fl *tFileList) save(filepath string) {
	file, err := os.Create(filepath)
	if err != nil {
		log.Panicln(err)
	}
	defer file.Close()

	_, err = io.Copy(file, strings.NewReader(fl.pack()))
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) load(filepath string) {
	fl.Log(Debug, "Loading \""+filepath+"\"...")
	file, err := os.Open(filepath)
	if err != nil {
		fl.Log(Error, "\""+filepath+"\" not found")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		key, entry := fl.parseLine(scanner.Text())
		if key != "" {
			fl.file[key] = entry
		}
	}

	err = scanner.Err()
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) parseLine(line string) (string, fileEntry) {
	// "?{/AMEDIATEKA/ANIMALS_2/SER_05620.mxf?}SER_05620.mxf?|13114515508?|2017-03-17 14:39:39 +0000 UTC"
	if !regExpLine.MatchString(line) {
		fl.Log(Error, "Wrong input in file list ("+line+")")
		return "", fileEntry{}
	}
	matches := regExpLine.FindStringSubmatch(line)
	key := matches[1]
	entry := fileEntry{}
	entry.Name = matches[2]
	entrySize, err := strconv.Atoi(matches[3])
	entry.Size = uint64(entrySize)
	entry.Time, err = time.Parse("2006-01-02 15:04:05 +0000 UTC", matches[4])
	if err != nil {
		fl.Log(Error, err)
		return "", fileEntry{}
	}
	return key, entry
}

// ILogger ...
type ILogger interface {
	Log(loglevel int, text ...interface{})
}

type logger struct {
	writers []tWriter
}

type tWriter struct {
	writer   io.Writer
	loglevel int
}

func newLogger() *logger {
	return &logger{}
}

func (l *logger) addLogger(loglevel int, writer io.Writer) {
	l.writers = append(l.writers, tWriter{writer, loglevel})
}

func (l *logger) Log(loglevel int, text ...interface{}) {
	for _, writer := range l.writers {
		if loglevel&writer.loglevel != 0 {
			_, err := writer.writer.Write([]byte(time.Now().Format("2006-01-02 15:04:05") + " " + logLeveltoStr(loglevel) + ": " + fmt.Sprint(text...) + "\n"))
			if err != nil {
				fmt.Println(err)
			}
		}
	}
	if loglevel&Panic != 0 {
		panic(fmt.Sprint(text...))
	}
}

// LogLevel flags
const (
	Quiet = 0
	Panic = 1 << iota
	Error
	Warning
	Notice
	Info
	Debug
)

func logLevelLeq(loglevel int) int {
	return loglevel - 1 | loglevel
}

func logLeveltoStr(loglevel int) string {
	s := []string{}
	if loglevel&Panic != 0 {
		s = append(s, "PNC")
	}
	if loglevel&Error != 0 {
		s = append(s, "ERR")
	}
	if loglevel&Warning != 0 {
		s = append(s, "WRN")
	}
	if loglevel&Notice != 0 {
		s = append(s, "NTC")
	}
	if loglevel&Info != 0 {
		s = append(s, "INF")
	}
	if loglevel&Debug != 0 {
		s = append(s, "DBG")
	}
	return strings.Join(s, "|")
}

func newFileWriter(path string) *os.File {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
	if err != nil {
		log.Panicln(err)
	}
	return file
}

// Global variables are set in private file.
// Ftp server address with port.
// var addr = ""
// var user = ""
// var pass = ""

var regExpLine = regexp.MustCompile(`\?\{(.*)\?\}(.*)\?\|(\d+)\?\|(.*)$`)
var logFilePath = "shuher.log"
var fileListPath = "shuherFileList.txt"
var watcherRootPath = "/AMEDIATEKA"
var fileMask = regexp.MustCompile(`^.*\.mxf$`)
var lastLine string
var longSleepTime = 15 * time.Minute
var shortSleepTime = 1 * time.Minute

func main() {
	// Create objects.
	fileWriter := newFileWriter(logFilePath)
	defer fileWriter.Close()
	logger := newLogger()
	logger.addLogger(logLevelLeq(Debug), fileWriter)
	logger.addLogger(logLevelLeq(Info), os.Stdout)
	ftpConn := newFtpConn()
	ftpConn.SetLogger(logger)
	fileList := newFileList()
	fileList.SetLogger(logger)
	// Load file list.
	fileList.load(fileListPath)
	// Properly close the connection on exit.
	defer ftpConn.quit()

	for {
		// Initialize the connection to the specified ftp server address.
		ftpConn.dial(addr)
		// Authenticate the client with specified user and password.
		ftpConn.login(user, pass)
		// Change directory to watcherRootPath.
		ftpConn.cd(watcherRootPath)
		// Walk the directory tree.
		if ftpConn.GetError() == nil {
			logger.Log(Info, "Looking for new files...")
			ftpConn.walk(fileList.file)
			fmt.Print(pad("", len(lastLine)) + "\r")
		}
		// Terminate the FTP connection.
		ftpConn.quit()
		// Remove deleted files from the fileList.
		fileList.clean()
		// Save new fileList.
		if ftpConn.GetError() == nil {
			fileList.save(fileListPath)
			// Wait for sleepTime before checking again.
			time.Sleep(longSleepTime)
		} else {
			// Wait for sleepTime before checking again.
			time.Sleep(shortSleepTime)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x86

import (
	"cmd/compile/internal/gc"
	"cmd/internal/obj"
	"cmd/internal/obj/x86"
)

func defframe(ptxt *obj.Prog) {
	// fill in argument size, stack size
	ptxt.To.Type = obj.TYPE_TEXTSIZE

	ptxt.To.Val = int32(gc.Rnd(gc.Curfn.Type.ArgWidth(), int64(gc.Widthptr)))
	frame := uint32(gc.Rnd(gc.Stksize+gc.Maxarg, int64(gc.Widthreg)))
	ptxt.To.Offset = int64(frame)

	// insert code to zero ambiguously live variables
	// so that the garbage collector only sees initialized values
	// when it looks for pointers.
	p := ptxt

	hi := int64(0)
	lo := hi
	ax := uint32(0)
	for _, n := range gc.Curfn.Func.Dcl {
		if !n.Name.Needzero {
			continue
		}
		if n.Class != gc.PAUTO {
			gc.Fatalf("needzero class %d", n.Class)
		}
		if n.Type.Width%int64(gc.Widthptr) != 0 || n.Xoffset%int64(gc.Widthptr) != 0 || n.Type.Width == 0 {
			gc.Fatalf("var %L has size %d offset %d", n, int(n.Type.Width), int(n.Xoffset))
		}
		if lo != hi && n.Xoffset+n.Type.Width == lo-int64(2*gc.Widthptr) {
			// merge with range we already have
			lo = n.Xoffset

			continue
		}

		// zero old range
		p = zerorange(p, int64(frame), lo, hi, &ax)

		// set new range
		hi = n.Xoffset + n.Type.Width

		lo = n.Xoffset
	}

	// zero final range
	zerorange(p, int64(frame), lo, hi, &ax)
}

func zerorange(p *obj.Prog, frame int64, lo int64, hi int64, ax *uint32) *obj.Prog {
	cnt := hi - lo
	if cnt == 0 {
		return p
	}
	if *ax == 0 {
		p = gc.Appendpp(p, x86.AMOVL, obj.TYPE_CONST, 0, 0, obj.TYPE_REG, x86.REG_AX, 0)
		*ax = 1
	}

	if cnt <= int64(4*gc.Widthreg) {
		for i := int64(0); i < cnt; i += int64(gc.Widthreg) {
			p = gc.Appendpp(p, x86.AMOVL, obj.TYPE_REG, x86.REG_AX, 0, obj.TYPE_MEM, x86.REG_SP, frame+lo+i)
		}
	} else if !gc.Nacl && cnt <= int64(128*gc.Widthreg) {
		p = gc.Appendpp(p, x86.ALEAL, obj.TYPE_MEM, x86.REG_SP, frame+lo, obj.TYPE_REG, x86.REG_DI, 0)
		p = gc.Appendpp(p, obj.ADUFFZERO, obj.TYPE_NONE, 0, 0, obj.TYPE_ADDR, 0, 1*(128-cnt/int64(gc.Widthreg)))
		p.To.Sym = gc.Linksym(gc.Pkglookup("duffzero", gc.Runtimepkg))
	} else {
		p = gc.Appendpp(p, x86.AMOVL, obj.TYPE_CONST, 0, cnt/int64(gc.Widthreg), obj.TYPE_REG, x86.REG_CX, 0)
		p = gc.Appendpp(p, x86.ALEAL, obj.TYPE_MEM, x86.REG_SP, frame+lo, obj.TYPE_REG, x86.REG_DI, 0)
		p = gc.Appendpp(p, x86.AREP, obj.TYPE_NONE, 0, 0, obj.TYPE_NONE, 0, 0)
		p = gc.Appendpp(p, x86.ASTOSL, obj.TYPE_NONE, 0, 0, obj.TYPE_NONE, 0, 0)
	}

	return p
}

func ginsnop() {
	p := gc.Prog(x86.AXCHGL)
	p.From.Type = obj.TYPE_REG
	p.From.Reg = x86.REG_AX
	p.To.Type = obj.TYPE_REG
	p.To.Reg = x86.REG_AX
}
<commit_msg>cmd/compile/internal/x86: fix zero-range merging logic<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x86

import (
	"cmd/compile/internal/gc"
	"cmd/internal/obj"
	"cmd/internal/obj/x86"
)

func defframe(ptxt *obj.Prog) {
	// fill in argument size, stack size
	ptxt.To.Type = obj.TYPE_TEXTSIZE

	ptxt.To.Val = int32(gc.Rnd(gc.Curfn.Type.ArgWidth(), int64(gc.Widthptr)))
	frame := uint32(gc.Rnd(gc.Stksize+gc.Maxarg, int64(gc.Widthreg)))
	ptxt.To.Offset = int64(frame)

	// insert code to zero ambiguously live variables
	// so that the garbage collector only sees initialized values
	// when it looks for pointers.
	p := ptxt

	hi := int64(0)
	lo := hi
	ax := uint32(0)
	for _, n := range gc.Curfn.Func.Dcl {
		if !n.Name.Needzero {
			continue
		}
		if n.Class != gc.PAUTO {
			gc.Fatalf("needzero class %d", n.Class)
		}
		if n.Type.Width%int64(gc.Widthptr) != 0 || n.Xoffset%int64(gc.Widthptr) != 0 || n.Type.Width == 0 {
			gc.Fatalf("var %L has size %d offset %d", n, int(n.Type.Width), int(n.Xoffset))
		}
		if lo != hi && n.Xoffset+n.Type.Width >= lo-int64(2*gc.Widthptr) {
			// merge with range we already have
			lo = n.Xoffset

			continue
		}

		// zero old range
		p = zerorange(p, int64(frame), lo, hi, &ax)

		// set new range
		hi = n.Xoffset + n.Type.Width

		lo = n.Xoffset
	}

	// zero final range
	zerorange(p, int64(frame), lo, hi, &ax)
}

func zerorange(p *obj.Prog, frame int64, lo int64, hi int64, ax *uint32) *obj.Prog {
	cnt := hi - lo
	if cnt == 0 {
		return p
	}
	if *ax == 0 {
		p = gc.Appendpp(p, x86.AMOVL, obj.TYPE_CONST, 0, 0, obj.TYPE_REG, x86.REG_AX, 0)
		*ax = 1
	}

	if cnt <= int64(4*gc.Widthreg) {
		for i := int64(0); i < cnt; i += int64(gc.Widthreg) {
			p = gc.Appendpp(p, x86.AMOVL, obj.TYPE_REG, x86.REG_AX, 0, obj.TYPE_MEM, x86.REG_SP, frame+lo+i)
		}
	} else if !gc.Nacl && cnt <= int64(128*gc.Widthreg) {
		p = gc.Appendpp(p, x86.ALEAL, obj.TYPE_MEM, x86.REG_SP, frame+lo, obj.TYPE_REG, x86.REG_DI, 0)
		p = gc.Appendpp(p, obj.ADUFFZERO, obj.TYPE_NONE, 0, 0, obj.TYPE_ADDR, 0, 1*(128-cnt/int64(gc.Widthreg)))
		p.To.Sym = gc.Linksym(gc.Pkglookup("duffzero", gc.Runtimepkg))
	} else {
		p = gc.Appendpp(p, x86.AMOVL, obj.TYPE_CONST, 0, cnt/int64(gc.Widthreg), obj.TYPE_REG, x86.REG_CX, 0)
		p = gc.Appendpp(p, x86.ALEAL, obj.TYPE_MEM, x86.REG_SP, frame+lo, obj.TYPE_REG, x86.REG_DI, 0)
		p = gc.Appendpp(p, x86.AREP, obj.TYPE_NONE, 0, 0, obj.TYPE_NONE, 0, 0)
		p = gc.Appendpp(p, x86.ASTOSL, obj.TYPE_NONE, 0, 0, obj.TYPE_NONE, 0, 0)
	}

	return p
}

func ginsnop() {
	p := gc.Prog(x86.AXCHGL)
	p.From.Type = obj.TYPE_REG
	p.From.Reg = x86.REG_AX
	p.To.Type = obj.TYPE_REG
	p.To.Reg = x86.REG_AX
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ld

import (
	"cmd/internal/obj"
	"cmd/internal/sys"
	"fmt"
	"strings"
	"unicode"
)

// deadcode marks all reachable symbols.
//
// The basis of the dead code elimination is a flood fill of symbols,
// following their relocations, beginning at *flagEntrySymbol.
//
// This flood fill is wrapped in logic for pruning unused methods.
// All methods are mentioned by relocations on their receiver's *rtype.
// These relocations are specially defined as R_METHODOFF by the compiler
// so we can detect and manipulated them here.
//
// There are three ways a method of a reachable type can be invoked:
//
//	1. direct call
//	2. through a reachable interface type
//	3. reflect.Value.Call, .Method, or reflect.Method.Func
//
// The first case is handled by the flood fill, a directly called method
// is marked as reachable.
//
// The second case is handled by decomposing all reachable interface
// types into method signatures. Each encountered method is compared
// against the interface method signatures, if it matches it is marked
// as reachable. This is extremely conservative, but easy and correct.
//
// The third case is handled by looking to see if any of:
//	- reflect.Value.Call is reachable
//	- reflect.Value.Method is reachable
// 	- reflect.Type.Method or MethodByName is called.
// If any of these happen, all bets are off and all exported methods
// of reachable types are marked reachable.
//
// Any unreached text symbols are removed from ctxt.Textp.
func deadcode(ctxt *Link) {
	if ctxt.Debugvlog != 0 {
		ctxt.Logf("%5.2f deadcode\n", obj.Cputime())
	}

	d := &deadcodepass{
		ctxt:        ctxt,
		ifaceMethod: make(map[methodsig]bool),
	}

	// First, flood fill any symbols directly reachable in the call
	// graph from *flagEntrySymbol. Ignore all methods not directly called.
	d.init()
	d.flood()

	callSym := ctxt.Syms.ROLookup("reflect.Value.Call", 0)
	methSym := ctxt.Syms.ROLookup("reflect.Value.Method", 0)
	reflectSeen := false

	if ctxt.DynlinkingGo() {
		// Exported methods may satisfy interfaces we don't know
		// about yet when dynamically linking.
		reflectSeen = true
	}

	for {
		if !reflectSeen {
			if d.reflectMethod || (callSym != nil && callSym.Attr.Reachable()) || (methSym != nil && methSym.Attr.Reachable()) {
				// Methods might be called via reflection. Give up on
				// static analysis, mark all exported methods of
				// all reachable types as reachable.
				reflectSeen = true
			}
		}

		// Mark all methods that could satisfy a discovered
		// interface as reachable. We recheck old marked interfaces
		// as new types (with new methods) may have been discovered
		// in the last pass.
		var rem []methodref
		for _, m := range d.markableMethods {
			if (reflectSeen && m.isExported()) || d.ifaceMethod[m.m] {
				d.markMethod(m)
			} else {
				rem = append(rem, m)
			}
		}
		d.markableMethods = rem

		if len(d.markQueue) == 0 {
			// No new work was discovered. Done.
			break
		}
		d.flood()
	}

	// Remove all remaining unreached R_METHODOFF relocations.
	for _, m := range d.markableMethods {
		for _, r := range m.r {
			d.cleanupReloc(r)
		}
	}

	if Buildmode != BuildmodeShared {
		// Keep a itablink if the symbol it points at is being kept.
		// (When BuildmodeShared, always keep itablinks.)
		for _, s := range ctxt.Syms.Allsym {
			if strings.HasPrefix(s.Name, "go.itablink.") {
				s.Attr.Set(AttrReachable, len(s.R) == 1 && s.R[0].Sym.Attr.Reachable())
			}
		}
	}

	// Remove dead text but keep file information (z symbols).
	textp := make([]*Symbol, 0, len(ctxt.Textp))
	for _, s := range ctxt.Textp {
		if s.Attr.Reachable() {
			textp = append(textp, s)
		}
	}
	ctxt.Textp = textp
}

var markextra = []string{
	"runtime.morestack",
	"runtime.morestackx",
	"runtime.morestack00",
	"runtime.morestack10",
	"runtime.morestack01",
	"runtime.morestack11",
	"runtime.morestack8",
	"runtime.morestack16",
	"runtime.morestack24",
	"runtime.morestack32",
	"runtime.morestack40",
	"runtime.morestack48",

	// on arm, lock in the div/mod helpers too
	"_div",
	"_divu",
	"_mod",
	"_modu",
}

// methodref holds the relocations from a receiver type symbol to its
// method. There are three relocations, one for each of the fields in
// the reflect.method struct: mtyp, ifn, and tfn.
type methodref struct {
	m   methodsig
	src *Symbol   // receiver type symbol
	r   [3]*Reloc // R_METHODOFF relocations to fields of runtime.method
}

func (m methodref) ifn() *Symbol { return m.r[1].Sym }

func (m methodref) isExported() bool {
	for _, r := range m.m {
		return unicode.IsUpper(r)
	}
	panic("methodref has no signature")
}

// deadcodepass holds state for the deadcode flood fill.
type deadcodepass struct {
	ctxt            *Link
	markQueue       []*Symbol          // symbols to flood fill in next pass
	ifaceMethod     map[methodsig]bool // methods declared in reached interfaces
	markableMethods []methodref        // methods of reached types
	reflectMethod   bool
}

func (d *deadcodepass) cleanupReloc(r *Reloc) {
	if r.Sym.Attr.Reachable() {
		r.Type = obj.R_ADDROFF
	} else {
		if d.ctxt.Debugvlog > 1 {
			d.ctxt.Logf("removing method %s\n", r.Sym.Name)
		}
		r.Sym = nil
		r.Siz = 0
	}
}

// mark appends a symbol to the mark queue for flood filling.
func (d *deadcodepass) mark(s, parent *Symbol) {
	if s == nil || s.Attr.Reachable() {
		return
	}
	if s.Attr.ReflectMethod() {
		d.reflectMethod = true
	}
	if *flagDumpDep {
		p := "_"
		if parent != nil {
			p = parent.Name
		}
		fmt.Printf("%s -> %s\n", p, s.Name)
	}
	s.Attr |= AttrReachable
	s.Reachparent = parent
	d.markQueue = append(d.markQueue, s)
}

// markMethod marks a method as reachable.
func (d *deadcodepass) markMethod(m methodref) {
	for _, r := range m.r {
		d.mark(r.Sym, m.src)
		r.Type = obj.R_ADDROFF
	}
}

// init marks all initial symbols as reachable.
// In a typical binary, this is *flagEntrySymbol.
func (d *deadcodepass) init() {
	var names []string

	if SysArch.Family == sys.ARM {
		// mark some functions that are only referenced after linker code editing
		if obj.GOARM == 5 {
			names = append(names, "_sfloat")
		}
		names = append(names, "runtime.read_tls_fallback")
	}

	if Buildmode == BuildmodeShared {
		// Mark all symbols defined in this library as reachable when
		// building a shared library.
		for _, s := range d.ctxt.Syms.Allsym {
			if s.Type != 0 && s.Type != obj.SDYNIMPORT {
				d.mark(s, nil)
			}
		}
	} else {
		// In a normal binary, start at main.main and the init
		// functions and mark what is reachable from there.
		names = append(names, *flagEntrySymbol)
		if *FlagLinkshared && (Buildmode == BuildmodeExe || Buildmode == BuildmodePIE) {
			names = append(names, "main.main", "main.init")
		} else if Buildmode == BuildmodePlugin {
			names = append(names, *flagPluginPath+".init", *flagPluginPath+".main", "go.plugin.tabs")

			// We don't keep the go.plugin.exports symbol,
			// but we do keep the symbols it refers to.
			exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0)
			if exports != nil {
				for _, r := range exports.R {
					d.mark(r.Sym, nil)
				}
			}
		}
		names = append(names, markextra...)
		for _, s := range dynexp {
			d.mark(s, nil)
		}
	}

	for _, name := range names {
		d.mark(d.ctxt.Syms.ROLookup(name, 0), nil)
	}
}

// flood flood fills symbols reachable from the markQueue symbols.
// As it goes, it collects methodref and interface method declarations.
func (d *deadcodepass) flood() {
	for len(d.markQueue) > 0 {
		s := d.markQueue[0]
		d.markQueue = d.markQueue[1:]
		if s.Type == obj.STEXT {
			if d.ctxt.Debugvlog > 1 {
				d.ctxt.Logf("marktext %s\n", s.Name)
			}
			if s.FuncInfo != nil {
				for _, a := range s.FuncInfo.Autom {
					d.mark(a.Gotype, s)
				}
			}

		}

		if strings.HasPrefix(s.Name, "type.") && s.Name[5] != '.' {
			if len(s.P) == 0 {
				// Probably a bug. The undefined symbol check
				// later will give a better error than deadcode.
				continue
			}
			if decodetypeKind(s)&kindMask == kindInterface {
				for _, sig := range decodeIfaceMethods(d.ctxt.Arch, s) {
					if d.ctxt.Debugvlog > 1 {
						d.ctxt.Logf("reached iface method: %s\n", sig)
					}
					d.ifaceMethod[sig] = true
				}
			}
		}

		mpos := 0 // 0-3, the R_METHODOFF relocs of runtime.uncommontype
		var methods []methodref
		for i := 0; i < len(s.R); i++ {
			r := &s.R[i]
			if r.Sym == nil {
				continue
			}
			if r.Type == obj.R_WEAKADDROFF {
				// An R_WEAKADDROFF relocation is not reason
				// enough to mark the pointed-to symbol as
				// reachable.
				continue
			}
			if r.Type != obj.R_METHODOFF {
				d.mark(r.Sym, s)
				continue
			}
			// Collect rtype pointers to methods for
			// later processing in deadcode.
			if mpos == 0 {
				m := methodref{src: s}
				m.r[0] = r
				methods = append(methods, m)
			} else {
				methods[len(methods)-1].r[mpos] = r
			}
			mpos++
			if mpos == len(methodref{}.r) {
				mpos = 0
			}
		}
		if len(methods) > 0 {
			// Decode runtime type information for type methods
			// to help work out which methods can be called
			// dynamically via interfaces.
			methodsigs := decodetypeMethods(d.ctxt.Arch, s)
			if len(methods) != len(methodsigs) {
				panic(fmt.Sprintf("%q has %d method relocations for %d methods", s.Name, len(methods), len(methodsigs)))
			}
			for i, m := range methodsigs {
				name := string(m)
				name = name[:strings.Index(name, "(")]
				if !strings.HasSuffix(methods[i].ifn().Name, name) {
					panic(fmt.Sprintf("%q relocation for %q does not match method %q", s.Name, methods[i].ifn().Name, name))
				}
				methods[i].m = m
			}
			d.markableMethods = append(d.markableMethods, methods...)
		}

		if s.FuncInfo != nil {
			for i := range s.FuncInfo.Funcdata {
				d.mark(s.FuncInfo.Funcdata[i], s)
			}
		}
		d.mark(s.Gotype, s)
		d.mark(s.Sub, s)
		d.mark(s.Outer, s)
	}
}
<commit_msg>cmd/link: eliminate markextra<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ld

import (
	"cmd/internal/obj"
	"cmd/internal/sys"
	"fmt"
	"strings"
	"unicode"
)

// deadcode marks all reachable symbols.
//
// The basis of the dead code elimination is a flood fill of symbols,
// following their relocations, beginning at *flagEntrySymbol.
//
// This flood fill is wrapped in logic for pruning unused methods.
// All methods are mentioned by relocations on their receiver's *rtype.
// These relocations are specially defined as R_METHODOFF by the compiler
// so we can detect and manipulated them here.
//
// There are three ways a method of a reachable type can be invoked:
//
//	1. direct call
//	2. through a reachable interface type
//	3. reflect.Value.Call, .Method, or reflect.Method.Func
//
// The first case is handled by the flood fill, a directly called method
// is marked as reachable.
//
// The second case is handled by decomposing all reachable interface
// types into method signatures. Each encountered method is compared
// against the interface method signatures, if it matches it is marked
// as reachable. This is extremely conservative, but easy and correct.
//
// The third case is handled by looking to see if any of:
//	- reflect.Value.Call is reachable
//	- reflect.Value.Method is reachable
// 	- reflect.Type.Method or MethodByName is called.
// If any of these happen, all bets are off and all exported methods
// of reachable types are marked reachable.
//
// Any unreached text symbols are removed from ctxt.Textp.
func deadcode(ctxt *Link) {
	if ctxt.Debugvlog != 0 {
		ctxt.Logf("%5.2f deadcode\n", obj.Cputime())
	}

	d := &deadcodepass{
		ctxt:        ctxt,
		ifaceMethod: make(map[methodsig]bool),
	}

	// First, flood fill any symbols directly reachable in the call
	// graph from *flagEntrySymbol. Ignore all methods not directly called.
	d.init()
	d.flood()

	callSym := ctxt.Syms.ROLookup("reflect.Value.Call", 0)
	methSym := ctxt.Syms.ROLookup("reflect.Value.Method", 0)
	reflectSeen := false

	if ctxt.DynlinkingGo() {
		// Exported methods may satisfy interfaces we don't know
		// about yet when dynamically linking.
		reflectSeen = true
	}

	for {
		if !reflectSeen {
			if d.reflectMethod || (callSym != nil && callSym.Attr.Reachable()) || (methSym != nil && methSym.Attr.Reachable()) {
				// Methods might be called via reflection. Give up on
				// static analysis, mark all exported methods of
				// all reachable types as reachable.
				reflectSeen = true
			}
		}

		// Mark all methods that could satisfy a discovered
		// interface as reachable. We recheck old marked interfaces
		// as new types (with new methods) may have been discovered
		// in the last pass.
		var rem []methodref
		for _, m := range d.markableMethods {
			if (reflectSeen && m.isExported()) || d.ifaceMethod[m.m] {
				d.markMethod(m)
			} else {
				rem = append(rem, m)
			}
		}
		d.markableMethods = rem

		if len(d.markQueue) == 0 {
			// No new work was discovered. Done.
			break
		}
		d.flood()
	}

	// Remove all remaining unreached R_METHODOFF relocations.
	for _, m := range d.markableMethods {
		for _, r := range m.r {
			d.cleanupReloc(r)
		}
	}

	if Buildmode != BuildmodeShared {
		// Keep a itablink if the symbol it points at is being kept.
		// (When BuildmodeShared, always keep itablinks.)
		for _, s := range ctxt.Syms.Allsym {
			if strings.HasPrefix(s.Name, "go.itablink.") {
				s.Attr.Set(AttrReachable, len(s.R) == 1 && s.R[0].Sym.Attr.Reachable())
			}
		}
	}

	// Remove dead text but keep file information (z symbols).
	textp := make([]*Symbol, 0, len(ctxt.Textp))
	for _, s := range ctxt.Textp {
		if s.Attr.Reachable() {
			textp = append(textp, s)
		}
	}
	ctxt.Textp = textp
}

// methodref holds the relocations from a receiver type symbol to its
// method. There are three relocations, one for each of the fields in
// the reflect.method struct: mtyp, ifn, and tfn.
type methodref struct {
	m   methodsig
	src *Symbol   // receiver type symbol
	r   [3]*Reloc // R_METHODOFF relocations to fields of runtime.method
}

func (m methodref) ifn() *Symbol { return m.r[1].Sym }

func (m methodref) isExported() bool {
	for _, r := range m.m {
		return unicode.IsUpper(r)
	}
	panic("methodref has no signature")
}

// deadcodepass holds state for the deadcode flood fill.
type deadcodepass struct {
	ctxt            *Link
	markQueue       []*Symbol          // symbols to flood fill in next pass
	ifaceMethod     map[methodsig]bool // methods declared in reached interfaces
	markableMethods []methodref        // methods of reached types
	reflectMethod   bool
}

func (d *deadcodepass) cleanupReloc(r *Reloc) {
	if r.Sym.Attr.Reachable() {
		r.Type = obj.R_ADDROFF
	} else {
		if d.ctxt.Debugvlog > 1 {
			d.ctxt.Logf("removing method %s\n", r.Sym.Name)
		}
		r.Sym = nil
		r.Siz = 0
	}
}

// mark appends a symbol to the mark queue for flood filling.
func (d *deadcodepass) mark(s, parent *Symbol) {
	if s == nil || s.Attr.Reachable() {
		return
	}
	if s.Attr.ReflectMethod() {
		d.reflectMethod = true
	}
	if *flagDumpDep {
		p := "_"
		if parent != nil {
			p = parent.Name
		}
		fmt.Printf("%s -> %s\n", p, s.Name)
	}
	s.Attr |= AttrReachable
	s.Reachparent = parent
	d.markQueue = append(d.markQueue, s)
}

// markMethod marks a method as reachable.
func (d *deadcodepass) markMethod(m methodref) {
	for _, r := range m.r {
		d.mark(r.Sym, m.src)
		r.Type = obj.R_ADDROFF
	}
}

// init marks all initial symbols as reachable.
// In a typical binary, this is *flagEntrySymbol.
func (d *deadcodepass) init() {
	var names []string

	if SysArch.Family == sys.ARM {
		// mark some functions that are only referenced after linker code editing
		names = append(names, "runtime.read_tls_fallback")
	}

	if Buildmode == BuildmodeShared {
		// Mark all symbols defined in this library as reachable when
		// building a shared library.
		for _, s := range d.ctxt.Syms.Allsym {
			if s.Type != 0 && s.Type != obj.SDYNIMPORT {
				d.mark(s, nil)
			}
		}
	} else {
		// In a normal binary, start at main.main and the init
		// functions and mark what is reachable from there.
		names = append(names, *flagEntrySymbol)
		if *FlagLinkshared && (Buildmode == BuildmodeExe || Buildmode == BuildmodePIE) {
			names = append(names, "main.main", "main.init")
		} else if Buildmode == BuildmodePlugin {
			names = append(names, *flagPluginPath+".init", *flagPluginPath+".main", "go.plugin.tabs")

			// We don't keep the go.plugin.exports symbol,
			// but we do keep the symbols it refers to.
			exports := d.ctxt.Syms.ROLookup("go.plugin.exports", 0)
			if exports != nil {
				for _, r := range exports.R {
					d.mark(r.Sym, nil)
				}
			}
		}
		for _, s := range dynexp {
			d.mark(s, nil)
		}
	}

	for _, name := range names {
		d.mark(d.ctxt.Syms.ROLookup(name, 0), nil)
	}
}

// flood flood fills symbols reachable from the markQueue symbols.
// As it goes, it collects methodref and interface method declarations.
func (d *deadcodepass) flood() {
	for len(d.markQueue) > 0 {
		s := d.markQueue[0]
		d.markQueue = d.markQueue[1:]
		if s.Type == obj.STEXT {
			if d.ctxt.Debugvlog > 1 {
				d.ctxt.Logf("marktext %s\n", s.Name)
			}
			if s.FuncInfo != nil {
				for _, a := range s.FuncInfo.Autom {
					d.mark(a.Gotype, s)
				}
			}

		}

		if strings.HasPrefix(s.Name, "type.") && s.Name[5] != '.' {
			if len(s.P) == 0 {
				// Probably a bug. The undefined symbol check
				// later will give a better error than deadcode.
				continue
			}
			if decodetypeKind(s)&kindMask == kindInterface {
				for _, sig := range decodeIfaceMethods(d.ctxt.Arch, s) {
					if d.ctxt.Debugvlog > 1 {
						d.ctxt.Logf("reached iface method: %s\n", sig)
					}
					d.ifaceMethod[sig] = true
				}
			}
		}

		mpos := 0 // 0-3, the R_METHODOFF relocs of runtime.uncommontype
		var methods []methodref
		for i := 0; i < len(s.R); i++ {
			r := &s.R[i]
			if r.Sym == nil {
				continue
			}
			if r.Type == obj.R_WEAKADDROFF {
				// An R_WEAKADDROFF relocation is not reason
				// enough to mark the pointed-to symbol as
				// reachable.
				continue
			}
			if r.Type != obj.R_METHODOFF {
				d.mark(r.Sym, s)
				continue
			}
			// Collect rtype pointers to methods for
			// later processing in deadcode.
			if mpos == 0 {
				m := methodref{src: s}
				m.r[0] = r
				methods = append(methods, m)
			} else {
				methods[len(methods)-1].r[mpos] = r
			}
			mpos++
			if mpos == len(methodref{}.r) {
				mpos = 0
			}
		}
		if len(methods) > 0 {
			// Decode runtime type information for type methods
			// to help work out which methods can be called
			// dynamically via interfaces.
			methodsigs := decodetypeMethods(d.ctxt.Arch, s)
			if len(methods) != len(methodsigs) {
				panic(fmt.Sprintf("%q has %d method relocations for %d methods", s.Name, len(methods), len(methodsigs)))
			}
			for i, m := range methodsigs {
				name := string(m)
				name = name[:strings.Index(name, "(")]
				if !strings.HasSuffix(methods[i].ifn().Name, name) {
					panic(fmt.Sprintf("%q relocation for %q does not match method %q", s.Name, methods[i].ifn().Name, name))
				}
				methods[i].m = m
			}
			d.markableMethods = append(d.markableMethods, methods...)
		}

		if s.FuncInfo != nil {
			for i := range s.FuncInfo.Funcdata {
				d.mark(s.FuncInfo.Funcdata[i], s)
			}
		}
		d.mark(s.Gotype, s)
		d.mark(s.Sub, s)
		d.mark(s.Outer, s)
	}
}
<|endoftext|>