<commit_before>package cwl

import (
	"fmt"
	//"github.com/davecgh/go-spew/spew"
	"github.com/MG-RAST/AWE/lib/shock"
	//"github.com/davecgh/go-spew/spew"
	"github.com/mitchellh/mapstructure"
	"net/url"
	"strings"
)

// http://www.commonwl.org/v1.0/Workflow.html#File
type File struct {
	CWLType_Impl   `yaml:"-"`
	Id             string         `yaml:"id,omitempty" json:"id"`
	Class          string         `yaml:"class,omitempty" json:"class"`
	Location       string         `yaml:"location,omitempty" json:"location"` // An IRI that identifies the file resource.
	Location_url   *url.URL       `yaml:"-"`                                  // only for internal purposes
	Path           string         `yaml:"path,omitempty" json:"path"`         // dirname + '/' + basename == path This field must be set by the implementation.
	Basename       string         `yaml:"basename,omitempty" json:"basename"` // dirname + '/' + basename == path // if not defined, take from location
	Dirname        string         `yaml:"dirname,omitempty" json:"dirname"`   // dirname + '/' + basename == path
	Nameroot       string         `yaml:"nameroot,omitempty" json:"nameroot"`
	Nameext        string         `yaml:"nameext,omitempty" json:"nameext"`
	Checksum       string         `yaml:"checksum,omitempty" json:"checksum"`
	Size           int32          `yaml:"size,omitempty" json:"size"`
	SecondaryFiles []CWL_location `yaml:"secondaryFiles,omitempty" json:"secondaryFiles"`
	Format         string         `yaml:"format,omitempty" json:"format"`
	Contents       string         `yaml:"contents,omitempty" json:"contents"`
	// Shock node
	Host   string `yaml:"-"`
	Node   string `yaml:"-"`
	Bearer string `yaml:"-"`
	Token  string `yaml:"-"`
}

func (f *File) GetClass() string    { return CWL_File }
func (f *File) GetId() string       { return f.Id }
func (f *File) SetId(id string)     { f.Id = id }
func (f *File) String() string      { return f.Path }
func (f *File) GetLocation() string { return f.Location } // for CWL_location
//func (f *File) Is_Array() bool      { return false }

func (f *File) Is_CommandInputParameterType() {} // for CommandInputParameterType

func NewFile(obj interface{}) (file File, err error) {

	file, err = MakeFile("unknown", obj)

	return
}

func MakeFile(id string, obj interface{}) (file File, err error) {
	file = File{}
	err = mapstructure.Decode(obj, &file)
	if err != nil {
		err = fmt.Errorf("(MakeFile) Could not convert File object %s", id)
		return
	}
	file.Class = CWL_File
	//fmt.Println("MakeFile input:")
	//spew.Dump(obj)
	//fmt.Println("MakeFile output:")
	//fmt.Printf("%+v\n", file)

	if file.Location != "" {

		// example shock://shock.metagenomics.anl.gov/node/92a76f64-d221-4947-9fd0-7106c3b9163a
		file_url, xerr := url.Parse(file.Location)
		if xerr != nil {
			err = fmt.Errorf("Error parsing url %s %s", file.Location, xerr.Error())

			return
		}

		file.Location_url = file_url
		scheme := strings.ToLower(file_url.Scheme)
		values := file_url.Query()

		// TODO add "node/uuid" matching to detect shock

		switch scheme {

		case "http":
			_, has_download := values["download"]
			//extract filename ?
			if has_download && false {

				array := strings.Split(file_url.Path, "/")
				if len(array) != 3 {
					err = fmt.Errorf("shock url cannot be parsed")
					return
				}
				if array[1] != "node" {
					err = fmt.Errorf("node missing in shock url")
					return
				}
				file.Node = array[2]

				file.Host = "http://" + file_url.Host
				shock_client := shock.ShockClient{Host: file.Host} // TODO Token: datatoken
				node, xerr := shock_client.Get_node(file.Node)

				if xerr != nil {
					err = fmt.Errorf("(Get_node) Could not get shock node (%s, %s): %s", file.Host, file.Node, xerr.Error())
					return
				}
				//fmt.Println("---node:")
				//spew.Dump(node)

				if file.Basename == "" {
					// use filename from shocknode
					if node.File.Name != "" {
						fmt.Printf("node.File.Name: %s\n", node.File.Name)
						file.Basename = node.File.Name
					} else {
						// if user does not specify a filename and shock does not, use node_id
						file.Basename = file.Node
					}

				}
			}
		}
	}
	if file.Id == "" {
		file.Id = id
	}
	return
}

type FileArray struct {
	CWLType_Impl
	Id   string `yaml:"id" json:"id"`
	Data []File
}

func (f *FileArray) GetClass() string { return CWL_File_array }

//func (f *FileArray) Is_Array() bool   { return true }

func (f *FileArray) Is_CWL_array_type() {}
func (f *FileArray) Get_Array() *[]File {
	return &f.Data
}

func (f *FileArray) GetId() string   { return f.Id }
func (f *FileArray) SetId(id string) { f.Id = id }

//func (f *FileArray) String() string      { return f.Path }
//func (f *FileArray) GetLocation() string { return f.Location } // for CWL_location

func (s *FileArray) Is_CommandInputParameterType() {} // for CommandInputParameterType
<commit_msg>remove unknown<commit_after>package cwl

import (
	"fmt"
	//"github.com/davecgh/go-spew/spew"
	"github.com/MG-RAST/AWE/lib/shock"
	//"github.com/davecgh/go-spew/spew"
	"github.com/mitchellh/mapstructure"
	"net/url"
	"strings"
)

// http://www.commonwl.org/v1.0/Workflow.html#File
type File struct {
	CWLType_Impl   `yaml:"-"`
	Id             string         `yaml:"id,omitempty" json:"id"`
	Class          string         `yaml:"class,omitempty" json:"class"`
	Location       string         `yaml:"location,omitempty" json:"location"` // An IRI that identifies the file resource.
	Location_url   *url.URL       `yaml:"-"`                                  // only for internal purposes
	Path           string         `yaml:"path,omitempty" json:"path"`         // dirname + '/' + basename == path This field must be set by the implementation.
	Basename       string         `yaml:"basename,omitempty" json:"basename"` // dirname + '/' + basename == path // if not defined, take from location
	Dirname        string         `yaml:"dirname,omitempty" json:"dirname"`   // dirname + '/' + basename == path
	Nameroot       string         `yaml:"nameroot,omitempty" json:"nameroot"`
	Nameext        string         `yaml:"nameext,omitempty" json:"nameext"`
	Checksum       string         `yaml:"checksum,omitempty" json:"checksum"`
	Size           int32          `yaml:"size,omitempty" json:"size"`
	SecondaryFiles []CWL_location `yaml:"secondaryFiles,omitempty" json:"secondaryFiles"`
	Format         string         `yaml:"format,omitempty" json:"format"`
	Contents       string         `yaml:"contents,omitempty" json:"contents"`
	// Shock node
	Host   string `yaml:"-"`
	Node   string `yaml:"-"`
	Bearer string `yaml:"-"`
	Token  string `yaml:"-"`
}

func (f *File) GetClass() string    { return CWL_File }
func (f *File) GetId() string       { return f.Id }
func (f *File) SetId(id string)     { f.Id = id }
func (f *File) String() string      { return f.Path }
func (f *File) GetLocation() string { return f.Location } // for CWL_location
//func (f *File) Is_Array() bool      { return false }

func (f *File) Is_CommandInputParameterType() {} // for CommandInputParameterType

func NewFile(obj interface{}) (file File, err error) {

	file, err = MakeFile("", obj)

	return
}

func MakeFile(id string, obj interface{}) (file File, err error) {
	file = File{}
	err = mapstructure.Decode(obj, &file)
	if err != nil {
		err = fmt.Errorf("(MakeFile) Could not convert File object %s", id)
		return
	}
	file.Class = CWL_File
	//fmt.Println("MakeFile input:")
	//spew.Dump(obj)
	//fmt.Println("MakeFile output:")
	//fmt.Printf("%+v\n", file)

	if file.Location != "" {

		// example shock://shock.metagenomics.anl.gov/node/92a76f64-d221-4947-9fd0-7106c3b9163a
		file_url, xerr := url.Parse(file.Location)
		if xerr != nil {
			err = fmt.Errorf("Error parsing url %s %s", file.Location, xerr.Error())

			return
		}

		file.Location_url = file_url
		scheme := strings.ToLower(file_url.Scheme)
		values := file_url.Query()

		// TODO add "node/uuid" matching to detect shock

		switch scheme {

		case "http":
			_, has_download := values["download"]
			//extract filename ?
			if has_download && false {

				array := strings.Split(file_url.Path, "/")
				if len(array) != 3 {
					err = fmt.Errorf("shock url cannot be parsed")
					return
				}
				if array[1] != "node" {
					err = fmt.Errorf("node missing in shock url")
					return
				}
				file.Node = array[2]

				file.Host = "http://" + file_url.Host
				shock_client := shock.ShockClient{Host: file.Host} // TODO Token: datatoken
				node, xerr := shock_client.Get_node(file.Node)

				if xerr != nil {
					err = fmt.Errorf("(Get_node) Could not get shock node (%s, %s): %s", file.Host, file.Node, xerr.Error())
					return
				}
				//fmt.Println("---node:")
				//spew.Dump(node)

				if file.Basename == "" {
					// use filename from shocknode
					if node.File.Name != "" {
						fmt.Printf("node.File.Name: %s\n", node.File.Name)
						file.Basename = node.File.Name
					} else {
						// if user does not specify a filename and shock does not, use node_id
						file.Basename = file.Node
					}

				}
			}
		}
	}
	if file.Id == "" {
		file.Id = id
	}
	return
}

type FileArray struct {
	CWLType_Impl
	Id   string `yaml:"id" json:"id"`
	Data []File
}

func (f *FileArray) GetClass() string { return CWL_File_array }

//func (f *FileArray) Is_Array() bool   { return true }

func (f *FileArray) Is_CWL_array_type() {}
func (f *FileArray) Get_Array() *[]File {
	return &f.Data
}

func (f *FileArray) GetId() string   { return f.Id }
func (f *FileArray) SetId(id string) { f.Id = id }

//func (f *FileArray) String() string      { return f.Path }
//func (f *FileArray) GetLocation() string { return f.Location } // for CWL_location

func (s *FileArray) Is_CommandInputParameterType() {} // for CommandInputParameterType
<|endoftext|>


<commit_before>package main

import (
	"container/heap"
	"fmt"
	"math/rand"
)

const seed = 0
const numNodes = 50000        // Total number of nodes to simulate
const minNumPeers = 10        // Each node should be connected to at least this many peers
const maxNumPeers = 15        // ... and at most this many
const latencyMS = int32(500)  // One way packet latency
const partTxMS = int32(10)    // Transmission time per peer of 4KB of data.
const sendQueueCapacity = 100 // Amount of messages to queue between peers.

func init() {
	rand.Seed(seed)
}

//-----------------------------------------------------------------------------

type Peer struct {
	node   *Node  // Pointer to node
	sent   int32  // Time of last packet send, including transmit time.
	remote int    // SomeNode.peers[x].node.peers[remote].node is SomeNode for all x.
	parts  []byte // [32]byte{} bitarray of received block pieces.
}

// Send a data event to the peer, or return false if queue is "full".
// Depending on how many event packets are "queued" for peer,
// the actual recvTime may be adjusted to be later.
func (p *Peer) sendEventData(event EventData) bool {
	desiredRecvTime := event.RecvTime()
	minRecvTime := p.sent + partTxMS + latencyMS
	if desiredRecvTime >= minRecvTime {
		p.node.sendEvent(event)
		p.sent += partTxMS
		return true
	} else {
		if (minRecvTime-desiredRecvTime)/partTxMS > sendQueueCapacity {
			return false
		} else {
			event.SetRecvTime(minRecvTime) // Adjust recvTime
			p.node.sendEvent(event)
			p.sent += partTxMS
			return true
		}
	}
}

// Since EventPart events are much smaller, we don't consider the transmit time,
// and assume that the sendQueue is always free.
func (p *Peer) sendEventParts(event EventParts) {
	p.node.sendEvent(event)
}

// Does the peer's .parts (as received by an EventParts event) contain part?
func (p *Peer) knownToHave(part uint8) bool {
	return p.parts[part/8]&(1<<(part%8)) > 0
}

//-----------------------------------------------------------------------------

type Node struct {
	index  int
	peers  []*Peer
	parts  []byte
	events *Heap
}

func (n *Node) sendEvent(event Event) {
	n.events.Push(event, event.RecvTime())
}

func (n *Node) recvEvent() Event {
	return n.events.Pop().(Event)
}

func (n *Node) receive(part uint8) bool {
	x := n.parts[part/8]
	nx := x | (1 << (part % 8))
	if x == nx {
		return false
	} else {
		n.parts[part/8] = nx
		return true
	}
}

// returns false if already connected, or remote node has too many connections.
func (n *Node) canConnectTo(node *Node) bool {
	if len(n.peers) > maxNumPeers {
		return false
	}
	for _, peer := range n.peers {
		if peer.node == node {
			return false
		}
	}
	return true
}

func (n *Node) isFull() bool {
	for _, part := range n.parts {
		if part != byte(0xff) {
			return false
		}
	}
	return true
}

func (n *Node) String() string {
	return fmt.Sprintf("{N:%d}", n.index)
}

//-----------------------------------------------------------------------------

type Event interface {
	RecvTime() int32
	SetRecvTime(int32)
}

type EventData struct {
	time int32 // time of receipt.
	part uint8
}

func (e EventData) RecvTime() int32 {
	return e.time
}

func (e EventData) SetRecvTime(time int32) {
	e.time = time
}

func (e EventData) String() string {
	return fmt.Sprintf("[%d:%d]", e.time, e.part)
}

type EventParts struct {
	time  int32 // time of receipt.
	src   int   // src node's peer index on destination node.
	parts []byte
}

func (e EventParts) RecvTime() int32 {
	return e.time
}

func (e EventParts) SetRecvTime(time int32) {
	e.time = time
}

func (e EventParts) String() string {
	return fmt.Sprintf("[%d:%d:%d]", e.time, e.src, e.parts)
}

//-----------------------------------------------------------------------------

func createNetwork() []*Node {
	nodes := make([]*Node, numNodes)
	for i := 0; i < numNodes; i++ {
		n := &Node{
			index:  i,
			peers:  []*Peer{},
			parts:  make([]byte, 32),
			events: NewHeap(),
		}
		nodes[i] = n
	}
	for i := 0; i < numNodes; i++ {
		n := nodes[i]
		for j := 0; j < minNumPeers; j++ {
			if len(n.peers) > j {
				// Already set, continue
				continue
			}
			pidx := rand.Intn(numNodes)
			for !n.canConnectTo(nodes[pidx]) {
				pidx = rand.Intn(numNodes)
			}
			// connect to nodes[pidx]
			remote := nodes[pidx]
			remote_j := len(remote.peers)
			n.peers = append(n.peers, &Peer{node: remote, remote: remote_j, parts: make([]byte, 32)})
			remote.peers = append(remote.peers, &Peer{node: n, remote: j, parts: make([]byte, 32)})
		}
	}
	return nodes
}

func printNodes(nodes []*Node) {
	for _, node := range nodes {
		peerStr := ""
		for _, peer := range node.peers {
			peerStr += fmt.Sprintf(" %v", peer.node.index)
		}
		fmt.Printf("[%v] peers: %v\n", node.index, peerStr)
	}
}

func countFull(nodes []*Node) (fullCount int) {
	for _, node := range nodes {
		if node.isFull() {
			fullCount += 1
		}
	}
	return fullCount
}

func main() {

	// Global vars
	nodes := createNetwork()
	timeMS := int32(0)
	proposer := nodes[0]
	for i := 0; i < 32; i++ {
		proposer.parts[i] = byte(0xff)
	}
	//printNodes(nodes[:])

	// The proposer sends parts to all of its peers.
	for i := 0; i < len(proposer.peers); i++ {
		timeMS := int32(0) // scoped
		peer := proposer.peers[i]
		for j := 0; j < 256; j++ {
			// Send each part to a peer, but each peer starts at a different offset.
			part := uint8((j + i*25) % 256)
			recvTime := timeMS + latencyMS + partTxMS
			event := EventData{
				time: recvTime,
				part: part,
			}
			peer.sendEventData(event)
			timeMS += partTxMS
		}
	}

	// Run simulation
	for {
		// Lets run the simulation for each user until endTimeMS
		// We use latencyMS/2 since causality has at least this much lag.
		endTimeMS := timeMS + latencyMS/2
		fmt.Printf("simulating until %v\n", endTimeMS)

		// Print out the network for debugging
		if true {
			for i := 40000; i < 40050; i++ {
				node := nodes[i]
				fmt.Printf("[%v] parts: %X\n", node.index, node.parts)
			}
		}

		for _, node := range nodes {

			// Iterate over the events of this node until event.time >= endTimeMS
			for {
				_event, ok := node.events.Peek().(Event)
				if !ok || _event.RecvTime() >= endTimeMS {
					break
				} else {
					node.events.Pop()
				}

				switch _event.(type) {
				case EventData:
					event := _event.(EventData)

					// Process this event
					if !node.receive(event.part) {
						// Already has this part, ignore this event.
						continue
					}

					// Let's iterate over peers & see which needs this piece.
					recvTime := event.time + latencyMS + partTxMS
					for _, peer := range node.peers {
						if peer.knownToHave(event.part) {
							continue
						}
						peer.sendEventData(EventData{
							time: recvTime,
							part: event.part,
						})
					}

				case EventParts:
					event := _event.(EventParts)
					node.peers[event.src].parts = event.parts
				}

			}
		}

		// If network is full, quit.
		if countFull(nodes) == numNodes {
			fmt.Printf("Done! took %v ms", timeMS)
			break
		}

		// Lets increment the timeMS now
		timeMS += latencyMS / 2

		// Send EventParts rather frequently. It's cheap.
		for _, node := range nodes {
			for _, peer := range node.peers {
				peer.sendEventParts(EventParts{
					time:  timeMS + latencyMS,
					src:   peer.remote,
					parts: node.parts,
				})
			}

			newParts := make([]byte, 32)
			copy(newParts, node.parts)
			node.parts = newParts
		}

	}
}

// ----------------------------------------------------------------------------

type Heap struct {
	pq priorityQueue
}

func NewHeap() *Heap {
	return &Heap{pq: make([]*pqItem, 0)}
}

func (h *Heap) Len() int {
	return len(h.pq)
}

func (h *Heap) Peek() interface{} {
	if len(h.pq) == 0 {
		return nil
	}
	return h.pq[0].value
}

func (h *Heap) Push(value interface{}, priority int32) {
	heap.Push(&h.pq, &pqItem{value: value, priority: priority})
}

func (h *Heap) Pop() interface{} {
	item := heap.Pop(&h.pq).(*pqItem)
	return item.value
}

/*
func main() {
    h := NewHeap()

    h.Push(String("msg1"), 1)
    h.Push(String("msg3"), 3)
    h.Push(String("msg2"), 2)

    fmt.Println(h.Pop())
    fmt.Println(h.Pop())
    fmt.Println(h.Pop())
}
*/

///////////////////////
// From: http://golang.org/pkg/container/heap/#example__priorityQueue

type pqItem struct {
	value    interface{}
	priority int32
	index    int
}

type priorityQueue []*pqItem

func (pq priorityQueue) Len() int { return len(pq) }

func (pq priorityQueue) Less(i, j int) bool {
	return pq[i].priority < pq[j].priority
}

func (pq priorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *priorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*pqItem)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *priorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

func (pq *priorityQueue) Update(item *pqItem, value interface{}, priority int32) {
	heap.Remove(pq, item.index)
	item.value = value
	item.priority = priority
	heap.Push(pq, item)
}
<commit_msg>with pull<commit_after>package main

import (
	"container/heap"
	"fmt"
	"math/rand"
    "strings"
)

const seed = 0
const numNodes = 30000        // Total number of nodes to simulate
const minNumPeers = 7         // Each node should be connected to at least this many peers
const maxNumPeers = 10        // ... and at most this many
const latencyMS = int32(500)  // One way packet latency
const partTxMS = int32(100)   // Transmission time per peer of 4KB of data.
const sendQueueCapacity = 5  // Amount of messages to queue between peers.

func init() {
	rand.Seed(seed)
}

//-----------------------------------------------------------------------------

type Peer struct {
	node   *Node  // Pointer to node
	sent   int32  // Time of last packet send, including transmit time.
	remote int    // SomeNode.peers[x].node.peers[remote].node is SomeNode for all x.
	parts  []byte // [32]byte{} bitarray of received block pieces.
}

// Send a data event to the peer, or return false if queue is "full".
// Depending on how many event packets are "queued" for peer,
// the actual recvTime may be adjusted to be later.
func (p *Peer) sendEventData(event EventData) bool {
	desiredRecvTime := event.RecvTime()
	minRecvTime := p.sent + partTxMS + latencyMS
	if desiredRecvTime >= minRecvTime {
		p.node.sendEvent(event)
		p.sent += partTxMS
		return true
	} else {
		if (minRecvTime-desiredRecvTime)/partTxMS > sendQueueCapacity {
			return false
		} else {
			event.SetRecvTime(minRecvTime) // Adjust recvTime
			p.node.sendEvent(event)
			p.sent += partTxMS
			return true
		}
	}
}

// Returns true if the sendQueue is not "full"
func (p *Peer) canSendData(now int32) bool {
    return (p.sent - now) < sendQueueCapacity
}

// Since EventPart events are much smaller, we don't consider the transmit time,
// and assume that the sendQueue is always free.
func (p *Peer) sendEventParts(event EventParts) {
	p.node.sendEvent(event)
}

// Does the peer's .parts (as received by an EventParts event) contain part?
func (p *Peer) knownToHave(part uint8) bool {
	return p.parts[part/8]&(1<<(part%8)) > 0
}

//-----------------------------------------------------------------------------

type Node struct {
	index  int
	peers  []*Peer
	parts  []byte
	events *Heap
}

func (n *Node) sendEvent(event Event) {
	n.events.Push(event, event.RecvTime())
}

func (n *Node) recvEvent() Event {
	return n.events.Pop().(Event)
}

func (n *Node) receive(part uint8) bool {
	x := n.parts[part/8]
	nx := x | (1 << (part % 8))
	if x == nx {
		return false
	} else {
		n.parts[part/8] = nx
		return true
	}
}

// returns false if already connected, or remote node has too many connections.
func (n *Node) canConnectTo(node *Node) bool {
	if len(n.peers) > maxNumPeers {
		return false
	}
	for _, peer := range n.peers {
		if peer.node == node {
			return false
		}
	}
	return true
}

func (n *Node) isFull() bool {
	for _, part := range n.parts {
		if part != byte(0xff) {
			return false
		}
	}
	return true
}

func (n *Node) pickRandomForPeer(peer *Peer) (part uint8, ok bool) {
    peerParts := peer.parts
    nodeParts := n.parts
    randStart := rand.Intn(32)
    for i:=0; i<32; i++ {
        bytei := uint8((i+randStart) % 32)
        nByte := nodeParts[bytei]
        pByte := peerParts[bytei]
        iHas  := nByte & ^pByte
        if iHas > 0 {
            randBitStart := rand.Intn(8)
            //fmt.Println("//--")
            for j:=0; j<8; j++ {
                biti := uint8((j+randBitStart) % 8)
                //fmt.Printf("%X %v %v %v\n", iHas, j, biti, randBitStart)
                if (iHas & (1<<biti)) > 0 {
                    return 8*bytei + biti, true
                }
            }
            panic("should not happen")
        }
    }
    return 0, false
}

func (n *Node) debug() {
    lines := []string{}
    lines = append(lines, n.String())
    lines = append(lines, fmt.Sprintf("events: %v, parts: %X", n.events.Len(), n.parts))
    for _, p := range n.peers {
        part, ok := n.pickRandomForPeer(p)
        lines = append(lines, fmt.Sprintf("peer sent: %v, parts: %X, (%v/%v)", p.sent, p.parts, part, ok))
    }
    fmt.Println("//---------------")
    fmt.Println(strings.Join(lines, "\n"))
    fmt.Println("//---------------")
}

func (n *Node) String() string {
	return fmt.Sprintf("{N:%d}", n.index)
}

//-----------------------------------------------------------------------------

type Event interface {
	RecvTime() int32
	SetRecvTime(int32)
}

type EventData struct {
	time int32 // time of receipt.
    src  int   // src node's peer index on destination node
	part uint8
}

func (e EventData) RecvTime() int32 {
	return e.time
}

func (e EventData) SetRecvTime(time int32) {
	e.time = time
}

func (e EventData) String() string {
	return fmt.Sprintf("[%d:%d:%d]", e.time, e.src, e.part)
}

type EventParts struct {
	time  int32 // time of receipt.
	src   int   // src node's peer index on destination node.
	parts []byte
}

func (e EventParts) RecvTime() int32 {
	return e.time
}

func (e EventParts) SetRecvTime(time int32) {
	e.time = time
}

func (e EventParts) String() string {
	return fmt.Sprintf("[%d:%d:%d]", e.time, e.src, e.parts)
}

//-----------------------------------------------------------------------------

func createNetwork() []*Node {
	nodes := make([]*Node, numNodes)
	for i := 0; i < numNodes; i++ {
		n := &Node{
			index:  i,
			peers:  []*Peer{},
			parts:  make([]byte, 32),
			events: NewHeap(),
		}
		nodes[i] = n
	}
	for i := 0; i < numNodes; i++ {
		n := nodes[i]
		for j := 0; j < minNumPeers; j++ {
			if len(n.peers) > j {
				// Already set, continue
				continue
			}
			pidx := rand.Intn(numNodes)
			for !n.canConnectTo(nodes[pidx]) {
				pidx = rand.Intn(numNodes)
			}
			// connect to nodes[pidx]
			remote := nodes[pidx]
			remote_j := len(remote.peers)
			n.peers = append(n.peers, &Peer{node: remote, remote: remote_j, parts: make([]byte, 32)})
			remote.peers = append(remote.peers, &Peer{node: n, remote: j, parts: make([]byte, 32)})
		}
	}
	return nodes
}

func printNodes(nodes []*Node) {
	for _, node := range nodes {
		peerStr := ""
		for _, peer := range node.peers {
			peerStr += fmt.Sprintf(" %v", peer.node.index)
		}
		fmt.Printf("[%v] peers: %v\n", node.index, peerStr)
	}
}

func countFull(nodes []*Node) (fullCount int) {
	for _, node := range nodes {
		if node.isFull() {
			fullCount += 1
		}
	}
	return fullCount
}

func main() {

	// Global vars
	nodes := createNetwork()
	timeMS := int32(0)
	proposer := nodes[0]
	for i := 0; i < 32; i++ {
		proposer.parts[i] = byte(0xff)
	}
	//printNodes(nodes[:])

	// The proposer sends parts to all of its peers.
	for i := 0; i < len(proposer.peers); i++ {
		timeMS := int32(0) // scoped
		peer := proposer.peers[i]
		for j := 0; j < 256; j++ {
			// Send each part to a peer, but each peer starts at a different offset.
			part := uint8((j + i*25) % 256)
			recvTime := timeMS + latencyMS + partTxMS
			event := EventData{
				time: recvTime,
                src:  peer.remote,
				part: part,
			}
			peer.sendEventData(event)
			timeMS += partTxMS
		}
	}

	// Run simulation
	for {
		// Lets run the simulation for each user until endTimeMS
		// We use latencyMS/2 since causality has at least this much lag.
		endTimeMS := timeMS + latencyMS/2
		fmt.Printf("simulating until %v\n", endTimeMS)

		// Print out the network for debugging
		if true {
			for i := 0; i < 40; i++ {
				node := nodes[i]
				fmt.Printf("[%v] parts: %X\n", node.index, node.parts)
			}
		}

		for _, node := range nodes {

			// Iterate over the events of this node until event.time >= endTimeMS
			for {
				_event, ok := node.events.Peek().(Event)
				if !ok || _event.RecvTime() >= endTimeMS {
					break
				} else {
					node.events.Pop()
				}

				switch _event.(type) {
				case EventData:
					event := _event.(EventData)

					// Process this event
					if !node.receive(event.part) {
						// Already has this part, ignore this event.
						continue
					}

					// Let's iterate over peers & see which needs this piece.
					for _, peer := range node.peers {
						if !peer.knownToHave(event.part) {
                            peer.sendEventData(EventData{
                                time: event.time + latencyMS + partTxMS,
                                src:  peer.remote,
                                part: event.part,
                            })
                        } else {
                            continue
                        }
					}

				case EventParts:
					event := _event.(EventParts)
					node.peers[event.src].parts = event.parts
                    peer := node.peers[event.src]

                    // Lets blast the peer with random parts.
                    randomSent := 0
                    randomSentErr := 0
                    for peer.canSendData(event.time) {
                        part, ok := node.pickRandomForPeer(peer)
                        if ok {
                            randomSent += 1
                            sent := peer.sendEventData(EventData{
                                time: event.time + latencyMS + partTxMS,
                                src:  peer.remote,
                                part: part,
                            })
                            if !sent {
                                randomSentErr += 1
                            }
                        } else {
                            break
                        }
                    }
                    /*
                    if randomSent > 0 {
                        fmt.Printf("radom sent: %v %v", randomSent, randomSentErr)
                    }
                    */
				}

			}
		}

		// If network is full, quit.
		if countFull(nodes) == numNodes {
			fmt.Printf("Done! took %v ms", timeMS)
			break
		}

		// Lets increment the timeMS now
		timeMS += latencyMS / 2

        // Debug
        if timeMS >= 25000 {
            nodes[1].debug()
            for e := nodes[1].events.Pop(); e != nil; e = nodes[1].events.Pop() {
                fmt.Println(e)
            }
            return
        }

		// Send EventParts rather frequently. It's cheap.
		for _, node := range nodes {
			for _, peer := range node.peers {
				peer.sendEventParts(EventParts{
					time:  timeMS + latencyMS,
					src:   peer.remote,
					parts: node.parts,
				})
			}

			newParts := make([]byte, 32)
			copy(newParts, node.parts)
			node.parts = newParts
		}

	}
}

// ----------------------------------------------------------------------------

type Heap struct {
	pq priorityQueue
}

func NewHeap() *Heap {
	return &Heap{pq: make([]*pqItem, 0)}
}

func (h *Heap) Len() int {
	return len(h.pq)
}

func (h *Heap) Peek() interface{} {
	if len(h.pq) == 0 {
		return nil
	}
	return h.pq[0].value
}

func (h *Heap) Push(value interface{}, priority int32) {
	heap.Push(&h.pq, &pqItem{value: value, priority: priority})
}

func (h *Heap) Pop() interface{} {
	item := heap.Pop(&h.pq).(*pqItem)
	return item.value
}

/*
func main() {
    h := NewHeap()

    h.Push(String("msg1"), 1)
    h.Push(String("msg3"), 3)
    h.Push(String("msg2"), 2)

    fmt.Println(h.Pop())
    fmt.Println(h.Pop())
    fmt.Println(h.Pop())
}
*/

///////////////////////
// From: http://golang.org/pkg/container/heap/#example__priorityQueue

type pqItem struct {
	value    interface{}
	priority int32
	index    int
}

type priorityQueue []*pqItem

func (pq priorityQueue) Len() int { return len(pq) }

func (pq priorityQueue) Less(i, j int) bool {
	return pq[i].priority < pq[j].priority
}

func (pq priorityQueue) Swap(i, j int) {
	pq[i], pq[j] = pq[j], pq[i]
	pq[i].index = i
	pq[j].index = j
}

func (pq *priorityQueue) Push(x interface{}) {
	n := len(*pq)
	item := x.(*pqItem)
	item.index = n
	*pq = append(*pq, item)
}

func (pq *priorityQueue) Pop() interface{} {
	old := *pq
	n := len(old)
	item := old[n-1]
	item.index = -1 // for safety
	*pq = old[0 : n-1]
	return item
}

func (pq *priorityQueue) Update(item *pqItem, value interface{}, priority int32) {
	heap.Remove(pq, item.index)
	item.value = value
	item.priority = priority
	heap.Push(pq, item)
}
<|endoftext|>


<commit_before>package slab

import (
	"testing"
)

func TestBasics(t *testing.T) {
	s := SynchronizedArena(NewSlabArena(1, 1024, 2))
	if s == nil {
		t.Errorf("expected new slab arena to work")
	}
	a := s.Alloc(1)
	if a == nil {
		t.Errorf("expected alloc to work")
	}
	if len(a) != 1 {
		t.Errorf("expected alloc to give right size buf")
	}
	if cap(a) != 1 + SLAB_MEMORY_FOOTER_LEN {
		t.Errorf("expected alloc cap to match algorithm, got: %v vs %v",
			cap(a), 1 + SLAB_MEMORY_FOOTER_LEN)
	}
	a[0] = 66
	s.DecRef(a)
	b := s.Alloc(1)
	if b == nil {
		t.Errorf("expected alloc to work")
	}
	if len(b) != 1 {
		t.Errorf("expected alloc to give right size buf")
	}
	if cap(b) != 1 + SLAB_MEMORY_FOOTER_LEN {
		t.Errorf("expected alloc cap to match algorithm, got: %v vs %v",
			cap(b), 1 + SLAB_MEMORY_FOOTER_LEN)
	}
	if b[0] != 66 {
		t.Errorf("expected alloc to return last freed buf")
	}
	s.AddRef(b)
	s.DecRef(b)
	s.DecRef(b)
	c := s.Alloc(1)
	if c[0] != 66 {
		t.Errorf("expected alloc to return last freed buf")
	}
}

func TestSlabClassGrowth(t *testing.T) {
	s := NewSlabArena(1, 8, 2).(*slabArena)
	expectSlabClasses := func(numSlabClasses int) {
		if len(s.slabClasses) != numSlabClasses {
			t.Errorf("expected %v slab classses, got: %v",
				numSlabClasses, len(s.slabClasses))
		}
	}
	expectSlabClasses(1)
	s.Alloc(1)
	expectSlabClasses(1)
	s.Alloc(1)
	expectSlabClasses(1)
	s.Alloc(2)
	expectSlabClasses(2)
	s.Alloc(1)
	s.Alloc(2)
	expectSlabClasses(2)
	s.Alloc(3)
	s.Alloc(4)
	expectSlabClasses(3)
	s.Alloc(5)
	s.Alloc(8)
	expectSlabClasses(4)
}

func TestDecRef(t *testing.T) {
	s := NewSlabArena(1, 8, 2).(*slabArena)
	expectSlabClasses := func(numSlabClasses int) {
		if len(s.slabClasses) != numSlabClasses {
			t.Errorf("expected %v slab classses, got: %v",
				numSlabClasses, len(s.slabClasses))
		}
	}
	a := make([][]byte, 128)
	for j := 0; j < 100; j++ {
		for i := 0; i < len(a); i++ {
			a[i] = s.Alloc(i % 8)
		}
		for i := 0; i < len(a); i++ {
			s.DecRef(a[i])
		}
	}
	expectSlabClasses(4)
}

func TestAddRef(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	a := s.Alloc(1)
	a[0] = 123
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	s.AddRef(a)
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	s.DecRef(a)
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	s.DecRef(a)
	if s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected 1 free chunk")
	}
	b := s.Alloc(1)
	if b[0] != 123 {
		t.Errorf("expected chunk to be reused")
	}
}

func TestLargeAlloc(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	if s.Alloc(2) != nil {
		t.Errorf("expected alloc larger than slab size to fail")
	}
}

func TestEmptyChunk(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	if sc.chunk(empty_chunkLoc) != nil {
		t.Errorf("expected empty chunk to not have a chunk()")
	}
}

func TestEmptyChunkMem(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	if sc.chunkMem(nil) != nil {
		t.Errorf("expected nil chunk to not have a chunk()")
	}
	if sc.chunkMem(&chunk{self:empty_chunkLoc}) != nil {
		t.Errorf("expected empty chunk to not have a chunk()")
	}
}

func TestAddRefOnAlreadyReleasedBuf(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	a := s.Alloc(1)
	s.DecRef(a)
	var err interface{}
	func() {
		defer func() { err = recover() }()
		s.AddRef(a)
	}()
	if err == nil {
		t.Errorf("expected panic on AddRef on already release buf")
	}
}

func TestDecRefOnAlreadyReleasedBuf(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	a := s.Alloc(1)
	s.DecRef(a)
	var err interface{}
	func() {
		defer func() { err = recover() }()
		s.DecRef(a)
	}()
	if err == nil {
		t.Errorf("expected panic on DecRef on already release buf")
	}
}

func TestPushFreeChunkOnReferencedChunk(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	var err interface{}
	func() {
		defer func() { err = recover() }()
		sc.pushFreeChunk(&chunk{refs:1})
	}()
	if err == nil {
		t.Errorf("expected panic when free'ing a ref-counted chunk")
	}
}

func TestPopFreeChunkOnFreeChunk(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	sc.chunkFree = empty_chunkLoc
	var err interface{}
	func() {
		defer func() { err = recover() }()
		sc.popFreeChunk()
	}()
	if err == nil {
		t.Errorf("expected panic when popFreeChunk() on free chunk")
	}
}

<commit_msg>TestPopFreeChunkOnReferencedFreeChunk().<commit_after>package slab

import (
	"testing"
)

func TestBasics(t *testing.T) {
	s := SynchronizedArena(NewSlabArena(1, 1024, 2))
	if s == nil {
		t.Errorf("expected new slab arena to work")
	}
	a := s.Alloc(1)
	if a == nil {
		t.Errorf("expected alloc to work")
	}
	if len(a) != 1 {
		t.Errorf("expected alloc to give right size buf")
	}
	if cap(a) != 1 + SLAB_MEMORY_FOOTER_LEN {
		t.Errorf("expected alloc cap to match algorithm, got: %v vs %v",
			cap(a), 1 + SLAB_MEMORY_FOOTER_LEN)
	}
	a[0] = 66
	s.DecRef(a)
	b := s.Alloc(1)
	if b == nil {
		t.Errorf("expected alloc to work")
	}
	if len(b) != 1 {
		t.Errorf("expected alloc to give right size buf")
	}
	if cap(b) != 1 + SLAB_MEMORY_FOOTER_LEN {
		t.Errorf("expected alloc cap to match algorithm, got: %v vs %v",
			cap(b), 1 + SLAB_MEMORY_FOOTER_LEN)
	}
	if b[0] != 66 {
		t.Errorf("expected alloc to return last freed buf")
	}
	s.AddRef(b)
	s.DecRef(b)
	s.DecRef(b)
	c := s.Alloc(1)
	if c[0] != 66 {
		t.Errorf("expected alloc to return last freed buf")
	}
}

func TestSlabClassGrowth(t *testing.T) {
	s := NewSlabArena(1, 8, 2).(*slabArena)
	expectSlabClasses := func(numSlabClasses int) {
		if len(s.slabClasses) != numSlabClasses {
			t.Errorf("expected %v slab classses, got: %v",
				numSlabClasses, len(s.slabClasses))
		}
	}
	expectSlabClasses(1)
	s.Alloc(1)
	expectSlabClasses(1)
	s.Alloc(1)
	expectSlabClasses(1)
	s.Alloc(2)
	expectSlabClasses(2)
	s.Alloc(1)
	s.Alloc(2)
	expectSlabClasses(2)
	s.Alloc(3)
	s.Alloc(4)
	expectSlabClasses(3)
	s.Alloc(5)
	s.Alloc(8)
	expectSlabClasses(4)
}

func TestDecRef(t *testing.T) {
	s := NewSlabArena(1, 8, 2).(*slabArena)
	expectSlabClasses := func(numSlabClasses int) {
		if len(s.slabClasses) != numSlabClasses {
			t.Errorf("expected %v slab classses, got: %v",
				numSlabClasses, len(s.slabClasses))
		}
	}
	a := make([][]byte, 128)
	for j := 0; j < 100; j++ {
		for i := 0; i < len(a); i++ {
			a[i] = s.Alloc(i % 8)
		}
		for i := 0; i < len(a); i++ {
			s.DecRef(a[i])
		}
	}
	expectSlabClasses(4)
}

func TestAddRef(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	a := s.Alloc(1)
	a[0] = 123
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	s.AddRef(a)
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	s.DecRef(a)
	if !s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected no free chunks")
	}
	s.DecRef(a)
	if s.slabClasses[0].chunkFree.isEmpty() {
		t.Errorf("expected 1 free chunk")
	}
	b := s.Alloc(1)
	if b[0] != 123 {
		t.Errorf("expected chunk to be reused")
	}
}

func TestLargeAlloc(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	if s.Alloc(2) != nil {
		t.Errorf("expected alloc larger than slab size to fail")
	}
}

func TestEmptyChunk(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	if sc.chunk(empty_chunkLoc) != nil {
		t.Errorf("expected empty chunk to not have a chunk()")
	}
}

func TestEmptyChunkMem(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	if sc.chunkMem(nil) != nil {
		t.Errorf("expected nil chunk to not have a chunk()")
	}
	if sc.chunkMem(&chunk{self:empty_chunkLoc}) != nil {
		t.Errorf("expected empty chunk to not have a chunk()")
	}
}

func TestAddRefOnAlreadyReleasedBuf(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	a := s.Alloc(1)
	s.DecRef(a)
	var err interface{}
	func() {
		defer func() { err = recover() }()
		s.AddRef(a)
	}()
	if err == nil {
		t.Errorf("expected panic on AddRef on already release buf")
	}
}

func TestDecRefOnAlreadyReleasedBuf(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	a := s.Alloc(1)
	s.DecRef(a)
	var err interface{}
	func() {
		defer func() { err = recover() }()
		s.DecRef(a)
	}()
	if err == nil {
		t.Errorf("expected panic on DecRef on already release buf")
	}
}

func TestPushFreeChunkOnReferencedChunk(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	var err interface{}
	func() {
		defer func() { err = recover() }()
		sc.pushFreeChunk(&chunk{refs:1})
	}()
	if err == nil {
		t.Errorf("expected panic when free'ing a ref-counted chunk")
	}
}

func TestPopFreeChunkOnFreeChunk(t *testing.T) {
	s := NewSlabArena(1, 1, 2).(*slabArena)
	sc := s.slabClasses[0]
	sc.chunkFree = empty_chunkLoc
	var err interface{}
	func() {
		defer func() { err = recover() }()
		sc.popFreeChunk()
	}()
	if err == nil {
		t.Errorf("expected panic when popFreeChunk() on free chunk")
	}
}

func TestPopFreeChunkOnReferencedFreeChunk(t *testing.T) {
	s := NewSlabArena(1, 1024, 2).(*slabArena)
	s.Alloc(1)
	sc := s.slabClasses[0]
	sc.chunk(sc.chunkFree).refs = 1
	var err interface{}
	func() {
		defer func() { err = recover() }()
		sc.popFreeChunk()
	}()
	if err == nil {
		t.Errorf("expected panic when popFreeChunk() on ref'ed chunk")
	}
}

<|endoftext|>


<commit_before>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package wire

import (
	"bytes"
	"encoding/binary"
	"io"

	"github.com/FactomProject/FactomCode/common"
	"github.com/agl/ed25519"
)

// MsgCommitEntry implements the Message interface and represents a factom
// Commit-Entry message.  It is used by client to commit the entry before
// revealing it.
type MsgCommitEntry struct {
	Version   int8
	MilliTime *[6]byte
	EntryHash *common.Hash
	Credits   uint8
	ECPubKey  *[32]byte
	Sig       *[64]byte
}

// BtcEncode encodes the receiver to w using the bitcoin protocol encoding.
// This is part of the Message interface implementation.
func (msg *MsgCommitEntry) BtcEncode(w io.Writer, pver uint32) error {
	// Version
	if err := writeElement(w, &msg.Version); err != nil {
		return err
	}

	//MilliTime
	if err := writeVarBytes(w, pver, msg.MilliTime[:]); err != nil {
		return err
	}

	//EntryHash
	if err := writeVarBytes(w, uint32(common.HASH_LENGTH), msg.EntryHash.Bytes); err != nil {
		return err
	}

	//Credits
	if err := writeElement(w, &msg.Credits); err != nil {
		return err
	}

	//ECPubKey
	if err := writeVarBytes(w, uint32(ed25519.PublicKeySize), msg.ECPubKey[:]); err != nil {
		return err
	}

	//Signature
	if err := writeVarBytes(w, uint32(ed25519.SignatureSize), msg.Sig[:]); err != nil {
		return err
	}

	return nil
}

// BtcDecode decodes r using the bitcoin protocol encoding into the receiver.
// This is part of the Message interface implementation.
func (msg *MsgCommitEntry) BtcDecode(r io.Reader, pver uint32) error {
	// Version
	if err := readElement(r, &msg.Version); err != nil {
		return err
	}

	// MilliTime
	if bytes, err := readVarBytes(r, pver, uint32(6), CmdCommitEntry); err != nil {
		return err
	} else {
		copy(msg.MilliTime[:], bytes)
	}

	// EntryHash
	if bytes, err := readVarBytes(r, pver, uint32(common.HASH_LENGTH),
		CmdCommitEntry); err != nil {
		return err
	} else {
		copy(msg.EntryHash.Bytes, bytes[:32])
	}

	// Credits
	if err := readElement(r, &msg.Credits); err != nil {
		return err
	}

	// ECPubKey
	if bytes, err := readVarBytes(r, pver, uint32(ed25519.PublicKeySize),
		CmdCommitEntry); err != nil {
		return err
	} else {
		msg.ECPubKey = new([32]byte)
		copy(msg.ECPubKey[:], bytes)
	}

	// Signature
	if bytes, err := readVarBytes(r, pver, uint32(ed25519.SignatureSize),
		CmdCommitEntry); err != nil {
		return err
	} else {
		msg.Sig = new([64]byte)
		copy(msg.Sig[:], bytes)
	}

	return nil
}

// Command returns the protocol command string for the message.  This is part
// of the Message interface implementation.
func (msg *MsgCommitEntry) Command() string {
	return CmdCommitEntry
}

// MaxPayloadLength returns the maximum length the payload can be for the
// receiver.  This is part of the Message interface implementation.
func (msg *MsgCommitEntry) MaxPayloadLength(pver uint32) uint32 {
	return MaxAppMsgPayload
}

// NewMsgCommitEntry returns a new bitcoin Commit Entry message that conforms to
// the Message interface.
func NewMsgCommitEntry() *MsgCommitEntry {
	m := new(MsgCommitEntry)
	m.MilliTime = new([6]byte)
	m.EntryHash = new(common.Hash)
	m.EntryHash.Bytes = make([]byte, 32)
	m.ECPubKey = new([32]byte)
	m.Sig = new([64]byte)

	return m
}

// Check whether the msg can pass the message level validations
// such as timestamp, signiture and etc
func (msg *MsgCommitEntry) IsValid() bool {
	// Verify signature (Version + MilliTime + EntryHash + Credits)
	buf := new(bytes.Buffer)
	binary.Write(buf, binary.BigEndian, msg.Version)
	buf.Write(msg.MilliTime[:])
	buf.Write(msg.EntryHash.Bytes)
	binary.Write(buf, binary.BigEndian, msg.Credits)

	return ed25519.Verify(msg.ECPubKey, buf.Bytes(), msg.Sig)
}

// Create a sha hash from the message binary (output of BtcEncode)
func (msg *MsgCommitEntry) Sha() (ShaHash, error) {

	buf := bytes.NewBuffer(nil)
	msg.BtcEncode(buf, ProtocolVersion)
	var sha ShaHash
	_ = sha.SetBytes(Sha256(buf.Bytes()))

	return sha, nil
}
<commit_msg>msgcommitentry<commit_after>// Copyright (c) 2013-2014 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package wire

import (
	"bytes"
	"io"

	"github.com/FactomProject/FactomCode/common"
)

// MsgCommitEntry implements the Message interface and represents a factom
// Commit-Entry message.  It is used by client to commit the entry before
// revealing it.
type MsgCommitEntry struct {
	CommitEntry *common.CommitEntry
}

// BtcEncode encodes the receiver to w using the bitcoin protocol encoding.
// This is part of the Message interface implementation.
func (msg *MsgCommitEntry) BtcEncode(w io.Writer, pver uint32) error {
	bytes, err := msg.CommitEntry.MarshalBinary()
	if err != nil {
		return err
	}

	err = writeVarBytes(w, pver, bytes)
	if err != nil {
		return err
	}

	return nil
}

// BtcDecode decodes r using the bitcoin protocol encoding into the receiver.
// This is part of the Message interface implementation.
func (msg *MsgCommitEntry) BtcDecode(r io.Reader, pver uint32) error {
	bytes, err := readVarBytes(r, pver, uint32(common.CommitEntrySize),
		CmdEntry)
	if err != nil {
		return err
	}

	msg.CommitEntry = new(common.CommitEntry)
	err = msg.CommitEntry.UnmarshalBinary(bytes)
	if err != nil {
		return err
	}

	return nil
}

// Command returns the protocol command string for the message.  This is part
// of the Message interface implementation.
func (msg *MsgCommitEntry) Command() string {
	return CmdCommitEntry
}

// MaxPayloadLength returns the maximum length the payload can be for the
// receiver.  This is part of the Message interface implementation.
func (msg *MsgCommitEntry) MaxPayloadLength(pver uint32) uint32 {
	return MaxAppMsgPayload
}

// NewMsgCommitEntry returns a new bitcoin Commit Entry message that conforms to
// the Message interface.
func NewMsgCommitEntry() *MsgCommitEntry {
	return &MsgCommitEntry{}
}

// Check whether the msg can pass the message level validations
// such as timestamp, signiture and etc
func (msg *MsgCommitEntry) IsValid() bool {
	return msg.CommitEntry.IsValid()
}

// Create a sha hash from the message binary (output of BtcEncode)
func (msg *MsgCommitEntry) Sha() (ShaHash, error) {

	buf := bytes.NewBuffer(nil)
	msg.BtcEncode(buf, ProtocolVersion)
	var sha ShaHash
	_ = sha.SetBytes(Sha256(buf.Bytes()))

	return sha, nil
}
<|endoftext|>


<commit_before>package rdstail

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/rds"
	"github.com/chrismrivera/backoff"
)

const (
	papertrailBackoffMaxWait  = time.Minute
	papertrailBackoffDeadline = time.Minute * 5
	// aws-sdk-go already offers retry functionality
)

func getMostRecentLogFile(r *rds.RDS, db string) (file *rds.DescribeDBLogFilesDetails, err error) {
	yesterday := time.Now().Add(-24 * time.Hour).Unix()
	file, err = getMostRecentLogFileSince(r, db, yesterday)
	if err != nil {
		return
	}

	if file == nil {
		lastWeek := time.Now().Add(-7 * 24 * time.Hour).Unix()
		file, err = getMostRecentLogFileSince(r, db, lastWeek)
		if err != nil {
			return
		}
	}

	if file == nil {
		file, err = getMostRecentLogFileSince(r, db, 0)
		if err != nil {
			return
		}
	}

	return
}

// getMostRecentLogFileSince returns the most recent log file whose lastWritten >= since.
func getMostRecentLogFileSince(r *rds.RDS, db string, since int64) (file *rds.DescribeDBLogFilesDetails, err error) {
	resp, err := describeLogFiles(r, db, since)
	if err != nil {
		return nil, err
	}
	for _, d := range resp {
		hasData := d.LastWritten != nil && d.LogFileName != nil
		isNewer := file == nil || file.LastWritten == nil || *d.LastWritten > *file.LastWritten
		if hasData && isNewer {
			file = d
		}
	}
	return
}

func describeLogFiles(r *rds.RDS, db string, since int64) (details []*rds.DescribeDBLogFilesDetails, err error) {
	req := &rds.DescribeDBLogFilesInput{
		DBInstanceIdentifier: aws.String(db),
	}
	if since != 0 {
		req.FileLastWritten = aws.Int64(since)
	}

	err = r.DescribeDBLogFilesPages(req, func(p *rds.DescribeDBLogFilesOutput, lastPage bool) bool {
		details = append(details, p.DescribeDBLogFiles...)
		return true
	})

	return
}

func tailLogFile(r *rds.RDS, db, name string, numLines int64, marker string) (string, string, error) {
	req := &rds.DownloadDBLogFilePortionInput{
		DBInstanceIdentifier: aws.String(db),
		LogFileName:          aws.String(name),
	}
	if numLines != 0 {
		req.NumberOfLines = aws.Int64(numLines)
	}
	if marker != "" {
		req.Marker = aws.String(marker)
	}

	var buf bytes.Buffer
	var markerPtr *string
	err := r.DownloadDBLogFilePortionPages(req, func(p *rds.DownloadDBLogFilePortionOutput, lastPage bool) bool {
		if p.LogFileData != nil {
			buf.WriteString(*p.LogFileData)
		}
		if lastPage {
			markerPtr = p.Marker
		}
		return true
	})

	marker = ""
	if markerPtr != nil {
		marker = *markerPtr
	}

	return buf.String(), marker, err
}

/// cmds

func Tail(r *rds.RDS, db string, numLines int64) error {
	logFile, err := getMostRecentLogFile(r, db)
	if err != nil {
		return nil
	}
	if logFile == nil {
		return errors.New("no log file found")
	}

	tail, _, err := tailLogFile(r, db, *logFile.LogFileName, numLines, "")
	if err != nil {
		return err
	}
	fmt.Println(tail)
	return nil
}

func Watch(r *rds.RDS, db string, rate time.Duration, callback func(string) error, stop <-chan struct{}) error {
	// Periodically check for new log files (unless there is a way to detect the file is done being written to)
	// Poll that log file, retaining the marker
	logFile, err := getMostRecentLogFile(r, db)
	if err != nil {
		return err
	}
	if logFile == nil {
		return errors.New("no log files")
	}

	// Get a marker for the end of the log file by requesting the most recent line
	lines, marker, err := tailLogFile(r, db, *logFile.LogFileName, 1, "")
	if err != nil {
		return err
	}

	t := time.NewTicker(rate)
	empty := 0
	const checkLogfileRate = 4
	for {
		select {
		case <-t.C:
			// If the logfile tail was empty n times, check for a newer log file
			if empty >= checkLogfileRate {
				empty = 0
				newLogFile, err := getMostRecentLogFileSince(r, db, *logFile.LastWritten)
				if err != nil {
					return err
				}

				// Ensure if we got a real new log file, and not the same file we are
				// already tailing. Reset the marker if its a real new log file only.
				if newLogFile != nil && *newLogFile.LogFileName != *logFile.LogFileName {
					logFile = newLogFile
					marker = ""
				}
			}

			lines, marker, err = tailLogFile(r, db, *logFile.LogFileName, 0, marker)
			if err != nil {
				return err
			}

			if lines == "" {
				empty++
			} else {
				empty = 0
				if err := callback(lines); err != nil {
					return err
				}
			}
		case <-stop:
			return nil
		}
	}

	return nil
}

func FeedPapertrail(r *rds.RDS, db string, rate time.Duration, papertrailHost, app, hostname string, stop <-chan struct{}) error {
	nameSegment := fmt.Sprintf(" %s %s: ", hostname, app)

	// Establish TLS connection with papertrail
	roots := x509.NewCertPool()
	ok := roots.AppendCertsFromPEM([]byte(papertrailPEM))
	if !ok {
		return errors.New("failed to parse papertrail root certificate")
	}

	conn, err := tls.Dial("tcp", papertrailHost, &tls.Config{
		RootCAs: roots,
	})
	if err != nil {
		return err
	}
	defer conn.Close()

	// watch with callback writing to the connection
	return Watch(r, db, rate, func(lines string) error {
		timestamp := time.Now().UTC().Format("2006-01-02T15:04:05")
    	        buf := bytes.Buffer{}
		buf.Reset()
		buf.WriteString(timestamp)
		buf.WriteString(nameSegment)
		buf.WriteString(lines)
		return backoff.Try(papertrailBackoffMaxWait, papertrailBackoffDeadline, func() error {
			_, err := conn.Write(buf.Bytes())
			return err
		})
	}, stop)
}
<commit_msg>Don't have to reset a buffer that is just initiated.<commit_after>package rdstail

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/rds"
	"github.com/chrismrivera/backoff"
)

const (
	papertrailBackoffMaxWait  = time.Minute
	papertrailBackoffDeadline = time.Minute * 5
	// aws-sdk-go already offers retry functionality
)

func getMostRecentLogFile(r *rds.RDS, db string) (file *rds.DescribeDBLogFilesDetails, err error) {
	yesterday := time.Now().Add(-24 * time.Hour).Unix()
	file, err = getMostRecentLogFileSince(r, db, yesterday)
	if err != nil {
		return
	}

	if file == nil {
		lastWeek := time.Now().Add(-7 * 24 * time.Hour).Unix()
		file, err = getMostRecentLogFileSince(r, db, lastWeek)
		if err != nil {
			return
		}
	}

	if file == nil {
		file, err = getMostRecentLogFileSince(r, db, 0)
		if err != nil {
			return
		}
	}

	return
}

// getMostRecentLogFileSince returns the most recent log file whose lastWritten >= since.
func getMostRecentLogFileSince(r *rds.RDS, db string, since int64) (file *rds.DescribeDBLogFilesDetails, err error) {
	resp, err := describeLogFiles(r, db, since)
	if err != nil {
		return nil, err
	}
	for _, d := range resp {
		hasData := d.LastWritten != nil && d.LogFileName != nil
		isNewer := file == nil || file.LastWritten == nil || *d.LastWritten > *file.LastWritten
		if hasData && isNewer {
			file = d
		}
	}
	return
}

func describeLogFiles(r *rds.RDS, db string, since int64) (details []*rds.DescribeDBLogFilesDetails, err error) {
	req := &rds.DescribeDBLogFilesInput{
		DBInstanceIdentifier: aws.String(db),
	}
	if since != 0 {
		req.FileLastWritten = aws.Int64(since)
	}

	err = r.DescribeDBLogFilesPages(req, func(p *rds.DescribeDBLogFilesOutput, lastPage bool) bool {
		details = append(details, p.DescribeDBLogFiles...)
		return true
	})

	return
}

func tailLogFile(r *rds.RDS, db, name string, numLines int64, marker string) (string, string, error) {
	req := &rds.DownloadDBLogFilePortionInput{
		DBInstanceIdentifier: aws.String(db),
		LogFileName:          aws.String(name),
	}
	if numLines != 0 {
		req.NumberOfLines = aws.Int64(numLines)
	}
	if marker != "" {
		req.Marker = aws.String(marker)
	}

	var buf bytes.Buffer
	var markerPtr *string
	err := r.DownloadDBLogFilePortionPages(req, func(p *rds.DownloadDBLogFilePortionOutput, lastPage bool) bool {
		if p.LogFileData != nil {
			buf.WriteString(*p.LogFileData)
		}
		if lastPage {
			markerPtr = p.Marker
		}
		return true
	})

	marker = ""
	if markerPtr != nil {
		marker = *markerPtr
	}

	return buf.String(), marker, err
}

/// cmds

func Tail(r *rds.RDS, db string, numLines int64) error {
	logFile, err := getMostRecentLogFile(r, db)
	if err != nil {
		return nil
	}
	if logFile == nil {
		return errors.New("no log file found")
	}

	tail, _, err := tailLogFile(r, db, *logFile.LogFileName, numLines, "")
	if err != nil {
		return err
	}
	fmt.Println(tail)
	return nil
}

func Watch(r *rds.RDS, db string, rate time.Duration, callback func(string) error, stop <-chan struct{}) error {
	// Periodically check for new log files (unless there is a way to detect the file is done being written to)
	// Poll that log file, retaining the marker
	logFile, err := getMostRecentLogFile(r, db)
	if err != nil {
		return err
	}
	if logFile == nil {
		return errors.New("no log files")
	}

	// Get a marker for the end of the log file by requesting the most recent line
	lines, marker, err := tailLogFile(r, db, *logFile.LogFileName, 1, "")
	if err != nil {
		return err
	}

	t := time.NewTicker(rate)
	empty := 0
	const checkLogfileRate = 4
	for {
		select {
		case <-t.C:
			// If the logfile tail was empty n times, check for a newer log file
			if empty >= checkLogfileRate {
				empty = 0
				newLogFile, err := getMostRecentLogFileSince(r, db, *logFile.LastWritten)
				if err != nil {
					return err
				}

				// Ensure if we got a real new log file, and not the same file we are
				// already tailing. Reset the marker if its a real new log file only.
				if newLogFile != nil && *newLogFile.LogFileName != *logFile.LogFileName {
					logFile = newLogFile
					marker = ""
				}
			}

			lines, marker, err = tailLogFile(r, db, *logFile.LogFileName, 0, marker)
			if err != nil {
				return err
			}

			if lines == "" {
				empty++
			} else {
				empty = 0
				if err := callback(lines); err != nil {
					return err
				}
			}
		case <-stop:
			return nil
		}
	}

	return nil
}

func FeedPapertrail(r *rds.RDS, db string, rate time.Duration, papertrailHost, app, hostname string, stop <-chan struct{}) error {
	nameSegment := fmt.Sprintf(" %s %s: ", hostname, app)

	// Establish TLS connection with papertrail
	roots := x509.NewCertPool()
	ok := roots.AppendCertsFromPEM([]byte(papertrailPEM))
	if !ok {
		return errors.New("failed to parse papertrail root certificate")
	}

	conn, err := tls.Dial("tcp", papertrailHost, &tls.Config{
		RootCAs: roots,
	})
	if err != nil {
		return err
	}
	defer conn.Close()

	// watch with callback writing to the connection
	return Watch(r, db, rate, func(lines string) error {
		timestamp := time.Now().UTC().Format("2006-01-02T15:04:05")
    	        buf := bytes.Buffer{}
		buf.WriteString(timestamp)
		buf.WriteString(nameSegment)
		buf.WriteString(lines)
		return backoff.Try(papertrailBackoffMaxWait, papertrailBackoffDeadline, func() error {
			_, err := conn.Write(buf.Bytes())
			return err
		})
	}, stop)
}
<|endoftext|>


<commit_before>package schemamagic

import (
	"fmt"

	"gopkg.in/jackc/pgx.v2"
)

// Table holds the table details as well as all the columns inside the table
type Table struct {
	Name          string
	DefaultSchema string
	Database      string
	Tx            *pgx.Tx
	Autocommit    bool
	Columns       []Column
}

// NewTable creates and returns an instance of a postgres table
func NewTable(t Table) *Table {
	table := new(Table)
	table.Name = t.Name
	table.DefaultSchema = t.DefaultSchema
	table.Database = t.Database
	table.Autocommit = t.Autocommit
	table.Columns = t.Columns
	table.Tx = t.Tx
	return table
}

// Append method accepts a column and appends it to the list of columns of the table
func (t *Table) Append(col Column) {
	t.Columns = append(t.Columns, col)
}

// Begin method initiates a DB transaction and checks if table (Name) exists in the DB. If it does, then it calls updateTable(). If it doesn't, it calls createTable(), and then updateTable()"""
func (t *Table) Begin() {
	log.Infoln("Operating on table --> ", t.Name)
	// Create the schema here
	schemaStatement := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", t.DefaultSchema)
	_, err := t.Tx.Exec(schemaStatement)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("Couldn't create schema --> ", t.DefaultSchema, " with error being --> ", err)
	}
	//  Check if table exists in the database
	presence := t.checkTableExistence()
	if !presence {
		// Table does not exist --> need to create it
		t.createTable()
	}
	// Loop over all the available columns and call updateTable() on each column
	for _, col := range t.Columns {
		log.Debugln("-----------------------------------------------")
		t.updateTable(col)
		log.Debugln("-----------------------------------------------")
	}
	if t.Autocommit {
		commitErr := t.Tx.Commit()
		if commitErr != nil {
			t.Tx.Rollback()
			log.Warningln("Couldn't commit changes to the TABLE --> ", t.Name, " with error being --> ", commitErr)
		}
	}
}

// checkTableExistence returns if the table already exists in the DB
func (t *Table) checkTableExistence() bool {
	var presence bool
	statement := fmt.Sprintf("SELECT exists(select 0 from pg_class where relname = '%s')", t.Name)
	err := t.Tx.QueryRow(statement).Scan(&presence)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("While querying for table existence, error is --> ", err)
	}
	log.Debugln("While checking for table existence, presence is ", presence)
	return presence
}

// createTable method creates the table in the particular DB
func (t *Table) createTable() {
	statement := fmt.Sprintf("CREATE TABLE %s()", t.Name)
	err := t.executeSQL(statement)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("While creating table --> ", t.Name, " error is --> ", err)
	}
}

// DropTable method drops the table from the DB
func (t *Table) DropTable() {
	presence := t.checkTableExistence()
	if presence {
		// Drop the table here
		log.Infoln("Trying to drop table --> ", t.Name)
		statement := fmt.Sprintf("DROP TABLE %s", t.Name)
		err := t.executeSQL(statement)
		if err != nil {
			t.Tx.Rollback()
			log.Warningln("While dropping table --> ", t.Name, " error is --> ", err)
		} else {
			log.Infoln("Successfully dropped table --> ", t.Name)
		}
	}
}

// updateTable alters the table by adding a new column to it, passed as the method parameter
func (t *Table) updateTable(col Column) {
	minNumberOfSteps := 0
	maxNumberOfSteps := 7
	columnPresence := t.checkColumnPresence(col.Name)
	if columnPresence {
		log.Debugln("Column --> ", col.Name, " already exists")
		columnDatatypeMatch := t.checkColumnDatatype(col)
		log.Debugln("Column --> ", col.Name, " datatype match value is --> ", columnDatatypeMatch)
		if columnDatatypeMatch {
			// If the datatype matches, pass, and SET minNumberOfSteps = 0 and maxNumberOfSteps = 0, since there should be no further execution, as the column is in its actual state
			minNumberOfSteps = 0
			maxNumberOfSteps = 0
		} else {
			// If the datatype does not match, then step=101, and set minNumberOfSteps to 1, since the step=1 has already been executed in the form of datatype modificaton
			minNumberOfSteps = 1
			statement, statementErr := col.prepareSQLStatement(101, t.Name)
			if statementErr != nil {
				err := t.executeSQL(statement)
				if err != nil {
					t.Tx.Rollback()
					log.Warningln("While executing SQL --> \n", statement, "\nerror is ", err)
				}
			}
		}
	} else {
		// Column does not exist
		log.Debugln("Column --> ", col.Name, " does not exist")
		minNumberOfSteps = 0
	}

	//  There are 4 steps involved here.

	// The first step is to call col.prepareSQLStatement with a step=1, which would return an SQL statement that would be used
	// to alter the table structure with a default set to NULL -> this is a very cheap operation,
	//  since the access exclusive lock would be acquired for a short time.

	//  The second step is to call col.prepareSQLStatement with a step=2, which would return an SQL statement that would be used
	//  to set the defaults to Column.default_value. In case Column.default_exists is False, then it returns an empty statement.
	//  This can be checked before executing the statement

	//  The third step is to call col.prepareSQLStatement with a step=3, which would update all the rows in the table with the default
	//  value.

	//  The fourth step is to call col.prepareSQLStatement with a step=3, which would return an SQL statement that would be used to
	//  alter the sequence start, in case the datatype is serial/bigserial. Similar to step=2, if it returns an empty statement,
	//  it either means that the datatype doesn't support a sequence, or the sequence needs to begin at 0.
	for step := minNumberOfSteps; step < maxNumberOfSteps; step++ {
		statement, statementErr := col.prepareSQLStatement(step+1, t.Name)
		log.Debugln("In steps, statement is \n", statement, " and error is ", statementErr)
		if statementErr == nil {
			err := t.executeSQL(statement)
			if err != nil {
				t.Tx.Rollback()
				log.Warningln("Statement --> ", statement, " could not be executed because of error --> ", err)
			}
		}
	}
}

// executeSQL executes the SQL query
func (t *Table) executeSQL(sql string) error {
	// var err error
	if sql != "" {
		_, err := t.Tx.Exec(sql)
		log.Debugln("Executing Statement --> \n", sql, " and error is ", err)
		return err
	}
	return nil
}

// checkColumnPresence checks if the column name passed is present in the current table
func (t *Table) checkColumnPresence(columnName string) bool {
	var presence bool
	statement := fmt.Sprintf("SELECT EXISTS(SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '%s' AND table_catalog = '%s' AND column_name = '%s')", t.Name, t.Database, columnName)
	err := t.Tx.QueryRow(statement).Scan(&presence)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("In checkColumnPresence, error for table --> ", t.Name, " and Column --> ", columnName, " is ", err)
	}
	return presence
}

// checkColumnDatatype checks if the column datatype of the column name passed is equal to the column datatype present in the table
func (t *Table) checkColumnDatatype(col Column) bool {
	columnName := col.Name
	columnDatatype := col.Datatype
	columnDefault := col.DefaultValue
	var (
		dbDatatype      string
		columnDefaultDB pgx.NullString
		presence        bool
	)
	statement := fmt.Sprintf("SELECT data_type, column_default FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '%s' AND table_catalog = '%s' AND column_name = '%s'", t.Name, t.Database, columnName)

	err := t.Tx.QueryRow(statement).Scan(&dbDatatype, &columnDefaultDB)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("While querying for column data type in table --> ", t.Name, " error is --> ", err)
	}

	log.Debugln("Datatype DB is ", dbDatatype, " and Column Default is ", columnDefaultDB, " and Column Datatype is ", columnDatatype)
	// Check if there are sequences here
	if columnDefaultDB.Valid {
		// This is a sequence
		if columnDatatype == "bigserial" && dbDatatype == "bigint" {
			presence = true
		} else if columnDatatype == "serial" && dbDatatype == "integer" {
			presence = true
		} else if columnDatatype == dbDatatype {
			if columnDefaultDB.String == columnDefault {
				presence = true
			} else {
				presence = false
			}
			// presence = true
		} else {
			presence = false
		}

	} else {
		// This is a non-sequence
		if columnDatatype == dbDatatype {
			presence = true
		} else {
			presence = false
		}
	}

	log.Debugln("In check column datatype, the value of presence is ", presence)
	return presence
}

func (t *Table) commit() {
	commitErr := t.Tx.Commit()
	if commitErr != nil {
		t.Tx.Rollback()
		log.Warningln("Couldn't commit table modifications --> error is ", commitErr)
	}
}
<commit_msg>Fixed error where md5 datatype wasn't being recognized (md5 != text).<commit_after>package schemamagic

import (
	"fmt"

	"gopkg.in/jackc/pgx.v2"
)

// Table holds the table details as well as all the columns inside the table
type Table struct {
	Name          string
	DefaultSchema string
	Database      string
	Tx            *pgx.Tx
	Autocommit    bool
	Columns       []Column
}

// NewTable creates and returns an instance of a postgres table
func NewTable(t Table) *Table {
	table := new(Table)
	table.Name = t.Name
	table.DefaultSchema = t.DefaultSchema
	table.Database = t.Database
	table.Autocommit = t.Autocommit
	table.Columns = t.Columns
	table.Tx = t.Tx
	return table
}

// Append method accepts a column and appends it to the list of columns of the table
func (t *Table) Append(col Column) {
	t.Columns = append(t.Columns, col)
}

// Begin method initiates a DB transaction and checks if table (Name) exists in the DB. If it does, then it calls updateTable(). If it doesn't, it calls createTable(), and then updateTable()"""
func (t *Table) Begin() {
	log.Infoln("Operating on table --> ", t.Name)
	// Create the schema here
	schemaStatement := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", t.DefaultSchema)
	_, err := t.Tx.Exec(schemaStatement)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("Couldn't create schema --> ", t.DefaultSchema, " with error being --> ", err)
	}
	//  Check if table exists in the database
	presence := t.checkTableExistence()
	if !presence {
		// Table does not exist --> need to create it
		t.createTable()
	}
	// Loop over all the available columns and call updateTable() on each column
	for _, col := range t.Columns {
		log.Debugln("-----------------------------------------------")
		t.updateTable(col)
		log.Debugln("-----------------------------------------------")
	}
	if t.Autocommit {
		commitErr := t.Tx.Commit()
		if commitErr != nil {
			t.Tx.Rollback()
			log.Warningln("Couldn't commit changes to the TABLE --> ", t.Name, " with error being --> ", commitErr)
		}
	}
}

// checkTableExistence returns if the table already exists in the DB
func (t *Table) checkTableExistence() bool {
	var presence bool
	statement := fmt.Sprintf("SELECT exists(select 0 from pg_class where relname = '%s')", t.Name)
	err := t.Tx.QueryRow(statement).Scan(&presence)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("While querying for table existence, error is --> ", err)
	}
	log.Debugln("While checking for table existence, presence is ", presence)
	return presence
}

// createTable method creates the table in the particular DB
func (t *Table) createTable() {
	statement := fmt.Sprintf("CREATE TABLE %s()", t.Name)
	err := t.executeSQL(statement)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("While creating table --> ", t.Name, " error is --> ", err)
	}
}

// DropTable method drops the table from the DB
func (t *Table) DropTable() {
	presence := t.checkTableExistence()
	if presence {
		// Drop the table here
		log.Infoln("Trying to drop table --> ", t.Name)
		statement := fmt.Sprintf("DROP TABLE %s", t.Name)
		err := t.executeSQL(statement)
		if err != nil {
			t.Tx.Rollback()
			log.Warningln("While dropping table --> ", t.Name, " error is --> ", err)
		} else {
			log.Infoln("Successfully dropped table --> ", t.Name)
		}
	}
}

// updateTable alters the table by adding a new column to it, passed as the method parameter
func (t *Table) updateTable(col Column) {
	minNumberOfSteps := 0
	maxNumberOfSteps := 7
	columnPresence := t.checkColumnPresence(col.Name)
	if columnPresence {
		log.Debugln("Column --> ", col.Name, " already exists")
		columnDatatypeMatch := t.checkColumnDatatype(col)
		log.Debugln("Column --> ", col.Name, " datatype match value is --> ", columnDatatypeMatch)
		if columnDatatypeMatch {
			// If the datatype matches, pass, and SET minNumberOfSteps = 0 and maxNumberOfSteps = 0, since there should be no further execution, as the column is in its actual state
			minNumberOfSteps = 0
			maxNumberOfSteps = 0
		} else {
			// If the datatype does not match, then step=101, and set minNumberOfSteps to 1, since the step=1 has already been executed in the form of datatype modificaton
			minNumberOfSteps = 1
			statement, statementErr := col.prepareSQLStatement(101, t.Name)
			if statementErr != nil {
				err := t.executeSQL(statement)
				if err != nil {
					t.Tx.Rollback()
					log.Warningln("While executing SQL --> \n", statement, "\nerror is ", err)
				}
			}
		}
	} else {
		// Column does not exist
		log.Debugln("Column --> ", col.Name, " does not exist")
		minNumberOfSteps = 0
	}

	//  There are 4 steps involved here.

	// The first step is to call col.prepareSQLStatement with a step=1, which would return an SQL statement that would be used
	// to alter the table structure with a default set to NULL -> this is a very cheap operation,
	//  since the access exclusive lock would be acquired for a short time.

	//  The second step is to call col.prepareSQLStatement with a step=2, which would return an SQL statement that would be used
	//  to set the defaults to Column.default_value. In case Column.default_exists is False, then it returns an empty statement.
	//  This can be checked before executing the statement

	//  The third step is to call col.prepareSQLStatement with a step=3, which would update all the rows in the table with the default
	//  value.

	//  The fourth step is to call col.prepareSQLStatement with a step=3, which would return an SQL statement that would be used to
	//  alter the sequence start, in case the datatype is serial/bigserial. Similar to step=2, if it returns an empty statement,
	//  it either means that the datatype doesn't support a sequence, or the sequence needs to begin at 0.
	for step := minNumberOfSteps; step < maxNumberOfSteps; step++ {
		statement, statementErr := col.prepareSQLStatement(step+1, t.Name)
		log.Debugln("In steps, statement is \n", statement, " and error is ", statementErr)
		if statementErr == nil {
			err := t.executeSQL(statement)
			if err != nil {
				t.Tx.Rollback()
				log.Warningln("Statement --> ", statement, " could not be executed because of error --> ", err)
			}
		}
	}
}

// executeSQL executes the SQL query
func (t *Table) executeSQL(sql string) error {
	// var err error
	if sql != "" {
		_, err := t.Tx.Exec(sql)
		log.Debugln("Executing Statement --> \n", sql, " and error is ", err)
		return err
	}
	return nil
}

// checkColumnPresence checks if the column name passed is present in the current table
func (t *Table) checkColumnPresence(columnName string) bool {
	var presence bool
	statement := fmt.Sprintf("SELECT EXISTS(SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '%s' AND table_catalog = '%s' AND column_name = '%s')", t.Name, t.Database, columnName)
	err := t.Tx.QueryRow(statement).Scan(&presence)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("In checkColumnPresence, error for table --> ", t.Name, " and Column --> ", columnName, " is ", err)
	}
	return presence
}

// checkColumnDatatype checks if the column datatype of the column name passed is equal to the column datatype present in the table
func (t *Table) checkColumnDatatype(col Column) bool {
	columnName := col.Name
	columnDatatype := col.Datatype
	columnDefault := col.DefaultValue
	var (
		dbDatatype      string
		columnDefaultDB pgx.NullString
		presence        bool
	)
	statement := fmt.Sprintf("SELECT data_type, column_default FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = '%s' AND table_catalog = '%s' AND column_name = '%s'", t.Name, t.Database, columnName)

	err := t.Tx.QueryRow(statement).Scan(&dbDatatype, &columnDefaultDB)
	if err != nil {
		t.Tx.Rollback()
		log.Warningln("While querying for column data type in table --> ", t.Name, " error is --> ", err)
	}

	log.Debugln("Datatype DB is ", dbDatatype, " and ColumnDefaultDB is ", columnDefaultDB, " and ColumnDefault is ", columnDefault, " and Column Datatype is ", columnDatatype)
	// Check if there are sequences here
	if columnDefaultDB.Valid {
		// This is a sequence
		if columnDatatype == "bigserial" && dbDatatype == "bigint" {
			presence = true
		} else if columnDatatype == "serial" && dbDatatype == "integer" {
			presence = true
		} else if columnDefaultDB.String == columnDefault {
			presence = true
		} else if columnDatatype == dbDatatype {
			presence = true
			// presence = true
		} else {
			presence = false
		}

	} else {
		// This is a non-sequence
		if columnDatatype == dbDatatype {
			presence = true
		} else {
			presence = false
		}
	}

	log.Debugln("In check column datatype, the value of presence is ", presence)
	return presence
}

func (t *Table) commit() {
	commitErr := t.Tx.Commit()
	if commitErr != nil {
		t.Tx.Rollback()
		log.Warningln("Couldn't commit table modifications --> error is ", commitErr)
	}
}
<|endoftext|>


<commit_before>package main

import "strconv"

func initRFixnum() *RObject {
	obj := &RObject{}
	obj.name = "RFixnum"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RFixnum method initialization
	obj.methods["new"] = &RMethod{gofunc: RFixnum_new}
	obj.methods["="] = &RMethod{gofunc: RFixnum_assign}
	obj.methods["+"] = &RMethod{gofunc: RFixnum_add}
	obj.methods["-"] = &RMethod{gofunc: RFixnum_sub}
	obj.methods["*"] = &RMethod{gofunc: RFixnum_mul}
	obj.methods["/"] = &RMethod{gofunc: RFixnum_div}
	obj.methods["to_s"] = &RMethod{gofunc: RFixnum_to_s}
	obj.methods["inspect"] = &RMethod{gofunc: RFixnum_to_s}
	obj.methods["times"] = &RMethod{gofunc: RFixnum_times}

	return obj
}

// Fixnum.new(int=0)
// v = [int64]
func RFixnum_new(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	val := int64(0)
	if len(v) == 1 {
		val = v[0].(int64)
	}

	obj := &RObject{}
	obj.class = vm.consts["RFixnum"]
	obj.val.fixnum = val
	if env == nil {
		obj.rev = vm.rev
	} else {
		obj.rev = env.transactionPC.rev
	}

	return obj
}

func RFixnum_assign(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	new_obj := env.transactionPC.objectSet[obj]

	if obj == new_obj {
		new_obj = RFixnum_new(vm, env, nil, v).(*RObject)
		env.transactionPC.objectSet[obj] = new_obj
	}

	new_obj.val.fixnum = v[0].(*RObject).val.fixnum

	return obj
}

func RFixnum_add(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum + v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_sub(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum - v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_mul(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum * v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_div(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum / v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_to_s(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	return RString_new(vm, env, nil, []Object{strconv.FormatInt(obj.val.fixnum, 10)})
}

// RFixnum.times(&block)
// Given: [RBlock]
// Block parameters: [i]
func RFixnum_times(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	obj = env.transactionPC.objectSet[obj]
	if v != nil && len(v) == 1 { // Given a single RBlock
		block := v[0].(*RObject)

		dummy_args := []Object{0}
		params := []*RObject{nil}

		for i := int64(0); i < obj.val.fixnum; i++ {
			// Prepare block arguments
			dummy_args[0] = i
			params[0] = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

			// Let VM handle all other stuff such as clean call frame
			vm.executeBlock(env, block, params)
		}
	}
	return obj
}
<commit_msg>Fixed crash on anonymous fixnum object creation<commit_after>package main

import "strconv"

func initRFixnum() *RObject {
	obj := &RObject{}
	obj.name = "RFixnum"
	obj.ivars = make(map[string]Object)
	obj.class = nil
	obj.methods = make(map[string]*RMethod)

	// RFixnum method initialization
	obj.methods["new"] = &RMethod{gofunc: RFixnum_new}
	obj.methods["="] = &RMethod{gofunc: RFixnum_assign}
	obj.methods["+"] = &RMethod{gofunc: RFixnum_add}
	obj.methods["-"] = &RMethod{gofunc: RFixnum_sub}
	obj.methods["*"] = &RMethod{gofunc: RFixnum_mul}
	obj.methods["/"] = &RMethod{gofunc: RFixnum_div}
	obj.methods["to_s"] = &RMethod{gofunc: RFixnum_to_s}
	obj.methods["inspect"] = &RMethod{gofunc: RFixnum_to_s}
	obj.methods["times"] = &RMethod{gofunc: RFixnum_times}

	return obj
}

// Fixnum.new(int=0)
// v = [int64]
func RFixnum_new(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	val := int64(0)
	if len(v) == 1 {
		val = v[0].(int64)
	}

	obj := &RObject{}
	obj.class = vm.consts["RFixnum"]
	obj.val.fixnum = val
	if env == nil {
		obj.rev = vm.rev
	} else {
		obj.rev = env.transactionPC.rev
	}

	return obj
}

func RFixnum_assign(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	new_obj := env.transactionPC.objectSet[obj]

	if obj == new_obj {
		new_obj = RFixnum_new(vm, env, nil, v).(*RObject)
		env.transactionPC.objectSet[obj] = new_obj
	}

	new_obj.val.fixnum = v[0].(*RObject).val.fixnum

	return obj
}

func RFixnum_add(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum + v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_sub(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum - v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_mul(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum * v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_div(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	dummy_args := []Object{obj.val.fixnum / v[0].(*RObject).val.fixnum}
	obj = RFixnum_new(vm, env, nil, dummy_args).(*RObject)

	return obj
}

func RFixnum_to_s(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	return RString_new(vm, env, nil, []Object{strconv.FormatInt(obj.val.fixnum, 10)})
}

// RFixnum.times(&block)
// Given: [RBlock]
// Block parameters: [i]
func RFixnum_times(vm *GobiesVM, env *ThreadEnv, receiver Object, v []Object) Object {
	obj := addRObjectToSet(receiver.(*RObject), env)
	obj = env.transactionPC.objectSet[obj]
	if v != nil && len(v) == 1 { // Given a single RBlock
		block := v[0].(*RObject)

		dummy_args := []Object{0}
		params := []*RObject{nil}

		for i := int64(0); i < obj.val.fixnum; i++ {
			// Prepare block arguments
			dummy_args[0] = i
			params[0] = RFixnum_new(vm, nil, nil, dummy_args).(*RObject)

			// Let VM handle all other stuff such as clean call frame
			vm.executeBlock(env, block, params)
		}
	}
	return obj
}
<|endoftext|>