<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The list package implements a doubly linked list.
//
// To iterate over a list (where l is a *List):
//	for e := l.Front(); e != nil; e = e.Next() {
//		// do something with e.Value
//	}
//
package list

// Element is an element in the linked list.
type Element struct {
	// Next and previous pointers in the doubly-linked list of elements.
	// The front of the list has prev = nil, and the back has next = nil.
	next, prev *Element

	// The list to which this element belongs.
	list *List

	// The contents of this list element.
	Value interface{}
}

// Next returns the next list element or nil.
func (e *Element) Next() *Element { return e.next }

// Prev returns the previous list element or nil.
func (e *Element) Prev() *Element { return e.prev }

// List represents a doubly linked list.
// The zero value for List is an empty list ready to use.
type List struct {
	front, back *Element
	len         int
}

// Init initializes or clears a List.
func (l *List) Init() *List {
	l.front = nil
	l.back = nil
	l.len = 0
	return l
}

// New returns an initialized list.
func New() *List { return new(List) }

// Front returns the first element in the list.
func (l *List) Front() *Element { return l.front }

// Back returns the last element in the list.
func (l *List) Back() *Element { return l.back }

// Remove removes the element from the list.
func (l *List) Remove(e *Element) {
	l.remove(e)
	e.list = nil // do what remove does not
}

// remove the element from the list, but do not clear the Element's list field.
// This is so that other List methods may use remove when relocating Elements
// without needing to restore the list field.
func (l *List) remove(e *Element) {
	if e.list != l {
		return
	}
	if e.prev == nil {
		l.front = e.next
	} else {
		e.prev.next = e.next
	}
	if e.next == nil {
		l.back = e.prev
	} else {
		e.next.prev = e.prev
	}

	e.prev = nil
	e.next = nil
	l.len--
}

func (l *List) insertBefore(e *Element, mark *Element) {
	if mark.prev == nil {
		// new front of the list
		l.front = e
	} else {
		mark.prev.next = e
	}
	e.prev = mark.prev
	mark.prev = e
	e.next = mark
	l.len++
}

func (l *List) insertAfter(e *Element, mark *Element) {
	if mark.next == nil {
		// new back of the list
		l.back = e
	} else {
		mark.next.prev = e
	}
	e.next = mark.next
	mark.next = e
	e.prev = mark
	l.len++
}

func (l *List) insertFront(e *Element) {
	if l.front == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertBefore(e, l.front)
}

func (l *List) insertBack(e *Element) {
	if l.back == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertAfter(e, l.back)
}

// PushFront inserts the value at the front of the list and returns a new Element containing the value.
func (l *List) PushFront(value interface{}) *Element {
	e := &Element{nil, nil, l, value}
	l.insertFront(e)
	return e
}

// PushBack inserts the value at the back of the list and returns a new Element containing the value.
func (l *List) PushBack(value interface{}) *Element {
	e := &Element{nil, nil, l, value}
	l.insertBack(e)
	return e
}

// InsertBefore inserts the value immediately before mark and returns a new Element containing the value.
func (l *List) InsertBefore(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertBefore(e, mark)
	return e
}

// InsertAfter inserts the value immediately after mark and returns a new Element containing the value.
func (l *List) InsertAfter(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertAfter(e, mark)
	return e
}

// MoveToFront moves the element to the front of the list.
func (l *List) MoveToFront(e *Element) {
	if e.list != l || l.front == e {
		return
	}
	l.remove(e)
	l.insertFront(e)
}

// MoveToBack moves the element to the back of the list.
func (l *List) MoveToBack(e *Element) {
	if e.list != l || l.back == e {
		return
	}
	l.remove(e)
	l.insertBack(e)
}

// Len returns the number of elements in the list.
func (l *List) Len() int { return l.len }

// PushBackList inserts each element of ol at the back of the list.
func (l *List) PushBackList(ol *List) {
	last := ol.Back()
	for e := ol.Front(); e != nil; e = e.Next() {
		l.PushBack(e.Value)
		if e == last {
			break
		}
	}
}

// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved.
func (l *List) PushFrontList(ol *List) {
	first := ol.Front()
	for e := ol.Back(); e != nil; e = e.Prev() {
		l.PushFront(e.Value)
		if e == first {
			break
		}
	}
}
<commit_msg>container/list: make Remove return Value of removed element. When it is known that there is already at least one element in the list, it is awkwardly verbose to use three lines and an extra variable declaration to remove the first or last item (a common case), rather than use a simple expression.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The list package implements a doubly linked list.
//
// To iterate over a list (where l is a *List):
//	for e := l.Front(); e != nil; e = e.Next() {
//		// do something with e.Value
//	}
//
package list

// Element is an element in the linked list.
type Element struct {
	// Next and previous pointers in the doubly-linked list of elements.
	// The front of the list has prev = nil, and the back has next = nil.
	next, prev *Element

	// The list to which this element belongs.
	list *List

	// The contents of this list element.
	Value interface{}
}

// Next returns the next list element or nil.
func (e *Element) Next() *Element { return e.next }

// Prev returns the previous list element or nil.
func (e *Element) Prev() *Element { return e.prev }

// List represents a doubly linked list.
// The zero value for List is an empty list ready to use.
type List struct {
	front, back *Element
	len         int
}

// Init initializes or clears a List.
func (l *List) Init() *List {
	l.front = nil
	l.back = nil
	l.len = 0
	return l
}

// New returns an initialized list.
func New() *List { return new(List) }

// Front returns the first element in the list.
func (l *List) Front() *Element { return l.front }

// Back returns the last element in the list.
func (l *List) Back() *Element { return l.back }

// Remove removes the element from the list
// and returns its Value.
func (l *List) Remove(e *Element) interface{} {
	l.remove(e)
	e.list = nil // do what remove does not
	return e.Value
}

// remove the element from the list, but do not clear the Element's list field.
// This is so that other List methods may use remove when relocating Elements
// without needing to restore the list field.
func (l *List) remove(e *Element) {
	if e.list != l {
		return
	}
	if e.prev == nil {
		l.front = e.next
	} else {
		e.prev.next = e.next
	}
	if e.next == nil {
		l.back = e.prev
	} else {
		e.next.prev = e.prev
	}

	e.prev = nil
	e.next = nil
	l.len--
}

func (l *List) insertBefore(e *Element, mark *Element) {
	if mark.prev == nil {
		// new front of the list
		l.front = e
	} else {
		mark.prev.next = e
	}
	e.prev = mark.prev
	mark.prev = e
	e.next = mark
	l.len++
}

func (l *List) insertAfter(e *Element, mark *Element) {
	if mark.next == nil {
		// new back of the list
		l.back = e
	} else {
		mark.next.prev = e
	}
	e.next = mark.next
	mark.next = e
	e.prev = mark
	l.len++
}

func (l *List) insertFront(e *Element) {
	if l.front == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertBefore(e, l.front)
}

func (l *List) insertBack(e *Element) {
	if l.back == nil {
		// empty list
		l.front, l.back = e, e
		e.prev, e.next = nil, nil
		l.len = 1
		return
	}
	l.insertAfter(e, l.back)
}

// PushFront inserts the value at the front of the list and returns a new Element containing the value.
func (l *List) PushFront(value interface{}) *Element {
	e := &Element{nil, nil, l, value}
	l.insertFront(e)
	return e
}

// PushBack inserts the value at the back of the list and returns a new Element containing the value.
func (l *List) PushBack(value interface{}) *Element {
	e := &Element{nil, nil, l, value}
	l.insertBack(e)
	return e
}

// InsertBefore inserts the value immediately before mark and returns a new Element containing the value.
func (l *List) InsertBefore(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertBefore(e, mark)
	return e
}

// InsertAfter inserts the value immediately after mark and returns a new Element containing the value.
func (l *List) InsertAfter(value interface{}, mark *Element) *Element {
	if mark.list != l {
		return nil
	}
	e := &Element{nil, nil, l, value}
	l.insertAfter(e, mark)
	return e
}

// MoveToFront moves the element to the front of the list.
func (l *List) MoveToFront(e *Element) {
	if e.list != l || l.front == e {
		return
	}
	l.remove(e)
	l.insertFront(e)
}

// MoveToBack moves the element to the back of the list.
func (l *List) MoveToBack(e *Element) {
	if e.list != l || l.back == e {
		return
	}
	l.remove(e)
	l.insertBack(e)
}

// Len returns the number of elements in the list.
func (l *List) Len() int { return l.len }

// PushBackList inserts each element of ol at the back of the list.
func (l *List) PushBackList(ol *List) {
	last := ol.Back()
	for e := ol.Front(); e != nil; e = e.Next() {
		l.PushBack(e.Value)
		if e == last {
			break
		}
	}
}

// PushFrontList inserts each element of ol at the front of the list. The ordering of the passed list is preserved.
func (l *List) PushFrontList(ol *List) {
	first := ol.Front()
	for e := ol.Back(); e != nil; e = e.Prev() {
		l.PushFront(e.Value)
		if e == first {
			break
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The ring package implements operations on circular lists.
package ring

// A Ring is an element of a circular list, or ring.
// Rings do not have a beginning or end; a pointer to any ring element
// serves as reference to the entire ring. Empty rings are represented
// as nil Ring pointers. The zero value for a Ring is a one-element
// ring with a nil Value.
//
type Ring struct {
	next, prev *Ring;
	Value interface{};  // for use by client; untouched by this library
}

func (r *Ring) init() *Ring {
	r.next = r;
	r.prev = r;
	return r;
}

// Next returns the next ring element. r must not be empty.
func (r *Ring) Next() *Ring {
	if r.next == nil {
		return r.init();
	}
	return r.next;
}

// Prev returns the previous ring element. r must not be empty.
func (r *Ring) Prev() *Ring {
	if r.next == nil {
		return r.init();
	}
	return r.prev;
}

// Move moves n % r.Len() elements backward (n < 0) or forward (n > 0)
// in the ring and returns that ring element. r must not be empty.
//
func (r *Ring) Move(n int) *Ring {
	if r.next == nil {
		return r.init();
	}
	switch {
	case n < 0:
		for ; n < 0; n++ {
			r = r.prev;
		}
	case n > 0:
		for ; n > 0; n-- {
			r = r.next;
		}
	}
	return r;
}

// New creates a ring of n elements.
func New(n int) *Ring {
	if n <= 0 {
		return nil;
	}
	r := new(Ring);
	p := r;
	for i := 1; i < n; i++ {
		p.next = &Ring{prev: p};
		p = p.next;
	}
	p.next = r;
	r.prev = p;
	return r;
}

// Link connects ring r with with ring s such that r.Next(1)
// becomes s and returns the original value for r.Next(1).
// r must not be empty.
//
// If r and s point to the same ring, linking
// them removes the elements between r and s from the ring.
// The removed elements form a subring and the result is a
// reference to that subring (if no elements were removed,
// the result is still the original value for r.Next(1),
// and not nil).
//
// If r and s point to different rings, linking
// them creates a single ring with the elements of s inserted
// after r. The result points to the element following the
// last element of s after insertion.
//
func (r *Ring) Link(s *Ring) *Ring {
	n := r.Next();
	if s != nil {
		p := s.Prev();
		// Note: Cannot use multiple assignment because
		// evaluation order of LHS is not specified.
		r.next = s;
		s.prev = r;
		n.prev = p;
		p.next = n;
	}
	return n;
}

// Unlink removes n % r.Len() elements from the ring r, starting
// at r.Next(). If n % r.Len() == 0, r remains unchanged.
// The result is the removed subring. r must not be empty.
//
func (r *Ring) Unlink(n int) *Ring {
	if n <= 0 {
		return nil;
	}
	return r.Link(r.Move(n + 1));
}

// Len computes the number of elements in ring r.
// It executes in time proportional to the number of elements.
//
func (r *Ring) Len() int {
	n := 0;
	if r != nil {
		n = 1;
		for p := r.Next(); p != r; p = p.next {
			n++;
		}
	}
	return n;
}

// Forward returns a channel for forward iteration through a ring.
// Iteration is undefined if the ring is changed during iteration.
//
func (r *Ring) Forward() <-chan *Ring {
	c := make(chan *Ring);
	go func() {
		if r != nil {
			c <- r;
			for p := r.Next(); p != r; p = p.next {
				c <- p;
			}
		}
		close(c);
	}();
	return c;
}

// Backward returns a channel for backward iteration through a ring.
// Iteration is undefined if the ring is changed during iteration.
//
func (r *Ring) Backward() <-chan *Ring {
	c := make(chan *Ring);
	go func() {
		if r != nil {
			c <- r;
			for p := r.Prev(); p != r; p = p.prev {
				c <- p;
			}
		}
		close(c);
	}();
	return c;
}
<commit_msg>Fixed typos in comments.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The ring package implements operations on circular lists.
package ring

// A Ring is an element of a circular list, or ring.
// Rings do not have a beginning or end; a pointer to any ring element
// serves as reference to the entire ring. Empty rings are represented
// as nil Ring pointers. The zero value for a Ring is a one-element
// ring with a nil Value.
//
type Ring struct {
	next, prev *Ring;
	Value interface{};  // for use by client; untouched by this library
}

func (r *Ring) init() *Ring {
	r.next = r;
	r.prev = r;
	return r;
}

// Next returns the next ring element. r must not be empty.
func (r *Ring) Next() *Ring {
	if r.next == nil {
		return r.init();
	}
	return r.next;
}

// Prev returns the previous ring element. r must not be empty.
func (r *Ring) Prev() *Ring {
	if r.next == nil {
		return r.init();
	}
	return r.prev;
}

// Move moves n % r.Len() elements backward (n < 0) or forward (n >= 0)
// in the ring and returns that ring element. r must not be empty.
//
func (r *Ring) Move(n int) *Ring {
	if r.next == nil {
		return r.init();
	}
	switch {
	case n < 0:
		for ; n < 0; n++ {
			r = r.prev;
		}
	case n > 0:
		for ; n > 0; n-- {
			r = r.next;
		}
	}
	return r;
}

// New creates a ring of n elements.
func New(n int) *Ring {
	if n <= 0 {
		return nil;
	}
	r := new(Ring);
	p := r;
	for i := 1; i < n; i++ {
		p.next = &Ring{prev: p};
		p = p.next;
	}
	p.next = r;
	r.prev = p;
	return r;
}

// Link connects ring r with with ring s such that r.Next()
// becomes s and returns the original value for r.Next().
// r must not be empty.
//
// If r and s point to the same ring, linking
// them removes the elements between r and s from the ring.
// The removed elements form a subring and the result is a
// reference to that subring (if no elements were removed,
// the result is still the original value for r.Next(),
// and not nil).
//
// If r and s point to different rings, linking
// them creates a single ring with the elements of s inserted
// after r. The result points to the element following the
// last element of s after insertion.
//
func (r *Ring) Link(s *Ring) *Ring {
	n := r.Next();
	if s != nil {
		p := s.Prev();
		// Note: Cannot use multiple assignment because
		// evaluation order of LHS is not specified.
		r.next = s;
		s.prev = r;
		n.prev = p;
		p.next = n;
	}
	return n;
}

// Unlink removes n % r.Len() elements from the ring r, starting
// at r.Next(). If n % r.Len() == 0, r remains unchanged.
// The result is the removed subring. r must not be empty.
//
func (r *Ring) Unlink(n int) *Ring {
	if n <= 0 {
		return nil;
	}
	return r.Link(r.Move(n + 1));
}

// Len computes the number of elements in ring r.
// It executes in time proportional to the number of elements.
//
func (r *Ring) Len() int {
	n := 0;
	if r != nil {
		n = 1;
		for p := r.Next(); p != r; p = p.next {
			n++;
		}
	}
	return n;
}

// Forward returns a channel for forward iteration through a ring.
// Iteration is undefined if the ring is changed during iteration.
//
func (r *Ring) Forward() <-chan *Ring {
	c := make(chan *Ring);
	go func() {
		if r != nil {
			c <- r;
			for p := r.Next(); p != r; p = p.next {
				c <- p;
			}
		}
		close(c);
	}();
	return c;
}

// Backward returns a channel for backward iteration through a ring.
// Iteration is undefined if the ring is changed during iteration.
//
func (r *Ring) Backward() <-chan *Ring {
	c := make(chan *Ring);
	go func() {
		if r != nil {
			c <- r;
			for p := r.Prev(); p != r; p = p.prev {
				c <- p;
			}
		}
		close(c);
	}();
	return c;
}
<|endoftext|>


<commit_before>package enproxy

import (
	"bufio"
	"io"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/idletiming"
)

const (
	X_ENPROXY_ID         = "X-Enproxy-Id"
	X_ENPROXY_DEST_ADDR  = "X-Enproxy-Dest-Addr"
	X_ENPROXY_EOF        = "X-Enproxy-EOF"
	X_ENPROXY_PROXY_HOST = "X-Enproxy-Proxy-Host"
	X_ENPROXY_OP         = "X-Enproxy-Op"

	OP_WRITE = "write"
	OP_READ  = "read"
)

var (
	log = golog.LoggerFor("enproxy")
)

var (
	defaultWriteFlushTimeout = 35 * time.Millisecond
	defaultReadFlushTimeout  = 35 * time.Millisecond
	defaultIdleTimeoutClient = 30 * time.Second
	defaultIdleTimeoutServer = 70 * time.Second

	// closeChannelDepth: controls depth of channels used for close processing.
	// Doesn't need to be particularly big, as it's just used to prevent
	// deadlocks on multiple calls to Close().
	closeChannelDepth = 20

	bodySize = 65536 // size of buffer used for request bodies

	oneSecond = 1 * time.Second
)

// Conn is a net.Conn that tunnels its data via an httpconn.Proxy using HTTP
// requests and responses.  It assumes that streaming requests are not supported
// by the underlying servers/proxies, and so uses a polling technique similar to
// the one used by meek, but different in that data is not encoded as JSON.
// https://trac.torproject.org/projects/tor/wiki/doc/AChildsGardenOfPluggableTransports#Undertheencryption.
//
// enproxy uses two parallel channels to send and receive data.  One channel
// handles writing data out by making sequential POST requests to the server
// which encapsulate the outbound data in their request bodies, while the other
// channel handles reading data by making GET requests and grabbing the data
// encapsulated in the response bodies.
//
// Write Channel:
//
//   1. Accept writes, piping these to the proxy as the body of an http POST
//   2. Continue to pipe the writes until the pause between consecutive writes
//      exceeds the IdleInterval, at which point we finish the request body. We
//      do this because it is assumed that intervening proxies (e.g. CloudFlare
//      CDN) do not allow streaming requests, so it is necessary to finish the
//      request for data to get flushed to the destination server.
//   3. After receiving a response to the POST request, return to step 1
//
// Read Channel:
//
//   1. Accept reads, issuing a new GET request if one is not already ongoing
//   2. Process read by grabbing data from the response to the GET request
//   3. Continue to accept reads, grabbing these from the response of the
//      existing GET request
//   4. Once the response to the GET request reaches EOF, return to step 1. This
//      will happen because the proxy periodically closes responses to make sure
//      intervening proxies don't time out.
//   5. If a response is received with a special header indicating a true EOF
//      from the destination server, return EOF to the reader
//
type Conn struct {
	// addr: the host:port of the destination server that we're trying to reach
	addr string

	tcpAddr *net.TCPAddr

	// config: configuration of this Conn
	config *Config

	// initialResponseCh: Self-reported FQDN of the proxy serving this connection
	// plus initial response from proxy.
	//
	// This allows us to guarantee we reach the same server in subsequent
	// requests, even if it was initially reached through a FQDN that may
	// resolve to different IPs in different DNS lookups (e.g. as in DNS round
	// robin).
	initialResponseCh chan hostWithResponse

	// id: unique identifier for this connection. This is used by the Proxy to
	// associate requests from this connection to the corresponding outbound
	// connection on the Proxy side.  It is populated using a type 4 UUID.
	id string

	/* Write processing */
	writeRequestsCh  chan []byte     // requests to write
	writeResponsesCh chan rwResponse // responses for writes
	doneWritingCh    chan bool
	rs               requestStrategy

	/* Request processing (for writes) */
	requestOutCh      chan *request // channel for next outgoing request body
	requestFinishedCh chan error
	doneRequestingCh  chan bool

	/* Read processing */
	readRequestsCh  chan []byte     // requests to read
	readResponsesCh chan rwResponse // responses for reads
	doneReadingCh   chan bool

	/* Fields for tracking activity/closed status */
	lastActivityTime  time.Time    // time of last read or write
	lastActivityMutex sync.RWMutex // mutex controlling access to lastActivityTime
	asyncErr          error        // error that occurred during asynchronous processing
	asyncErrMutex     sync.RWMutex // mutex guarding asyncErr
	asyncErrCh        chan error   // channel used to interrupted any waiting reads/writes with an async error
	closed            bool         // whether or not this Conn is closed
	closedMutex       sync.RWMutex // mutex controlling access to closed flag

	/* Track current response */
	resp *http.Response // the current response being used to read data
}

// Config configures a Conn
type Config struct {
	// DialProxy: function to open a connection to the proxy
	DialProxy dialFunc

	// NewRequest: function to create a new request to the proxy
	NewRequest newRequestFunc

	// FlushTimeout: how long to let writes idle before writing out a
	// request to the proxy.  Defaults to 15 milliseconds.
	FlushTimeout time.Duration

	// IdleTimeout: how long to wait before closing an idle connection, defaults
	// to 30 seconds on the client and 70 seconds on the server proxy.
	//
	// For clients, the value should be set lower than the proxy's idle timeout
	// so that enproxy redials before the active connection is closed. The value
	// should be set higher than the maximum possible time between the proxy
	// receiving the last data from a request and the proxy returning the first
	// data of the response, otherwise the connection will be closed in the
	// middle of processing a request.
	IdleTimeout time.Duration

	// BufferRequests: if true, requests to the proxy will be buffered and sent
	// with identity encoding.  If false, they'll be streamed with chunked
	// encoding.
	BufferRequests bool
}

// dialFunc is a function that dials an address (e.g. the upstream proxy)
type dialFunc func(addr string) (net.Conn, error)

// newRequestFunc is a function that builds a new request to the upstream proxy
type newRequestFunc func(host string, method string, body io.Reader) (*http.Request, error)

// rwResponse is a response to a read or write
type rwResponse struct {
	n   int
	err error
}

type connInfo struct {
	conn        *idletiming.IdleTimingConn
	bufReader   *bufio.Reader
	closed      bool
	closedMutex sync.Mutex
}

type hostWithResponse struct {
	proxyHost string
	proxyConn *connInfo
	resp      *http.Response
	err       error
}

// Write() implements the function from net.Conn
func (c *Conn) Write(b []byte) (n int, err error) {
	err = c.getAsyncErr()
	if err != nil {
		return
	}

	if c.submitWrite(b) {
		select {
		case res, ok := <-c.writeResponsesCh:
			if !ok {
				return 0, io.EOF
			} else {
				return res.n, res.err
			}
		case err := <-c.asyncErrCh:
			return 0, err
		}
	} else {
		return 0, io.EOF
	}
}

// Read() implements the function from net.Conn
func (c *Conn) Read(b []byte) (n int, err error) {
	err = c.getAsyncErr()
	if err != nil {
		return
	}

	if c.submitRead(b) {
		select {
		case res, ok := <-c.readResponsesCh:
			if !ok {
				return 0, io.EOF
			} else {
				return res.n, res.err
			}
		case err := <-c.asyncErrCh:
			return 0, err
		}
	} else {
		return 0, io.EOF
	}
}

func (c *Conn) fail(err error) {
	c.asyncErrMutex.Lock()
	if c.asyncErr != nil {
		c.asyncErr = err
	}
	c.asyncErrMutex.Unlock()

	// Let any waiting readers or writers know about the error
	for i := 0; i < 2; i++ {
		select {
		case c.asyncErrCh <- err:
			// submitted okay
		default:
			// channel full, continue
		}
	}
}

func (c *Conn) getAsyncErr() error {
	c.asyncErrMutex.RLock()
	err := c.asyncErr
	c.asyncErrMutex.RUnlock()
	return err
}

// Close() implements the function from net.Conn
func (c *Conn) Close() error {
	c.closedMutex.Lock()
	defer c.closedMutex.Unlock()
	if !c.closed {
		close(c.writeRequestsCh)
		close(c.readRequestsCh)
		<-c.doneReadingCh
		<-c.doneWritingCh
		<-c.doneRequestingCh
		c.closed = true
	}
	return nil
}

// LocalAddr() is not implemented
func (c *Conn) LocalAddr() net.Addr {
	panic("LocalAddr() not implemented")
}

// RemoteAddr() is not implemented
func (c *Conn) RemoteAddr() net.Addr {
	return c.tcpAddr
}

// SetDeadline() is currently unimplemented.
func (c *Conn) SetDeadline(t time.Time) error {
	log.Tracef("SetDeadline not implemented")
	return nil
}

// SetReadDeadline() is currently unimplemented.
func (c *Conn) SetReadDeadline(t time.Time) error {
	log.Tracef("SetReadDeadline not implemented")
	return nil
}

// SetWriteDeadline() is currently unimplemented.
func (c *Conn) SetWriteDeadline(t time.Time) error {
	log.Tracef("SetWriteDeadline not implemented")
	return nil
}
<commit_msg>Closing connection on fail<commit_after>package enproxy

import (
	"bufio"
	"io"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/getlantern/golog"
	"github.com/getlantern/idletiming"
)

const (
	X_ENPROXY_ID         = "X-Enproxy-Id"
	X_ENPROXY_DEST_ADDR  = "X-Enproxy-Dest-Addr"
	X_ENPROXY_EOF        = "X-Enproxy-EOF"
	X_ENPROXY_PROXY_HOST = "X-Enproxy-Proxy-Host"
	X_ENPROXY_OP         = "X-Enproxy-Op"

	OP_WRITE = "write"
	OP_READ  = "read"
)

var (
	log = golog.LoggerFor("enproxy")
)

var (
	defaultWriteFlushTimeout = 35 * time.Millisecond
	defaultReadFlushTimeout  = 35 * time.Millisecond
	defaultIdleTimeoutClient = 30 * time.Second
	defaultIdleTimeoutServer = 70 * time.Second

	// closeChannelDepth: controls depth of channels used for close processing.
	// Doesn't need to be particularly big, as it's just used to prevent
	// deadlocks on multiple calls to Close().
	closeChannelDepth = 20

	bodySize = 65536 // size of buffer used for request bodies

	oneSecond = 1 * time.Second
)

// Conn is a net.Conn that tunnels its data via an httpconn.Proxy using HTTP
// requests and responses.  It assumes that streaming requests are not supported
// by the underlying servers/proxies, and so uses a polling technique similar to
// the one used by meek, but different in that data is not encoded as JSON.
// https://trac.torproject.org/projects/tor/wiki/doc/AChildsGardenOfPluggableTransports#Undertheencryption.
//
// enproxy uses two parallel channels to send and receive data.  One channel
// handles writing data out by making sequential POST requests to the server
// which encapsulate the outbound data in their request bodies, while the other
// channel handles reading data by making GET requests and grabbing the data
// encapsulated in the response bodies.
//
// Write Channel:
//
//   1. Accept writes, piping these to the proxy as the body of an http POST
//   2. Continue to pipe the writes until the pause between consecutive writes
//      exceeds the IdleInterval, at which point we finish the request body. We
//      do this because it is assumed that intervening proxies (e.g. CloudFlare
//      CDN) do not allow streaming requests, so it is necessary to finish the
//      request for data to get flushed to the destination server.
//   3. After receiving a response to the POST request, return to step 1
//
// Read Channel:
//
//   1. Accept reads, issuing a new GET request if one is not already ongoing
//   2. Process read by grabbing data from the response to the GET request
//   3. Continue to accept reads, grabbing these from the response of the
//      existing GET request
//   4. Once the response to the GET request reaches EOF, return to step 1. This
//      will happen because the proxy periodically closes responses to make sure
//      intervening proxies don't time out.
//   5. If a response is received with a special header indicating a true EOF
//      from the destination server, return EOF to the reader
//
type Conn struct {
	// addr: the host:port of the destination server that we're trying to reach
	addr string

	tcpAddr *net.TCPAddr

	// config: configuration of this Conn
	config *Config

	// initialResponseCh: Self-reported FQDN of the proxy serving this connection
	// plus initial response from proxy.
	//
	// This allows us to guarantee we reach the same server in subsequent
	// requests, even if it was initially reached through a FQDN that may
	// resolve to different IPs in different DNS lookups (e.g. as in DNS round
	// robin).
	initialResponseCh chan hostWithResponse

	// id: unique identifier for this connection. This is used by the Proxy to
	// associate requests from this connection to the corresponding outbound
	// connection on the Proxy side.  It is populated using a type 4 UUID.
	id string

	/* Write processing */
	writeRequestsCh  chan []byte     // requests to write
	writeResponsesCh chan rwResponse // responses for writes
	doneWritingCh    chan bool
	rs               requestStrategy

	/* Request processing (for writes) */
	requestOutCh      chan *request // channel for next outgoing request body
	requestFinishedCh chan error
	doneRequestingCh  chan bool

	/* Read processing */
	readRequestsCh  chan []byte     // requests to read
	readResponsesCh chan rwResponse // responses for reads
	doneReadingCh   chan bool

	/* Fields for tracking activity/closed status */
	lastActivityTime  time.Time    // time of last read or write
	lastActivityMutex sync.RWMutex // mutex controlling access to lastActivityTime
	asyncErr          error        // error that occurred during asynchronous processing
	asyncErrMutex     sync.RWMutex // mutex guarding asyncErr
	asyncErrCh        chan error   // channel used to interrupted any waiting reads/writes with an async error
	closed            bool         // whether or not this Conn is closed
	closedMutex       sync.RWMutex // mutex controlling access to closed flag

	/* Track current response */
	resp *http.Response // the current response being used to read data
}

// Config configures a Conn
type Config struct {
	// DialProxy: function to open a connection to the proxy
	DialProxy dialFunc

	// NewRequest: function to create a new request to the proxy
	NewRequest newRequestFunc

	// FlushTimeout: how long to let writes idle before writing out a
	// request to the proxy.  Defaults to 15 milliseconds.
	FlushTimeout time.Duration

	// IdleTimeout: how long to wait before closing an idle connection, defaults
	// to 30 seconds on the client and 70 seconds on the server proxy.
	//
	// For clients, the value should be set lower than the proxy's idle timeout
	// so that enproxy redials before the active connection is closed. The value
	// should be set higher than the maximum possible time between the proxy
	// receiving the last data from a request and the proxy returning the first
	// data of the response, otherwise the connection will be closed in the
	// middle of processing a request.
	IdleTimeout time.Duration

	// BufferRequests: if true, requests to the proxy will be buffered and sent
	// with identity encoding.  If false, they'll be streamed with chunked
	// encoding.
	BufferRequests bool
}

// dialFunc is a function that dials an address (e.g. the upstream proxy)
type dialFunc func(addr string) (net.Conn, error)

// newRequestFunc is a function that builds a new request to the upstream proxy
type newRequestFunc func(host string, method string, body io.Reader) (*http.Request, error)

// rwResponse is a response to a read or write
type rwResponse struct {
	n   int
	err error
}

type connInfo struct {
	conn        *idletiming.IdleTimingConn
	bufReader   *bufio.Reader
	closed      bool
	closedMutex sync.Mutex
}

type hostWithResponse struct {
	proxyHost string
	proxyConn *connInfo
	resp      *http.Response
	err       error
}

// Write() implements the function from net.Conn
func (c *Conn) Write(b []byte) (n int, err error) {
	err = c.getAsyncErr()
	if err != nil {
		return
	}

	if c.submitWrite(b) {
		select {
		case res, ok := <-c.writeResponsesCh:
			if !ok {
				return 0, io.EOF
			} else {
				return res.n, res.err
			}
		case err := <-c.asyncErrCh:
			return 0, err
		}
	} else {
		return 0, io.EOF
	}
}

// Read() implements the function from net.Conn
func (c *Conn) Read(b []byte) (n int, err error) {
	err = c.getAsyncErr()
	if err != nil {
		return
	}

	if c.submitRead(b) {
		select {
		case res, ok := <-c.readResponsesCh:
			if !ok {
				return 0, io.EOF
			} else {
				return res.n, res.err
			}
		case err := <-c.asyncErrCh:
			return 0, err
		}
	} else {
		return 0, io.EOF
	}
}

func (c *Conn) fail(err error) {
	c.asyncErrMutex.Lock()
	if c.asyncErr != nil {
		c.asyncErr = err
	}
	c.asyncErrMutex.Unlock()

	// Let any waiting readers or writers know about the error
	for i := 0; i < 2; i++ {
		select {
		case c.asyncErrCh <- err:
			// submitted okay
		default:
			// channel full, continue
		}
	}

	go c.Close()
}

func (c *Conn) getAsyncErr() error {
	c.asyncErrMutex.RLock()
	err := c.asyncErr
	c.asyncErrMutex.RUnlock()
	return err
}

// Close() implements the function from net.Conn
func (c *Conn) Close() error {
	c.closedMutex.Lock()
	defer c.closedMutex.Unlock()
	if !c.closed {
		close(c.writeRequestsCh)
		close(c.readRequestsCh)
		<-c.doneReadingCh
		<-c.doneWritingCh
		<-c.doneRequestingCh
		c.closed = true
	}
	return nil
}

// LocalAddr() is not implemented
func (c *Conn) LocalAddr() net.Addr {
	panic("LocalAddr() not implemented")
}

// RemoteAddr() is not implemented
func (c *Conn) RemoteAddr() net.Addr {
	return c.tcpAddr
}

// SetDeadline() is currently unimplemented.
func (c *Conn) SetDeadline(t time.Time) error {
	log.Tracef("SetDeadline not implemented")
	return nil
}

// SetReadDeadline() is currently unimplemented.
func (c *Conn) SetReadDeadline(t time.Time) error {
	log.Tracef("SetReadDeadline not implemented")
	return nil
}

// SetWriteDeadline() is currently unimplemented.
func (c *Conn) SetWriteDeadline(t time.Time) error {
	log.Tracef("SetWriteDeadline not implemented")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!plan9

package syslog

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"testing"
	"time"
)

func runPktSyslog(c net.PacketConn, done chan<- string) {
	var buf [4096]byte
	var rcvd string
	ct := 0
	for {
		var n int
		var err error

		c.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
		n, _, err = c.ReadFrom(buf[:])
		rcvd += string(buf[:n])
		if err != nil {
			if oe, ok := err.(*net.OpError); ok {
				if ct < 3 && oe.Temporary() {
					ct++
					continue
				}
			}
			break
		}
	}
	c.Close()
	done <- rcvd
}

var crashy = false

func runStreamSyslog(l net.Listener, done chan<- string, wg *sync.WaitGroup) {
	for {
		var c net.Conn
		var err error
		if c, err = l.Accept(); err != nil {
			return
		}
		wg.Add(1)
		go func(c net.Conn) {
			defer wg.Done()
			c.SetReadDeadline(time.Now().Add(5 * time.Second))
			b := bufio.NewReader(c)
			for ct := 1; !crashy || ct&7 != 0; ct++ {
				s, err := b.ReadString('\n')
				if err != nil {
					break
				}
				done <- s
			}
			c.Close()
		}(c)
	}
}

func startServer(n, la string, done chan<- string) (addr string, sock io.Closer, wg *sync.WaitGroup) {
	if n == "udp" || n == "tcp" {
		la = "127.0.0.1:0"
	} else {
		// unix and unixgram: choose an address if none given
		if la == "" {
			// use ioutil.TempFile to get a name that is unique
			f, err := ioutil.TempFile("", "syslogtest")
			if err != nil {
				log.Fatal("TempFile: ", err)
			}
			f.Close()
			la = f.Name()
		}
		os.Remove(la)
	}

	wg = new(sync.WaitGroup)
	if n == "udp" || n == "unixgram" {
		l, e := net.ListenPacket(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.LocalAddr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runPktSyslog(l, done)
		}()
	} else {
		l, e := net.Listen(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.Addr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runStreamSyslog(l, done, wg)
		}()
	}
	return
}

func TestWithSimulated(t *testing.T) {
	msg := "Test 123"
	transport := []string{"unix", "unixgram", "udp", "tcp"}

	for _, tr := range transport {
		done := make(chan string)
		addr, _, _ := startServer(tr, "", done)
		if tr == "unix" || tr == "unixgram" {
			defer os.Remove(addr)
		}
		s, err := Dial(tr, addr, LOG_INFO|LOG_USER, "syslog_test")
		if err != nil {
			t.Fatalf("Dial() failed: %v", err)
		}
		err = s.Info(msg)
		if err != nil {
			t.Fatalf("log failed: %v", err)
		}
		check(t, msg, <-done)
		s.Close()
	}
}

func TestFlap(t *testing.T) {
	net := "unix"
	done := make(chan string)
	addr, sock, _ := startServer(net, "", done)
	defer os.Remove(addr)
	defer sock.Close()

	s, err := Dial(net, addr, LOG_INFO|LOG_USER, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %v", err)
	}
	msg := "Moo 2"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	// restart the server
	_, sock2, _ := startServer(net, addr, done)
	defer sock2.Close()

	// and try retransmitting
	msg = "Moo 3"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	s.Close()
}

func TestNew(t *testing.T) {
	if LOG_LOCAL7 != 23<<3 {
		t.Fatalf("LOG_LOCAL7 has wrong value")
	}
	if testing.Short() {
		// Depends on syslog daemon running, and sometimes it's not.
		t.Skip("skipping syslog test during -short")
	}

	s, err := New(LOG_INFO|LOG_USER, "the_tag")
	if err != nil {
		t.Fatalf("New() failed: %s", err)
	}
	// Don't send any messages.
	s.Close()
}

func TestNewLogger(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := NewLogger(LOG_USER|LOG_INFO, 0)
	if f == nil {
		t.Error(err)
	}
}

func TestDial(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := Dial("", "", (LOG_LOCAL7|LOG_DEBUG)+1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	f, err = Dial("", "", -1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	l, err := Dial("", "", LOG_USER|LOG_ERR, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %s", err)
	}
	l.Close()
}

func check(t *testing.T, in, out string) {
	tmpl := fmt.Sprintf("<%d>%%s %%s syslog_test[%%d]: %s\n", LOG_USER+LOG_INFO, in)
	if hostname, err := os.Hostname(); err != nil {
		t.Error("Error retrieving hostname")
	} else {
		var parsedHostname, timestamp string
		var pid int
		if n, err := fmt.Sscanf(out, tmpl, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
			t.Errorf("Got %q, does not match template %q (%d %s)", out, tmpl, n, err)
		}
	}
}

func TestWrite(t *testing.T) {
	tests := []struct {
		pri Priority
		pre string
		msg string
		exp string
	}{
		{LOG_USER | LOG_ERR, "syslog_test", "", "%s %s syslog_test[%d]: \n"},
		{LOG_USER | LOG_ERR, "syslog_test", "write test", "%s %s syslog_test[%d]: write test\n"},
		// Write should not add \n if there already is one
		{LOG_USER | LOG_ERR, "syslog_test", "write test 2\n", "%s %s syslog_test[%d]: write test 2\n"},
	}

	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		for _, test := range tests {
			done := make(chan string)
			addr, sock, _ := startServer("udp", "", done)
			defer sock.Close()
			l, err := Dial("udp", addr, test.pri, test.pre)
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer l.Close()
			_, err = io.WriteString(l, test.msg)
			if err != nil {
				t.Fatalf("WriteString() failed: %v", err)
			}
			rcvd := <-done
			test.exp = fmt.Sprintf("<%d>", test.pri) + test.exp
			var parsedHostname, timestamp string
			var pid int
			if n, err := fmt.Sscanf(rcvd, test.exp, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
				t.Errorf("s.Info() = '%q', didn't match '%q' (%d %s)", rcvd, test.exp, n, err)
			}
		}
	}
}

func TestConcurrentWrite(t *testing.T) {
	addr, sock, _ := startServer("udp", "", make(chan string))
	defer sock.Close()
	w, err := Dial("udp", addr, LOG_USER|LOG_ERR, "how's it going?")
	if err != nil {
		t.Fatalf("syslog.Dial() failed: %v", err)
	}
	var wg sync.WaitGroup
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			err := w.Info("test")
			if err != nil {
				t.Errorf("Info() failed: %v", err)
				return
			}
		}()
	}
	wg.Wait()
}

func TestConcurrentReconnect(t *testing.T) {
	crashy = true
	defer func() { crashy = false }()

	const N = 10
	const M = 100
	net := "unix"
	done := make(chan string, N*M)
	addr, sock, srvWG := startServer(net, "", done)
	defer os.Remove(addr)

	// count all the messages arriving
	count := make(chan int)
	go func() {
		ct := 0
		for _ = range done {
			ct++
			// we are looking for 500 out of 1000 events
			// here because lots of log messages are lost
			// in buffers (kernel and/or bufio)
			if ct > N*M/2 {
				break
			}
		}
		count <- ct
	}()

	var wg sync.WaitGroup
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			defer wg.Done()
			w, err := Dial(net, addr, LOG_USER|LOG_ERR, "tag")
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer w.Close()
			for i := 0; i < M; i++ {
				err := w.Info("test")
				if err != nil {
					t.Errorf("Info() failed: %v", err)
					return
				}
			}
		}()
	}
	wg.Wait()
	sock.Close()
	srvWG.Wait()
	close(done)

	select {
	case <-count:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout in concurrent reconnect")
	}
}
<commit_msg>syslog: fix data race on 'crashy' in test function<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!plan9

package syslog

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"sync"
	"testing"
	"time"
)

func runPktSyslog(c net.PacketConn, done chan<- string) {
	var buf [4096]byte
	var rcvd string
	ct := 0
	for {
		var n int
		var err error

		c.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
		n, _, err = c.ReadFrom(buf[:])
		rcvd += string(buf[:n])
		if err != nil {
			if oe, ok := err.(*net.OpError); ok {
				if ct < 3 && oe.Temporary() {
					ct++
					continue
				}
			}
			break
		}
	}
	c.Close()
	done <- rcvd
}

var crashy = false

func runStreamSyslog(l net.Listener, done chan<- string, wg *sync.WaitGroup) {
	for {
		var c net.Conn
		var err error
		if c, err = l.Accept(); err != nil {
			return
		}
		wg.Add(1)
		go func(c net.Conn) {
			defer wg.Done()
			c.SetReadDeadline(time.Now().Add(5 * time.Second))
			b := bufio.NewReader(c)
			for ct := 1; !crashy || ct&7 != 0; ct++ {
				s, err := b.ReadString('\n')
				if err != nil {
					break
				}
				done <- s
			}
			c.Close()
		}(c)
	}
}

func startServer(n, la string, done chan<- string) (addr string, sock io.Closer, wg *sync.WaitGroup) {
	if n == "udp" || n == "tcp" {
		la = "127.0.0.1:0"
	} else {
		// unix and unixgram: choose an address if none given
		if la == "" {
			// use ioutil.TempFile to get a name that is unique
			f, err := ioutil.TempFile("", "syslogtest")
			if err != nil {
				log.Fatal("TempFile: ", err)
			}
			f.Close()
			la = f.Name()
		}
		os.Remove(la)
	}

	wg = new(sync.WaitGroup)
	if n == "udp" || n == "unixgram" {
		l, e := net.ListenPacket(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.LocalAddr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runPktSyslog(l, done)
		}()
	} else {
		l, e := net.Listen(n, la)
		if e != nil {
			log.Fatalf("startServer failed: %v", e)
		}
		addr = l.Addr().String()
		sock = l
		wg.Add(1)
		go func() {
			defer wg.Done()
			runStreamSyslog(l, done, wg)
		}()
	}
	return
}

func TestWithSimulated(t *testing.T) {
	msg := "Test 123"
	transport := []string{"unix", "unixgram", "udp", "tcp"}

	for _, tr := range transport {
		done := make(chan string)
		addr, sock, srvWG := startServer(tr, "", done)
		defer srvWG.Wait()
		defer sock.Close()
		if tr == "unix" || tr == "unixgram" {
			defer os.Remove(addr)
		}
		s, err := Dial(tr, addr, LOG_INFO|LOG_USER, "syslog_test")
		if err != nil {
			t.Fatalf("Dial() failed: %v", err)
		}
		err = s.Info(msg)
		if err != nil {
			t.Fatalf("log failed: %v", err)
		}
		check(t, msg, <-done)
		s.Close()
	}
}

func TestFlap(t *testing.T) {
	net := "unix"
	done := make(chan string)
	addr, sock, srvWG := startServer(net, "", done)
	defer srvWG.Wait()
	defer os.Remove(addr)
	defer sock.Close()

	s, err := Dial(net, addr, LOG_INFO|LOG_USER, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %v", err)
	}
	msg := "Moo 2"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	// restart the server
	_, sock2, srvWG2 := startServer(net, addr, done)
	defer srvWG2.Wait()
	defer sock2.Close()

	// and try retransmitting
	msg = "Moo 3"
	err = s.Info(msg)
	if err != nil {
		t.Fatalf("log failed: %v", err)
	}
	check(t, msg, <-done)

	s.Close()
}

func TestNew(t *testing.T) {
	if LOG_LOCAL7 != 23<<3 {
		t.Fatalf("LOG_LOCAL7 has wrong value")
	}
	if testing.Short() {
		// Depends on syslog daemon running, and sometimes it's not.
		t.Skip("skipping syslog test during -short")
	}

	s, err := New(LOG_INFO|LOG_USER, "the_tag")
	if err != nil {
		t.Fatalf("New() failed: %s", err)
	}
	// Don't send any messages.
	s.Close()
}

func TestNewLogger(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := NewLogger(LOG_USER|LOG_INFO, 0)
	if f == nil {
		t.Error(err)
	}
}

func TestDial(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping syslog test during -short")
	}
	f, err := Dial("", "", (LOG_LOCAL7|LOG_DEBUG)+1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	f, err = Dial("", "", -1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	l, err := Dial("", "", LOG_USER|LOG_ERR, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %s", err)
	}
	l.Close()
}

func check(t *testing.T, in, out string) {
	tmpl := fmt.Sprintf("<%d>%%s %%s syslog_test[%%d]: %s\n", LOG_USER+LOG_INFO, in)
	if hostname, err := os.Hostname(); err != nil {
		t.Error("Error retrieving hostname")
	} else {
		var parsedHostname, timestamp string
		var pid int
		if n, err := fmt.Sscanf(out, tmpl, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
			t.Errorf("Got %q, does not match template %q (%d %s)", out, tmpl, n, err)
		}
	}
}

func TestWrite(t *testing.T) {
	tests := []struct {
		pri Priority
		pre string
		msg string
		exp string
	}{
		{LOG_USER | LOG_ERR, "syslog_test", "", "%s %s syslog_test[%d]: \n"},
		{LOG_USER | LOG_ERR, "syslog_test", "write test", "%s %s syslog_test[%d]: write test\n"},
		// Write should not add \n if there already is one
		{LOG_USER | LOG_ERR, "syslog_test", "write test 2\n", "%s %s syslog_test[%d]: write test 2\n"},
	}

	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		for _, test := range tests {
			done := make(chan string)
			addr, sock, srvWG := startServer("udp", "", done)
			defer srvWG.Wait()
			defer sock.Close()
			l, err := Dial("udp", addr, test.pri, test.pre)
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer l.Close()
			_, err = io.WriteString(l, test.msg)
			if err != nil {
				t.Fatalf("WriteString() failed: %v", err)
			}
			rcvd := <-done
			test.exp = fmt.Sprintf("<%d>", test.pri) + test.exp
			var parsedHostname, timestamp string
			var pid int
			if n, err := fmt.Sscanf(rcvd, test.exp, &timestamp, &parsedHostname, &pid); n != 3 || err != nil || hostname != parsedHostname {
				t.Errorf("s.Info() = '%q', didn't match '%q' (%d %s)", rcvd, test.exp, n, err)
			}
		}
	}
}

func TestConcurrentWrite(t *testing.T) {
	addr, sock, srvWG := startServer("udp", "", make(chan string, 1))
	defer srvWG.Wait()
	defer sock.Close()
	w, err := Dial("udp", addr, LOG_USER|LOG_ERR, "how's it going?")
	if err != nil {
		t.Fatalf("syslog.Dial() failed: %v", err)
	}
	var wg sync.WaitGroup
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			err := w.Info("test")
			if err != nil {
				t.Errorf("Info() failed: %v", err)
				return
			}
		}()
	}
	wg.Wait()
}

func TestConcurrentReconnect(t *testing.T) {
	crashy = true
	defer func() { crashy = false }()

	const N = 10
	const M = 100
	net := "unix"
	done := make(chan string, N*M)
	addr, sock, srvWG := startServer(net, "", done)
	defer os.Remove(addr)

	// count all the messages arriving
	count := make(chan int)
	go func() {
		ct := 0
		for _ = range done {
			ct++
			// we are looking for 500 out of 1000 events
			// here because lots of log messages are lost
			// in buffers (kernel and/or bufio)
			if ct > N*M/2 {
				break
			}
		}
		count <- ct
	}()

	var wg sync.WaitGroup
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			defer wg.Done()
			w, err := Dial(net, addr, LOG_USER|LOG_ERR, "tag")
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %v", err)
			}
			defer w.Close()
			for i := 0; i < M; i++ {
				err := w.Info("test")
				if err != nil {
					t.Errorf("Info() failed: %v", err)
					return
				}
			}
		}()
	}
	wg.Wait()
	sock.Close()
	srvWG.Wait()
	close(done)

	select {
	case <-count:
	case <-time.After(100 * time.Millisecond):
		t.Error("timeout in concurrent reconnect")
	}
}
<|endoftext|>


<commit_before>// Package web implements set of middlewares and API for rendering components
// in http flow.
//
// Example usage of handlers/middlewares:
//
// 	// Construct API from middlewares
// 	chain.UseC(UnmarshalFromQuery("GET"))
// 	chain.UseC(UnmarshalFromBody("POST"))
// 	chain.UseC(CompileFromCtx)
// 	chain.UseC(RenderFromCtx)
// 	http.ListenAndServe(":8080", chain.HandlerCtx(ctx, xhandler.HandlerFuncC(WriteRendered)))
//
package web

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/rs/xhandler"

	"bitbucket.org/moovie/util/httputil"
	"bitbucket.org/moovie/util/stringslice"
	"bitbucket.org/moovie/util/template"

	"github.com/crackcomm/renderer/pkg/renderer"
)

// Middleware - HTTP Middleware function.
type Middleware func(next xhandler.HandlerC) xhandler.HandlerC

// UnmarshalFromRequest - Unmarshals component using `UnmarshalFromQuery` on `GET`
// method and `` on `POST` method.
func UnmarshalFromRequest() Middleware {
	get, post := UnmarshalFromQuery("GET"), UnmarshalFromBody("POST")
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			if r.Method == "GET" {
				get(next).ServeHTTPC(ctx, w, r)
			} else if r.Method == "POST" {
				post(next).ServeHTTPC(ctx, w, r)
			}
		})
	}
}

// UnmarshalFromQuery - Unmarshals component from `json` query on certain methods.
// Stores result in context to be retrieved with `renderer.ComponentFromCtx`.
func UnmarshalFromQuery(methods ...string) Middleware {
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			if len(methods) != 0 && !stringslice.Contain(methods, r.Method) {
				next.ServeHTTPC(ctx, w, r)
				return
			}

			// Read component from request
			c, err := readComponent(r)
			if err != nil {
				httputil.WriteError(w, r, http.StatusBadRequest, err.Error())
				return
			}

			// Create a context with component and move to next handler
			ctx = renderer.ComponentCtx(ctx, c)
			next.ServeHTTPC(ctx, w, r)
		})
	}
}

func readComponent(r *http.Request) (c *renderer.Component, err error) {
	c = new(renderer.Component)
	if b := r.URL.Query().Get("json"); b != "" {
		err = json.Unmarshal([]byte(b), c)
		return
	}
	c.Name = r.URL.Query().Get("name")
	if c.Name == "" {
		return nil, errors.New("no component in request")
	}
	c.Main = r.URL.Query().Get("main")
	c.Extends = r.URL.Query().Get("extends")
	if b := r.URL.Query().Get("styles"); b != "" {
		c.Styles = strings.Split(b, ",")
	}
	if b := r.URL.Query().Get("scripts"); b != "" {
		c.Scripts = strings.Split(b, ",")
	}
	if b := r.URL.Query().Get("require"); b != "" {
		c.Require = make(map[string]renderer.Component)
		err = json.Unmarshal([]byte(b), &c.Require)
		if err != nil {
			return
		}
	}
	if b := r.URL.Query().Get("context"); b != "" {
		c.Context = make(template.Context)
		err = json.Unmarshal([]byte(b), &c.Context)
		if err != nil {
			return
		}
	}
	if b := r.URL.Query().Get("with"); b != "" {
		c.With = make(map[string]string)
		err = json.Unmarshal([]byte(b), &c.With)
		if err != nil {
			return
		}
	}
	return
}

// UnmarshalFromBody - Unmarshals component from request bodyCompileFromCtx() on certain methods.
// Stores result in context to be retrieved with `renderer.ComponentFromCtx`.
func UnmarshalFromBody(methods ...string) Middleware {
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			if !stringslice.Contain(methods, r.Method) {
				next.ServeHTTPC(ctx, w, r)
				return
			}
			c := new(renderer.Component)
			err := json.NewDecoder(r.Body).Decode(c)
			if err != nil {
				httputil.WriteError(w, r, http.StatusBadRequest, err.Error())
				return
			}
			ctx = renderer.ComponentCtx(ctx, c)
			next.ServeHTTPC(ctx, w, r)
		})
	}
}

// CompileFromCtx - Compiles component from context.
// Stores result in context to be retrieved with `renderer.ComponentFromCtx`.
func CompileFromCtx(next xhandler.HandlerC) xhandler.HandlerC {
	return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		c, ok := renderer.ComponentFromCtx(ctx)
		if !ok {
			httputil.WriteError(w, r, http.StatusBadRequest, "no component set")
			return
		}
		compiler, ok := renderer.CompilerFromCtx(ctx)
		if !ok {
			httputil.WriteError(w, r, http.StatusInternalServerError, "compiler not found")
			return
		}
		compiled, err := compiler.CompileFromStorage(c)
		if err != nil {
			httputil.WriteError(w, r, http.StatusExpectationFailed, fmt.Sprintf("compile error: %v", err))
			return
		}
		ctx = renderer.CompiledCtx(ctx, compiled)
		next.ServeHTTPC(ctx, w, r)
	})
}

// RenderFromCtx - Renders compiled component from context.
// Stores result in context to be retrieved with `renderer.RenderedFromCtx`.
func RenderFromCtx(next xhandler.HandlerC) xhandler.HandlerC {
	return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		c, ok := renderer.CompiledFromCtx(ctx)
		if !ok {
			httputil.WriteError(w, r, http.StatusBadRequest, "component not compiled")
			return
		}
		t, _ := renderer.TemplateCtx(ctx)
		res, err := renderer.Render(c, t)
		if err != nil {
			httputil.WriteError(w, r, http.StatusExpectationFailed, fmt.Sprintf("render error: %v", err))
			return
		}
		ctx = renderer.RenderedCtx(ctx, res)
		next.ServeHTTPC(ctx, w, r)
	})
}

// WriteRendered - Writes rendered component from context to response writer.
// Depending on `Accept` header, it will write json or plain html body.
func WriteRendered(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if strings.Contains(r.Header.Get("Accept"), "application/json") {
		WriteRenderedJSON(ctx, w, r)
	} else {
		WriteRenderedHTML(ctx, w, r)
	}
}

// WriteRenderedJSON - Writes rendered component from context to response writer.
func WriteRenderedJSON(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	res, ok := renderer.RenderedFromCtx(ctx)
	if !ok {
		httputil.WriteError(w, r, http.StatusBadRequest, "component not rendered")
		return
	}
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(res); err != nil {
		glog.Warningf("[api] response encode error: %v", err)
	}
}

// WriteRenderedHTML - Writes rendered component from context to response writer.
func WriteRenderedHTML(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	res, ok := renderer.RenderedFromCtx(ctx)
	if !ok {
		httputil.WriteError(w, r, http.StatusBadRequest, "component not rendered")
		return
	}
	w.Header().Set("Content-Type", "text/html")
	body, err := renderer.RenderHTML(res)
	if err != nil {
		http.Error(w, fmt.Sprintf("html error: %v", err), http.StatusExpectationFailed)
		return
	}
	w.Write([]byte(body))
}
<commit_msg>component setting middleware + use compile from ctx method<commit_after>// Package web implements set of middlewares and API for rendering components
// in http flow.
//
// Example usage of handlers/middlewares:
//
// 	// Construct API from middlewares
// 	chain.UseC(UnmarshalFromQuery("GET"))
// 	chain.UseC(UnmarshalFromBody("POST"))
// 	chain.UseC(CompileFromCtx)
// 	chain.UseC(RenderFromCtx)
// 	http.ListenAndServe(":8080", chain.HandlerCtx(ctx, xhandler.HandlerFuncC(WriteRendered)))
//
package web

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"strings"

	"github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/rs/xhandler"

	"bitbucket.org/moovie/util/httputil"
	"bitbucket.org/moovie/util/stringslice"
	"bitbucket.org/moovie/util/template"

	"github.com/crackcomm/renderer/pkg/renderer"
)

// Middleware - HTTP Middleware function.
type Middleware func(next xhandler.HandlerC) xhandler.HandlerC

// ComponentMiddleware - Creates a middleware that sets given component in ctx.
func ComponentMiddleware(c *renderer.Component) Middleware {
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			ctx = renderer.ComponentCtx(ctx, c)
			next.ServeHTTPC(ctx, w, r)
		})
	}
}

// UnmarshalFromRequest - Unmarshals component using `UnmarshalFromQuery` on `GET`
// method and `` on `POST` method.
func UnmarshalFromRequest() Middleware {
	get, post := UnmarshalFromQuery("GET"), UnmarshalFromBody("POST")
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			if r.Method == "GET" {
				get(next).ServeHTTPC(ctx, w, r)
			} else if r.Method == "POST" {
				post(next).ServeHTTPC(ctx, w, r)
			}
		})
	}
}

// UnmarshalFromQuery - Unmarshals component from `json` query on certain methods.
// Stores result in context to be retrieved with `renderer.ComponentFromCtx`.
func UnmarshalFromQuery(methods ...string) Middleware {
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			if len(methods) != 0 && !stringslice.Contain(methods, r.Method) {
				next.ServeHTTPC(ctx, w, r)
				return
			}

			// Read component from request
			c, err := readComponent(r)
			if err != nil {
				httputil.WriteError(w, r, http.StatusBadRequest, err.Error())
				return
			}

			// Create a context with component and move to next handler
			ctx = renderer.ComponentCtx(ctx, c)
			next.ServeHTTPC(ctx, w, r)
		})
	}
}

func readComponent(r *http.Request) (c *renderer.Component, err error) {
	c = new(renderer.Component)
	if b := r.URL.Query().Get("json"); b != "" {
		err = json.Unmarshal([]byte(b), c)
		return
	}
	c.Name = r.URL.Query().Get("name")
	if c.Name == "" {
		return nil, errors.New("no component in request")
	}
	c.Main = r.URL.Query().Get("main")
	c.Extends = r.URL.Query().Get("extends")
	if b := r.URL.Query().Get("styles"); b != "" {
		c.Styles = strings.Split(b, ",")
	}
	if b := r.URL.Query().Get("scripts"); b != "" {
		c.Scripts = strings.Split(b, ",")
	}
	if b := r.URL.Query().Get("require"); b != "" {
		c.Require = make(map[string]renderer.Component)
		err = json.Unmarshal([]byte(b), &c.Require)
		if err != nil {
			return
		}
	}
	if b := r.URL.Query().Get("context"); b != "" {
		c.Context = make(template.Context)
		err = json.Unmarshal([]byte(b), &c.Context)
		if err != nil {
			return
		}
	}
	if b := r.URL.Query().Get("with"); b != "" {
		c.With = make(map[string]string)
		err = json.Unmarshal([]byte(b), &c.With)
		if err != nil {
			return
		}
	}
	return
}

// UnmarshalFromBody - Unmarshals component from request bodyCompileFromCtx() on certain methods.
// Stores result in context to be retrieved with `renderer.ComponentFromCtx`.
func UnmarshalFromBody(methods ...string) Middleware {
	return func(next xhandler.HandlerC) xhandler.HandlerC {
		return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
			if !stringslice.Contain(methods, r.Method) {
				next.ServeHTTPC(ctx, w, r)
				return
			}
			c := new(renderer.Component)
			err := json.NewDecoder(r.Body).Decode(c)
			if err != nil {
				httputil.WriteError(w, r, http.StatusBadRequest, err.Error())
				return
			}
			ctx = renderer.ComponentCtx(ctx, c)
			next.ServeHTTPC(ctx, w, r)
		})
	}
}

// CompileFromCtx - Compiles component from context.
// Stores result in context to be retrieved with `renderer.ComponentFromCtx`.
func CompileFromCtx(next xhandler.HandlerC) xhandler.HandlerC {
	return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		compiled, err := renderer.CompileFromCtx(ctx)
		if err != nil {
			httputil.WriteError(w, r, http.StatusExpectationFailed, fmt.Sprintf("compile error: %v", err))
			return
		}
		ctx = renderer.CompiledCtx(ctx, compiled)
		next.ServeHTTPC(ctx, w, r)
	})
}

// RenderFromCtx - Renders compiled component from context.
// Stores result in context to be retrieved with `renderer.RenderedFromCtx`.
func RenderFromCtx(next xhandler.HandlerC) xhandler.HandlerC {
	return xhandler.HandlerFuncC(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		c, ok := renderer.CompiledFromCtx(ctx)
		if !ok {
			httputil.WriteError(w, r, http.StatusBadRequest, "component not compiled")
			return
		}
		t, _ := renderer.TemplateCtx(ctx)
		res, err := renderer.Render(c, t)
		if err != nil {
			httputil.WriteError(w, r, http.StatusExpectationFailed, fmt.Sprintf("render error: %v", err))
			return
		}
		ctx = renderer.RenderedCtx(ctx, res)
		next.ServeHTTPC(ctx, w, r)
	})
}

// WriteRendered - Writes rendered component from context to response writer.
// Depending on `Accept` header, it will write json or plain html body.
func WriteRendered(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if strings.Contains(r.Header.Get("Accept"), "application/json") {
		WriteRenderedJSON(ctx, w, r)
	} else {
		WriteRenderedHTML(ctx, w, r)
	}
}

// WriteRenderedJSON - Writes rendered component from context to response writer.
func WriteRenderedJSON(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	res, ok := renderer.RenderedFromCtx(ctx)
	if !ok {
		httputil.WriteError(w, r, http.StatusBadRequest, "component not rendered")
		return
	}
	w.Header().Set("Content-Type", "application/json")
	if err := json.NewEncoder(w).Encode(res); err != nil {
		glog.Warningf("[api] response encode error: %v", err)
	}
}

// WriteRenderedHTML - Writes rendered component from context to response writer.
func WriteRenderedHTML(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	res, ok := renderer.RenderedFromCtx(ctx)
	if !ok {
		httputil.WriteError(w, r, http.StatusBadRequest, "component not rendered")
		return
	}
	w.Header().Set("Content-Type", "text/html")
	body, err := renderer.RenderHTML(res)
	if err != nil {
		http.Error(w, fmt.Sprintf("html error: %v", err), http.StatusExpectationFailed)
		return
	}
	w.Write([]byte(body))
}
<|endoftext|>


<commit_before>package replication

import (
	"bytes"
	"fmt"
	"io"
	"os"

	"github.com/juju/errors"
)

type BinlogParser struct {
	format *FormatDescriptionEvent

	tables map[uint64]*TableMapEvent

	// for rawMode, we only parse FormatDescriptionEvent and RotateEvent
	rawMode bool

	parseTime bool
}

func NewBinlogParser() *BinlogParser {
	p := new(BinlogParser)

	p.tables = make(map[uint64]*TableMapEvent)

	return p
}

func (p *BinlogParser) Reset() {
	p.format = nil
}

type OnEventFunc func(*BinlogEvent) error

func (p *BinlogParser) ParseFile(name string, offset int64, onEvent OnEventFunc) error {
	f, err := os.Open(name)
	if err != nil {
		return errors.Trace(err)
	}
	defer f.Close()

	b := make([]byte, 4)
	if _, err = f.Read(b); err != nil {
		return errors.Trace(err)
	} else if !bytes.Equal(b, BinLogFileHeader) {
		return errors.Errorf("%s is not a valid binlog file, head 4 bytes must fe'bin' ", name)
	}

	if offset < 4 {
		offset = 4
	}

	if _, err = f.Seek(offset, os.SEEK_SET); err != nil {
		return errors.Errorf("seek %s to %d error %v", name, offset, err)
	}

	return p.ParseReader(f, onEvent)
}

func (p *BinlogParser) ParseReader(r io.Reader, onEvent OnEventFunc) error {
	p.Reset()

	var err error
	var n int64

	for {
		headBuf := make([]byte, EventHeaderSize)

		if _, err = io.ReadFull(r, headBuf); err == io.EOF {
			return nil
		} else if err != nil {
			return errors.Trace(err)
		}

		var h *EventHeader
		h, err = p.parseHeader(headBuf)
		if err != nil {
			return errors.Trace(err)
		}

		if h.EventSize <= uint32(EventHeaderSize) {
			return errors.Errorf("invalid event header, event size is %d, too small", h.EventSize)

		}

		var buf bytes.Buffer
		if n, err = io.CopyN(&buf, r, int64(h.EventSize)-int64(EventHeaderSize)); err != nil {
			return errors.Errorf("get event body err %v, need %d - %d, but got %d", err, h.EventSize, EventHeaderSize, n)
		}

		data := buf.Bytes()
		rawData := data

		eventLen := int(h.EventSize) - EventHeaderSize

		if len(data) != eventLen {
			return errors.Errorf("invalid data size %d in event %s, less event length %d", len(data), h.EventType, eventLen)
		}

		var e Event
		e, err = p.parseEvent(h, data)
		if err != nil {
			break
		}

		if err = onEvent(&BinlogEvent{rawData, h, e}); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (p *BinlogParser) SetRawMode(mode bool) {
	p.rawMode = mode
}

func (p *BinlogParser) SetParseTime(parseTime bool) {
	p.parseTime = parseTime
}

func (p *BinlogParser) parseHeader(data []byte) (*EventHeader, error) {
	h := new(EventHeader)
	err := h.Decode(data)
	if err != nil {
		return nil, err
	}

	return h, nil
}

func (p *BinlogParser) parseEvent(h *EventHeader, data []byte) (Event, error) {
	var e Event

	if h.EventType == FORMAT_DESCRIPTION_EVENT {
		p.format = &FormatDescriptionEvent{}
		e = p.format
	} else {
		if p.format != nil && p.format.ChecksumAlgorithm == BINLOG_CHECKSUM_ALG_CRC32 {
			data = data[0 : len(data)-4]
		}

		if h.EventType == ROTATE_EVENT {
			e = &RotateEvent{}
		} else if !p.rawMode {
			switch h.EventType {
			case QUERY_EVENT:
				e = &QueryEvent{}
			case XID_EVENT:
				e = &XIDEvent{}
			case TABLE_MAP_EVENT:
				te := &TableMapEvent{}
				if p.format.EventTypeHeaderLengths[TABLE_MAP_EVENT-1] == 6 {
					te.tableIDSize = 4
				} else {
					te.tableIDSize = 6
				}
				e = te
			case WRITE_ROWS_EVENTv0,
				UPDATE_ROWS_EVENTv0,
				DELETE_ROWS_EVENTv0,
				WRITE_ROWS_EVENTv1,
				DELETE_ROWS_EVENTv1,
				UPDATE_ROWS_EVENTv1,
				WRITE_ROWS_EVENTv2,
				UPDATE_ROWS_EVENTv2,
				DELETE_ROWS_EVENTv2:
				e = p.newRowsEvent(h)
			case ROWS_QUERY_EVENT:
				e = &RowsQueryEvent{}
			case GTID_EVENT:
				e = &GTIDEvent{}
			case BEGIN_LOAD_QUERY_EVENT:
				e = &BeginLoadQueryEvent{}
			case EXECUTE_LOAD_QUERY_EVENT:
				e = &ExecuteLoadQueryEvent{}
			case MARIADB_ANNOTATE_ROWS_EVENT:
				e = &MariadbAnnotaeRowsEvent{}
			case MARIADB_BINLOG_CHECKPOINT_EVENT:
				e = &MariadbBinlogCheckPointEvent{}
			case MARIADB_GTID_LIST_EVENT:
				e = &MariadbGTIDListEvent{}
			case MARIADB_GTID_EVENT:
				ee := &MariadbGTIDEvent{}
				ee.GTID.ServerID = h.ServerID
				e = ee
			default:
				e = &GenericEvent{}
			}
		} else {
			e = &GenericEvent{}
		}
	}

	if err := e.Decode(data); err != nil {
		return nil, &EventError{h, err.Error(), data}
	}

	if te, ok := e.(*TableMapEvent); ok {
		p.tables[te.TableID] = te
	}

	if re, ok := e.(*RowsEvent); ok {
		if (re.Flags & RowsEventStmtEndFlag) > 0 {
			// Refer https://github.com/alibaba/canal/blob/38cc81b7dab29b51371096fb6763ca3a8432ffee/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java#L176
			p.tables = make(map[uint64]*TableMapEvent)
		}
	}

	return e, nil
}

// Given the bytes for a a binary log event: return the decoded event.
// With the exception of the FORMAT_DESCRIPTION_EVENT event type
// there must have previously been passed a FORMAT_DESCRIPTION_EVENT
// into the parser for this to work properly on any given event.
// Passing a new FORMAT_DESCRIPTION_EVENT into the parser will replace
// an existing one.
func (p *BinlogParser) Parse(data []byte) (*BinlogEvent, error) {
	rawData := data

	h, err := p.parseHeader(data)

	if err != nil {
		return nil, err
	}

	data = data[EventHeaderSize:]
	eventLen := int(h.EventSize) - EventHeaderSize

	if len(data) != eventLen {
		return nil, fmt.Errorf("invalid data size %d in event %s, less event length %d", len(data), h.EventType, eventLen)
	}

	e, err := p.parseEvent(h, data)
	if err != nil {
		return nil, err
	}

	return &BinlogEvent{rawData, h, e}, nil
}

func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent {
	e := &RowsEvent{}
	if p.format.EventTypeHeaderLengths[h.EventType-1] == 6 {
		e.tableIDSize = 4
	} else {
		e.tableIDSize = 6
	}

	e.needBitmap2 = false
	e.tables = p.tables
	e.parseTime = p.parseTime

	switch h.EventType {
	case WRITE_ROWS_EVENTv0:
		e.Version = 0
	case UPDATE_ROWS_EVENTv0:
		e.Version = 0
	case DELETE_ROWS_EVENTv0:
		e.Version = 0
	case WRITE_ROWS_EVENTv1:
		e.Version = 1
	case DELETE_ROWS_EVENTv1:
		e.Version = 1
	case UPDATE_ROWS_EVENTv1:
		e.Version = 1
		e.needBitmap2 = true
	case WRITE_ROWS_EVENTv2:
		e.Version = 2
	case UPDATE_ROWS_EVENTv2:
		e.Version = 2
		e.needBitmap2 = true
	case DELETE_ROWS_EVENTv2:
		e.Version = 2
	}

	return e
}
<commit_msg>Stop doing a Reset() in ParseReader. It is unnecessary and limiting (#133)<commit_after>package replication

import (
	"bytes"
	"fmt"
	"io"
	"os"

	"github.com/juju/errors"
)

type BinlogParser struct {
	format *FormatDescriptionEvent

	tables map[uint64]*TableMapEvent

	// for rawMode, we only parse FormatDescriptionEvent and RotateEvent
	rawMode bool

	parseTime bool
}

func NewBinlogParser() *BinlogParser {
	p := new(BinlogParser)

	p.tables = make(map[uint64]*TableMapEvent)

	return p
}

func (p *BinlogParser) Reset() {
	p.format = nil
}

type OnEventFunc func(*BinlogEvent) error

func (p *BinlogParser) ParseFile(name string, offset int64, onEvent OnEventFunc) error {
	f, err := os.Open(name)
	if err != nil {
		return errors.Trace(err)
	}
	defer f.Close()

	b := make([]byte, 4)
	if _, err = f.Read(b); err != nil {
		return errors.Trace(err)
	} else if !bytes.Equal(b, BinLogFileHeader) {
		return errors.Errorf("%s is not a valid binlog file, head 4 bytes must fe'bin' ", name)
	}

	if offset < 4 {
		offset = 4
	}

	if _, err = f.Seek(offset, os.SEEK_SET); err != nil {
		return errors.Errorf("seek %s to %d error %v", name, offset, err)
	}

	return p.ParseReader(f, onEvent)
}

func (p *BinlogParser) ParseReader(r io.Reader, onEvent OnEventFunc) error {
	var err error
	var n int64

	for {
		headBuf := make([]byte, EventHeaderSize)

		if _, err = io.ReadFull(r, headBuf); err == io.EOF {
			return nil
		} else if err != nil {
			return errors.Trace(err)
		}

		var h *EventHeader
		h, err = p.parseHeader(headBuf)
		if err != nil {
			return errors.Trace(err)
		}

		if h.EventSize <= uint32(EventHeaderSize) {
			return errors.Errorf("invalid event header, event size is %d, too small", h.EventSize)

		}

		var buf bytes.Buffer
		if n, err = io.CopyN(&buf, r, int64(h.EventSize)-int64(EventHeaderSize)); err != nil {
			return errors.Errorf("get event body err %v, need %d - %d, but got %d", err, h.EventSize, EventHeaderSize, n)
		}

		data := buf.Bytes()
		rawData := data

		eventLen := int(h.EventSize) - EventHeaderSize

		if len(data) != eventLen {
			return errors.Errorf("invalid data size %d in event %s, less event length %d", len(data), h.EventType, eventLen)
		}

		var e Event
		e, err = p.parseEvent(h, data)
		if err != nil {
			break
		}

		if err = onEvent(&BinlogEvent{rawData, h, e}); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (p *BinlogParser) SetRawMode(mode bool) {
	p.rawMode = mode
}

func (p *BinlogParser) SetParseTime(parseTime bool) {
	p.parseTime = parseTime
}

func (p *BinlogParser) parseHeader(data []byte) (*EventHeader, error) {
	h := new(EventHeader)
	err := h.Decode(data)
	if err != nil {
		return nil, err
	}

	return h, nil
}

func (p *BinlogParser) parseEvent(h *EventHeader, data []byte) (Event, error) {
	var e Event

	if h.EventType == FORMAT_DESCRIPTION_EVENT {
		p.format = &FormatDescriptionEvent{}
		e = p.format
	} else {
		if p.format != nil && p.format.ChecksumAlgorithm == BINLOG_CHECKSUM_ALG_CRC32 {
			data = data[0 : len(data)-4]
		}

		if h.EventType == ROTATE_EVENT {
			e = &RotateEvent{}
		} else if !p.rawMode {
			switch h.EventType {
			case QUERY_EVENT:
				e = &QueryEvent{}
			case XID_EVENT:
				e = &XIDEvent{}
			case TABLE_MAP_EVENT:
				te := &TableMapEvent{}
				if p.format.EventTypeHeaderLengths[TABLE_MAP_EVENT-1] == 6 {
					te.tableIDSize = 4
				} else {
					te.tableIDSize = 6
				}
				e = te
			case WRITE_ROWS_EVENTv0,
				UPDATE_ROWS_EVENTv0,
				DELETE_ROWS_EVENTv0,
				WRITE_ROWS_EVENTv1,
				DELETE_ROWS_EVENTv1,
				UPDATE_ROWS_EVENTv1,
				WRITE_ROWS_EVENTv2,
				UPDATE_ROWS_EVENTv2,
				DELETE_ROWS_EVENTv2:
				e = p.newRowsEvent(h)
			case ROWS_QUERY_EVENT:
				e = &RowsQueryEvent{}
			case GTID_EVENT:
				e = &GTIDEvent{}
			case BEGIN_LOAD_QUERY_EVENT:
				e = &BeginLoadQueryEvent{}
			case EXECUTE_LOAD_QUERY_EVENT:
				e = &ExecuteLoadQueryEvent{}
			case MARIADB_ANNOTATE_ROWS_EVENT:
				e = &MariadbAnnotaeRowsEvent{}
			case MARIADB_BINLOG_CHECKPOINT_EVENT:
				e = &MariadbBinlogCheckPointEvent{}
			case MARIADB_GTID_LIST_EVENT:
				e = &MariadbGTIDListEvent{}
			case MARIADB_GTID_EVENT:
				ee := &MariadbGTIDEvent{}
				ee.GTID.ServerID = h.ServerID
				e = ee
			default:
				e = &GenericEvent{}
			}
		} else {
			e = &GenericEvent{}
		}
	}

	if err := e.Decode(data); err != nil {
		return nil, &EventError{h, err.Error(), data}
	}

	if te, ok := e.(*TableMapEvent); ok {
		p.tables[te.TableID] = te
	}

	if re, ok := e.(*RowsEvent); ok {
		if (re.Flags & RowsEventStmtEndFlag) > 0 {
			// Refer https://github.com/alibaba/canal/blob/38cc81b7dab29b51371096fb6763ca3a8432ffee/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogEvent.java#L176
			p.tables = make(map[uint64]*TableMapEvent)
		}
	}

	return e, nil
}

// Given the bytes for a a binary log event: return the decoded event.
// With the exception of the FORMAT_DESCRIPTION_EVENT event type
// there must have previously been passed a FORMAT_DESCRIPTION_EVENT
// into the parser for this to work properly on any given event.
// Passing a new FORMAT_DESCRIPTION_EVENT into the parser will replace
// an existing one.
func (p *BinlogParser) Parse(data []byte) (*BinlogEvent, error) {
	rawData := data

	h, err := p.parseHeader(data)

	if err != nil {
		return nil, err
	}

	data = data[EventHeaderSize:]
	eventLen := int(h.EventSize) - EventHeaderSize

	if len(data) != eventLen {
		return nil, fmt.Errorf("invalid data size %d in event %s, less event length %d", len(data), h.EventType, eventLen)
	}

	e, err := p.parseEvent(h, data)
	if err != nil {
		return nil, err
	}

	return &BinlogEvent{rawData, h, e}, nil
}

func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent {
	e := &RowsEvent{}
	if p.format.EventTypeHeaderLengths[h.EventType-1] == 6 {
		e.tableIDSize = 4
	} else {
		e.tableIDSize = 6
	}

	e.needBitmap2 = false
	e.tables = p.tables
	e.parseTime = p.parseTime

	switch h.EventType {
	case WRITE_ROWS_EVENTv0:
		e.Version = 0
	case UPDATE_ROWS_EVENTv0:
		e.Version = 0
	case DELETE_ROWS_EVENTv0:
		e.Version = 0
	case WRITE_ROWS_EVENTv1:
		e.Version = 1
	case DELETE_ROWS_EVENTv1:
		e.Version = 1
	case UPDATE_ROWS_EVENTv1:
		e.Version = 1
		e.needBitmap2 = true
	case WRITE_ROWS_EVENTv2:
		e.Version = 2
	case UPDATE_ROWS_EVENTv2:
		e.Version = 2
		e.needBitmap2 = true
	case DELETE_ROWS_EVENTv2:
		e.Version = 2
	}

	return e
}
<|endoftext|>


<commit_before>package gofury

import (
	"github.com/valyala/fasthttp"
	"fmt"
	"encoding/json"
)

type HealthCheckers []HealthChecker

// HealthChecker interface.
type HealthChecker interface {
	CheckHealth() bool
	HealthCheckerName() string
}

func HealthCheck(ctx *fasthttp.RequestCtx, healthChecks HealthCheckers) {
	output, err := json.MarshalIndent(healthChecks, "", "  ")
	if err == nil {
		ctx.SetContentType("application/json")
		fmt.Fprintln(ctx, string(output))
	} else {
		ctx.SetStatusCode(fasthttp.StatusInternalServerError)
	}
}
<commit_msg>error handling refactor<commit_after>package gofury

import (
	"github.com/valyala/fasthttp"
	"fmt"
	"encoding/json"
)

type HealthCheckers []HealthChecker

// HealthChecker interface.
type HealthChecker interface {
	CheckHealth() bool
	HealthCheckerName() string
}

func HealthCheck(ctx *fasthttp.RequestCtx, healthChecks HealthCheckers) {
	ctx.SetContentType("application/json")
	output, err := json.MarshalIndent(healthChecks, "", "    ")
	if err != nil {
		ctx.SetStatusCode(fasthttp.StatusInternalServerError)
		return
	}
	ctx.SetStatusCode(fasthttp.StatusOK)
	fmt.Fprintln(ctx, string(output))
}
<|endoftext|>


<commit_before>package heartbeat

import (
	"sync"
	"testing"
	"time"
)

func TestHeartbeat(t *testing.T) {
	d := time.Millisecond * 50
	msg := "Test message"
	fired := false
	cleanup := func() {
		if m := recover(); m != nil {
			fired = true
			if msg != m {
				t.Errorf("Override panic message not set. Expected %s, got %s\n", msg, m)
			}
		}
	}
	h := Heartbeat(d, msg, cleanup)
	time.Sleep(2 * d)
	if !fired {
		t.Errorf("Heartbeat timer didn't fire")
	}
	h = Heartbeat(d, msg, nil)
	tt := time.NewTicker(d / 2)
	i := 0
	for {
		select {
		case <-tt.C:
			i++
			if i < 4 {
				h(false)
			} else if i == 4 {
				h(true)
			} else if i == 10 {
				return
			}
		}
	}
	// check that we can call h(true) multiple times
	h(true)
}

func TestConcurrentAccess(t *testing.T) {
	hb := Heartbeat(time.Minute, "", nil)
	wg := sync.WaitGroup{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			for j := 0; j < 10000; j++ {
				hb(false)
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func ExampleHeartbeat_typical() {
	heartbeatExpiration := time.Second * 1
	callback := Heartbeat(heartbeatExpiration, "", nil)
	for i := 0; i < 5; i++ {
		time.Sleep(heartbeatExpiration / 2) // simulate some processing
		callback(false)                     // keep the heartbeat from expiring
	}
	callback(true) // cancel the heartbeat
}

func ExampleHeartbeat_cleanup() {
	heartbeatExpiration := time.Second * 1
	panicMessage := "Sample message"
	heartbeatFired := false
	cleanup := func() {
		if msg := recover(); msg != nil {
			// do whatever cleanup needs to be done for an expired heartbeat
			// msg will be "Sample message"
			heartbeatFired = true
		}
	}
	callback := Heartbeat(heartbeatExpiration, panicMessage, cleanup)
	time.Sleep(heartbeatExpiration * 2) // wait for the heartbeat to expire
	// heartbeatFired == true now
	callback(true) // no need to call this for an expired heartbeat, but it doesn't hurt
}

func ExampleHeartbeat_noCatch() {
	heartbeatExpiration := time.Second * 1
	panicMessage := "Sample message"
	Heartbeat(heartbeatExpiration, panicMessage, nil)
	time.Sleep(heartbeatExpiration * 2) // wait for the heartbeat to expire
	// we'll never get to here, because the heartbeat will have panic()'d
}
<commit_msg>fix race condition in TestHeartbeat<commit_after>package heartbeat

import (
	"sync"
	"testing"
	"time"
)

func TestHeartbeat(t *testing.T) {
	d := time.Millisecond * 50
	msg := "Test message"
	fired := make(chan struct{})
	cleanup := func() {
		if m := recover(); m != nil {
			if msg != m {
				t.Errorf("Override panic message not set. Expected %s, got %s\n", msg, m)
			}
			close(fired)
		}
	}
	_ = Heartbeat(d, msg, cleanup)
	select {
	case <-fired:
	case <-time.After(2 * d):
		t.Errorf("Heartbeat timer didn't fire")
	}
	h := Heartbeat(d, msg, nil)
	tt := time.NewTicker(d / 2)
	i := 0
	for {
		select {
		case <-tt.C:
			i++
			if i < 4 {
				h(false)
			} else if i == 4 {
				h(true)
			} else if i == 10 {
				return
			}
		}
	}
	// check that we can call h(true) multiple times
	h(true)
}

func TestConcurrentAccess(t *testing.T) {
	hb := Heartbeat(time.Minute, "", nil)
	wg := sync.WaitGroup{}
	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func() {
			for j := 0; j < 10000; j++ {
				hb(false)
			}
			wg.Done()
		}()
	}
	wg.Wait()
}

func ExampleHeartbeat_typical() {
	heartbeatExpiration := time.Second * 1
	callback := Heartbeat(heartbeatExpiration, "", nil)
	for i := 0; i < 5; i++ {
		time.Sleep(heartbeatExpiration / 2) // simulate some processing
		callback(false)                     // keep the heartbeat from expiring
	}
	callback(true) // cancel the heartbeat
}

func ExampleHeartbeat_cleanup() {
	heartbeatExpiration := time.Second * 1
	panicMessage := "Sample message"
	heartbeatFired := false
	cleanup := func() {
		if msg := recover(); msg != nil {
			// do whatever cleanup needs to be done for an expired heartbeat
			// msg will be "Sample message"
			heartbeatFired = true
		}
	}
	callback := Heartbeat(heartbeatExpiration, panicMessage, cleanup)
	time.Sleep(heartbeatExpiration * 2) // wait for the heartbeat to expire
	// heartbeatFired == true now
	callback(true) // no need to call this for an expired heartbeat, but it doesn't hurt
}

func ExampleHeartbeat_noCatch() {
	heartbeatExpiration := time.Second * 1
	panicMessage := "Sample message"
	Heartbeat(heartbeatExpiration, panicMessage, nil)
	time.Sleep(heartbeatExpiration * 2) // wait for the heartbeat to expire
	// we'll never get to here, because the heartbeat will have panic()'d
}
<|endoftext|>


<commit_before>package xorm

import (
	"reflect"

	"github.com/coscms/xorm/core"
)

// =====================================
// 定义ResultSet
// =====================================
func NewResultSet() *ResultSet {
	return &ResultSet{
		Fields:    make([]string, 0),
		Values:    make([]string, 0),
		NameIndex: make(map[string]int),
		Length:    0,
	}
}

type ResultSet struct {
	Fields    []string
	Values    []string
	NameIndex map[string]int
	Length    int
}

func (r *ResultSet) Get(index int) string {
	if index >= r.Length {
		return ""
	}
	return r.Values[index]
}

func (r *ResultSet) GetByName(name string) string {
	if index, ok := r.NameIndex[name]; ok {
		return r.Get(index)
	}
	return ""
}

func (r *ResultSet) Set(index int, value string) bool {
	if index >= r.Length {
		return false
	}
	r.Values[index] = value
	return true
}

func (r *ResultSet) SetByName(name string, value string) bool {
	if index, ok := r.NameIndex[name]; ok {
		return r.Set(index, value)
	} else {
		r.NameIndex[name] = len(r.Values)
		r.Fields = append(r.Fields, name)
		r.Values = append(r.Values, value)
		r.Length = len(r.Values)
	}
	return true
}

// =====================================
// 增加Session结构体中的方法
// =====================================
func (session *Session) queryRows(sqlStr string, paramStr ...interface{}) (rows *core.Rows, err error) {
	session.queryPreprocess(&sqlStr, paramStr...)

	if session.IsAutoCommit {
		return session.innerQueryRows(session.DB(), sqlStr, paramStr...)
	}
	return session.txQueryRows(session.Tx, sqlStr, paramStr...)
}

func (session *Session) txQueryRows(tx *core.Tx, sqlStr string, params ...interface{}) (rows *core.Rows, err error) {
	rows, err = tx.Query(sqlStr, params...)
	if err != nil {
		return nil, err
	}
	return
}

func (session *Session) innerQueryRows(db *core.DB, sqlStr string, params ...interface{}) (rows *core.Rows, err error) {
	stmt, rows, err := session.Engine.LogSQLQueryTime(sqlStr, params, func() (*core.Stmt, *core.Rows, error) {
		stmt, err := db.Prepare(sqlStr)
		if err != nil {
			return stmt, nil, err
		}
		rows, err := stmt.Query(params...)

		return stmt, rows, err
	})
	if stmt != nil {
		defer stmt.Close()
	}
	if err != nil {
		return nil, err
	}
	return
}

/**
 * Exec a raw sql and return records as []*ResultSet
 * @param  string					SQL
 * @param  ...interface{}			params
 * @return []*ResultSet,error
 * @author AdamShen (swh@admpub.com)
 */
func (session *Session) Q(sqlStr string, paramStr ...interface{}) (resultsSlice []*ResultSet, err error) {

	defer session.resetStatement()
	if session.IsAutoClose {
		defer session.Close()
	}

	resultsSlice = make([]*ResultSet, 0)
	rows, err := session.queryRows(sqlStr, paramStr...)
	if rows != nil && err == nil {
		resultsSlice, err = rows2ResultSetSlice(rows)
	}
	if rows != nil {
		defer rows.Close()
	}
	return
}

/**
 * 逐行执行回调函数
 * @param  func(*core.Rows) callback		callback func
 * @param  string sqlStr 					SQL
 * @param  ...interface{} paramStr			params
 * @return error
 * @author AdamShen (swh@admpub.com)
 * @example
 * QCallback(func(rows *core.Rows){
 * 	if err := rows.Scan(bean); err != nil {
 *		return
 *	}
 *	//.....
 * },"SELECT * FROM shop WHERE type=?","vip")
 */
func (session *Session) QCallback(callback func(*core.Rows), sqlStr string, paramStr ...interface{}) (err error) {

	defer session.resetStatement()
	if session.IsAutoClose {
		defer session.Close()
	}

	rows, err := session.queryRows(sqlStr, paramStr...)
	if rows != nil && err == nil {
		for rows.Next() {
			callback(rows)
		}
	}
	if rows != nil {
		defer rows.Close()
	}
	return
}

// =====================================
// 函数
// =====================================
func rows2ResultSetSlice(rows *core.Rows) (resultsSlice []*ResultSet, err error) {
	fields, err := rows.Columns()
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		result, err := row2ResultSet(rows, fields)
		if err != nil {
			return nil, err
		}
		resultsSlice = append(resultsSlice, result)
	}

	return resultsSlice, nil
}

func row2ResultSet(rows *core.Rows, fields []string) (result *ResultSet, err error) {
	//result := make(map[string]string)
	result = NewResultSet()
	getRowByRows(rows, fields, func(data string, index int, fieldName string) {
		//result[fieldName] = data
		result.NameIndex[fieldName] = len(result.Fields)
		result.Fields = append(result.Fields, fieldName)
		result.Values = append(result.Values, data)
	})
	result.Length = len(result.Values)
	return result, nil
}

//获取一行中每一列数据
func getRowByRows(rows *core.Rows, fields []string, fn func(data string, index int, fieldName string)) (err error) {
	length := len(fields)
	scanResultContainers := make([]interface{}, length)
	for i := 0; i < length; i++ {
		var scanResultContainer interface{}
		scanResultContainers[i] = &scanResultContainer
	}
	if err := rows.Scan(scanResultContainers...); err != nil {
		return err
	}
	for ii, key := range fields {
		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
		//if row is null then ignore
		if rawValue.Interface() == nil {
			continue
		}
		if data, err := value2String(&rawValue); err == nil {
			fn(data, ii, key)
		} else {
			return err
		}
	}
	return nil
}

//根据core.Rows来查询结果
func getResultSliceByRows(rows *core.Rows, erre error) (resultsSlice []map[string][]byte, err error) {
	resultsSlice = make([]map[string][]byte, 0)
	if rows != nil && erre == nil {
		resultsSlice, err = rows2maps(rows)
	}
	if rows != nil {
		defer rows.Close()
	}
	return
}
<commit_msg>update<commit_after>package xorm

import (
	"reflect"

	"github.com/coscms/xorm/core"
)

// =====================================
// 定义ResultSet
// =====================================
func NewResultSet() *ResultSet {
	return &ResultSet{
		Fields:    make([]string, 0),
		Values:    make([]string, 0),
		NameIndex: make(map[string]int),
		Length:    0,
	}
}

type ResultSet struct {
	Fields    []string
	Values    []string
	NameIndex map[string]int
	Length    int
}

func (r *ResultSet) Get(index int) string {
	if index >= r.Length {
		return ""
	}
	return r.Values[index]
}

func (r *ResultSet) GetByName(name string) string {
	if index, ok := r.NameIndex[name]; ok {
		return r.Get(index)
	}
	return ""
}

func (r *ResultSet) Set(index int, value string) bool {
	if index >= r.Length {
		return false
	}
	r.Values[index] = value
	return true
}

func (r *ResultSet) SetByName(name string, value string) bool {
	if index, ok := r.NameIndex[name]; ok {
		return r.Set(index, value)
	} else {
		r.NameIndex[name] = len(r.Values)
		r.Fields = append(r.Fields, name)
		r.Values = append(r.Values, value)
		r.Length = len(r.Values)
	}
	return true
}

// =====================================
// 增加Session结构体中的方法
// =====================================
func (session *Session) queryRows(sqlStr string, paramStr ...interface{}) (rows *core.Rows, err error) {
	session.queryPreprocess(&sqlStr, paramStr...)

	if session.IsAutoCommit {
		return session.innerQueryRows(session.DB(), sqlStr, paramStr...)
	}
	return session.txQueryRows(session.Tx, sqlStr, paramStr...)
}

func (session *Session) txQueryRows(tx *core.Tx, sqlStr string, params ...interface{}) (rows *core.Rows, err error) {
	rows, err = tx.Query(sqlStr, params...)
	if err != nil {
		return nil, err
	}
	return
}

func (session *Session) innerQueryRows(db *core.DB, sqlStr string, params ...interface{}) (rows *core.Rows, err error) {
	stmt, rows, err := session.Engine.LogSQLQueryTime(sqlStr, params, func() (*core.Stmt, *core.Rows, error) {
		stmt, err := db.Prepare(sqlStr)
		if err != nil {
			return stmt, nil, err
		}
		rows, err := stmt.Query(params...)

		return stmt, rows, err
	})
	if stmt != nil {
		defer stmt.Close()
	}
	if err != nil {
		return nil, err
	}
	return
}

/**
 * Exec a raw sql and return records as []*ResultSet
 * @param  string					SQL
 * @param  ...interface{}			params
 * @return []*ResultSet,error
 * @author AdamShen (swh@admpub.com)
 */
func (session *Session) Q(sqlStr string, paramStr ...interface{}) (resultsSlice []*ResultSet, err error) {

	defer session.resetStatement()
	if session.IsAutoClose {
		defer session.Close()
	}

	resultsSlice = make([]*ResultSet, 0)
	rows, err := session.queryRows(sqlStr, paramStr...)
	if rows != nil && err == nil {
		resultsSlice, err = rows2ResultSetSlice(rows)
	}
	if rows != nil {
		defer rows.Close()
	}
	return
}

/**
 * 逐行执行回调函数
 * @param  func(*core.Rows) callback		callback func
 * @param  string sqlStr 					SQL
 * @param  ...interface{} paramStr			params
 * @return error
 * @author AdamShen (swh@admpub.com)
 * @example
 * QCallback(func(rows *core.Rows){
 * 	if err := rows.Scan(bean); err != nil {
 *		return
 *	}
 *	//.....
 * },"SELECT * FROM shop WHERE type=?","vip")
 */
func (session *Session) QCallback(callback func(*core.Rows), sqlStr string, paramStr ...interface{}) (err error) {

	defer session.resetStatement()
	if session.IsAutoClose {
		defer session.Close()
	}

	rows, err := session.queryRows(sqlStr, paramStr...)
	if rows != nil && err == nil {
		for rows.Next() {
			callback(rows)
		}
	}
	if rows != nil {
		defer rows.Close()
	}
	return
}

// =====================================
// 函数
// =====================================
func rows2ResultSetSlice(rows *core.Rows) (resultsSlice []*ResultSet, err error) {
	fields, err := rows.Columns()
	if err != nil {
		return nil, err
	}
	for rows.Next() {
		result, err := row2ResultSet(rows, fields)
		if err != nil {
			return nil, err
		}
		resultsSlice = append(resultsSlice, result)
	}

	return resultsSlice, nil
}

func row2ResultSet(rows *core.Rows, fields []string) (result *ResultSet, err error) {
	//result := make(map[string]string)
	result = NewResultSet()
	LineProcessing(rows, fields, func(data string, index int, fieldName string) {
		//result[fieldName] = data
		result.NameIndex[fieldName] = len(result.Fields)
		result.Fields = append(result.Fields, fieldName)
		result.Values = append(result.Values, data)
	})
	result.Length = len(result.Values)
	return result, nil
}

//获取一行中每一列数据
func LineProcessing(rows *core.Rows, fields []string, fn func(data string, index int, fieldName string)) (err error) {
	length := len(fields)
	scanResultContainers := make([]interface{}, length)
	for i := 0; i < length; i++ {
		var scanResultContainer interface{}
		scanResultContainers[i] = &scanResultContainer
	}
	if err := rows.Scan(scanResultContainers...); err != nil {
		return err
	}
	for ii, key := range fields {
		rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
		//if row is null then ignore
		if rawValue.Interface() == nil {
			continue
		}
		if data, err := value2String(&rawValue); err == nil {
			fn(data, ii, key)
		} else {
			return err
		}
	}
	return nil
}

//根据core.Rows来查询结果
func getResultSliceByRows(rows *core.Rows, erre error) (resultsSlice []map[string][]byte, err error) {
	resultsSlice = make([]map[string][]byte, 0)
	if rows != nil && erre == nil {
		resultsSlice, err = rows2maps(rows)
	}
	if rows != nil {
		defer rows.Close()
	}
	return
}
<|endoftext|>


<commit_before>package gofakeit

import (
	"math/rand"
)

// Check if in lib
func dataCheck(dataVal []string) bool {
	var checkOk bool

	_, checkOk = Data[dataVal[0]]
	if len(dataVal) == 2 && checkOk {
		_, checkOk = Data[dataVal[0]][dataVal[1]]
	}

	return checkOk
}

// Check if in lib
func intDataCheck(dataVal []string) bool {
	var checkOk bool

	_, checkOk = IntData[dataVal[0]]
	if len(dataVal) == 2 && checkOk {
		_, checkOk = IntData[dataVal[0]][dataVal[1]]
	}

	return checkOk
}

// Get Random Value
func getRandValue(dataVal []string) string {
	if !dataCheck(dataVal) {
		return ""
	}
	return Data[dataVal[0]][dataVal[1]][rand.Intn(len(Data[dataVal[0]][dataVal[1]]))]
}

// Get Random Integer Value
func getRandIntValue(dataVal []string) int {
	if !intDataCheck(dataVal) {
		return 0
	}
	return IntData[dataVal[0]][dataVal[1]][rand.Intn(len(IntData[dataVal[0]][dataVal[1]]))]
}

// Replace # with numbers
func replaceWithNumbers(str string) string {
	bytestr := []byte(str)
	hashtag := []byte("#")[0]
	numbers := []byte("0123456789")
	for i := 0; i < len(bytestr); i++ {
		if bytestr[i] == hashtag {
			bytestr[i] = numbers[rand.Intn(9)]
		}
	}
	if bytestr[0] == []byte("0")[0] {
		bytestr[0] = numbers[rand.Intn(8)+1]
	}

	return string(bytestr)
}

// Replace ? with letters
func replaceWithLetters(str string) string {
	bytestr := []byte(str)
	question := []byte("?")[0]
	letters := []byte("abcdefghijklmnopqrstuvwxyz")
	for i := 0; i < len(bytestr); i++ {
		if bytestr[i] == question {
			bytestr[i] = letters[rand.Intn(26)]
		}
	}

	return string(bytestr)
}

// Generate random letter
func randLetter() string {
	return string([]byte("abcdefghijklmnopqrstuvwxyz")[rand.Intn(26)])
}

// Generate random integer between min and max
func randIntRange(min, max int) int {
	if min == max {
		return min
	}
	return rand.Intn((max+1)-min) + min
}

func randFloatRange(min, max float64) float64 {
	if min == max {
		return min
	}
	return rand.Float64()*(max-min) + min
}
<commit_msg>misc - updated grabbing data from sub package<commit_after>package gofakeit

import (
	"math/rand"

	"github.com/brianvoe/gofakeit/data"
)

// Check if in lib
func dataCheck(dataVal []string) bool {
	var checkOk bool

	_, checkOk = data.Data[dataVal[0]]
	if len(dataVal) == 2 && checkOk {
		_, checkOk = data.Data[dataVal[0]][dataVal[1]]
	}

	return checkOk
}

// Check if in lib
func intDataCheck(dataVal []string) bool {
	var checkOk bool

	_, checkOk = data.IntData[dataVal[0]]
	if len(dataVal) == 2 && checkOk {
		_, checkOk = data.IntData[dataVal[0]][dataVal[1]]
	}

	return checkOk
}

// Get Random Value
func getRandValue(dataVal []string) string {
	if !dataCheck(dataVal) {
		return ""
	}
	return data.Data[dataVal[0]][dataVal[1]][rand.Intn(len(data.Data[dataVal[0]][dataVal[1]]))]
}

// Get Random Integer Value
func getRandIntValue(dataVal []string) int {
	if !intDataCheck(dataVal) {
		return 0
	}
	return data.IntData[dataVal[0]][dataVal[1]][rand.Intn(len(data.IntData[dataVal[0]][dataVal[1]]))]
}

// Replace # with numbers
func replaceWithNumbers(str string) string {
	bytestr := []byte(str)
	hashtag := []byte("#")[0]
	numbers := []byte("0123456789")
	for i := 0; i < len(bytestr); i++ {
		if bytestr[i] == hashtag {
			bytestr[i] = numbers[rand.Intn(9)]
		}
	}
	if bytestr[0] == []byte("0")[0] {
		bytestr[0] = numbers[rand.Intn(8)+1]
	}

	return string(bytestr)
}

// Replace ? with letters
func replaceWithLetters(str string) string {
	bytestr := []byte(str)
	question := []byte("?")[0]
	letters := []byte("abcdefghijklmnopqrstuvwxyz")
	for i := 0; i < len(bytestr); i++ {
		if bytestr[i] == question {
			bytestr[i] = letters[rand.Intn(26)]
		}
	}

	return string(bytestr)
}

// Generate random letter
func randLetter() string {
	return string([]byte("abcdefghijklmnopqrstuvwxyz")[rand.Intn(26)])
}

// Generate random integer between min and max
func randIntRange(min, max int) int {
	if min == max {
		return min
	}
	return rand.Intn((max+1)-min) + min
}

func randFloatRange(min, max float64) float64 {
	if min == max {
		return min
	}
	return rand.Float64()*(max-min) + min
}
<|endoftext|>