<commit_before>package swp

import (
	"fmt"
	"sync"
	"time"
)

// AckStatus conveys info from the receiver to the sender when an Ack is received.
type AckStatus struct {
	OnlyUpdateFlowCtrl  bool // don't send ack, just update flow info.
	AckNum              Seqno
	AckCameWithPacket   Seqno
	AvailReaderBytesCap int64 // for sender throttling/flow-control
	AvailReaderMsgCap   int64 // for sender throttling/flow-control
}

// SenderState tracks the sender's sliding window state.
// To avoid circulate deadlocks, the Sender never talks
// directly to the RecvState. The RecvState will
// tell the Sender stuff on GotAck.
type SenderState struct {
	Net              Network
	Inbox            string
	Dest             string
	LastAckRec       Seqno
	LastFrameSent    Seqno
	Txq              []*TxqSlot
	SenderWindowSize Seqno
	mut              sync.Mutex
	Timeout          time.Duration

	// the main goroutine safe way to request
	// sending a packet:
	BlockingSend chan *Packet
	GotAck       chan AckStatus

	ReqStop     chan bool
	Done        chan bool
	SendHistory []*Packet
	SendSz      int64
	//	slotsAvail   int64
	SendAck      chan *Packet
	DiscardCount int64

	LastSendTime      time.Time
	KeepAliveInterval time.Duration
	keepAlive         <-chan time.Time

	SentButNotAcked map[Seqno]*TxqSlot

	// flow control params
	// last seen from our downstream
	// receiver, we throttle ourselves
	// based on these. For example, our
	// effective usable window is
	//   min(LastSeenAvailReaderMsgCap, slotsAvail)
	// for count of messages we can send.
	LastSeenAvailReaderBytesCap int64
	LastSeenAvailReaderMsgCap   int64

	// EffectiveWindow = AdvertisedWindow - (LastByteSent - LastByteAcked)
	EffectiveWindow int64

	// do synchronized access via GetFlow()
	// and UpdateFlow(s.Net)
	FlowCt         FlowCtrl
	TotalBytesSent int64
}

func NewSenderState(net Network, sendSz int64, timeout time.Duration,
	inbox string, destInbox string) *SenderState {
	s := &SenderState{
		Net:              net,
		Inbox:            inbox,
		Dest:             destInbox,
		SenderWindowSize: Seqno(sendSz),
		Txq:              make([]*TxqSlot, sendSz),
		Timeout:          timeout,
		LastFrameSent:    -1,
		LastAckRec:       -1,
		ReqStop:          make(chan bool),
		Done:             make(chan bool),
		SendHistory:      make([]*Packet, 0),
		BlockingSend:     make(chan *Packet),
		SendSz:           sendSz,
		GotAck:           make(chan AckStatus),
		SendAck:          make(chan *Packet),
		SentButNotAcked:  make(map[Seqno]*TxqSlot),

		// send keepalives (important especially for resuming flow from a
		// stopped state) at least this often:
		KeepAliveInterval: 100 * time.Millisecond,
		FlowCt: FlowCtrl{flow: Flow{
			ReservedByteCap: 64 * 1024,
			ReservedMsgCap:  32,
		}},
		// don't start fast, as we could overwhelm
		// the receiver. Instead start very slowly,
		// allowing 2 messages so our 003 reorder test runs.
		// It will get updated after the first ack
		// or keep alive.
		LastSeenAvailReaderMsgCap:   2,
		LastSeenAvailReaderBytesCap: 1024 * 1024,
	}
	for i := range s.Txq {
		s.Txq[i] = &TxqSlot{}
	}
	return s
}

func (s *SenderState) ComputeInflight() (bytesInflight int64, msgInflight int64) {
	for _, slot := range s.SentButNotAcked {
		msgInflight++
		bytesInflight += int64(len(slot.Pack.Data))
	}
	return
}

// Start initiates the SenderState goroutine, which manages
// sends, timeouts, and resends
func (s *SenderState) Start() {

	go func() {
		//s.slotsAvail = s.SendSz

		var acceptSend chan *Packet

		// check for expired timers at wakeFreq
		wakeFreq := s.Timeout / 2

		// send keepalives (for resuming flow from a
		// stopped state) at least this often:
		s.keepAlive = time.After(s.KeepAliveInterval)

		regularIntervalWakeup := time.After(wakeFreq)

	sendloop:
		for {
			//q("%v top of sendloop, sender LAR: %v, LFS: %v \n",
			//	s.Inbox, s.LastAckRec, s.LastFrameSent)

			// do we have capacity to accept a send?
			// do a conditional receive. Start by
			// assuming no:
			acceptSend = nil

			// then check if we can set acceptSend.
			//
			// We accept a packet for sending if flow control info
			// from the receiver allows it.
			//
			bytesInflight, msgInflight := s.ComputeInflight()
			q("%v bytesInflight = %v", s.Inbox, bytesInflight)
			q("%v msgInflight = %v", s.Inbox, msgInflight)

			if s.LastSeenAvailReaderMsgCap-msgInflight > 0 {
				p("%v flow-control: okay to send. s.LastSeenAvailReaderMsgCap: %v > msgInflight: %v",
					s.Inbox, s.LastSeenAvailReaderMsgCap, msgInflight)
				acceptSend = s.BlockingSend
			} else {
				p("%v flow-control kicked in: not sending. s.LastSeenAvailReaderMsgCap = %v <= msgInflight %v", s.Inbox, s.LastSeenAvailReaderMsgCap, msgInflight)
			}

			p("%v top of sender select loop", s.Inbox)
			select {
			case <-s.keepAlive:
				p("%v keepAlive at %v", s.Inbox, time.Now())
				s.doKeepAlive()

			case <-regularIntervalWakeup:
				p("%v regularIntervalWakeup at %v", s.Inbox, time.Now())

				// have any of our packets timed-out and need to be
				// sent again?
				retry := []*TxqSlot{}
				for _, slot := range s.SentButNotAcked {
					if slot.RetryDeadline.Before(time.Now()) {
						retry = append(retry, slot)
					}
				}
			doRetryLoop:
				for _, slot := range retry {
					if slot.Pack == nil {
						//q("retry loop, slot = %#v", slot)
					}
					_, ok := s.SentButNotAcked[slot.Pack.SeqNum]
					if !ok {
						//q("already acked and gone from SentButNotAcked, so skip SeqNum %v and PopTop",
						//	slot.Pack.SeqNum)
						continue doRetryLoop
					}
					if slot.Pack.SeqNum <= s.LastAckRec {
						//q("already acked; is <= s.LastAckRecv (%v), so skip SeqNum %v and PopTop",
						//	s.LastAckRec, slot.Pack.SeqNum)
						continue doRetryLoop
					}

					// reset deadline and resend
					slot.RetryDeadline = time.Now().Add(s.Timeout)

					flow := s.FlowCt.UpdateFlow(s.Inbox, s.Net, -1, -1)
					slot.Pack.AvailReaderBytesCap = flow.AvailReaderBytesCap
					slot.Pack.AvailReaderMsgCap = flow.AvailReaderMsgCap
					p("%v doing retry Net.Send() for pack = '%#v' of paydirt '%s'",
						s.Inbox, slot.Pack, string(slot.Pack.Data))
					err := s.Net.Send(slot.Pack, "retry")
					panicOn(err)
				}
				regularIntervalWakeup = time.After(wakeFreq)

			case <-s.ReqStop:
				close(s.Done)
				return
			case pack := <-acceptSend:
				p("%v got <-acceptSend pack: '%#v'", s.Inbox, pack)
				s.doOrigDataSend(pack)

			case a := <-s.GotAck:
				// ack received - do sender side stuff
				//
				p("%v sender GotAck a: %#v", s.Inbox, a)
				//
				// flow control: respect a.AvailReaderBytesCap
				// and a.AvailReaderMsgCap info that we have
				// received from this ack
				//
				p("%v sender GotAck, updating s.LastSeenAvailReaderMsgCap %v -> %v",
					s.Inbox, s.LastSeenAvailReaderMsgCap, a.AvailReaderMsgCap)
				s.LastSeenAvailReaderBytesCap = a.AvailReaderBytesCap
				s.LastSeenAvailReaderMsgCap = a.AvailReaderMsgCap

				// need to update our map of SentButNotAcked
				// and remove everything before AckNum, which is cumulative.
				for _, slot := range s.SentButNotAcked {
					if slot.Pack.SeqNum <= a.AckNum {
						delete(s.SentButNotAcked, a.AckNum)
					}
				}

				if a.OnlyUpdateFlowCtrl {
					// it wasn't an Ack, just updated flow info
					// from a received data message.
					q("%s sender Gotack: just updated flow control, continuing sendloop", s.Inbox)
					continue sendloop
				}
				q("%s sender Gotack: more than just flowcontrol...", s.Inbox)
				delete(s.SentButNotAcked, a.AckNum)
				if !InWindow(a.AckNum, s.LastAckRec+1, s.LastFrameSent) {
					//q("%v a.AckNum = %v outside sender's window [%v, %v], dropping it.",
					//	s.Inbox, a.AckNum, s.LastAckRec+1, s.LastFrameSent)
					s.DiscardCount++
					continue sendloop
				}
				q("%v packet.AckNum = %v inside sender's window, keeping it.", s.Inbox, a.AckNum)
				for {
					s.LastAckRec++

					// do this before changing slot, since we point into slot.
					delete(s.SentButNotAcked, s.LastAckRec)

					slot := s.Txq[s.LastAckRec%s.SenderWindowSize]
					q("%v ... slot = %#v", s.Inbox, slot)

					// release the send slot
					//s.slotsAvail++
					if s.LastAckRec == a.AckNum {
						q("%v s.LastAskRec[%v] matches a.AckNum[%v], breaking",
							s.Inbox, s.LastAckRec, a.AckNum)
						break
					}
					q("%v s.LastAskRec[%v] != a.AckNum[%v], looping",
						s.Inbox, s.LastAckRec, a.AckNum)
				}
			case ackPack := <-s.SendAck:
				// request to send an ack:
				// don't go though the BlockingSend protocol; since
				// could effectively livelock us.
				p("%v doing Net.Send() SendAck request on ackPack: '%#v'",
					s.Inbox, ackPack)
				err := s.Net.Send(ackPack, "SendAck/ackPack")
				panicOn(err)
			}
		}
	}()
}

// Stop the SenderState componennt
func (s *SenderState) Stop() {
	s.mut.Lock()
	select {
	case <-s.ReqStop:
	default:
		close(s.ReqStop)
	}
	s.mut.Unlock()
	<-s.Done
}

// for first time sends of data, not retries or acks.
func (s *SenderState) doOrigDataSend(pack *Packet) {
	//s.slotsAvail--
	//q("%v sender in acceptSend, now %v slotsAvail", s.Inbox, s.slotsAvail)

	s.LastFrameSent++
	//q("%v LastFrameSent is now %v", s.Inbox, s.LastFrameSent)

	s.TotalBytesSent += int64(len(pack.Data))
	pack.CumulBytesTransmitted = s.TotalBytesSent

	//p("%v SenderState.doOrigDataSend is decrementing LastSeenAvailReaderBytesCap %v -> %v",
	//	s.LastSeenAvailReaderBytesCap, s.LastSeenAvailReaderBytesCap-1)
	//	s.LastSeenAvailReaderBytesCap -= int64(len(pack.Data))
	//	s.LastSeenAvailReaderMsgCap--

	lfs := s.LastFrameSent
	pos := lfs % s.SenderWindowSize
	slot := s.Txq[pos]

	pack.SeqNum = lfs
	if pack.From != s.Inbox {
		pack.From = s.Inbox
	}
	pack.From = s.Inbox
	slot.Pack = pack
	// data sends get stored in SentButNotAcked
	s.SentButNotAcked[lfs] = slot

	now := time.Now()
	s.SendHistory = append(s.SendHistory, pack)
	slot.RetryDeadline = now.Add(s.Timeout)
	s.LastSendTime = now

	flow := s.FlowCt.UpdateFlow(s.Inbox+":sender", s.Net, -1, -1)
	q("%v doSend(), flow = '%#v'", s.Inbox, flow)
	pack.AvailReaderBytesCap = flow.AvailReaderBytesCap
	pack.AvailReaderMsgCap = flow.AvailReaderMsgCap
	err := s.Net.Send(slot.Pack, fmt.Sprintf("doSend() for %v", s.Inbox))
	panicOn(err)
}

func (s *SenderState) doKeepAlive() {
	if time.Since(s.LastSendTime) < s.KeepAliveInterval {
		return
	}
	flow := s.FlowCt.UpdateFlow(s.Inbox+":sender", s.Net, -1, -1)
	q("%v doKeepAlive(), flow = '%#v'", s.Inbox, flow)
	// send a packet with no data, to elicit an ack
	// with a new advertised window. This is
	// *not* an ack, because we need it to be
	// acked itself so we get any updated
	// flow control info from the other end.
	s.LastSendTime = time.Now()
	kap := &Packet{
		From:                s.Inbox,
		Dest:                s.Dest,
		SeqNum:              -777, // => keepalive
		KeepAlive:           true,
		AvailReaderBytesCap: flow.AvailReaderBytesCap,
		AvailReaderMsgCap:   flow.AvailReaderMsgCap,
	}
	q("%v doing keepalive Net.Send()", s.Inbox)
	err := s.Net.Send(kap, fmt.Sprintf("keepalive from %v", s.Inbox))
	panicOn(err)

	s.keepAlive = time.After(s.KeepAliveInterval)
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>byte capacity now enforced as well<commit_after>package swp

import (
	"fmt"
	"sync"
	"time"
)

// AckStatus conveys info from the receiver to the sender when an Ack is received.
type AckStatus struct {
	OnlyUpdateFlowCtrl  bool // don't send ack, just update flow info.
	AckNum              Seqno
	AckCameWithPacket   Seqno
	AvailReaderBytesCap int64 // for sender throttling/flow-control
	AvailReaderMsgCap   int64 // for sender throttling/flow-control
}

// SenderState tracks the sender's sliding window state.
// To avoid circulate deadlocks, the Sender never talks
// directly to the RecvState. The RecvState will
// tell the Sender stuff on GotAck.
type SenderState struct {
	Net              Network
	Inbox            string
	Dest             string
	LastAckRec       Seqno
	LastFrameSent    Seqno
	Txq              []*TxqSlot
	SenderWindowSize Seqno
	mut              sync.Mutex
	Timeout          time.Duration

	// the main goroutine safe way to request
	// sending a packet:
	BlockingSend chan *Packet
	GotAck       chan AckStatus

	ReqStop      chan bool
	Done         chan bool
	SendHistory  []*Packet
	SendSz       int64
	SendAck      chan *Packet
	DiscardCount int64

	LastSendTime      time.Time
	KeepAliveInterval time.Duration
	keepAlive         <-chan time.Time

	SentButNotAcked map[Seqno]*TxqSlot

	// flow control params
	// last seen from our downstream
	// receiver, we throttle ourselves
	// based on these.
	LastSeenAvailReaderBytesCap int64
	LastSeenAvailReaderMsgCap   int64

	// EffectiveWindow = AdvertisedWindow - (LastByteSent - LastByteAcked)
	EffectiveWindow int64

	// do synchronized access via GetFlow()
	// and UpdateFlow(s.Net)
	FlowCt         FlowCtrl
	TotalBytesSent int64
}

func NewSenderState(net Network, sendSz int64, timeout time.Duration,
	inbox string, destInbox string) *SenderState {
	s := &SenderState{
		Net:              net,
		Inbox:            inbox,
		Dest:             destInbox,
		SenderWindowSize: Seqno(sendSz),
		Txq:              make([]*TxqSlot, sendSz),
		Timeout:          timeout,
		LastFrameSent:    -1,
		LastAckRec:       -1,
		ReqStop:          make(chan bool),
		Done:             make(chan bool),
		SendHistory:      make([]*Packet, 0),
		BlockingSend:     make(chan *Packet),
		SendSz:           sendSz,
		GotAck:           make(chan AckStatus),
		SendAck:          make(chan *Packet),
		SentButNotAcked:  make(map[Seqno]*TxqSlot),

		// send keepalives (important especially for resuming flow from a
		// stopped state) at least this often:
		KeepAliveInterval: 100 * time.Millisecond,
		FlowCt: FlowCtrl{flow: Flow{
			ReservedByteCap: 64 * 1024,
			ReservedMsgCap:  32,
		}},
		// don't start fast, as we could overwhelm
		// the receiver. Instead start very slowly,
		// allowing 2 messages so our 003 reorder test runs.
		// It will get updated after the first ack
		// or keep alive.
		LastSeenAvailReaderMsgCap:   2,
		LastSeenAvailReaderBytesCap: 1024 * 1024,
	}
	for i := range s.Txq {
		s.Txq[i] = &TxqSlot{}
	}
	return s
}

func (s *SenderState) ComputeInflight() (bytesInflight int64, msgInflight int64) {
	for _, slot := range s.SentButNotAcked {
		msgInflight++
		bytesInflight += int64(len(slot.Pack.Data))
	}
	return
}

// Start initiates the SenderState goroutine, which manages
// sends, timeouts, and resends
func (s *SenderState) Start() {

	go func() {

		var acceptSend chan *Packet

		// check for expired timers at wakeFreq
		wakeFreq := s.Timeout / 2

		// send keepalives (for resuming flow from a
		// stopped state) at least this often:
		s.keepAlive = time.After(s.KeepAliveInterval)

		regularIntervalWakeup := time.After(wakeFreq)

	sendloop:
		for {
			//q("%v top of sendloop, sender LAR: %v, LFS: %v \n",
			//	s.Inbox, s.LastAckRec, s.LastFrameSent)

			// does the downstream reader have capacity to accept a send?
			// Block any new sends if so. We do a conditional receive. Start by
			// assuming no:
			acceptSend = nil

			// then check if we can set acceptSend.
			//
			// We accept a packet for sending if flow control info
			// from the receiver allows it.
			//
			bytesInflight, msgInflight := s.ComputeInflight()
			q("%v bytesInflight = %v", s.Inbox, bytesInflight)
			q("%v msgInflight = %v", s.Inbox, msgInflight)

			if s.LastSeenAvailReaderMsgCap-msgInflight > 0 &&
				s.LastSeenAvailReaderBytesCap-bytesInflight > 0 {
				p("%v flow-control: okay to send. s.LastSeenAvailReaderMsgCap: %v > msgInflight: %v",
					s.Inbox, s.LastSeenAvailReaderMsgCap, msgInflight)
				acceptSend = s.BlockingSend
			} else {
				p("%v flow-control kicked in: not sending. s.LastSeenAvailReaderMsgCap = %v,"+
					" msgInflight=%v, s.LastSeenAvailReaderBytesCap=%v bytesInflight=%v",
					s.Inbox, s.LastSeenAvailReaderMsgCap, msgInflight,
					s.LastSeenAvailReaderBytesCap, bytesInflight)
			}

			p("%v top of sender select loop", s.Inbox)
			select {
			case <-s.keepAlive:
				p("%v keepAlive at %v", s.Inbox, time.Now())
				s.doKeepAlive()

			case <-regularIntervalWakeup:
				p("%v regularIntervalWakeup at %v", s.Inbox, time.Now())

				// have any of our packets timed-out and need to be
				// sent again?
				retry := []*TxqSlot{}
				for _, slot := range s.SentButNotAcked {
					if slot.RetryDeadline.Before(time.Now()) {
						retry = append(retry, slot)
					}
				}
			doRetryLoop:
				for _, slot := range retry {
					if slot.Pack == nil {
						//q("retry loop, slot = %#v", slot)
					}
					_, ok := s.SentButNotAcked[slot.Pack.SeqNum]
					if !ok {
						//q("already acked and gone from SentButNotAcked, so skip SeqNum %v and PopTop",
						//	slot.Pack.SeqNum)
						continue doRetryLoop
					}
					if slot.Pack.SeqNum <= s.LastAckRec {
						//q("already acked; is <= s.LastAckRecv (%v), so skip SeqNum %v and PopTop",
						//	s.LastAckRec, slot.Pack.SeqNum)
						continue doRetryLoop
					}

					// reset deadline and resend
					slot.RetryDeadline = time.Now().Add(s.Timeout)

					flow := s.FlowCt.UpdateFlow(s.Inbox, s.Net, -1, -1)
					slot.Pack.AvailReaderBytesCap = flow.AvailReaderBytesCap
					slot.Pack.AvailReaderMsgCap = flow.AvailReaderMsgCap
					p("%v doing retry Net.Send() for pack = '%#v' of paydirt '%s'",
						s.Inbox, slot.Pack, string(slot.Pack.Data))
					err := s.Net.Send(slot.Pack, "retry")
					panicOn(err)
				}
				regularIntervalWakeup = time.After(wakeFreq)

			case <-s.ReqStop:
				close(s.Done)
				return
			case pack := <-acceptSend:
				p("%v got <-acceptSend pack: '%#v'", s.Inbox, pack)
				s.doOrigDataSend(pack)

			case a := <-s.GotAck:
				// ack received - do sender side stuff
				//
				p("%v sender GotAck a: %#v", s.Inbox, a)
				//
				// flow control: respect a.AvailReaderBytesCap
				// and a.AvailReaderMsgCap info that we have
				// received from this ack
				//
				p("%v sender GotAck, updating s.LastSeenAvailReaderMsgCap %v -> %v",
					s.Inbox, s.LastSeenAvailReaderMsgCap, a.AvailReaderMsgCap)
				s.LastSeenAvailReaderBytesCap = a.AvailReaderBytesCap
				s.LastSeenAvailReaderMsgCap = a.AvailReaderMsgCap

				// need to update our map of SentButNotAcked
				// and remove everything before AckNum, which is cumulative.
				for _, slot := range s.SentButNotAcked {
					if slot.Pack.SeqNum <= a.AckNum {
						delete(s.SentButNotAcked, a.AckNum)
					}
				}

				if a.OnlyUpdateFlowCtrl {
					// it wasn't an Ack, just updated flow info
					// from a received data message.
					q("%s sender Gotack: just updated flow control, continuing sendloop", s.Inbox)
					continue sendloop
				}
				q("%s sender Gotack: more than just flowcontrol...", s.Inbox)
				delete(s.SentButNotAcked, a.AckNum)
				if !InWindow(a.AckNum, s.LastAckRec+1, s.LastFrameSent) {
					//q("%v a.AckNum = %v outside sender's window [%v, %v], dropping it.",
					//	s.Inbox, a.AckNum, s.LastAckRec+1, s.LastFrameSent)
					s.DiscardCount++
					continue sendloop
				}
				q("%v packet.AckNum = %v inside sender's window, keeping it.", s.Inbox, a.AckNum)
				for {
					s.LastAckRec++

					// do this before changing slot, since we point into slot.
					delete(s.SentButNotAcked, s.LastAckRec)

					slot := s.Txq[s.LastAckRec%s.SenderWindowSize]
					q("%v ... slot = %#v", s.Inbox, slot)

					// release the send slot
					//s.slotsAvail++
					if s.LastAckRec == a.AckNum {
						q("%v s.LastAskRec[%v] matches a.AckNum[%v], breaking",
							s.Inbox, s.LastAckRec, a.AckNum)
						break
					}
					q("%v s.LastAskRec[%v] != a.AckNum[%v], looping",
						s.Inbox, s.LastAckRec, a.AckNum)
				}
			case ackPack := <-s.SendAck:
				// request to send an ack:
				// don't go though the BlockingSend protocol; since
				// could effectively livelock us.
				p("%v doing Net.Send() SendAck request on ackPack: '%#v'",
					s.Inbox, ackPack)
				err := s.Net.Send(ackPack, "SendAck/ackPack")
				panicOn(err)
			}
		}
	}()
}

// Stop the SenderState componennt
func (s *SenderState) Stop() {
	s.mut.Lock()
	select {
	case <-s.ReqStop:
	default:
		close(s.ReqStop)
	}
	s.mut.Unlock()
	<-s.Done
}

// for first time sends of data, not retries or acks.
func (s *SenderState) doOrigDataSend(pack *Packet) {
	//s.slotsAvail--
	//q("%v sender in acceptSend, now %v slotsAvail", s.Inbox, s.slotsAvail)

	s.LastFrameSent++
	//q("%v LastFrameSent is now %v", s.Inbox, s.LastFrameSent)

	s.TotalBytesSent += int64(len(pack.Data))
	pack.CumulBytesTransmitted = s.TotalBytesSent

	//p("%v SenderState.doOrigDataSend is decrementing LastSeenAvailReaderBytesCap %v -> %v",
	//	s.LastSeenAvailReaderBytesCap, s.LastSeenAvailReaderBytesCap-1)
	//	s.LastSeenAvailReaderBytesCap -= int64(len(pack.Data))
	//	s.LastSeenAvailReaderMsgCap--

	lfs := s.LastFrameSent
	pos := lfs % s.SenderWindowSize
	slot := s.Txq[pos]

	pack.SeqNum = lfs
	if pack.From != s.Inbox {
		pack.From = s.Inbox
	}
	pack.From = s.Inbox
	slot.Pack = pack
	// data sends get stored in SentButNotAcked
	s.SentButNotAcked[lfs] = slot

	now := time.Now()
	s.SendHistory = append(s.SendHistory, pack)
	slot.RetryDeadline = now.Add(s.Timeout)
	s.LastSendTime = now

	flow := s.FlowCt.UpdateFlow(s.Inbox+":sender", s.Net, -1, -1)
	q("%v doSend(), flow = '%#v'", s.Inbox, flow)
	pack.AvailReaderBytesCap = flow.AvailReaderBytesCap
	pack.AvailReaderMsgCap = flow.AvailReaderMsgCap
	err := s.Net.Send(slot.Pack, fmt.Sprintf("doSend() for %v", s.Inbox))
	panicOn(err)
}

func (s *SenderState) doKeepAlive() {
	if time.Since(s.LastSendTime) < s.KeepAliveInterval {
		return
	}
	flow := s.FlowCt.UpdateFlow(s.Inbox+":sender", s.Net, -1, -1)
	q("%v doKeepAlive(), flow = '%#v'", s.Inbox, flow)
	// send a packet with no data, to elicit an ack
	// with a new advertised window. This is
	// *not* an ack, because we need it to be
	// acked itself so we get any updated
	// flow control info from the other end.
	s.LastSendTime = time.Now()
	kap := &Packet{
		From:                s.Inbox,
		Dest:                s.Dest,
		SeqNum:              -777, // => keepalive
		KeepAlive:           true,
		AvailReaderBytesCap: flow.AvailReaderBytesCap,
		AvailReaderMsgCap:   flow.AvailReaderMsgCap,
	}
	q("%v doing keepalive Net.Send()", s.Inbox)
	err := s.Net.Send(kap, fmt.Sprintf("keepalive from %v", s.Inbox))
	panicOn(err)

	s.keepAlive = time.After(s.KeepAliveInterval)
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package eventmaster

import (
	"encoding/json"
	"net/http"
	"path/filepath"
	"time"

	assetfs "github.com/elazarl/go-bindata-assetfs"
	"github.com/julienschmidt/httprouter"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	log "github.com/sirupsen/logrus"

	"github.com/wish/eventmaster/jh"
	"github.com/wish/eventmaster/metrics"
	tmpl "github.com/wish/eventmaster/templates"
	"github.com/wish/eventmaster/ui"
)

// Server implements http.Handler for the eventmaster http server.
type Server struct {
	store *EventStore

	handler http.Handler

	ui        http.FileSystem
	templates TemplateGetter
}

// ServeHTTP dispatches to the underlying router.
func (srv *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	srv.handler.ServeHTTP(w, req)
}

// NewServer returns a ready-to-use Server that uses store, and the appropriate
// static and templates facilities.
//
// If static or templates are non-empty then files are served from those
// locations (useful for development). Otherwise the server uses embedded
// static assets.
func NewServer(store *EventStore, static, templates string) *Server {
	// Handle static files either embedded (empty static) or off the filesystem (during dev work)
	var fs http.FileSystem
	switch static {
	case "":
		fs = &assetfs.AssetFS{
			Asset:     ui.Asset,
			AssetDir:  ui.AssetDir,
			AssetInfo: ui.AssetInfo,
		}
	default:
		if p, d := filepath.Split(static); d == "ui" {
			static = p
		}
		fs = http.Dir(static)
	}

	var t TemplateGetter
	switch templates {
	case "":
		t = NewAssetTemplate(tmpl.Asset)
	default:
		t = Disk{Root: templates}
	}

	srv := &Server{
		store:     store,
		ui:        fs,
		templates: t,
	}

	srv.handler = registerRoutes(srv)

	return srv
}

func registerRoutes(srv *Server) http.Handler {
	r := httprouter.New()

	// API endpoints
	r.POST("/v1/event", latency("/v1/event", jh.Adapter(srv.addEvent)))
	r.GET("/v1/event", latency("/v1/event", jh.Adapter(srv.getEvent)))
	r.GET("/v1/event/:id", latency("/v1/event", jh.Adapter(srv.getEventByID)))
	r.POST("/v1/topic", latency("/v1/topic", jh.Adapter(srv.addTopic)))
	r.PUT("/v1/topic/:name", latency("/v1/topic", jh.Adapter(srv.updateTopic)))
	r.GET("/v1/topic", latency("/v1/topic", jh.Adapter(srv.getTopic)))
	r.DELETE("/v1/topic/:name", latency("/v1/topic", jh.Adapter(srv.deleteTopic)))
	r.POST("/v1/dc", latency("/v1/dc", jh.Adapter(srv.addDC)))
	r.PUT("/v1/dc/:name", latency("/v1/dc", jh.Adapter(srv.updateDC)))
	r.GET("/v1/dc", latency("/v1/dc", jh.Adapter(srv.getDC)))

	r.GET("/v1/health", latency("/v1/health", jh.Adapter(srv.healthCheck)))

	// GitHub webhook endpoint
	r.POST("/v1/github_event", latency("/v1/github_event", jh.Adapter(srv.gitHubEvent)))

	// UI endpoints
	r.GET("/", latency("/", srv.HandleMainPage))
	r.GET("/add_event", latency("/add_event", srv.HandleCreatePage))
	r.GET("/topic", latency("/topic", srv.HandleTopicPage))
	r.GET("/dc", latency("/dc", srv.HandleDCPage))
	r.GET("/event", latency("/event", srv.HandleGetEventPage))

	// grafana datasource endpoints
	r.GET("/grafana", latency("/grafana", cors(srv.grafanaOK)))
	r.GET("/grafana/", latency("/grafana/", cors(srv.grafanaOK)))
	r.OPTIONS("/grafana/*route", latency("/grafana", cors(srv.grafanaOK)))
	r.POST("/grafana/annotations", latency("/grafana/annotations", cors(srv.grafanaAnnotations)))
	r.POST("/grafana/search", latency("/grafana/search", cors(srv.grafanaSearch)))

	r.Handler("GET", "/metrics", promhttp.Handler())

	r.Handler("GET", "/ui/*filepath", http.FileServer(srv.ui))

	r.GET("/version/", latency("/version/", srv.version))

	r.PanicHandler = func(w http.ResponseWriter, req *http.Request, i interface{}) {
		metrics.HTTPStatus("panic", http.StatusInternalServerError)

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusInternalServerError)
		e := struct {
			E string `json:"error"`
		}{"server panicked"}
		if err := json.NewEncoder(w).Encode(&e); err != nil {
			log.Printf("json encode: %v", err)
		}
	}

	r.NotFound = func(w http.ResponseWriter, req *http.Request) {
		log.Printf("not found: %+v", req.URL.Path)
		metrics.HTTPStatus(http.StatusText(http.StatusNotFound), http.StatusNotFound)

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusNotFound)
		e := struct {
			E string `json:"error"`
		}{"not found"}
		if err := json.NewEncoder(w).Encode(&e); err != nil {
			log.Printf("json encode: %v", err)
		}
	}

	r.MethodNotAllowed = func(w http.ResponseWriter, req *http.Request) {
		metrics.HTTPStatus(http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusMethodNotAllowed)
		e := struct {
			E string `json:"error"`
		}{"method not allowed"}
		if err := json.NewEncoder(w).Encode(&e); err != nil {
			log.Printf("json encode: %v", err)
		}
	}

	return r
}

func latency(prefix string, h httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		log.Infof("request received for %s", prefix)
		start := time.Now()
		defer func() {
			metrics.HTTPLatency(prefix, start)
		}()

		lw := NewStatusRecorder(w)
		h(lw, req, ps)

		metrics.HTTPStatus(prefix, lw.Status())
	}
}

func (srv *Server) healthCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) (interface{}, error) {
	return nil, nil
}
<commit_msg>Add logging about originating host and HTTP method used (#81)<commit_after>package eventmaster

import (
	"encoding/json"
	"net/http"
	"path/filepath"
	"time"

	assetfs "github.com/elazarl/go-bindata-assetfs"
	"github.com/julienschmidt/httprouter"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	log "github.com/sirupsen/logrus"

	"github.com/wish/eventmaster/jh"
	"github.com/wish/eventmaster/metrics"
	tmpl "github.com/wish/eventmaster/templates"
	"github.com/wish/eventmaster/ui"
)

// Server implements http.Handler for the eventmaster http server.
type Server struct {
	store *EventStore

	handler http.Handler

	ui        http.FileSystem
	templates TemplateGetter
}

// ServeHTTP dispatches to the underlying router.
func (srv *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	srv.handler.ServeHTTP(w, req)
}

// NewServer returns a ready-to-use Server that uses store, and the appropriate
// static and templates facilities.
//
// If static or templates are non-empty then files are served from those
// locations (useful for development). Otherwise the server uses embedded
// static assets.
func NewServer(store *EventStore, static, templates string) *Server {
	// Handle static files either embedded (empty static) or off the filesystem (during dev work)
	var fs http.FileSystem
	switch static {
	case "":
		fs = &assetfs.AssetFS{
			Asset:     ui.Asset,
			AssetDir:  ui.AssetDir,
			AssetInfo: ui.AssetInfo,
		}
	default:
		if p, d := filepath.Split(static); d == "ui" {
			static = p
		}
		fs = http.Dir(static)
	}

	var t TemplateGetter
	switch templates {
	case "":
		t = NewAssetTemplate(tmpl.Asset)
	default:
		t = Disk{Root: templates}
	}

	srv := &Server{
		store:     store,
		ui:        fs,
		templates: t,
	}

	srv.handler = registerRoutes(srv)

	return srv
}

func registerRoutes(srv *Server) http.Handler {
	r := httprouter.New()

	// API endpoints
	r.POST("/v1/event", latency("/v1/event", jh.Adapter(srv.addEvent)))
	r.GET("/v1/event", latency("/v1/event", jh.Adapter(srv.getEvent)))
	r.GET("/v1/event/:id", latency("/v1/event", jh.Adapter(srv.getEventByID)))
	r.POST("/v1/topic", latency("/v1/topic", jh.Adapter(srv.addTopic)))
	r.PUT("/v1/topic/:name", latency("/v1/topic", jh.Adapter(srv.updateTopic)))
	r.GET("/v1/topic", latency("/v1/topic", jh.Adapter(srv.getTopic)))
	r.DELETE("/v1/topic/:name", latency("/v1/topic", jh.Adapter(srv.deleteTopic)))
	r.POST("/v1/dc", latency("/v1/dc", jh.Adapter(srv.addDC)))
	r.PUT("/v1/dc/:name", latency("/v1/dc", jh.Adapter(srv.updateDC)))
	r.GET("/v1/dc", latency("/v1/dc", jh.Adapter(srv.getDC)))

	r.GET("/v1/health", latency("/v1/health", jh.Adapter(srv.healthCheck)))

	// GitHub webhook endpoint
	r.POST("/v1/github_event", latency("/v1/github_event", jh.Adapter(srv.gitHubEvent)))

	// UI endpoints
	r.GET("/", latency("/", srv.HandleMainPage))
	r.GET("/add_event", latency("/add_event", srv.HandleCreatePage))
	r.GET("/topic", latency("/topic", srv.HandleTopicPage))
	r.GET("/dc", latency("/dc", srv.HandleDCPage))
	r.GET("/event", latency("/event", srv.HandleGetEventPage))

	// grafana datasource endpoints
	r.GET("/grafana", latency("/grafana", cors(srv.grafanaOK)))
	r.GET("/grafana/", latency("/grafana/", cors(srv.grafanaOK)))
	r.OPTIONS("/grafana/*route", latency("/grafana", cors(srv.grafanaOK)))
	r.POST("/grafana/annotations", latency("/grafana/annotations", cors(srv.grafanaAnnotations)))
	r.POST("/grafana/search", latency("/grafana/search", cors(srv.grafanaSearch)))

	r.Handler("GET", "/metrics", promhttp.Handler())

	r.Handler("GET", "/ui/*filepath", http.FileServer(srv.ui))

	r.GET("/version/", latency("/version/", srv.version))

	r.PanicHandler = func(w http.ResponseWriter, req *http.Request, i interface{}) {
		metrics.HTTPStatus("panic", http.StatusInternalServerError)

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusInternalServerError)
		e := struct {
			E string `json:"error"`
		}{"server panicked"}
		if err := json.NewEncoder(w).Encode(&e); err != nil {
			log.Printf("json encode: %v", err)
		}
	}

	r.NotFound = func(w http.ResponseWriter, req *http.Request) {
		log.Printf("not found: %+v", req.URL.Path)
		metrics.HTTPStatus(http.StatusText(http.StatusNotFound), http.StatusNotFound)

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusNotFound)
		e := struct {
			E string `json:"error"`
		}{"not found"}
		if err := json.NewEncoder(w).Encode(&e); err != nil {
			log.Printf("json encode: %v", err)
		}
	}

	r.MethodNotAllowed = func(w http.ResponseWriter, req *http.Request) {
		metrics.HTTPStatus(http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)

		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusMethodNotAllowed)
		e := struct {
			E string `json:"error"`
		}{"method not allowed"}
		if err := json.NewEncoder(w).Encode(&e); err != nil {
			log.Printf("json encode: %v", err)
		}
	}

	return r
}

func latency(prefix string, h httprouter.Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
		log.Infof("%s request received for %s from %s", req.Method, prefix, req.RemoteAddr)
		start := time.Now()
		defer func() {
			metrics.HTTPLatency(prefix, start)
		}()

		lw := NewStatusRecorder(w)
		h(lw, req, ps)

		metrics.HTTPStatus(prefix, lw.Status())
	}
}

func (srv *Server) healthCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) (interface{}, error) {
	return nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"strconv"
	"time"

	"runtime"

	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
)

const (
	// StopAfterFailedInserts is the number of failed inserts after which we will stop fetching from Instagram.
	// Inserts fail when the image already exists, 50 was chosen to allow for images that are fetched out of order.
	StopAfterFailedInserts = 50
)

func ping(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
	err := p.Ping()
	if err != nil {
		http.Error(w, "Ping failed", 500)
	}
	fmt.Fprint(w, "pong")
}

func stats(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	cnt, err := notificationCount()
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
	fmt.Fprint(w, "Notifications: ", string(cnt))
}

func fetch(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	sid := p.ByName("sid")
	subId, _ := strconv.Atoi(sid)
	stopAfter := parseStopAfter(r)
	counter := fetchMediaForSubscription(subId, stopAfter)
	fmt.Fprintf(w, "Fetch completed, fetched %d\n", counter)
}

// Parse stopAfter param to int or return default
func parseStopAfter(r *http.Request) int {
	r.ParseForm()
	stopAfter := StopAfterFailedInserts
	v := r.FormValue("stopAfter")
	if v != "" {
		sa, err := strconv.Atoi(v)
		if err == nil {
			stopAfter = sa
		}
	}
	return stopAfter
}

func fetchAll(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	stopAfter := parseStopAfter(r)
	fetchedForSub := fetchMediaForAllSubscriptions(stopAfter)
	fmt.Fprintf(w, "Fetch completed, fetched %d\n", fetchedForSub)
}

func fetchMediaForSubscription(sid int, stopAfter int) int {
	sub := getSubscription(sid)
	failed := 0
	counter := 0
	fetchQueue := make(chan *Media)
	stop := make(chan struct{})
	go fetchMedia(sub, fetchQueue, stop)
	for m := range fetchQueue {
		new, err := insert(*m)
		if err != nil {
			fmt.Printf("Error encountered %v when inserting\n", err)
		}
		if !new {
			failed++
			if failed == stopAfter {
				stop <- struct{}{}
			}
		} else {
			failed = 0
			counter++
		}
	}
	return counter
}

func fetchMediaForAllSubscriptions(stopAfter int) map[int]int {
	subs := getSubscriptions()
	fmt.Printf("Found %v subscriptions\n", len(subs))
	counts := make(map[int]int)
	for _, sub := range subs {
		fmt.Printf("%v : Starting fetch for %v id %v\n", time.Now(), sub.Name, sub.ID)
		counts[sub.ID] = fetchMediaForSubscription(sub.ID, stopAfter)
		fmt.Printf("%v : Finished fetch for %v id %v\n", time.Now(), sub.Name, sub.ID)
	}
	return counts
}

func initAPI() *negroni.Negroni {
	n := negroni.New(
		negroni.NewRecovery(),
		negroni.NewLogger(), negroni.NewStatic(http.Dir("ui")))
	router := httprouter.New()
	router.GET("/ping", ping)
	router.GET("/stats", stats)
	router.GET("/fetch/:sid", fetch)
	router.GET("/fetch-all", fetchAll)
	n.UseHandler(router)
	return n
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db := initDb()
	defer db.Close()

	initClient()

	n := initAPI()

	host := getEnvOrDefault("SPOTO_HOST", "localhost")
	port := getEnvOrDefault("SPOTO_PORT", "3000")
	bindTo := fmt.Sprintf("%s:%s", host, port)

	n.Run(bindTo)
}

func getEnvOrDefault(key, def string) string {
	env := os.Getenv(key)
	if len(env) == 0 {
		env = def
	}
	return env
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Add date time to logs<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"time"

	"runtime"

	"github.com/codegangsta/negroni"
	"github.com/julienschmidt/httprouter"
)

const (
	// StopAfterFailedInserts is the number of failed inserts after which we will stop fetching from Instagram.
	// Inserts fail when the image already exists, 50 was chosen to allow for images that are fetched out of order.
	StopAfterFailedInserts = 50
)

func ping(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) {
	err := p.Ping()
	if err != nil {
		http.Error(w, "Ping failed", 500)
	}
	fmt.Fprint(w, "pong")
}

func stats(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	cnt, err := notificationCount()
	if err != nil {
		http.Error(w, err.Error(), 500)
	}
	fmt.Fprint(w, "Notifications: ", string(cnt))
}

func fetch(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	sid := p.ByName("sid")
	subId, _ := strconv.Atoi(sid)
	stopAfter := parseStopAfter(r)
	counter := fetchMediaForSubscription(subId, stopAfter)
	fmt.Fprintf(w, "Fetch completed, fetched %d\n", counter)
}

// Parse stopAfter param to int or return default
func parseStopAfter(r *http.Request) int {
	r.ParseForm()
	stopAfter := StopAfterFailedInserts
	v := r.FormValue("stopAfter")
	if v != "" {
		sa, err := strconv.Atoi(v)
		if err == nil {
			stopAfter = sa
		}
	}
	return stopAfter
}

func fetchAll(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
	stopAfter := parseStopAfter(r)
	fetchedForSub := fetchMediaForAllSubscriptions(stopAfter)
	fmt.Fprintf(w, "Fetch completed, fetched %d\n", fetchedForSub)
}

func fetchMediaForSubscription(sid int, stopAfter int) int {
	sub := getSubscription(sid)
	failed := 0
	counter := 0
	fetchQueue := make(chan *Media)
	stop := make(chan struct{})
	go fetchMedia(sub, fetchQueue, stop)
	for m := range fetchQueue {
		new, err := insert(*m)
		if err != nil {
			fmt.Printf("Error encountered %v when inserting\n", err)
		}
		if !new {
			failed++
			if failed == stopAfter {
				stop <- struct{}{}
			}
		} else {
			failed = 0
			counter++
		}
	}
	return counter
}

func fetchMediaForAllSubscriptions(stopAfter int) map[int]int {
	subs := getSubscriptions()
	fmt.Printf("Found %v subscriptions\n", len(subs))
	counts := make(map[int]int)
	for _, sub := range subs {
		fmt.Printf("%v : Starting fetch for %v id %v\n", time.Now(), sub.Name, sub.ID)
		counts[sub.ID] = fetchMediaForSubscription(sub.ID, stopAfter)
		fmt.Printf("%v : Finished fetch for %v id %v\n", time.Now(), sub.Name, sub.ID)
	}
	return counts
}

func initAPI() *negroni.Negroni {
	n := negroni.New(
		negroni.NewRecovery(),
		&negroni.Logger{log.New(os.Stdout, time.Now().Format(time.RFC3339)+" ", 0)},
		negroni.NewStatic(http.Dir("ui")))
	router := httprouter.New()
	router.GET("/ping", ping)
	router.GET("/stats", stats)
	router.GET("/fetch/:sid", fetch)
	router.GET("/fetch-all", fetchAll)
	n.UseHandler(router)
	return n
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db := initDb()
	defer db.Close()

	initClient()

	n := initAPI()

	host := getEnvOrDefault("SPOTO_HOST", "localhost")
	port := getEnvOrDefault("SPOTO_PORT", "3000")
	bindTo := fmt.Sprintf("%s:%s", host, port)

	n.Run(bindTo)
}

func getEnvOrDefault(key, def string) string {
	env := os.Getenv(key)
	if len(env) == 0 {
		env = def
	}
	return env
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Igor Dolzhikov. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package spawn

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"regexp"
	"sort"
	"time"

	"github.com/takama/router"
)

const (

	// VERSION - current version of the service
	VERSION = "0.1.7"

	// DATE - revision date of the service
	DATE = "2015-03-29T11:30:17Z"

	// MaxSignals - maximum count of update signals
	MaxSignals = 1000

	// MaxJobs - maximum count of update jobs for every bundle
	MaxJobs = 100000

	// DefaultTimeout is a timeout for the worker's response
	DefaultTimeout time.Duration = 10

	// HTTP methods, which should be queued
	protocolHTTP = "http"
	methodPOST   = "POST"
	methodPUT    = "PUT"
	methodDELETE = "DELETE"

	// Job signals
	responseSignal = iota
	nodeJobSignal
)

// simplest logger, which initialized during starts of the application
var (
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile)
)

// Server Record
type Server struct {

	// Server name/description
	Name string

	// Node Bundle contains the Node records
	Nodes *NodeBundle
	// contains filtered or unexported fields

	// Embeded router
	*router.Router

	// Response signal channel
	response chan struct{}

	// responseTimeout is a timeout for worker's response
	responseTimeout time.Duration

	// job signal channel
	job chan int

	// quit signal channel
	quit chan struct{}

	// round robin mode
	roundRobin bool

	// nodes will queried according to priority
	byPriority bool

	// nodes health check
	check HealthCheck

	// Queue Bundle contains the queue records
	queues *queueBundle
}

// HealthCheck contains parameters which used for checking node
type HealthCheck struct {

	// health check time of the node in seconds
	Seconds time.Duration `json:"seconds"`

	// url which will be checked
	URL string `json:"url"`

	// regexp pattern for extended check analyze
	Pattern string `json:"regexp"`
}

// NewServer creates a new server which contains the nodes/queues
func NewServer(name string) (*Server, error) {

	// Init the router
	r := router.New()
	r.PanicHandler = panicHandler
	r.NotFound = notFound
	r.Logger = logger
	r.CustomHandler = baseHandler

	// Init the Server
	server := &Server{
		Name:            name,
		Router:          r,
		responseTimeout: DefaultTimeout,
		response:        make(chan struct{}),
		job:             make(chan int, MaxSignals),
		quit:            make(chan struct{}),
	}

	// Create and init nodes bundle
	server.Nodes = &NodeBundle{
		Server:  server,
		records: make(map[string]map[uint64]Node),
	}

	// Create and init queues bundle
	server.queues = &queueBundle{records: make(map[string]*queue)}

	return server, nil
}

// Run the server, init the handlers, init the specified modes
// If handler RequestHandler is not defined used default handler
// with standard handling of the requests/responses
func (server *Server) Run(
	hostPort, apiHostPort string,
	handler RequestHandler,
	nodes []Node,
	roundRobin, byPriority bool,
	check HealthCheck,
) (status string, err error) {

	// if used round-robin mode
	if roundRobin {
		stdlog.Println(server.Name, "will used 'round-robin' mode")
		server.roundRobin = roundRobin
	}

	// if used by-priority mode
	if byPriority {
		stdlog.Println("Nodes will queried according to priority")
		server.byPriority = byPriority
	}

	// Init the Nodes update channel
	server.Nodes.update = make(chan nodeJob, MaxJobs)

	// Starts the worker which manage server's jobs
	go server.manage()

	// Init the Nodes settings
	if !server.Nodes.SetAll(nodes) {
		status = server.Name + " is not loaded"
		err = errors.New("The nodes settings in config have incorrect values")
		return
	}

	// Init a health check settings
	server.check = check

	// The info handler returns a system status of the application
	server.GET("/info", infoHandler)

	// Lists methods, which display how to use API
	server.GET("/list", displayAllMethods)
	server.GET("/list/nodes", displayAllNodeMethods)
	server.GET("/list/nodes/get", displayGetNodeMethods)
	server.GET("/list/nodes/set", displaySetNodeMethods)
	server.GET("/list/nodes/delete", displayDeleteNodeMethods)

	// Init API methods for the Nodes
	server.GET("/nodes/:host/:port", server.Nodes.getRecord)
	server.GET("/nodes/:host", server.Nodes.getAllRecordsByHost)
	server.GET("/nodes", server.Nodes.getAllRecords)
	server.PUT("/nodes/:host/:port", server.Nodes.putRecord)
	server.PUT("/nodes", server.Nodes.putAllRecords)
	server.DELETE("/nodes/:host/:port", server.Nodes.deleteRecord)
	server.DELETE("/nodes/:host", server.Nodes.deleteAllRecordsByHost)
	server.DELETE("/nodes", server.Nodes.deleteAllRecords)

	go server.Listen(apiHostPort)
	go func() {
		p := new(proxy)
		if handler != nil {
			p.handler = handler
		} else {
			p.handler = server.proxyHandler
		}
		if err := http.ListenAndServe(hostPort, p); err != nil {
			errlog.Fatal(err)
		}
	}()

	status = server.Name + " loaded successfully"

	return
}

// Shutdown closes the server graceful
func (server *Server) Shutdown() (status string, err error) {

	// Set timer to wait one minute
	timeout := time.NewTimer(time.Second * 30)

	// a unwanted response sweeps if exist
	for {
		select {
		case <-server.response:
			continue
		default:
		}
		break
	}

	// sends a 'quit' signal
	server.quit <- struct{}{}

	status = server.Name + " connections closed"
	select {

	// Exit by timeout if jobs did not done
	case <-timeout.C:
		err = errors.New("timeout")
		return
	// Exit after all jobs done
	case <-server.response:
		return
	}
}

// Manage routine which manage all jobs
func (server *Server) manage() {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in Manage routine", recovery)
			// Recover routine
			go server.manage()
		} else {
			stdlog.Println("Manage routine stoped")
			server.response <- struct{}{}
		}
	}()
	for {
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		default:
		}
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		case <-server.quit:
			return
		}
	}
}

// Do updates depended by the signal
func (server *Server) doJob(signal int) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in do job routine", recovery)
		}
	}()
	switch signal {
	case responseSignal:
		server.response <- struct{}{}
	case nodeJobSignal:
		server.Nodes.updateRecords()
		server.Nodes.InitRing()
	}
}

// proxyHandler manages all requests/responses
func (server *Server) proxyHandler(request *http.Request) *http.Response {

	// Add "X-Forwarded-For" to repost remote host IP
	if remoteHost, _, err := net.SplitHostPort(request.RemoteAddr); err == nil {
		request.Header.Add("X-Forwarded-For", remoteHost)
	}
	// Use HTTP scheme
	request.URL.Scheme = protocolHTTP

	// If requests should not be queued, get result immediately
	if request.Method != methodPOST &&
		request.Method != methodPUT &&
		request.Method != methodDELETE {

		return server.processGET(request)
	}

	return server.processUpdate(request)
}

// call 'GET' request to the node using defined mode
func (server *Server) processGET(request *http.Request) *http.Response {
	if server.roundRobin {

		// Use round robin to get data from the host
		for count := 0; count < server.Nodes.ring.Len(); count++ {
			if node, ok := server.Nodes.CurrentFromRing(); ok &&
				node.Active && !node.Maintenance {

				// The host is active and is not in maintenance
				request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// Prepare next host
				server.Nodes.TwistRing()

				if server.checkNode(request.URL.Host) {
					response, err := http.DefaultTransport.RoundTrip(request)
					if err == nil {
						// If response is sucess, return
						return response
					}
					errlog.Println(err)
				}
			} else {

				// Use next host if not active or maintenance mode
				server.Nodes.TwistRing()
			}
		}
	} else {

		// If is not round robin mode, use first registered host
		if nodes, total := server.Nodes.GetAll(); total > 0 {
			if server.byPriority {
				sort.Sort(byPriority(nodes))
			}
			for _, node := range nodes {
				if node.Active && !node.Maintenance {

					// The host is active and is not in maintenance
					request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)
					if server.checkNode(request.URL.Host) {
						response, err := http.DefaultTransport.RoundTrip(request)
						if err == nil {
							// If response is sucess, return
							return response
						}
						errlog.Println(err)
					}
				}
			}
		}
	}

	stdlog.Println("Warning: no one of the nodes is active")

	return nil
}

// call 'PUT', 'POST', 'DELETE' request to the node
func (server *Server) processUpdate(request *http.Request) *http.Response {
	// grab update request
	proxyRequestData, err := httputil.DumpRequest(request, true)
	if err != nil {

		// if unsuccessful, return nil response
		errlog.Println(err)
		return nil
	}
	var host string
	var response *http.Response
	if nodes, total := server.Nodes.GetAll(); total > 0 {
		answer := make(chan *http.Response, total)
		for _, node := range nodes {
			if node.Active {

				host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// create new queue job
				job := &queueJob{
					query:  make(chan []byte, 1),
					answer: answer,
				}
				job.query <- proxyRequestData

				queue, _ := server.queues.check(host)
				queue.jobs <- job
				queue.task <- doJobTask
			}
		}
		timeout := time.NewTimer(time.Second * server.responseTimeout)
		for {
			select {
			case response = <-answer:
				return response
			case <-timeout.C:
				return response
			}
		}
	}
	return response
}

// worker receive a data from the queue and send it to the node
func (server *Server) worker(q *queue) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in worker routine", recovery)
			// the worker recovers again
			go server.worker(q)
		} else {
			q.response <- struct{}{}
			stdlog.Println("Worker closed for", q.id)
		}
	}()
	stdlog.Println("Worker started for", q.id)
	for {
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		default:
		}
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		case <-q.quit:
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
}

func (server *Server) doUpdate(q *queue) {
	// check the node
	for {
		if server.checkNode(q.id) {
			break
		}
		stdlog.Println("Node", q.id, "does not ready for updates")
		stdlog.Println("try again in", server.check.Seconds, "seconds")
		timeout := time.NewTimer(time.Second * server.check.Seconds)
		select {
		//  Repeat by timeout
		case <-timeout.C:
			continue
		case <-q.quit:
			q.task <- doJobTask
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
	// if the node is alive, post data
	job := <-q.jobs
	data := <-job.query
	if response, err := dispatchRequest(q.id, data); err != nil {

		// Job does not done
		errlog.Println(err)

	} else {

		// job done
		job.answer <- response
		job.done = true
	}
}

// check the node
func (server *Server) checkNode(host string) bool {
	response, err := http.Get(protocolHTTP + "://" + host + server.check.URL)
	if err != nil {
		stdlog.Println(host, err)
		return false
	}

	defer response.Body.Close()
	// if pattern does not exist, should be true
	if server.check.Pattern == "" {
		return true
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		stdlog.Println(host, err)
		return false
	}
	// check of regexp pattern
	valid := regexp.MustCompile(server.check.Pattern)
	return valid.MatchString(string(data))
}

// Reproduce request to specified node and capture response
func dispatchRequest(host string, data []byte) (*http.Response, error) {
	reader := bufio.NewReader(bytes.NewBuffer(data))
	request, err := http.ReadRequest(reader)
	if err != nil {
		stdlog.Println(host, err)
		return nil, err
	}
	request.Body = ioutil.NopCloser(reader)
	request.URL.Scheme = protocolHTTP
	request.URL.Host = host

	response, err := http.DefaultTransport.RoundTrip(request)
	if err != nil {
		stdlog.Println(host, err)
		return nil, err
	}
	return response, nil
}
<commit_msg>Bumped version number to 0.1.8<commit_after>// Copyright 2015 Igor Dolzhikov. All rights reserved.
// Use of this source code is governed by a license
// that can be found in the LICENSE file.

package spawn

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"regexp"
	"sort"
	"time"

	"github.com/takama/router"
)

const (

	// VERSION - current version of the service
	VERSION = "0.1.8"

	// DATE - revision date of the service
	DATE = "2015-04-02T11:52:17Z"

	// MaxSignals - maximum count of update signals
	MaxSignals = 1000

	// MaxJobs - maximum count of update jobs for every bundle
	MaxJobs = 100000

	// DefaultTimeout is a timeout for the worker's response
	DefaultTimeout time.Duration = 10

	// HTTP methods, which should be queued
	protocolHTTP = "http"
	methodPOST   = "POST"
	methodPUT    = "PUT"
	methodDELETE = "DELETE"

	// Job signals
	responseSignal = iota
	nodeJobSignal
)

// simplest logger, which initialized during starts of the application
var (
	stdlog = log.New(os.Stdout, "", log.Ldate|log.Ltime)
	errlog = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lshortfile)
)

// Server Record
type Server struct {

	// Server name/description
	Name string

	// Node Bundle contains the Node records
	Nodes *NodeBundle
	// contains filtered or unexported fields

	// Embeded router
	*router.Router

	// Response signal channel
	response chan struct{}

	// responseTimeout is a timeout for worker's response
	responseTimeout time.Duration

	// job signal channel
	job chan int

	// quit signal channel
	quit chan struct{}

	// round robin mode
	roundRobin bool

	// nodes will queried according to priority
	byPriority bool

	// nodes health check
	check HealthCheck

	// Queue Bundle contains the queue records
	queues *queueBundle
}

// HealthCheck contains parameters which used for checking node
type HealthCheck struct {

	// health check time of the node in seconds
	Seconds time.Duration `json:"seconds"`

	// url which will be checked
	URL string `json:"url"`

	// regexp pattern for extended check analyze
	Pattern string `json:"regexp"`
}

// NewServer creates a new server which contains the nodes/queues
func NewServer(name string) (*Server, error) {

	// Init the router
	r := router.New()
	r.PanicHandler = panicHandler
	r.NotFound = notFound
	r.Logger = logger
	r.CustomHandler = baseHandler

	// Init the Server
	server := &Server{
		Name:            name,
		Router:          r,
		responseTimeout: DefaultTimeout,
		response:        make(chan struct{}),
		job:             make(chan int, MaxSignals),
		quit:            make(chan struct{}),
	}

	// Create and init nodes bundle
	server.Nodes = &NodeBundle{
		Server:  server,
		records: make(map[string]map[uint64]Node),
	}

	// Create and init queues bundle
	server.queues = &queueBundle{records: make(map[string]*queue)}

	return server, nil
}

// Run the server, init the handlers, init the specified modes
// If handler RequestHandler is not defined used default handler
// with standard handling of the requests/responses
func (server *Server) Run(
	hostPort, apiHostPort string,
	handler RequestHandler,
	nodes []Node,
	roundRobin, byPriority bool,
	check HealthCheck,
) (status string, err error) {

	// if used round-robin mode
	if roundRobin {
		stdlog.Println(server.Name, "will used 'round-robin' mode")
		server.roundRobin = roundRobin
	}

	// if used by-priority mode
	if byPriority {
		stdlog.Println("Nodes will queried according to priority")
		server.byPriority = byPriority
	}

	// Init the Nodes update channel
	server.Nodes.update = make(chan nodeJob, MaxJobs)

	// Starts the worker which manage server's jobs
	go server.manage()

	// Init the Nodes settings
	if !server.Nodes.SetAll(nodes) {
		status = server.Name + " is not loaded"
		err = errors.New("The nodes settings in config have incorrect values")
		return
	}

	// Init a health check settings
	server.check = check

	// The info handler returns a system status of the application
	server.GET("/info", infoHandler)

	// Lists methods, which display how to use API
	server.GET("/list", displayAllMethods)
	server.GET("/list/nodes", displayAllNodeMethods)
	server.GET("/list/nodes/get", displayGetNodeMethods)
	server.GET("/list/nodes/set", displaySetNodeMethods)
	server.GET("/list/nodes/delete", displayDeleteNodeMethods)

	// Init API methods for the Nodes
	server.GET("/nodes/:host/:port", server.Nodes.getRecord)
	server.GET("/nodes/:host", server.Nodes.getAllRecordsByHost)
	server.GET("/nodes", server.Nodes.getAllRecords)
	server.PUT("/nodes/:host/:port", server.Nodes.putRecord)
	server.PUT("/nodes", server.Nodes.putAllRecords)
	server.DELETE("/nodes/:host/:port", server.Nodes.deleteRecord)
	server.DELETE("/nodes/:host", server.Nodes.deleteAllRecordsByHost)
	server.DELETE("/nodes", server.Nodes.deleteAllRecords)

	go server.Listen(apiHostPort)
	go func() {
		p := new(proxy)
		if handler != nil {
			p.handler = handler
		} else {
			p.handler = server.proxyHandler
		}
		if err := http.ListenAndServe(hostPort, p); err != nil {
			errlog.Fatal(err)
		}
	}()

	status = server.Name + " loaded successfully"

	return
}

// Shutdown closes the server graceful
func (server *Server) Shutdown() (status string, err error) {

	// Set timer to wait one minute
	timeout := time.NewTimer(time.Second * 30)

	// a unwanted response sweeps if exist
	for {
		select {
		case <-server.response:
			continue
		default:
		}
		break
	}

	// sends a 'quit' signal
	server.quit <- struct{}{}

	status = server.Name + " connections closed"
	select {

	// Exit by timeout if jobs did not done
	case <-timeout.C:
		err = errors.New("timeout")
		return
	// Exit after all jobs done
	case <-server.response:
		return
	}
}

// Manage routine which manage all jobs
func (server *Server) manage() {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in Manage routine", recovery)
			// Recover routine
			go server.manage()
		} else {
			stdlog.Println("Manage routine stoped")
			server.response <- struct{}{}
		}
	}()
	for {
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		default:
		}
		select {
		case job := <-server.job:
			server.doJob(job)
			continue
		case <-server.quit:
			return
		}
	}
}

// Do updates depended by the signal
func (server *Server) doJob(signal int) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in do job routine", recovery)
		}
	}()
	switch signal {
	case responseSignal:
		server.response <- struct{}{}
	case nodeJobSignal:
		server.Nodes.updateRecords()
		server.Nodes.InitRing()
	}
}

// proxyHandler manages all requests/responses
func (server *Server) proxyHandler(request *http.Request) *http.Response {

	// Add "X-Forwarded-For" to repost remote host IP
	if remoteHost, _, err := net.SplitHostPort(request.RemoteAddr); err == nil {
		request.Header.Add("X-Forwarded-For", remoteHost)
	}
	// Use HTTP scheme
	request.URL.Scheme = protocolHTTP

	// If requests should not be queued, get result immediately
	if request.Method != methodPOST &&
		request.Method != methodPUT &&
		request.Method != methodDELETE {

		return server.processGET(request)
	}

	return server.processUpdate(request)
}

// call 'GET' request to the node using defined mode
func (server *Server) processGET(request *http.Request) *http.Response {
	if server.roundRobin {

		// Use round robin to get data from the host
		for count := 0; count < server.Nodes.ring.Len(); count++ {
			if node, ok := server.Nodes.CurrentFromRing(); ok &&
				node.Active && !node.Maintenance {

				// The host is active and is not in maintenance
				request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// Prepare next host
				server.Nodes.TwistRing()

				if server.checkNode(request.URL.Host) {
					response, err := http.DefaultTransport.RoundTrip(request)
					if err == nil {
						// If response is sucess, return
						return response
					}
					errlog.Println(err)
				}
			} else {

				// Use next host if not active or maintenance mode
				server.Nodes.TwistRing()
			}
		}
	} else {

		// If is not round robin mode, use first registered host
		if nodes, total := server.Nodes.GetAll(); total > 0 {
			if server.byPriority {
				sort.Sort(byPriority(nodes))
			}
			for _, node := range nodes {
				if node.Active && !node.Maintenance {

					// The host is active and is not in maintenance
					request.URL.Host = fmt.Sprintf("%s:%d", node.Host, node.Port)
					if server.checkNode(request.URL.Host) {
						response, err := http.DefaultTransport.RoundTrip(request)
						if err == nil {
							// If response is sucess, return
							return response
						}
						errlog.Println(err)
					}
				}
			}
		}
	}

	stdlog.Println("Warning: no one of the nodes is active")

	return nil
}

// call 'PUT', 'POST', 'DELETE' request to the node
func (server *Server) processUpdate(request *http.Request) *http.Response {
	// grab update request
	proxyRequestData, err := httputil.DumpRequest(request, true)
	if err != nil {

		// if unsuccessful, return nil response
		errlog.Println(err)
		return nil
	}
	var host string
	var response *http.Response
	if nodes, total := server.Nodes.GetAll(); total > 0 {
		answer := make(chan *http.Response, total)
		for _, node := range nodes {
			if node.Active {

				host = fmt.Sprintf("%s:%d", node.Host, node.Port)

				// create new queue job
				job := &queueJob{
					query:  make(chan []byte, 1),
					answer: answer,
				}
				job.query <- proxyRequestData

				queue, _ := server.queues.check(host)
				queue.jobs <- job
				queue.task <- doJobTask
			}
		}
		timeout := time.NewTimer(time.Second * server.responseTimeout)
		for {
			select {
			case response = <-answer:
				return response
			case <-timeout.C:
				return response
			}
		}
	}
	return response
}

// worker receive a data from the queue and send it to the node
func (server *Server) worker(q *queue) {
	defer func() {
		if recovery := recover(); recovery != nil {
			errlog.Println("Recovered in worker routine", recovery)
			// the worker recovers again
			go server.worker(q)
		} else {
			q.response <- struct{}{}
			stdlog.Println("Worker closed for", q.id)
		}
	}()
	stdlog.Println("Worker started for", q.id)
	for {
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		default:
		}
		select {
		case task := <-q.task:
			switch task {
			case doJobTask:
				server.doUpdate(q)
			}
			continue
		case <-q.quit:
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
}

func (server *Server) doUpdate(q *queue) {
	// check the node
	for {
		if server.checkNode(q.id) {
			break
		}
		stdlog.Println("Node", q.id, "does not ready for updates")
		stdlog.Println("try again in", server.check.Seconds, "seconds")
		timeout := time.NewTimer(time.Second * server.check.Seconds)
		select {
		//  Repeat by timeout
		case <-timeout.C:
			continue
		case <-q.quit:
			q.task <- doJobTask
			return
		case <-q.ask:
			q.response <- struct{}{}
		}
	}
	// if the node is alive, post data
	job := <-q.jobs
	data := <-job.query
	if response, err := dispatchRequest(q.id, data); err != nil {

		// Job does not done
		errlog.Println(err)

	} else {

		// job done
		job.answer <- response
		job.done = true
	}
}

// check the node
func (server *Server) checkNode(host string) bool {
	response, err := http.Get(protocolHTTP + "://" + host + server.check.URL)
	if err != nil {
		stdlog.Println(host, err)
		return false
	}

	defer response.Body.Close()
	// if pattern does not exist, should be true
	if server.check.Pattern == "" {
		return true
	}

	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		stdlog.Println(host, err)
		return false
	}
	// check of regexp pattern
	valid := regexp.MustCompile(server.check.Pattern)
	return valid.MatchString(string(data))
}

// Reproduce request to specified node and capture response
func dispatchRequest(host string, data []byte) (*http.Response, error) {
	reader := bufio.NewReader(bytes.NewBuffer(data))
	request, err := http.ReadRequest(reader)
	if err != nil {
		stdlog.Println(host, err)
		return nil, err
	}
	request.Body = ioutil.NopCloser(reader)
	request.URL.Scheme = protocolHTTP
	request.URL.Host = host

	response, err := http.DefaultTransport.RoundTrip(request)
	if err != nil {
		stdlog.Println(host, err)
		return nil, err
	}
	return response, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path"
)

const (
	googleGeocodeURL = "http://maps.google.com/maps/api/geocode/json"
	forecastURL      = "https://api.forecast.io/forecast/your-api-key-here/"
	rootAssetPath    = "react-weather"
)

type Location struct {
	Lat float64
	Lng float64
}

type GeocodeResponse struct {
	Results []struct {
		FormattedAddress string `json:"formatted_address"`
		Geometry         struct {
			Location Location
		}
	}
}

type WeatherResponse struct {
	Address  string      `json:"formattedAddress"`
	Location Location    `json:"location"`
	Weather  interface{} `json:"data"`
}

func main() {
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/weather/", weatherHandler)
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(path.Join(rootAssetPath, "www/static")))))
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(rootAssetPath, "www/index.html"))
}

func weatherHandler(w http.ResponseWriter, r *http.Request) {
	if address, ok := r.URL.Query()["address"]; ok {
		address := address[0]
		result, err := geocode(address)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		geocoded := result.Results[0]
		loc := geocoded.Geometry.Location
		weather, err := loadWeather(loc)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		resp := &WeatherResponse{Address: geocoded.FormattedAddress, Location: loc, Weather: weather}
		enc := json.NewEncoder(w)
		if err := enc.Encode(&resp); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
	} else {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
	}
}

func geocode(address string) (res *GeocodeResponse, err error) {
	u, _ := url.Parse(googleGeocodeURL)
	q := url.Values{}
	q.Add("address", address)
	u.RawQuery = q.Encode()
	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	res = &GeocodeResponse{}
	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
		return nil, err
	}
	if len(res.Results) == 0 {
		return nil, errors.New("No Results")
	}
	return res, nil
}

func loadWeather(loc Location) (data interface{}, err error) {
	u := fmt.Sprintf("%s%f,%f", forecastURL, loc.Lat, loc.Lng)
	parsed, _ := url.Parse(u)
	q := url.Values{}
	q.Add("exclude", "minutely,daily")
	q.Add("units", "si")
	parsed.RawQuery = q.Encode()
	resp, err := http.Get(parsed.String())
	if err != nil {
		return nil, err
	}
	var res interface{}
	if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
		return nil, err
	}
	return res, nil
}
<commit_msg>Update rootAssetPath<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"path"
)

const (
	googleGeocodeURL = "http://maps.google.com/maps/api/geocode/json"
	forecastURL      = "https://api.forecast.io/forecast/your-api-key-here/"
	rootAssetPath    = "/path/to/react-weather"
)

type Location struct {
	Lat float64
	Lng float64
}

type GeocodeResponse struct {
	Results []struct {
		FormattedAddress string `json:"formatted_address"`
		Geometry         struct {
			Location Location
		}
	}
}

type WeatherResponse struct {
	Address  string      `json:"formattedAddress"`
	Location Location    `json:"location"`
	Weather  interface{} `json:"data"`
}

func main() {
	http.HandleFunc("/", indexHandler)
	http.HandleFunc("/weather/", weatherHandler)
	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir(path.Join(rootAssetPath, "www/static")))))
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(rootAssetPath, "www/index.html"))
}

func weatherHandler(w http.ResponseWriter, r *http.Request) {
	if address, ok := r.URL.Query()["address"]; ok {
		address := address[0]
		result, err := geocode(address)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		geocoded := result.Results[0]
		loc := geocoded.Geometry.Location
		weather, err := loadWeather(loc)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		resp := &WeatherResponse{Address: geocoded.FormattedAddress, Location: loc, Weather: weather}
		enc := json.NewEncoder(w)
		if err := enc.Encode(&resp); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
	} else {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
	}
}

func geocode(address string) (res *GeocodeResponse, err error) {
	u, _ := url.Parse(googleGeocodeURL)
	q := url.Values{}
	q.Add("address", address)
	u.RawQuery = q.Encode()
	resp, err := http.Get(u.String())
	if err != nil {
		return nil, err
	}
	res = &GeocodeResponse{}
	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
		return nil, err
	}
	if len(res.Results) == 0 {
		return nil, errors.New("No Results")
	}
	return res, nil
}

func loadWeather(loc Location) (data interface{}, err error) {
	u := fmt.Sprintf("%s%f,%f", forecastURL, loc.Lat, loc.Lng)
	parsed, _ := url.Parse(u)
	q := url.Values{}
	q.Add("exclude", "minutely,daily")
	q.Add("units", "si")
	parsed.RawQuery = q.Encode()
	resp, err := http.Get(parsed.String())
	if err != nil {
		return nil, err
	}
	var res interface{}
	if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
		return nil, err
	}
	return res, nil
}
<|endoftext|>


<commit_before>// Package bla provides ...
package bla

import (
	"bytes"
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const Version = "0.1 alpha"

func New() {

	flag.Parse()
	LoadConfig(*configPath)

	server := &Server{
		Docs:        make(map[string]*Doc, 0),
		sortedDocs:  make([]*Doc, 0),
		DocLock:     &sync.RWMutex{},
		Tags:        make(map[string][]*Doc, 0),
		TagLock:     &sync.RWMutex{},
		StaticFiles: make([]string, 0),
		Version:     Version,
		StopWatch:   make(chan bool)}

	server.Reset()
	if err := server.LoadTempalte(); err != nil {
		log.Fatal(err)
	}
	server.LoadStaticFiles()
	server.LoadAllDocs()
	if err := server.SaveAllDocs(); err != nil {
		log.Fatal(err)
	}
	server.MakeHome()
	log.Print(Cfg)
	server.Watch()
	server.ConfigWatch()

	admin := map[string]http.HandlerFunc{
		".add":  server.Add,
		".edit": server.Edit,
		".quit": server.Quit,
	}
	for k, v := range admin {
		http.HandleFunc(path.Join(Cfg.BasePath, k), server.auth(v))
	}

	http.Handle("/", http.FileServer(http.Dir(Cfg.PublicPath)))

	if Cfg.TLSKeyFile != "" && Cfg.TLSCertFile != "" {
		log.Fatal(http.ListenAndServeTLS(Cfg.Addr, Cfg.TLSCertFile, Cfg.TLSKeyFile, nil))
	} else {
		log.Fatal(http.ListenAndServe(Cfg.Addr, nil))
	}
}

func (s *Server) auth(orig http.HandlerFunc) http.HandlerFunc {

	return func(w http.ResponseWriter, r *http.Request) {

		if u, p, ok := r.BasicAuth(); ok {
			if u == Cfg.Username && p == Cfg.Password {
				orig(w, r)
				return
			}
		}

		w.Header().Set("WWW-Authenticate", `Basic realm="`+Cfg.BaseURL+`"`)
		w.WriteHeader(401)
		w.Write([]byte("401 Unauthorized\n"))
	}
}

func (s *Server) Quit(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Location", strings.Replace(Cfg.BaseURL, "://", "://log:out@", 1))
	w.WriteHeader(301)
}

func (s *Server) Add(w http.ResponseWriter, r *http.Request) {

	switch r.Method {
	case "GET":
		rd := s.newRootData()
		if err := s.template.add.ExecuteTemplate(w, "root", rd); err != nil {
			log.Print(err)
			w.Write([]byte(err.Error()))
			w.WriteHeader(500)
		}
	case "POST":
		r.ParseForm()
		content := r.FormValue("mce_0")
		parsed, err := goquery.NewDocumentFromReader(bytes.NewBuffer([]byte(content)))
		if err != nil {
			log.Printf("Parse failed:%v", err)
			w.WriteHeader(500)
			return
		}
		header := parsed.Find("h1").First().Text()
		if header == "" {
			log.Print("Header not existed", header)
			w.WriteHeader(500)
			return

		}

		f, err := os.Create(filepath.Join(Cfg.ContentPath, header+".html"))
		if err != nil {
			log.Print(err)
			w.WriteHeader(500)
			return
		}

		f.WriteString(content)
		f.Close()

		w.Header().Set("Location", Cfg.BasePath)
		w.WriteHeader(301)

	default:
		w.WriteHeader(403)

	}
}

func (s *Server) Edit(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		doc := r.URL.Query().Get("doc")
		if doc == "" {

			w.Write([]byte("no doc query"))
			w.WriteHeader(403)
			return
		}

		fp := filepath.Join(Cfg.ContentPath, doc)
		if _, err := os.Stat(fp); os.IsNotExist(err) {
			w.Write([]byte("no such doc"))
			w.WriteHeader(404)
			return
		}

		f, err := os.Open(fp)
		if err != nil {
			w.Write([]byte(err.Error()))
			w.WriteHeader(500)
			return
		}
		defer f.Close()

		rd := s.newRootData()
		parsed, _ := goquery.NewDocumentFromReader(f)

		rd.Docs = append(rd.Docs, &Doc{Parsed: parsed})
		if err := s.template.add.ExecuteTemplate(w, "root", rd); err != nil {
			log.Print(err)
			w.Write([]byte(err.Error()))
			w.WriteHeader(500)
			f.Close()
			return
		}

		bak, err := os.Create(fp + ".bak")
		io.Copy(bak, f)
		bak.Close()
		f.Close()
	}
}

func (s *Server) Reset() {

	if err := os.RemoveAll(Cfg.PublicPath); err != nil {
		log.Print(err)
	}
	os.MkdirAll(Cfg.PublicPath, 0755)

	if Cfg.Favicon != "" {
		fav, _ := filepath.Abs(Cfg.Favicon)
		dest, _ := filepath.Abs(filepath.Join(Cfg.PublicPath, "favicon.ico"))

		if err := os.Symlink(fav, dest); err != nil {
			log.Print(err)
		} else {
			log.Print("Rebuild link:", fav, " -> ", dest)
		}
	}

	uploads, _ := filepath.Abs(Cfg.UploadPath)
	uploadBase := filepath.Base(uploads)
	dest, _ := filepath.Abs(filepath.Join(Cfg.PublicPath, Cfg.BasePath, uploadBase))

	os.MkdirAll(filepath.Join(Cfg.PublicPath, Cfg.BasePath), 0755)

	if err := os.Symlink(uploads, dest); err != nil {
		log.Print(err)
	} else {
		log.Print("Rebuild link:", uploads, " -> ", dest)
	}
}

func (s *Server) LoadStaticFiles() {

	log.Print("Rebuild from: ", Cfg.TemplatePath)
	orig, err := filepath.Abs(filepath.Join(Cfg.TemplatePath, "asset"))
	dest, err := filepath.Abs(filepath.Join(Cfg.PublicPath, Cfg.BasePath, "asset"))
	if err != nil {
		log.Print(err)
	}

	log.Print("Rebuild link:", orig, " -> ", dest)
	if err := os.Symlink(orig, dest); err != nil {
		log.Print(err)
	}
	log.Print("Loading Asset files")
	filepath.Walk(orig,
		func(path string, info os.FileInfo, e error) (err error) {

			if info == nil || !info.Mode().IsRegular() {
				return
			}
			if !(filepath.Ext(path) == ".css" || filepath.Ext(path) == ".js") {
				return
			}
			log.Print("|- ", path)
			s.StaticFiles = append(s.StaticFiles, path)
			return
		})
}

func (s *Server) LoadTempalte() (err error) {

	var funcMap = template.FuncMap{
		"base": path.Base,
		"ext":  filepath.Ext,
		"now":  time.Now,
	}
	root := filepath.Join(Cfg.TemplatePath, "root.tmpl")
	parsed := func(fname string) (t *template.Template) {
		t, err = template.New("").Funcs(funcMap).ParseFiles(root, filepath.Join(Cfg.TemplatePath, fname))
		if err != nil {
			log.Fatal(err)
		}
		return t
	}

	s.template.home = parsed("home.tmpl")
	s.template.doc = parsed("doc.tmpl")
	s.template.index = parsed("index.tmpl")

	s.template.add = parsed("add.tmpl")
	return
}

type Server struct {
	Docs        map[string]*Doc
	sortedDocs  []*Doc
	Tags        map[string][]*Doc
	StaticFiles []string
	template    struct{ home, doc, index, add *template.Template }
	Version     string
	StopWatch   chan bool
	DocLock     *sync.RWMutex
	TagLock     *sync.RWMutex
}
<commit_msg>san doc path<commit_after>// Package bla provides ...
package bla

import (
	"bytes"
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/PuerkitoBio/goquery"
)

const Version = "0.1 alpha"

func New() {

	flag.Parse()
	LoadConfig(*configPath)

	server := &Server{
		Docs:        make(map[string]*Doc, 0),
		sortedDocs:  make([]*Doc, 0),
		DocLock:     &sync.RWMutex{},
		Tags:        make(map[string][]*Doc, 0),
		TagLock:     &sync.RWMutex{},
		StaticFiles: make([]string, 0),
		Version:     Version,
		StopWatch:   make(chan bool)}

	server.Reset()
	if err := server.LoadTempalte(); err != nil {
		log.Fatal(err)
	}
	server.LoadStaticFiles()
	server.LoadAllDocs()
	if err := server.SaveAllDocs(); err != nil {
		log.Fatal(err)
	}
	server.MakeHome()
	log.Print(Cfg)
	server.Watch()
	server.ConfigWatch()

	admin := map[string]http.HandlerFunc{
		".add":  server.Add,
		".edit": server.Edit,
		".quit": server.Quit,
	}
	for k, v := range admin {
		http.HandleFunc(path.Join(Cfg.BasePath, k), server.auth(v))
	}

	http.Handle("/", http.FileServer(http.Dir(Cfg.PublicPath)))

	if Cfg.TLSKeyFile != "" && Cfg.TLSCertFile != "" {
		log.Fatal(http.ListenAndServeTLS(Cfg.Addr, Cfg.TLSCertFile, Cfg.TLSKeyFile, nil))
	} else {
		log.Fatal(http.ListenAndServe(Cfg.Addr, nil))
	}
}

func (s *Server) auth(orig http.HandlerFunc) http.HandlerFunc {

	return func(w http.ResponseWriter, r *http.Request) {

		if u, p, ok := r.BasicAuth(); ok {
			if u == Cfg.Username && p == Cfg.Password {
				orig(w, r)
				return
			}
		}

		w.Header().Set("WWW-Authenticate", `Basic realm="`+Cfg.BaseURL+`"`)
		w.WriteHeader(401)
		w.Write([]byte("401 Unauthorized\n"))
	}
}

func (s *Server) Quit(w http.ResponseWriter, r *http.Request) {

	w.Header().Set("Location", strings.Replace(Cfg.BaseURL, "://", "://log:out@", 1))
	w.WriteHeader(301)
}

func (s *Server) Add(w http.ResponseWriter, r *http.Request) {

	switch r.Method {
	case "GET":
		rd := s.newRootData()
		if err := s.template.add.ExecuteTemplate(w, "root", rd); err != nil {
			log.Print(err)
			w.Write([]byte(err.Error()))
			w.WriteHeader(500)
		}
	case "POST":
		r.ParseForm()
		content := r.FormValue("mce_0")
		parsed, err := goquery.NewDocumentFromReader(bytes.NewBuffer([]byte(content)))
		if err != nil {
			log.Printf("Parse failed:%v", err)
			w.WriteHeader(500)
			return
		}
		header := parsed.Find("h1").First().Text()
		if header == "" {
			log.Print("Header not existed", header)
			w.WriteHeader(500)
			return

		}

		f, err := os.Create(filepath.Join(Cfg.ContentPath, santiSpace(header)+".html"))
		if err != nil {
			log.Print(err)
			w.WriteHeader(500)
			return
		}

		f.WriteString(content)
		f.Close()

		w.Header().Set("Location", Cfg.BasePath)
		w.WriteHeader(301)

	default:
		w.WriteHeader(403)

	}
}

func (s *Server) Edit(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		doc := r.URL.Query().Get("doc")
		if doc == "" {

			w.Write([]byte("no doc query"))
			w.WriteHeader(403)
			return
		}

		fp := filepath.Join(Cfg.ContentPath, doc+".html")
		if _, err := os.Stat(fp); os.IsNotExist(err) {
			w.Write([]byte("no such doc"))
			w.WriteHeader(404)
			return
		}

		f, err := os.Open(fp)
		if err != nil {
			w.Write([]byte(err.Error()))
			w.WriteHeader(500)
			return
		}
		defer f.Close()

		rd := s.newRootData()
		parsed, _ := goquery.NewDocumentFromReader(f)

		rd.Docs = append(rd.Docs, &Doc{Parsed: parsed})
		if err := s.template.add.ExecuteTemplate(w, "root", rd); err != nil {
			log.Print(err)
			w.Write([]byte(err.Error()))
			w.WriteHeader(500)
			f.Close()
			return
		}

		bak, err := os.Create(fp + ".bak")
		io.Copy(bak, f)
		bak.Close()
		f.Close()
	}
}

func (s *Server) Reset() {

	if err := os.RemoveAll(Cfg.PublicPath); err != nil {
		log.Print(err)
	}
	os.MkdirAll(Cfg.PublicPath, 0755)

	if Cfg.Favicon != "" {
		fav, _ := filepath.Abs(Cfg.Favicon)
		dest, _ := filepath.Abs(filepath.Join(Cfg.PublicPath, "favicon.ico"))

		if err := os.Symlink(fav, dest); err != nil {
			log.Print(err)
		} else {
			log.Print("Rebuild link:", fav, " -> ", dest)
		}
	}

	uploads, _ := filepath.Abs(Cfg.UploadPath)
	uploadBase := filepath.Base(uploads)
	dest, _ := filepath.Abs(filepath.Join(Cfg.PublicPath, Cfg.BasePath, uploadBase))

	os.MkdirAll(filepath.Join(Cfg.PublicPath, Cfg.BasePath), 0755)

	if err := os.Symlink(uploads, dest); err != nil {
		log.Print(err)
	} else {
		log.Print("Rebuild link:", uploads, " -> ", dest)
	}
}

func (s *Server) LoadStaticFiles() {

	log.Print("Rebuild from: ", Cfg.TemplatePath)
	orig, err := filepath.Abs(filepath.Join(Cfg.TemplatePath, "asset"))
	dest, err := filepath.Abs(filepath.Join(Cfg.PublicPath, Cfg.BasePath, "asset"))
	if err != nil {
		log.Print(err)
	}

	log.Print("Rebuild link:", orig, " -> ", dest)
	if err := os.Symlink(orig, dest); err != nil {
		log.Print(err)
	}
	log.Print("Loading Asset files")
	filepath.Walk(orig,
		func(path string, info os.FileInfo, e error) (err error) {

			if info == nil || !info.Mode().IsRegular() {
				return
			}
			if !(filepath.Ext(path) == ".css" || filepath.Ext(path) == ".js") {
				return
			}
			log.Print("|- ", path)
			s.StaticFiles = append(s.StaticFiles, path)
			return
		})
}

func (s *Server) LoadTempalte() (err error) {

	var funcMap = template.FuncMap{
		"base": path.Base,
		"ext":  filepath.Ext,
		"now":  time.Now,
	}
	root := filepath.Join(Cfg.TemplatePath, "root.tmpl")
	parsed := func(fname string) (t *template.Template) {
		t, err = template.New("").Funcs(funcMap).ParseFiles(root, filepath.Join(Cfg.TemplatePath, fname))
		if err != nil {
			log.Fatal(err)
		}
		return t
	}

	s.template.home = parsed("home.tmpl")
	s.template.doc = parsed("doc.tmpl")
	s.template.index = parsed("index.tmpl")

	s.template.add = parsed("add.tmpl")
	return
}

type Server struct {
	Docs        map[string]*Doc
	sortedDocs  []*Doc
	Tags        map[string][]*Doc
	StaticFiles []string
	template    struct{ home, doc, index, add *template.Template }
	Version     string
	StopWatch   chan bool
	DocLock     *sync.RWMutex
	TagLock     *sync.RWMutex
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/cespare/memstats"
)

type Server struct {
	addr string
	db   *DB
}

func NewServer(dir, addr string, chunkSize uint64, expiry time.Duration) (*Server, error) {
	db, err := OpenDB(chunkSize, expiry, dir)
	if err != nil {
		return nil, err
	}
	return &Server{
		addr: addr,
		db:   db,
	}, nil
}

type Request struct {
	Type RequestType
	Key  []byte
	Val  []byte
	Resp chan *Response
	Err  error // If there was an error reading the request, only this field is set.
}

type RequestType uint8

const (
	RequestSet RequestType = iota + 1
	RequestGet
	RequestPing
	RequestInfo
)

type RedisType uint8

const (
	RedisErr RedisType = iota + 1
	RedisString
	RedisBulk
)

type Response struct {
	Type RedisType
	Msg  []byte
}

func (s *Server) Stop() error {
	return s.db.Close()
}

func (s *Server) Start() error {
	l, err := net.Listen("tcp", s.addr)
	if err != nil {
		return err
	}
	return s.loop(l)
}

func (s *Server) loop(l net.Listener) error {
	for {
		c, err := l.Accept()
		if err != nil {
			if e, ok := err.(net.Error); ok && e.Temporary() {
				delay := 10 * time.Millisecond
				log.Printf("Accept error: %s; retrying in %s", e, delay)
				time.Sleep(delay)
				continue
			}
			return err
		}
		go s.HandleConn(c)
	}
}

func when(ch chan *Response, pred bool) chan *Response {
	if pred {
		return ch
	}
	return nil
}

func head(q []*Response) *Response {
	if len(q) == 0 {
		return nil
	}
	return q[0]
}

func (s *Server) HandleConn(c net.Conn) {
	log.Printf("Client connected from %s", c.RemoteAddr())

	// readErr and writeErr are how the request reader and response writer goroutines can notify the other that
	// the client (or connection) broke/disconnected.
	// These signal chans are only closed.
	readErr := make(chan struct{})
	writeErr := make(chan struct{})

	// This request goroutine reads requests and sends them into this goroutine to be handled and buffered;
	// Responses are sent off to the response goroutine.
	// This is necessary for Redis pipelining to work.
	requests := make(chan *Request)
	responses := make(chan *Response)

	go s.readRequests(c, requests, readErr, writeErr)
	go s.writeResponses(c, responses, readErr, writeErr)

	var responseQueue []*Response

reqLoop:
	for {
		resp := &Response{Type: RedisString}
		select {
		case r := <-requests:
			if r.Err != nil {
				resp.Msg = []byte(r.Err.Error())
				resp.Type = RedisErr
			} else {
				switch r.Type {
				case RequestSet:
					if _, err := s.db.Put(r.Key, r.Val); err != nil {
						if e, ok := err.(FatalDBError); ok {
							log.Println("Fatal DB error:", e)
							if err := s.Stop(); err != nil {
								log.Println("Error while shutting down:", err)
							}
							os.Exit(1)
						}
						resp = ResponseFromError(err)
						break
					}
					resp.Msg = []byte("OK")
				case RequestGet:
					v, _, err := s.db.Get(r.Key)
					switch err {
					case nil:
						resp.Type = RedisBulk
						resp.Msg = v
					case ErrKeyNotExist:
						// Redis null val
						resp.Type = RedisBulk
					default:
						resp = ResponseFromError(err)
					}
				case RequestPing:
					resp.Msg = []byte("PONG")
				case RequestInfo:
					resp.Msg = s.db.Info()
				default:
					panic("unexpected request type")
				}
			}
			responseQueue = append(responseQueue, resp)
		case when(responses, len(responseQueue) > 0) <- head(responseQueue):
			responseQueue = responseQueue[1:]
		case <-readErr:
			break reqLoop
		case <-writeErr:
			break reqLoop
		}
	}

	log.Printf("Client disconnected from %s", c.RemoteAddr())
	c.Close()
}

func (s *Server) readRequests(c net.Conn, requests chan<- *Request, readErr, writeErr chan struct{}) {
	br := bufio.NewReader(c)
	for {
		var r Request
		if err := r.Parse(br); err != nil {
			if _, ok := err.(net.Error); ok {
				close(readErr)
				return
			}
			r.Err = err
		}
		requests <- &r
		select {
		case <-writeErr:
			return
		default:
		}
	}
}

func (s *Server) writeResponses(c net.Conn, responses <-chan *Response, readErr, writeErr chan struct{}) {
	for {
		select {
		case resp := <-responses:
			if err := resp.Write(c); err != nil {
				close(writeErr)
				return
			}
		case <-readErr:
			return
		}
	}
}

func ResponseFromError(err error) *Response {
	return &Response{
		Type: RedisErr,
		Msg:  []byte(err.Error()),
	}
}

var (
	ErrMalformedRequest    = errors.New("malformed request")
	ErrUnrecognizedCommand = errors.New("unrecognized command")
	ErrWrongNumArgs        = errors.New("wrong number of arguments for command")
)

func (r *Request) Parse(br *bufio.Reader) error {
	array, err := parseRedisArrayBulkString(br)
	if err != nil {
		return err
	}
	if len(array) == 0 {
		return ErrMalformedRequest
	}
	switch strings.ToLower(array[0]) {
	case "set":
		if len(array) != 3 {
			return ErrWrongNumArgs
		}
		r.Type = RequestSet
		r.Key = []byte(array[1])
		r.Val = []byte(array[2])
	case "get":
		if len(array) != 2 {
			return ErrWrongNumArgs
		}
		r.Type = RequestGet
		r.Key = []byte(array[1])
	case "ping":
		if len(array) != 1 {
			return ErrWrongNumArgs
		}
		r.Type = RequestPing
	case "info":
		if len(array) != 1 {
			return ErrWrongNumArgs
		}
		r.Type = RequestInfo
	default:
		return ErrUnrecognizedCommand
	}
	return nil
}

func (r *Response) Write(w io.Writer) error {
	var msg []byte
	switch r.Type {
	case RedisErr:
		msg = append([]byte("-"), r.Msg...)
		msg = append(msg, "\r\n"...)
	case RedisString:
		msg = append([]byte("+"), r.Msg...)
		msg = append(msg, "\r\n"...)
	case RedisBulk:
		if r.Msg == nil {
			msg = []byte("$-1\r\n")
			break
		}
		msg = append([]byte{'$'}, strconv.Itoa(len(r.Msg))...)
		msg = append(msg, "\r\n"...)
		msg = append(msg, r.Msg...)
		msg = append(msg, "\r\n"...)
	default:
		panic("unexpected response type")
	}
	_, err := w.Write(msg)
	return err
}

func main() {
	var (
		addr      = flag.String("addr", "localhost:5533", "Listen addr")
		dir       = flag.String("dir", "db", "DB directory")
		chunkSize = flag.Uint64("chunksize", 100e6, "Max size for chunks")
		expiry    = flag.Duration("expiry", time.Hour, "How long data persists before expiring")
	)
	flag.Parse()

	log.Println("Now listening on", *addr)
	server, err := NewServer(*dir, *addr, *chunkSize, *expiry)
	if err != nil {
		log.Fatal(err)
	}

	go func() {
		var diff memstats.Diff
		stats := new(memstats.Stats)
		stats.Collect()
		for _ = range time.Tick(time.Minute) {
			stats.Collect()
			if stats.ReadDiff(&diff) {
				fmt.Println("------------- MEMSTATS -----------")
				fmt.Println(&diff)
			}
		}
	}()

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		log.Printf("Caught signal (%v); shutting down...", <-c)
		if err := server.Stop(); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}()

	if err := server.Start(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Revert "Print memstats"<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type Server struct {
	addr string
	db   *DB
}

func NewServer(dir, addr string, chunkSize uint64, expiry time.Duration) (*Server, error) {
	db, err := OpenDB(chunkSize, expiry, dir)
	if err != nil {
		return nil, err
	}
	return &Server{
		addr: addr,
		db:   db,
	}, nil
}

type Request struct {
	Type RequestType
	Key  []byte
	Val  []byte
	Resp chan *Response
	Err  error // If there was an error reading the request, only this field is set.
}

type RequestType uint8

const (
	RequestSet RequestType = iota + 1
	RequestGet
	RequestPing
	RequestInfo
)

type RedisType uint8

const (
	RedisErr RedisType = iota + 1
	RedisString
	RedisBulk
)

type Response struct {
	Type RedisType
	Msg  []byte
}

func (s *Server) Stop() error {
	return s.db.Close()
}

func (s *Server) Start() error {
	l, err := net.Listen("tcp", s.addr)
	if err != nil {
		return err
	}
	return s.loop(l)
}

func (s *Server) loop(l net.Listener) error {
	for {
		c, err := l.Accept()
		if err != nil {
			if e, ok := err.(net.Error); ok && e.Temporary() {
				delay := 10 * time.Millisecond
				log.Printf("Accept error: %s; retrying in %s", e, delay)
				time.Sleep(delay)
				continue
			}
			return err
		}
		go s.HandleConn(c)
	}
}

func when(ch chan *Response, pred bool) chan *Response {
	if pred {
		return ch
	}
	return nil
}

func head(q []*Response) *Response {
	if len(q) == 0 {
		return nil
	}
	return q[0]
}

func (s *Server) HandleConn(c net.Conn) {
	log.Printf("Client connected from %s", c.RemoteAddr())

	// readErr and writeErr are how the request reader and response writer goroutines can notify the other that
	// the client (or connection) broke/disconnected.
	// These signal chans are only closed.
	readErr := make(chan struct{})
	writeErr := make(chan struct{})

	// This request goroutine reads requests and sends them into this goroutine to be handled and buffered;
	// Responses are sent off to the response goroutine.
	// This is necessary for Redis pipelining to work.
	requests := make(chan *Request)
	responses := make(chan *Response)

	go s.readRequests(c, requests, readErr, writeErr)
	go s.writeResponses(c, responses, readErr, writeErr)

	var responseQueue []*Response

reqLoop:
	for {
		resp := &Response{Type: RedisString}
		select {
		case r := <-requests:
			if r.Err != nil {
				resp.Msg = []byte(r.Err.Error())
				resp.Type = RedisErr
			} else {
				switch r.Type {
				case RequestSet:
					if _, err := s.db.Put(r.Key, r.Val); err != nil {
						if e, ok := err.(FatalDBError); ok {
							log.Println("Fatal DB error:", e)
							if err := s.Stop(); err != nil {
								log.Println("Error while shutting down:", err)
							}
							os.Exit(1)
						}
						resp = ResponseFromError(err)
						break
					}
					resp.Msg = []byte("OK")
				case RequestGet:
					v, _, err := s.db.Get(r.Key)
					switch err {
					case nil:
						resp.Type = RedisBulk
						resp.Msg = v
					case ErrKeyNotExist:
						// Redis null val
						resp.Type = RedisBulk
					default:
						resp = ResponseFromError(err)
					}
				case RequestPing:
					resp.Msg = []byte("PONG")
				case RequestInfo:
					resp.Msg = s.db.Info()
				default:
					panic("unexpected request type")
				}
			}
			responseQueue = append(responseQueue, resp)
		case when(responses, len(responseQueue) > 0) <- head(responseQueue):
			responseQueue = responseQueue[1:]
		case <-readErr:
			break reqLoop
		case <-writeErr:
			break reqLoop
		}
	}

	log.Printf("Client disconnected from %s", c.RemoteAddr())
	c.Close()
}

func (s *Server) readRequests(c net.Conn, requests chan<- *Request, readErr, writeErr chan struct{}) {
	br := bufio.NewReader(c)
	for {
		var r Request
		if err := r.Parse(br); err != nil {
			if _, ok := err.(net.Error); ok {
				close(readErr)
				return
			}
			r.Err = err
		}
		requests <- &r
		select {
		case <-writeErr:
			return
		default:
		}
	}
}

func (s *Server) writeResponses(c net.Conn, responses <-chan *Response, readErr, writeErr chan struct{}) {
	for {
		select {
		case resp := <-responses:
			if err := resp.Write(c); err != nil {
				close(writeErr)
				return
			}
		case <-readErr:
			return
		}
	}
}

func ResponseFromError(err error) *Response {
	return &Response{
		Type: RedisErr,
		Msg:  []byte(err.Error()),
	}
}

var (
	ErrMalformedRequest    = errors.New("malformed request")
	ErrUnrecognizedCommand = errors.New("unrecognized command")
	ErrWrongNumArgs        = errors.New("wrong number of arguments for command")
)

func (r *Request) Parse(br *bufio.Reader) error {
	array, err := parseRedisArrayBulkString(br)
	if err != nil {
		return err
	}
	if len(array) == 0 {
		return ErrMalformedRequest
	}
	switch strings.ToLower(array[0]) {
	case "set":
		if len(array) != 3 {
			return ErrWrongNumArgs
		}
		r.Type = RequestSet
		r.Key = []byte(array[1])
		r.Val = []byte(array[2])
	case "get":
		if len(array) != 2 {
			return ErrWrongNumArgs
		}
		r.Type = RequestGet
		r.Key = []byte(array[1])
	case "ping":
		if len(array) != 1 {
			return ErrWrongNumArgs
		}
		r.Type = RequestPing
	case "info":
		if len(array) != 1 {
			return ErrWrongNumArgs
		}
		r.Type = RequestInfo
	default:
		return ErrUnrecognizedCommand
	}
	return nil
}

func (r *Response) Write(w io.Writer) error {
	var msg []byte
	switch r.Type {
	case RedisErr:
		msg = append([]byte("-"), r.Msg...)
		msg = append(msg, "\r\n"...)
	case RedisString:
		msg = append([]byte("+"), r.Msg...)
		msg = append(msg, "\r\n"...)
	case RedisBulk:
		if r.Msg == nil {
			msg = []byte("$-1\r\n")
			break
		}
		msg = append([]byte{'$'}, strconv.Itoa(len(r.Msg))...)
		msg = append(msg, "\r\n"...)
		msg = append(msg, r.Msg...)
		msg = append(msg, "\r\n"...)
	default:
		panic("unexpected response type")
	}
	_, err := w.Write(msg)
	return err
}

func main() {
	var (
		addr      = flag.String("addr", "localhost:5533", "Listen addr")
		dir       = flag.String("dir", "db", "DB directory")
		chunkSize = flag.Uint64("chunksize", 100e6, "Max size for chunks")
		expiry    = flag.Duration("expiry", time.Hour, "How long data persists before expiring")
	)
	flag.Parse()

	log.Println("Now listening on", *addr)
	server, err := NewServer(*dir, *addr, *chunkSize, *expiry)
	if err != nil {
		log.Fatal(err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		log.Printf("Caught signal (%v); shutting down...", <-c)
		if err := server.Stop(); err != nil {
			log.Fatal(err)
		}
		os.Exit(0)
	}()

	if err := server.Start(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>