<commit_before>package serfer

import (
	"net"
	"testing"

	"github.com/hashicorp/serf/serf"
	log "github.com/mgutz/logxi/v1"
	"github.com/stretchr/testify/suite"
)

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestEventHandlerSuite(t *testing.T) {
	suite.Run(t, new(EventHandlerTestSuite))
}

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type EventHandlerTestSuite struct {
	suite.Suite
	Handler SerfEventHandler
	Member  serf.Member
}

// Make sure that Handler and Member are set before each test
func (suite *EventHandlerTestSuite) SetupTest() {
	suite.Handler = SerfEventHandler{
		Logger: &log.NullLogger{},
	}

	suite.Member = serf.Member{
		Name:        "",
		Addr:        net.ParseIP("127.0.0.1"),
		Port:        9022,
		Tags:        make(map[string]string),
		Status:      serf.StatusAlive,
		ProtocolMin: serf.ProtocolVersionMin,
		ProtocolMax: serf.ProtocolVersionMax,
		ProtocolCur: serf.ProtocolVersionMax,
		DelegateMin: serf.ProtocolVersionMin,
		DelegateMax: serf.ProtocolVersionMax,
		DelegateCur: serf.ProtocolVersionMax,
	}
}

// Test NodeJoin events are processed properly
func (suite *EventHandlerTestSuite) TestNodeJoined() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberJoin,
		[]serf.Member{suite.Member},
	}

	// Add NodeJoined handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeJoined = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeLeave messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeLeave() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberLeave,
		[]serf.Member{suite.Member},
	}

	// Add NodeLeft handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeLeft = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeFailed messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeFailed() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberFailed,
		[]serf.Member{suite.Member},
	}

	// Add NodeFailed handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeFailed = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeReaped messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeReaped() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberReap,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeReaped = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test UserEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestUserEvent() {

	// Create Member Event
	evt := serf.UserEvent{
		LTime:    serf.LamportTime(0),
		Name:     "Event",
		Payload:  make([]byte, 0),
		Coalesce: false,
	}

	// Add UserEvent handler
	m := &MockUserEventHandler{}
	m.On("HandleUserEvent", evt).Return()
	suite.Handler.UserEvent = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleUserEvent", evt)
	r.AssertNotCalled(suite.T(), "Reconcile", evt)
}

// Test NodeUpdated messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeUpdated() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberUpdate,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeUpdated = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test QueryEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestQueryEvent() {

	// Create Query
	query := serf.Query{
		LTime:   serf.LamportTime(0),
		Name:    "Event",
		Payload: make([]byte, 0),
	}

	// Add UserEvent handler
	m := &MockQueryEventHandler{}
	m.On("HandleQueryEvent", query).Return()
	suite.Handler.QueryHandler = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", query).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(&query)
	m.AssertCalled(suite.T(), "HandleQueryEvent", query)
	r.AssertNotCalled(suite.T(), "Reconcile", query)
}

// Test nil messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestNilEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	suite.Handler.HandleEvent(nil)
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}

// Test unknown messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestUnknownEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	t1 := &MockEvent{Name: "UnknownType", Type: serf.EventType(-1)}
	t1.On("EventType").Return()
	suite.Handler.HandleEvent(t1)

	// Test Assertions
	t1.AssertCalled(suite.T(), "EventType")
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}
<commit_msg>Update tests for unknown events, leader elections, user events and non leader events<commit_after>package serfer

import (
	"net"
	"testing"

	"github.com/hashicorp/serf/serf"
	log "github.com/mgutz/logxi/v1"
	"github.com/stretchr/testify/suite"
)

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestEventHandlerSuite(t *testing.T) {
	suite.Run(t, new(EventHandlerTestSuite))
}

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type EventHandlerTestSuite struct {
	suite.Suite
	Prefix  string
	Handler SerfEventHandler
	Member  serf.Member
}

// Make sure that Handler and Member are set before each test
func (suite *EventHandlerTestSuite) SetupTest() {
	suite.Prefix = "serfer"
	suite.Handler = SerfEventHandler{
		ServicePrefix:     suite.Prefix,
		ReconcileOnJoin:   true,
		ReconcileOnFail:   true,
		ReconcileOnLeave:  true,
		ReconcileOnUpdate: true,
		ReconcileOnReap:   true,
		IsLeader: func() bool {
			return true
		},
		IsLeaderEvent: func(name string) bool {
			return name == suite.Prefix+":new-leader"
		},
		Logger: &log.NullLogger{},
	}

	suite.Member = serf.Member{
		Name:        "",
		Addr:        net.ParseIP("127.0.0.1"),
		Port:        9022,
		Tags:        make(map[string]string),
		Status:      serf.StatusAlive,
		ProtocolMin: serf.ProtocolVersionMin,
		ProtocolMax: serf.ProtocolVersionMax,
		ProtocolCur: serf.ProtocolVersionMax,
		DelegateMin: serf.ProtocolVersionMin,
		DelegateMax: serf.ProtocolVersionMax,
		DelegateCur: serf.ProtocolVersionMax,
	}
}

// Test NodeJoin events are processed properly
func (suite *EventHandlerTestSuite) TestNodeJoined() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberJoin,
		[]serf.Member{suite.Member},
	}

	// Add NodeJoined handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeJoined = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", suite.Member).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", suite.Member)
}

// Test NodeLeave messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeLeave() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberLeave,
		[]serf.Member{suite.Member},
	}

	// Add NodeLeft handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeLeft = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", suite.Member).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", suite.Member)
}

// Test NodeFailed messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeFailed() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberFailed,
		[]serf.Member{suite.Member},
	}

	// Add NodeFailed handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeFailed = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", suite.Member).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", suite.Member)
}

// Test NodeReaped messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeReaped() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberReap,
		[]serf.Member{suite.Member},
	}
	newMember := suite.Member
	newMember.Status = StatusReap

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeReaped = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", newMember).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", newMember)
}

// Test NodeUpdated messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeUpdated() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberUpdate,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeUpdated = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", suite.Member).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", suite.Member)
}

// Test QueryEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestQueryEvent() {

	// Create Query
	query := serf.Query{
		LTime:   serf.LamportTime(0),
		Name:    "Event",
		Payload: make([]byte, 0),
	}

	// Add UserEvent handler
	m := &MockQueryEventHandler{}
	m.On("HandleQueryEvent", query).Return()
	suite.Handler.QueryHandler = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", query).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(&query)
	m.AssertCalled(suite.T(), "HandleQueryEvent", query)
	r.AssertNotCalled(suite.T(), "Reconcile", query)
}

// Test nil messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestNilEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	suite.Handler.HandleEvent(nil)
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}

// Test unknown messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestUnknownEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	t1 := &MockEvent{Name: "UnknownType", Type: serf.EventType(-1)}
	t1.On("EventType").Return()
	suite.Handler.HandleEvent(t1)

	// Test Assertions
	t1.AssertCalled(suite.T(), "EventType")
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}

// Test leader election events are dispatched properly
func (suite *EventHandlerTestSuite) TestUserEvent_LeaderElection() {

	// Create Member Event
	evt := serf.UserEvent{
		LTime:    serf.LamportTime(0),
		Name:     suite.Prefix + ":new-leader",
		Payload:  make([]byte, 0),
		Coalesce: false,
	}

	// Add UserEvent handler
	m := &MockUserEventHandler{}
	m.On("HandleUserEvent", evt).Return()
	suite.Handler.LeaderElectionHandler = m

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleUserEvent", evt)
}

// Test unknown user events are dispatched properly
func (suite *EventHandlerTestSuite) TestUserEvent_UnknownEvent() {

	// Create Member Event
	evt := serf.UserEvent{
		LTime:    serf.LamportTime(0),
		Name:     "unk",
		Payload:  make([]byte, 0),
		Coalesce: false,
	}

	// Add UserEvent handler
	m := &MockUserEventHandler{}
	m.On("HandleUserEvent", evt).Return()
	suite.Handler.UnknownEventHandler = m

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleUserEvent", evt)
}

// Test user events are dispatched properly
func (suite *EventHandlerTestSuite) TestUserEvent() {

	// Create Member Event
	evt := serf.UserEvent{
		LTime:    serf.LamportTime(0),
		Name:     suite.Prefix + ":user-event",
		Payload:  make([]byte, 0),
		Coalesce: false,
	}
	modified := evt
	modified.Name = "user-event"

	// Add UserEvent handler
	m := &MockUserEventHandler{}
	m.On("HandleUserEvent", modified).Return()
	suite.Handler.UserEvent = m

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleUserEvent", modified)
}

// Test non leader event
func (suite *EventHandlerTestSuite) TestNonLeaderEvetn() {
	var called bool
	suite.Handler.IsLeader = func() bool {
		called = true
		return false
	}

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberUpdate,
		[]serf.Member{suite.Member},
	}

	// Add Reconciler
	r := &MockReconciler{}
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	suite.True(called, "IsLeader should have been called")
	r.AssertNotCalled(suite.T(), "Reconcile")
}
<|endoftext|>


<commit_before>package handlers

import (
	"github.com/byuoitav/sony-control-microservice/helpers"
	"github.com/labstack/echo"
)

func PowerOn(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "tvpower")
	if err != nil {
		return err
	}

	return nil
}

func Standby(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "poweroff")
	if err != nil {
		return err
	}

	return nil
}

func SwitchInput(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), context.Param("port")) // Input must be hdmi1, hdmi2, etc.
	if err != nil {
		return err
	}

	return nil
}

func VolumeUp(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "volumeup")
	if err != nil {
		return err
	}

	return nil
}

func VolumeDown(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "volumedown")
	if err != nil {
		return err
	}

	return nil
}

func VolumeMute(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "mute")
	if err != nil {
		return err
	}

	return nil
}

func BlankDisplay(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "pictureoff")
	if err != nil {
		return err
	}

	return nil
}

func UnblankDisplay(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "pictureoff")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Changing the command that is sent on power on to prevent accidental turn-off<commit_after>package handlers

import (
	"github.com/byuoitav/sony-control-microservice/helpers"
	"github.com/labstack/echo"
)

func PowerOn(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "wakeup")
	if err != nil {
		return err
	}

	return nil
}

func Standby(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "poweroff")
	if err != nil {
		return err
	}

	return nil
}

func SwitchInput(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), context.Param("port")) // Input must be hdmi1, hdmi2, etc.
	if err != nil {
		return err
	}

	return nil
}

func VolumeUp(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "volumeup")
	if err != nil {
		return err
	}

	return nil
}

func VolumeDown(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "volumedown")
	if err != nil {
		return err
	}

	return nil
}

func VolumeMute(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "mute")
	if err != nil {
		return err
	}

	return nil
}

func BlankDisplay(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "pictureoff")
	if err != nil {
		return err
	}

	return nil
}

func UnblankDisplay(context echo.Context) error {
	_, err := helpers.SendCommand(context.Param("address"), "pictureoff")
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// http://www.w3schools.com/rss/default.asp
// http://www.tutorialspoint.com/rss/what-is-atom.htm
// http://stackoverflow.com/questions/6619717/what-is-the-difference-between-rss-and-atom-feeds
// http://stackoverflow.com/questions/16309944/atom-to-rss-feeds-converter
package tmp

import (
	"net/http"
	"io/ioutil"
	"encoding/xml"
	"html/template"
	"log"
)

// http://www.w3schools.com/rss/rss_syntax.asp
// http://www.w3schools.com/rss/rss_channel.asp
type Rss2 struct {
	XMLName		xml.Name	`xml:"rss"`
	Version		string		`xml:"version,attr"`
	// Required
	Title		string		`xml:"channel>title"`
	Link		string		`xml:"channel>link"`
	Description	string		`xml:"channel>description"`
	// Optional
	PubDate		string		`xml:"channel>pubDate"`
	ItemList	[]Item		`xml:"channel>item"`
}

// http://www.w3schools.com/rss/rss_item.asp
type Item struct {
	// Required
	Title		string		`xml:"title"`
	Link		string		`xml:"link"`
	Description	template.HTML	`xml:"description"`
	// Optional
	PubDate		string		`xml:"pubDate"`
	Comments	string		`xml:"comments"`
	// for reader only
	IsRead		bool
	ReadDatetime	string
}

// http://en.wikipedia.org/wiki/Atom_(standard)
// http://golang.org/src/pkg/encoding/xml/
type Atom1 struct {
	XMLName		xml.Name	`xml:"http://www.w3.org/2005/Atom feed"`
	Title		string		`xml:"title"`
	Subtitle	string		`xml:"subtitle"`
	Id		string		`xml:"id"`
	Link		Link		`xml:"link"`
	Author		Author		`xml:"author"`
	EntryList	[]Entry		`xml:"entry"`
}

type Link struct {
	Href		string		`xml:"href,attr"`
}

type Author struct {
	Name		string		`xml:"name"`
	Email		string		`xml:"email"`
}

type Entry struct {
	Title		string		`xml:"title"`
	Summary		string		`xml:"summary"`
	Id		string		`xml:"id"`
	Link		Link		`xml:"link"`
	Author		Author		`xml:"author"`
}

func atom1ToRss2(a Atom1) Rss2 {
	r := Rss2{
		Title: a.Title,
		Link: a.Link.Href,
		Description: a.Subtitle,
	}
	return r
}

const atomErrStr = "expected element type <rss> but have <feed>"

func parseAtom(content []byte) (Rss2, bool){
	a := Atom1{}
	err := xml.Unmarshal(content, &a)
	if err != nil {
		log.Println(err)
		return Rss2{}, false
	}
	return atom1ToRss2(a), true
}

func parseSeedContent(content []byte) (Rss2, bool) {
	v := Rss2{}
	err := xml.Unmarshal(content, &v)
	if err != nil {
		if err.Error() == atomErrStr {
			// try Atom 1.0
			return parseAtom(content)
		}
		log.Println(err)
		return v, false
	}

	if v.Version == "2.0" {
		// RSS 2.0
		return v, true
	}

	log.Println("not RSS 2.0")
	return v, false
}

func GetSeed(url string) (Rss2, bool) {
	resp, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return Rss2{}, false
	}
	defer resp.Body.Close()

	body, err2 := ioutil.ReadAll(resp.Body)
	if err2 != nil {
		log.Println(err2)
		return Rss2{}, false
	}

	return parseSeedContent(body)
}

<commit_msg>atom 1.0 to rss 2.0<commit_after>// http://www.w3schools.com/rss/default.asp
// http://www.tutorialspoint.com/rss/what-is-atom.htm
// http://stackoverflow.com/questions/6619717/what-is-the-difference-between-rss-and-atom-feeds
// http://stackoverflow.com/questions/16309944/atom-to-rss-feeds-converter
package tmp

import (
	"net/http"
	"io/ioutil"
	"encoding/xml"
	"html/template"
	"log"
)

// http://www.w3schools.com/rss/rss_syntax.asp
// http://www.w3schools.com/rss/rss_channel.asp
type Rss2 struct {
	XMLName		xml.Name	`xml:"rss"`
	Version		string		`xml:"version,attr"`
	// Required
	Title		string		`xml:"channel>title"`
	Link		string		`xml:"channel>link"`
	Description	string		`xml:"channel>description"`
	// Optional
	PubDate		string		`xml:"channel>pubDate"`
	ItemList	[]Item		`xml:"channel>item"`
}

// http://www.w3schools.com/rss/rss_item.asp
type Item struct {
	// Required
	Title		string		`xml:"title"`
	Link		string		`xml:"link"`
	Description	template.HTML	`xml:"description"`
	// Optional
	PubDate		string		`xml:"pubDate"`
	Comments	string		`xml:"comments"`
	// for reader only
	IsRead		bool
	ReadDatetime	string
}

// http://en.wikipedia.org/wiki/Atom_(standard)
// http://golang.org/src/pkg/encoding/xml/
type Atom1 struct {
	XMLName		xml.Name	`xml:"http://www.w3.org/2005/Atom feed"`
	Title		string		`xml:"title"`
	Subtitle	string		`xml:"subtitle"`
	Id		string		`xml:"id"`
	Updated		string		`xml:"updated"`
	Rights		string		`xml:"rights"`
	Link		Link		`xml:"link"`
	Author		Author		`xml:"author"`
	EntryList	[]Entry		`xml:"entry"`
}

type Link struct {
	Href		string		`xml:"href,attr"`
}

type Author struct {
	Name		string		`xml:"name"`
	Email		string		`xml:"email"`
}

type Entry struct {
	Title		string		`xml:"title"`
	Summary		string		`xml:"summary"`
	Content		string		`xml:"content"`
	Id		string		`xml:"id"`
	Updated		string		`xml:"updated"`
	Link		Link		`xml:"link"`
	Author		Author		`xml:"author"`
}

func atom1ToRss2(a Atom1) Rss2 {
	r := Rss2{
		Title: a.Title,
		Link: a.Link.Href,
		Description: a.Subtitle,
		PubDate: a.Updated,
	}
	r.ItemList = make([]Item, len(a.EntryList))
	for i, entry := range a.EntryList {
		r.ItemList[i].Title = entry.Title
		r.ItemList[i].Link = entry.Link.Href
		if entry.Content == "" {
			r.ItemList[i].Description = template.HTML(entry.Summary)
		} else {
			r.ItemList[i].Description = template.HTML(entry.Content)
		}
	}
	return r
}

const atomErrStr = "expected element type <rss> but have <feed>"

func parseAtom(content []byte) (Rss2, bool){
	a := Atom1{}
	err := xml.Unmarshal(content, &a)
	if err != nil {
		log.Println(err)
		return Rss2{}, false
	}
	return atom1ToRss2(a), true
}

func parseSeedContent(content []byte) (Rss2, bool) {
	v := Rss2{}
	err := xml.Unmarshal(content, &v)
	if err != nil {
		if err.Error() == atomErrStr {
			// try Atom 1.0
			return parseAtom(content)
		}
		log.Println(err)
		return v, false
	}

	if v.Version == "2.0" {
		// RSS 2.0
		return v, true
	}

	log.Println("not RSS 2.0")
	return v, false
}

func GetSeed(url string) (Rss2, bool) {
	resp, err := http.Get(url)
	if err != nil {
		log.Println(err)
		return Rss2{}, false
	}
	defer resp.Body.Close()

	body, err2 := ioutil.ReadAll(resp.Body)
	if err2 != nil {
		log.Println(err2)
		return Rss2{}, false
	}

	return parseSeedContent(body)
}

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/dustin/httputil"
	"github.com/dustin/seriesly/timelib"
)

var min = flag.String("min", "", "minimum timestamp (RFC3339)")

func maybeFatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func setupDb(u string) {
	req, err := http.NewRequest("PUT", u, nil)
	maybeFatal(err)
	res, err := http.DefaultClient.Do(req)
	maybeFatal(err)
	res.Body.Close()
}

func sendOne(u, k string, body []byte) {
	resp, err := http.DefaultClient.Post(u+"?ts="+k,
		"application/json", bytes.NewReader(body))
	maybeFatal(err)
	defer resp.Body.Close()
	if resp.StatusCode >= 300 || resp.StatusCode < 200 {
		log.Fatalf("HTTP Error on %v: %v", k, err)
	}
}

func parseMinTime() time.Time {
	tm, err := timelib.ParseTime(*min)
	if err != nil {
		tm = time.Time{}
	}
	return tm
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatalf("Usage:  gzip -dc backup.gz | %v http://seriesly:3133/dbname",
			os.Args[0])
	}

	httputil.InitHTTPTracker(false)

	u := flag.Arg(0)
	setupDb(u)

	minTime := parseMinTime()

	t := time.Tick(5 * time.Second)
	i := 0

	d := json.NewDecoder(os.Stdin)
	for {
		kv := map[string]*json.RawMessage{}

		err := d.Decode(&kv)
		if err == io.EOF {
			log.Printf("Done!")
			break
		}
		maybeFatal(err)

		var latestKey string
		for k, v := range kv {
			if !minTime.IsZero() {
				thist, err := timelib.ParseTime(k)
				if err == nil && minTime.After(thist) {
					continue
				}
			}
			body := []byte(*v)
			sendOne(u, k, body)
			latestKey = k
		}

		i++
		select {
		case <-t:
			log.Printf("Processed %v items, latest was %v", i, latestKey)
		default:
		}
	}
}
<commit_msg>Log final loaded info<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/dustin/httputil"
	"github.com/dustin/seriesly/timelib"
)

var min = flag.String("min", "", "minimum timestamp (RFC3339)")

func maybeFatal(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func setupDb(u string) {
	req, err := http.NewRequest("PUT", u, nil)
	maybeFatal(err)
	res, err := http.DefaultClient.Do(req)
	maybeFatal(err)
	res.Body.Close()
}

func sendOne(u, k string, body []byte) {
	resp, err := http.DefaultClient.Post(u+"?ts="+k,
		"application/json", bytes.NewReader(body))
	maybeFatal(err)
	defer resp.Body.Close()
	if resp.StatusCode >= 300 || resp.StatusCode < 200 {
		log.Fatalf("HTTP Error on %v: %v", k, err)
	}
}

func parseMinTime() time.Time {
	tm, err := timelib.ParseTime(*min)
	if err != nil {
		tm = time.Time{}
	}
	return tm
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatalf("Usage:  gzip -dc backup.gz | %v http://seriesly:3133/dbname",
			os.Args[0])
	}

	httputil.InitHTTPTracker(false)

	u := flag.Arg(0)
	setupDb(u)

	minTime := parseMinTime()

	t := time.Tick(5 * time.Second)
	i := 0
	var latestKey string

	d := json.NewDecoder(os.Stdin)
	for {
		kv := map[string]*json.RawMessage{}

		err := d.Decode(&kv)
		if err == io.EOF {
			log.Printf("Done!  Processed %v items. Last was %v",
				i, latestKey)
			break
		}
		maybeFatal(err)

		for k, v := range kv {
			if !minTime.IsZero() {
				thist, err := timelib.ParseTime(k)
				if err == nil && minTime.After(thist) {
					continue
				}
			}
			body := []byte(*v)
			sendOne(u, k, body)
			latestKey = k
		}

		i++
		select {
		case <-t:
			log.Printf("Processed %v items, latest was %v", i, latestKey)
		default:
		}
	}
}
<|endoftext|>


<commit_before>/*

kasper is a lightweight Kafka stream processing library.

*/

package kasper

import (
	"log"
	"sync"
	"time"
	"strconv"

	"github.com/Shopify/sarama"
)

// TopicProcessor describes kafka topic processor
type TopicProcessor struct {
	config              *TopicProcessorConfig
	containerID         int
	client              sarama.Client
	producer            sarama.SyncProducer
	offsetManager       sarama.OffsetManager
	partitionProcessors map[int32]*partitionProcessor
	inputTopics         []string
	partitions          []int
	shutdown            chan struct{}
	waitGroup           sync.WaitGroup
	batchingEnabled     bool
	batchSize           int
	batchWaitDuration   time.Duration

	incomingMessageCount        Counter
	outgoingMessageCount        Counter
	messagesBehindHighWaterMark Gauge
}

// MessageProcessor describes kafka message processor
type MessageProcessor interface {
	// Process message from Kafka input topics.
	// This is the function where you perform all needed actions, like
	// population KV storage or producing Kafka output messages
	Process(IncomingMessage, Sender, Coordinator)
}

// TBD
type BatchMessageProcessor interface {
	// TBD
	ProcessBatch([]*IncomingMessage, Sender, Coordinator)
}

// NewTopicProcessor creates a new TopicProcessor with the given config.
// It requires a factory function that creates MessageProcessor instances and a container id.
// The container id must be a number between 0 and config.ContainerCount - 1.
func NewTopicProcessor(config *TopicProcessorConfig, makeProcessor func() MessageProcessor, containerID int) *TopicProcessor {
	mustHaveValidConfig(config, containerID)
	inputTopics := config.InputTopics
	client, partitions, offsetManager := mustSetupClient(config, containerID)
	partitionProcessors := make(map[int32]*partitionProcessor, len(partitions))
	requiredAcks := config.Config.RequiredAcks
	producer := mustSetupProducer(config.BrokerList, config.producerClientID(containerID), requiredAcks)
	topicProcessor := TopicProcessor{
		config:              config,
		containerID:         containerID,
		client:              client,
		producer:            producer,
		offsetManager:       offsetManager,
		partitionProcessors: partitionProcessors,
		inputTopics:         inputTopics,
		partitions:          partitions,
		shutdown:            make(chan struct{}),
		waitGroup:           sync.WaitGroup{},
		batchingEnabled:     false,
	}
	setupMetrics(&topicProcessor, config.Config.MetricsProvider)
	for _, partition := range partitions {
		processor := makeProcessor()
		partitionProcessors[int32(partition)] = newPartitionProcessor(&topicProcessor, processor, partition)
	}
	return &topicProcessor
}

// TBD
type BatchingOpts struct {
	makeProcessor     func() BatchMessageProcessor
	batchSize         int
	batchWaitDuration time.Duration
}

// TBD
func NewBatchTopicProcessor(config *TopicProcessorConfig, opts BatchingOpts, containerID int) *TopicProcessor {
	mustHaveValidConfig(config, containerID)
	inputTopics := config.InputTopics
	client, partitions, offsetManager := mustSetupClient(config, containerID)
	partitionProcessors := make(map[int32]*partitionProcessor, len(partitions))
	producer := mustSetupProducer(config.BrokerList, config.producerClientID(containerID), config.Config.RequiredAcks)
	topicProcessor := TopicProcessor{
		config:              config,
		containerID:         containerID,
		client:              client,
		producer:            producer,
		offsetManager:       offsetManager,
		partitionProcessors: partitionProcessors,
		inputTopics:         inputTopics,
		partitions:          partitions,
		shutdown:            make(chan struct{}),
		waitGroup:           sync.WaitGroup{},
		batchingEnabled:     true,
		batchSize:           opts.batchSize,
		batchWaitDuration:   opts.batchWaitDuration,
	}
	setupMetrics(&topicProcessor, config.Config.MetricsProvider)
	for _, partition := range partitions {
		processor := opts.makeProcessor()
		partitionProcessors[int32(partition)] = newBatchPartitionProcessor(&topicProcessor, processor, partition)
	}
	return &topicProcessor
}

func setupMetrics(tp *TopicProcessor, provider MetricsProvider) {
	tp.incomingMessageCount = provider.NewCounter("incoming_message_count", "Number of incoming messages received", "topic", "partition")
	tp.outgoingMessageCount = provider.NewCounter("outgoing_message_count", "Number of outgoing messages sent", "topic", "partition")
	tp.messagesBehindHighWaterMark = provider.NewGauge("messages_behind_high_water_mark_count", "Number of messages remaining to consume on the topic/partition", "topic", "partition")
}

func mustHaveValidConfig(config *TopicProcessorConfig, containerID int) {
	if containerID < 0 || containerID >= config.ContainerCount {
		log.Fatalf("ContainerID expected to be between 0 and %d, got: %d", config.ContainerCount-1, containerID)
	}
	for _, topic := range config.InputTopics {
		_, ok := config.TopicSerdes[topic]
		if !ok {
			log.Fatalf("Could not find Serde for topic '%s'", topic)
		}
	}
}

func mustSetupClient(config *TopicProcessorConfig, containerID int) (sarama.Client, []int, sarama.OffsetManager) {
	saramaConfig := sarama.NewConfig()
	saramaConfig.Consumer.Offsets.Initial = sarama.OffsetOldest // TODO: make this configurable
	client, err := sarama.NewClient(config.BrokerList, saramaConfig)
	if err != nil {
		log.Fatal(err)
	}
	partitions := config.partitionsForContainer(containerID)
	for _, partition := range partitions {
		_, ok := config.PartitionToContainerID[partition]
		if !ok {
			log.Print("Could not find PartitionToContainerID mapping for partition ", partition)
		}
	}
	offsetManager, err := sarama.NewOffsetManagerFromClient(config.kafkaConsumerGroup(), client)
	if err != nil {
		log.Fatal(err)
	}
	return client, partitions, offsetManager
}

// Start launches a deferred routine for topic processing.
func (tp *TopicProcessor) Start() {
	tp.waitGroup.Add(1)
	go func() {
		defer tp.waitGroup.Done()
		tp.runLoop()
	}()
}

// Shutdown safely shuts down topic processing, waiting for unfinished jobs
func (tp *TopicProcessor) Shutdown() {
	close(tp.shutdown)
	tp.waitGroup.Wait()
}

func (tp *TopicProcessor) runLoop() {
	consumerChan := tp.getConsumerMessagesChan()
	metricsTicker := time.NewTicker(tp.config.Config.MetricsUpdateInterval)
	var batchTickerChan <-chan time.Time
	var batchTicker *time.Ticker

	if tp.batchingEnabled {
		batchTicker = time.NewTicker(tp.batchWaitDuration)
		batchTickerChan = batchTicker.C
	} else {
		batchTickerChan = make(<-chan time.Time)
	}

	batches := make(map[int][]*sarama.ConsumerMessage)
	lengths := make(map[int]int)

	for _, partition := range tp.partitions {
		batches[partition] = make([]*sarama.ConsumerMessage, tp.batchSize)
		lengths[partition] = 0
	}

	for {
		select {
		case consumerMessage := <-consumerChan:
			if tp.batchingEnabled {
				partition := int(consumerMessage.Partition)
				batches[partition][lengths[partition]] = consumerMessage
				lengths[partition]++
				if lengths[partition] == tp.batchSize {
					tp.processConsumerMessageBatch(batches[partition], partition)
					lengths[partition] = 0
				}
			} else {
				tp.processConsumerMessage(consumerMessage)
			}
		case <-metricsTicker.C:
			tp.onMetricsTick()
		case <-batchTickerChan:
			for _, partition := range tp.partitions {
				tp.processConsumerMessageBatch(batches[partition][0:lengths[partition]], partition)
				lengths[partition] = 0
			}
		case <-tp.shutdown:
			tp.onShutdown(metricsTicker, batchTicker)
			return
		}
	}
}

func (tp *TopicProcessor) processConsumerMessage(consumerMessage *sarama.ConsumerMessage) {
	tp.incomingMessageCount.Inc(consumerMessage.Topic, strconv.Itoa(int(consumerMessage.Partition)))
	pp := tp.partitionProcessors[consumerMessage.Partition]
	producerMessages := pp.process(consumerMessage)
	err := tp.producer.SendMessages(producerMessages)
	if err != nil {
		tp.onProducerError(err)
	}
	pp.markOffsets(consumerMessage)
	for _, message := range producerMessages {
		tp.outgoingMessageCount.Inc(message.Topic, strconv.Itoa(int(message.Partition)))
	}
}

func (tp *TopicProcessor) processConsumerMessageBatch(messages []*sarama.ConsumerMessage, partition int) {
	for _, message := range messages {
		tp.incomingMessageCount.Inc(message.Topic, strconv.Itoa(int(message.Partition)))
	}
	pp := tp.partitionProcessors[int32(partition)]
	producerMessages := pp.processBatch(messages)
	err := tp.producer.SendMessages(producerMessages)
	if err != nil {
		tp.onProducerError(err)
	}
	pp.markOffsetsForBatch(messages)
	for _, message := range producerMessages {
		tp.outgoingMessageCount.Inc(message.Topic, strconv.Itoa(int(message.Partition)))
	}
}

func (tp *TopicProcessor) onShutdown(tickers ...*time.Ticker) {
	for _, ticker := range tickers {
		if ticker != nil {
			ticker.Stop()
		}
	}
	for _, pp := range tp.partitionProcessors {
		pp.onShutdown()
	}
	err := tp.producer.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = tp.client.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func (tp *TopicProcessor) getConsumerMessagesChan() <-chan *sarama.ConsumerMessage {
	consumerMessagesChan := make(chan *sarama.ConsumerMessage)
	for _, ch := range tp.consumerMessageChannels() {
		tp.waitGroup.Add(1)
		go func(c <-chan *sarama.ConsumerMessage) {
			defer tp.waitGroup.Done()
			for msg := range c {
				select {
				case consumerMessagesChan <- msg:
					continue
				case <-tp.shutdown:
					return
				}

			}
		}(ch)
	}
	return consumerMessagesChan
}

func (tp *TopicProcessor) onProducerError(err error) {
	log.Fatal(err)
}

func (tp *TopicProcessor) onMetricsTick() {
	for _, pp := range tp.partitionProcessors {
		pp.onMetricsTick()
	}
}

func (tp *TopicProcessor) consumerMessageChannels() []<-chan *sarama.ConsumerMessage {
	var chans []<-chan *sarama.ConsumerMessage
	for _, partitionProcessor := range tp.partitionProcessors {
		partitionChannels := partitionProcessor.consumerMessageChannels()
		chans = append(chans, partitionChannels...)
	}
	return chans
}

func mustSetupProducer(brokers []string, producerClientID string, requiredAcks sarama.RequiredAcks) sarama.SyncProducer {
	saramaConfig := sarama.NewConfig()
	saramaConfig.ClientID = producerClientID
	saramaConfig.Producer.Return.Successes = true
	saramaConfig.Producer.Partitioner = sarama.NewManualPartitioner
	saramaConfig.Producer.RequiredAcks = requiredAcks

	producer, err := sarama.NewSyncProducer(brokers, saramaConfig)
	if err != nil {
		log.Fatal(err)
	}

	return producer
}
<commit_msg>Don't process empty batch<commit_after>/*

kasper is a lightweight Kafka stream processing library.

*/

package kasper

import (
	"log"
	"sync"
	"time"
	"strconv"

	"github.com/Shopify/sarama"
)

// TopicProcessor describes kafka topic processor
type TopicProcessor struct {
	config              *TopicProcessorConfig
	containerID         int
	client              sarama.Client
	producer            sarama.SyncProducer
	offsetManager       sarama.OffsetManager
	partitionProcessors map[int32]*partitionProcessor
	inputTopics         []string
	partitions          []int
	shutdown            chan struct{}
	waitGroup           sync.WaitGroup
	batchingEnabled     bool
	batchSize           int
	batchWaitDuration   time.Duration

	incomingMessageCount        Counter
	outgoingMessageCount        Counter
	messagesBehindHighWaterMark Gauge
}

// MessageProcessor describes kafka message processor
type MessageProcessor interface {
	// Process message from Kafka input topics.
	// This is the function where you perform all needed actions, like
	// population KV storage or producing Kafka output messages
	Process(IncomingMessage, Sender, Coordinator)
}

// TBD
type BatchMessageProcessor interface {
	// TBD
	ProcessBatch([]*IncomingMessage, Sender, Coordinator)
}

// NewTopicProcessor creates a new TopicProcessor with the given config.
// It requires a factory function that creates MessageProcessor instances and a container id.
// The container id must be a number between 0 and config.ContainerCount - 1.
func NewTopicProcessor(config *TopicProcessorConfig, makeProcessor func() MessageProcessor, containerID int) *TopicProcessor {
	mustHaveValidConfig(config, containerID)
	inputTopics := config.InputTopics
	client, partitions, offsetManager := mustSetupClient(config, containerID)
	partitionProcessors := make(map[int32]*partitionProcessor, len(partitions))
	requiredAcks := config.Config.RequiredAcks
	producer := mustSetupProducer(config.BrokerList, config.producerClientID(containerID), requiredAcks)
	topicProcessor := TopicProcessor{
		config:              config,
		containerID:         containerID,
		client:              client,
		producer:            producer,
		offsetManager:       offsetManager,
		partitionProcessors: partitionProcessors,
		inputTopics:         inputTopics,
		partitions:          partitions,
		shutdown:            make(chan struct{}),
		waitGroup:           sync.WaitGroup{},
		batchingEnabled:     false,
	}
	setupMetrics(&topicProcessor, config.Config.MetricsProvider)
	for _, partition := range partitions {
		processor := makeProcessor()
		partitionProcessors[int32(partition)] = newPartitionProcessor(&topicProcessor, processor, partition)
	}
	return &topicProcessor
}

// TBD
type BatchingOpts struct {
	makeProcessor     func() BatchMessageProcessor
	batchSize         int
	batchWaitDuration time.Duration
}

// TBD
func NewBatchTopicProcessor(config *TopicProcessorConfig, opts BatchingOpts, containerID int) *TopicProcessor {
	mustHaveValidConfig(config, containerID)
	inputTopics := config.InputTopics
	client, partitions, offsetManager := mustSetupClient(config, containerID)
	partitionProcessors := make(map[int32]*partitionProcessor, len(partitions))
	producer := mustSetupProducer(config.BrokerList, config.producerClientID(containerID), config.Config.RequiredAcks)
	topicProcessor := TopicProcessor{
		config:              config,
		containerID:         containerID,
		client:              client,
		producer:            producer,
		offsetManager:       offsetManager,
		partitionProcessors: partitionProcessors,
		inputTopics:         inputTopics,
		partitions:          partitions,
		shutdown:            make(chan struct{}),
		waitGroup:           sync.WaitGroup{},
		batchingEnabled:     true,
		batchSize:           opts.batchSize,
		batchWaitDuration:   opts.batchWaitDuration,
	}
	setupMetrics(&topicProcessor, config.Config.MetricsProvider)
	for _, partition := range partitions {
		processor := opts.makeProcessor()
		partitionProcessors[int32(partition)] = newBatchPartitionProcessor(&topicProcessor, processor, partition)
	}
	return &topicProcessor
}

func setupMetrics(tp *TopicProcessor, provider MetricsProvider) {
	tp.incomingMessageCount = provider.NewCounter("incoming_message_count", "Number of incoming messages received", "topic", "partition")
	tp.outgoingMessageCount = provider.NewCounter("outgoing_message_count", "Number of outgoing messages sent", "topic", "partition")
	tp.messagesBehindHighWaterMark = provider.NewGauge("messages_behind_high_water_mark_count", "Number of messages remaining to consume on the topic/partition", "topic", "partition")
}

func mustHaveValidConfig(config *TopicProcessorConfig, containerID int) {
	if containerID < 0 || containerID >= config.ContainerCount {
		log.Fatalf("ContainerID expected to be between 0 and %d, got: %d", config.ContainerCount-1, containerID)
	}
	for _, topic := range config.InputTopics {
		_, ok := config.TopicSerdes[topic]
		if !ok {
			log.Fatalf("Could not find Serde for topic '%s'", topic)
		}
	}
}

func mustSetupClient(config *TopicProcessorConfig, containerID int) (sarama.Client, []int, sarama.OffsetManager) {
	saramaConfig := sarama.NewConfig()
	saramaConfig.Consumer.Offsets.Initial = sarama.OffsetOldest // TODO: make this configurable
	client, err := sarama.NewClient(config.BrokerList, saramaConfig)
	if err != nil {
		log.Fatal(err)
	}
	partitions := config.partitionsForContainer(containerID)
	for _, partition := range partitions {
		_, ok := config.PartitionToContainerID[partition]
		if !ok {
			log.Print("Could not find PartitionToContainerID mapping for partition ", partition)
		}
	}
	offsetManager, err := sarama.NewOffsetManagerFromClient(config.kafkaConsumerGroup(), client)
	if err != nil {
		log.Fatal(err)
	}
	return client, partitions, offsetManager
}

// Start launches a deferred routine for topic processing.
func (tp *TopicProcessor) Start() {
	tp.waitGroup.Add(1)
	go func() {
		defer tp.waitGroup.Done()
		tp.runLoop()
	}()
}

// Shutdown safely shuts down topic processing, waiting for unfinished jobs
func (tp *TopicProcessor) Shutdown() {
	close(tp.shutdown)
	tp.waitGroup.Wait()
}

func (tp *TopicProcessor) runLoop() {
	consumerChan := tp.getConsumerMessagesChan()
	metricsTicker := time.NewTicker(tp.config.Config.MetricsUpdateInterval)
	var batchTickerChan <-chan time.Time
	var batchTicker *time.Ticker

	if tp.batchingEnabled {
		batchTicker = time.NewTicker(tp.batchWaitDuration)
		batchTickerChan = batchTicker.C
	} else {
		batchTickerChan = make(<-chan time.Time)
	}

	batches := make(map[int][]*sarama.ConsumerMessage)
	lengths := make(map[int]int)

	for _, partition := range tp.partitions {
		batches[partition] = make([]*sarama.ConsumerMessage, tp.batchSize)
		lengths[partition] = 0
	}

	for {
		select {
		case consumerMessage := <-consumerChan:
			if tp.batchingEnabled {
				partition := int(consumerMessage.Partition)
				batches[partition][lengths[partition]] = consumerMessage
				lengths[partition]++
				if lengths[partition] == tp.batchSize {
					tp.processConsumerMessageBatch(batches[partition], partition)
					lengths[partition] = 0
				}
			} else {
				tp.processConsumerMessage(consumerMessage)
			}
		case <-metricsTicker.C:
			tp.onMetricsTick()
		case <-batchTickerChan:
			for _, partition := range tp.partitions {
				if lengths[partition] == 0 {
					continue
				}
				tp.processConsumerMessageBatch(batches[partition][0:lengths[partition]], partition)
				lengths[partition] = 0
			}
		case <-tp.shutdown:
			tp.onShutdown(metricsTicker, batchTicker)
			return
		}
	}
}

func (tp *TopicProcessor) processConsumerMessage(consumerMessage *sarama.ConsumerMessage) {
	tp.incomingMessageCount.Inc(consumerMessage.Topic, strconv.Itoa(int(consumerMessage.Partition)))
	pp := tp.partitionProcessors[consumerMessage.Partition]
	producerMessages := pp.process(consumerMessage)
	err := tp.producer.SendMessages(producerMessages)
	if err != nil {
		tp.onProducerError(err)
	}
	pp.markOffsets(consumerMessage)
	for _, message := range producerMessages {
		tp.outgoingMessageCount.Inc(message.Topic, strconv.Itoa(int(message.Partition)))
	}
}

func (tp *TopicProcessor) processConsumerMessageBatch(messages []*sarama.ConsumerMessage, partition int) {
	for _, message := range messages {
		tp.incomingMessageCount.Inc(message.Topic, strconv.Itoa(int(message.Partition)))
	}
	pp := tp.partitionProcessors[int32(partition)]
	producerMessages := pp.processBatch(messages)
	err := tp.producer.SendMessages(producerMessages)
	if err != nil {
		tp.onProducerError(err)
	}
	pp.markOffsetsForBatch(messages)
	for _, message := range producerMessages {
		tp.outgoingMessageCount.Inc(message.Topic, strconv.Itoa(int(message.Partition)))
	}
}

func (tp *TopicProcessor) onShutdown(tickers ...*time.Ticker) {
	for _, ticker := range tickers {
		if ticker != nil {
			ticker.Stop()
		}
	}
	for _, pp := range tp.partitionProcessors {
		pp.onShutdown()
	}
	err := tp.producer.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = tp.client.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func (tp *TopicProcessor) getConsumerMessagesChan() <-chan *sarama.ConsumerMessage {
	consumerMessagesChan := make(chan *sarama.ConsumerMessage)
	for _, ch := range tp.consumerMessageChannels() {
		tp.waitGroup.Add(1)
		go func(c <-chan *sarama.ConsumerMessage) {
			defer tp.waitGroup.Done()
			for msg := range c {
				select {
				case consumerMessagesChan <- msg:
					continue
				case <-tp.shutdown:
					return
				}

			}
		}(ch)
	}
	return consumerMessagesChan
}

func (tp *TopicProcessor) onProducerError(err error) {
	log.Fatal(err)
}

func (tp *TopicProcessor) onMetricsTick() {
	for _, pp := range tp.partitionProcessors {
		pp.onMetricsTick()
	}
}

func (tp *TopicProcessor) consumerMessageChannels() []<-chan *sarama.ConsumerMessage {
	var chans []<-chan *sarama.ConsumerMessage
	for _, partitionProcessor := range tp.partitionProcessors {
		partitionChannels := partitionProcessor.consumerMessageChannels()
		chans = append(chans, partitionChannels...)
	}
	return chans
}

func mustSetupProducer(brokers []string, producerClientID string, requiredAcks sarama.RequiredAcks) sarama.SyncProducer {
	saramaConfig := sarama.NewConfig()
	saramaConfig.ClientID = producerClientID
	saramaConfig.Producer.Return.Successes = true
	saramaConfig.Producer.Partitioner = sarama.NewManualPartitioner
	saramaConfig.Producer.RequiredAcks = requiredAcks

	producer, err := sarama.NewSyncProducer(brokers, saramaConfig)
	if err != nil {
		log.Fatal(err)
	}

	return producer
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package errors

import (
	"fmt"
	"io"
	"net/http"
)

var _ error = ErrUnexpectedStatus{}

// ErrUnexpectedStatus is returned if a registry API request returned with unexpected HTTP status
type ErrUnexpectedStatus struct {
	Status                    string
	StatusCode                int
	Body                      []byte
	RequestURL, RequestMethod string
}

func (e ErrUnexpectedStatus) Error() string {
	return fmt.Sprintf("unexpected status: %s", e.Status)
}

// NewUnexpectedStatusErr creates an ErrUnexpectedStatus from HTTP response
func NewUnexpectedStatusErr(resp *http.Response) error {
	var b []byte
	if resp.Body != nil {
		b, _ = io.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB
	}
	err := ErrUnexpectedStatus{
		Body:          b,
		Status:        resp.Status,
		StatusCode:    resp.StatusCode,
		RequestMethod: resp.Request.Method,
	}
	if resp.Request.URL != nil {
		err.RequestURL = resp.Request.URL.String()
	}
	return err
}
<commit_msg>Improve ErrUnexpectedStatus default string<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package errors

import (
	"fmt"
	"io"
	"net/http"
)

var _ error = ErrUnexpectedStatus{}

// ErrUnexpectedStatus is returned if a registry API request returned with unexpected HTTP status
type ErrUnexpectedStatus struct {
	Status                    string
	StatusCode                int
	Body                      []byte
	RequestURL, RequestMethod string
}

func (e ErrUnexpectedStatus) Error() string {
	return fmt.Sprintf("unexpected status from %s request to %s: %s", e.RequestMethod, e.RequestURL, e.Status)
}

// NewUnexpectedStatusErr creates an ErrUnexpectedStatus from HTTP response
func NewUnexpectedStatusErr(resp *http.Response) error {
	var b []byte
	if resp.Body != nil {
		b, _ = io.ReadAll(io.LimitReader(resp.Body, 64000)) // 64KB
	}
	err := ErrUnexpectedStatus{
		Body:          b,
		Status:        resp.Status,
		StatusCode:    resp.StatusCode,
		RequestMethod: resp.Request.Method,
	}
	if resp.Request.URL != nil {
		err.RequestURL = resp.Request.URL.String()
	}
	return err
}
<|endoftext|>


<commit_before>package fslock_test

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/utils/fslock"
)

func Test(t *testing.T) {
	TestingT(t)
}

type fslockSuite struct{}

var _ = Suite(fslockSuite{})

func (fslockSuite) SetUpSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Millisecond)
}

func (fslockSuite) TearDownSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Second)
}

// This test also happens to test that locks can get created when the parent
// lock directory doesn't exist.
func (fslockSuite) TestValidNamesLockDir(c *C) {

	for _, name := range []string{
		"a",
		"longer",
		"longer-with.special-characters",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, IsNil)
	}
}

func (fslockSuite) TestInvalidNames(c *C) {

	for _, name := range []string{
		"NoCapitals",
		"no+plus",
		"no/slash",
		"no\\backslash",
		"no$dollar",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, ErrorMatches, "Invalid lock name .*")
	}
}

func (fslockSuite) TestNewLockWithExistingDir(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	_, err = fslock.NewLock(dir, "special")
	c.Assert(err, IsNil)
}

func (fslockSuite) TestNewLockWithExistingFileInPlace(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	path := path.Join(dir, "locks")
	err = ioutil.WriteFile(path, []byte("foo"), 0644)
	c.Assert(err, IsNil)

	_, err = fslock.NewLock(path, "special")
	c.Assert(err, ErrorMatches, `.* not a directory`)
}

func (fslockSuite) TestIsLockHeldBasics(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)

	err = lock.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, true)

	err = lock.Unlock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestIsLockHeldTwoLocks(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock2.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestLockBlocks(c *C) {

	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	acquired := make(chan struct{})
	err = lock1.Lock()
	c.Assert(err, IsNil)

	go func() {
		lock2.Lock()
		acquired <- struct{}{}
		close(acquired)
	}()

	// Waiting for something not to happen is inherently hard...
	select {
	case <-acquired:
		c.Fatalf("Unexpected lock acquisition")
	case <-time.After(50 * time.Millisecond):
		// all good
	}

	err = lock1.Unlock()
	c.Assert(err, IsNil)

	select {
	case <-acquired:
		// all good
	case <-time.After(50 * time.Millisecond):
		c.Fatalf("Expected lock acquisition")
	}

	c.Assert(lock2.IsLockHeld(), Equals, true)
}

func (fslockSuite) TestLockWithTimeoutUnlocked(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, IsNil)
}

func (fslockSuite) TestLockWithTimeoutLocked(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	err = lock2.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, Equals, fslock.ErrTimeout)
}

func (fslockSuite) TestUnlock(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.Unlock()
	c.Assert(err, Equals, fslock.ErrLockNotHeld)
}

func (fslockSuite) TestIsLocked(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	c.Assert(lock1.IsLocked(), Equals, true)
	c.Assert(lock2.IsLocked(), Equals, true)
}

func (fslockSuite) TestBreakLock(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	err = lock2.BreakLock()
	c.Assert(err, IsNil)
	c.Assert(lock2.IsLocked(), Equals, false)

	// Normally locks are broken due to client crashes, not duration.
	err = lock1.Unlock()
	c.Assert(err, Equals, fslock.ErrLockNotHeld)
}

func (fslockSuite) TestMessage(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	c.Assert(lock.GetMessage(), Equals, "")

	err = lock.SetMessage("my message")
	c.Assert(err, Equals, fslock.ErrLockNotHeld)

	err = lock.Lock()
	c.Assert(err, IsNil)

	err = lock.SetMessage("my message")
	c.Assert(err, IsNil)
	c.Assert(lock.GetMessage(), Equals, "my message")

	// Unlocking removes the message.
	err = lock.Unlock()
	c.Assert(err, IsNil)
	c.Assert(lock.GetMessage(), Equals, "")
}

func (fslockSuite) TestMessageAcrossLocks(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)
	err = lock1.SetMessage("very busy")
	c.Assert(err, IsNil)

	c.Assert(lock2.GetMessage(), Equals, "very busy")
}
<commit_msg>Test breaking a lock that isn't there.<commit_after>package fslock_test

import (
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/utils/fslock"
)

func Test(t *testing.T) {
	TestingT(t)
}

type fslockSuite struct{}

var _ = Suite(fslockSuite{})

func (fslockSuite) SetUpSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Millisecond)
}

func (fslockSuite) TearDownSuite(c *C) {
	fslock.SetLockWaitDelay(1 * time.Second)
}

// This test also happens to test that locks can get created when the parent
// lock directory doesn't exist.
func (fslockSuite) TestValidNamesLockDir(c *C) {

	for _, name := range []string{
		"a",
		"longer",
		"longer-with.special-characters",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, IsNil)
	}
}

func (fslockSuite) TestInvalidNames(c *C) {

	for _, name := range []string{
		"NoCapitals",
		"no+plus",
		"no/slash",
		"no\\backslash",
		"no$dollar",
	} {
		dir := c.MkDir()
		_, err := fslock.NewLock(dir, name)
		c.Assert(err, ErrorMatches, "Invalid lock name .*")
	}
}

func (fslockSuite) TestNewLockWithExistingDir(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	_, err = fslock.NewLock(dir, "special")
	c.Assert(err, IsNil)
}

func (fslockSuite) TestNewLockWithExistingFileInPlace(c *C) {
	dir := c.MkDir()
	err := os.MkdirAll(dir, 0755)
	c.Assert(err, IsNil)
	path := path.Join(dir, "locks")
	err = ioutil.WriteFile(path, []byte("foo"), 0644)
	c.Assert(err, IsNil)

	_, err = fslock.NewLock(path, "special")
	c.Assert(err, ErrorMatches, `.* not a directory`)
}

func (fslockSuite) TestIsLockHeldBasics(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)

	err = lock.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, true)

	err = lock.Unlock()
	c.Assert(err, IsNil)
	c.Assert(lock.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestIsLockHeldTwoLocks(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)
	c.Assert(lock2.IsLockHeld(), Equals, false)
}

func (fslockSuite) TestLockBlocks(c *C) {

	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	acquired := make(chan struct{})
	err = lock1.Lock()
	c.Assert(err, IsNil)

	go func() {
		lock2.Lock()
		acquired <- struct{}{}
		close(acquired)
	}()

	// Waiting for something not to happen is inherently hard...
	select {
	case <-acquired:
		c.Fatalf("Unexpected lock acquisition")
	case <-time.After(50 * time.Millisecond):
		// all good
	}

	err = lock1.Unlock()
	c.Assert(err, IsNil)

	select {
	case <-acquired:
		// all good
	case <-time.After(50 * time.Millisecond):
		c.Fatalf("Expected lock acquisition")
	}

	c.Assert(lock2.IsLockHeld(), Equals, true)
}

func (fslockSuite) TestLockWithTimeoutUnlocked(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, IsNil)
}

func (fslockSuite) TestLockWithTimeoutLocked(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	err = lock2.LockWithTimeout(10 * time.Millisecond)
	c.Assert(err, Equals, fslock.ErrTimeout)
}

func (fslockSuite) TestUnlock(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock.Unlock()
	c.Assert(err, Equals, fslock.ErrLockNotHeld)
}

func (fslockSuite) TestIsLocked(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	c.Assert(lock1.IsLocked(), Equals, true)
	c.Assert(lock2.IsLocked(), Equals, true)
}

func (fslockSuite) TestBreakLock(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)

	err = lock2.BreakLock()
	c.Assert(err, IsNil)
	c.Assert(lock2.IsLocked(), Equals, false)

	// Normally locks are broken due to client crashes, not duration.
	err = lock1.Unlock()
	c.Assert(err, Equals, fslock.ErrLockNotHeld)

	// Breaking a non-existant isn't an error
	err = lock2.BreakLock()
	c.Assert(err, IsNil)
}

func (fslockSuite) TestMessage(c *C) {
	dir := c.MkDir()
	lock, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	c.Assert(lock.GetMessage(), Equals, "")

	err = lock.SetMessage("my message")
	c.Assert(err, Equals, fslock.ErrLockNotHeld)

	err = lock.Lock()
	c.Assert(err, IsNil)

	err = lock.SetMessage("my message")
	c.Assert(err, IsNil)
	c.Assert(lock.GetMessage(), Equals, "my message")

	// Unlocking removes the message.
	err = lock.Unlock()
	c.Assert(err, IsNil)
	c.Assert(lock.GetMessage(), Equals, "")
}

func (fslockSuite) TestMessageAcrossLocks(c *C) {
	dir := c.MkDir()
	lock1, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)
	lock2, err := fslock.NewLock(dir, "testing")
	c.Assert(err, IsNil)

	err = lock1.Lock()
	c.Assert(err, IsNil)
	err = lock1.SetMessage("very busy")
	c.Assert(err, IsNil)

	c.Assert(lock2.GetMessage(), Equals, "very busy")
}
<|endoftext|>


<commit_before>package couch

import (
    "strings"
    "fmt"
    "os"
    "json"
    "bytes"
    "http"
    "net"
    "io/ioutil"
)

//
// Helper and utility functions (private)
//

// Replaces all instances of from with to in s (quite inefficient right now)
func replace(s, from, to string) string {
    toks := strings.SplitAfter(s, from, 0)
    newstr := ""
    for i, tok := range toks {
        if i < len(toks)-1 {
            if !strings.HasSuffix(tok, from) {
                panic("problem in replace")
            }
            newtok := tok[0 : len(tok)-len(from)]
            newstr = newstr + newtok + to
        } else {
            newstr = newstr + tok
        }
    }
    return newstr
}

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (string, os.Error) {
    buf := new(bytes.Buffer)
    if err := json.Marshal(buf, p); err != nil {
        return "", err
    }
    return buf.String(), nil
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) os.Error {
    if ok, errtok := json.Unmarshal(s, p); !ok {
        return os.NewError(fmt.Sprintf("error unmarshaling: %s", errtok))
    }
    return nil
}

// Since the json pkg doesn't handle fields beginning with _, we need to
// convert "_id" and "_rev" to "Id" and "Rev" to extract that data.
func temp_hack_go_to_json(json_str string) string {
    json_str = replace(json_str, `"Id"`, `"_id"`)
    json_str = replace(json_str, `"Rev"`, `"_rev"`)
    return json_str
}

func temp_hack_json_to_go(json_str string) string {
    json_str = replace(json_str, `"_id"`, `"Id"`)
    json_str = replace(json_str, `"_rev"`, `"Rev"`)
    return json_str
}

type IdAndRev struct {
    Id  string
    Rev string
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    // this assumes the temp replacement hack has already been applied
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

//
// Database object + public methods
//

type Database struct {
    Host string
    Port string
    Name string
}

func (p Database) BaseURL() string {
    return fmt.Sprintf("http://%s:%s", p.Host, p.Port)
}

func (p Database) DBURL() string {
    return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Test whether CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
    url := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs")
    s := url_to_string(url)
    if len(s) > 0 {
        return true
    }
    return false
}

type DatabaseInfo struct {
    Db_name string
    // other stuff too, ignore for now
}

// Test whether specified database exists in specified CouchDB instance
func (p Database) Exists() bool {
    di := new(DatabaseInfo)
    if err := from_JSON(url_to_string(p.DBURL()), di); err != nil {
        return false
    }
    if di.Db_name != p.Name {
        return false
    }
    return true
}

func NewDatabase(host, port, name string) (Database, os.Error) {
    db := Database{host, port, name}
    if !db.Running() {
        return db, os.NewError("CouchDB not running")
    }
    if !db.Exists() {
        // TODO create database
    }
    return db, nil
}

type InsertResponse struct {
    Ok  bool
    Id  string
    Rev string
}

// Inserts document to CouchDB, returning id and rev on success.
func (p Database) Insert(d interface{}) (string, string, os.Error) {
    json_str, err := to_JSON(d)
    if err != nil {
        return "", "", err
    }
    json_str = temp_hack_go_to_json(json_str)

    r, err := http.Post(p.DBURL(), "application/json", bytes.NewBufferString(json_str))
    if err != nil {
        return "", "", err
    }

    b, err := ioutil.ReadAll(r.Body)
    r.Body.Close()
    if err != nil {
        return "", "", err
    }

    ir := new(InsertResponse)
    if err := from_JSON(string(b), ir); err != nil {
        return "", "", err
    }

    if !ir.Ok {
        return "", "", os.NewError("CouchDB returned not-OK")
    }

    return ir.Id, ir.Rev, nil
}

// Unmarshals the document matching id to the given interface, returning rev.
func (p Database) Retrieve(id string, d interface{}) (string, os.Error) {
    if len(id) <= 0 {
        return "", os.NewError("no id specified")
    }

    json_str := url_to_string(fmt.Sprintf("%s/%s", p.DBURL(), id))
    json_str = temp_hack_json_to_go(json_str)
    _, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }

    return rev, from_JSON(json_str, d)
}

// Edits the given document, which must specify both Id and Rev fields, and
// returns the new revision.
func (p Database) Edit(d interface{}) (string, os.Error) {
    _, rev, err := p.Insert(d)
    return rev, err
}

// Deletes document given by id and rev.
func (p Database) Delete(id, rev string) os.Error {
    // Set up request
    var req http.Request
    req.Method = "DELETE"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string{
        "Content-Type": "application/json",
        "If-Match":     rev,
    }
    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(fmt.Sprintf("%s/%s", p.DBURL(), id))

    // Make connection
    conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
    if err != nil {
        return err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return err
    }

    // Read response
    r, err := http_conn.Read()
    if r == nil {
        return os.NewError("no response")
    }
    if err != nil {
        return err
    }
    data, _ := ioutil.ReadAll(r.Body)
    r.Body.Close()
    ir := new(InsertResponse)
    if err := from_JSON(string(data), ir); err != nil {
        return err
    }
    if !ir.Ok {
        return os.NewError("CouchDB returned not-OK")
    }

    return nil
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view/key combo.
// view should be eg. "_design/my_foo/_view/my_bar"
func (p Database) QueryByView(view, key string) ([]string, os.Error) {
    if len(view) <= 0 || len(key) <= 0 {
        return make([]string, 0), os.NewError("empty view or key")
    }

    parameters := fmt.Sprintf(`key="%s"`, http.URLEscape(key))
    full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0), err
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids, nil
}
<commit_msg>Create DB if it doesn't exist in NewDatabase<commit_after>package couch

import (
    "strings"
    "fmt"
    "os"
    "json"
    "bytes"
    "http"
    "net"
    "io/ioutil"
)

//
// Helper and utility functions (private)
//

// Replaces all instances of from with to in s (quite inefficient right now)
func replace(s, from, to string) string {
    toks := strings.SplitAfter(s, from, 0)
    newstr := ""
    for i, tok := range toks {
        if i < len(toks)-1 {
            if !strings.HasSuffix(tok, from) {
                panic("problem in replace")
            }
            newtok := tok[0 : len(tok)-len(from)]
            newstr = newstr + newtok + to
        } else {
            newstr = newstr + tok
        }
    }
    return newstr
}

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (string, os.Error) {
    buf := new(bytes.Buffer)
    if err := json.Marshal(buf, p); err != nil {
        return "", err
    }
    return buf.String(), nil
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) os.Error {
    if ok, errtok := json.Unmarshal(s, p); !ok {
        return os.NewError(fmt.Sprintf("error unmarshaling: %s", errtok))
    }
    return nil
}

// Since the json pkg doesn't handle fields beginning with _, we need to
// convert "_id" and "_rev" to "Id" and "Rev" to extract that data.
func temp_hack_go_to_json(json_str string) string {
    json_str = replace(json_str, `"Id"`, `"_id"`)
    json_str = replace(json_str, `"Rev"`, `"_rev"`)
    return json_str
}

func temp_hack_json_to_go(json_str string) string {
    json_str = replace(json_str, `"_id"`, `"Id"`)
    json_str = replace(json_str, `"_rev"`, `"Rev"`)
    return json_str
}

type IdAndRev struct {
    Id  string
    Rev string
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    // this assumes the temp replacement hack has already been applied
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

type CreateResponse struct {
    Ok bool
}

func (p Database) create_database() os.Error {
    // Set up request
    var req http.Request
    req.Method = "PUT"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string{
        "Content-Type": "application/json",
    }
    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(p.DBURL())

    // Make connection
    conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
    if err != nil {
        return err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return err
    }

    // Read response
    r, err := http_conn.Read()
    if r == nil {
        return os.NewError("no response")
    }
    if err != nil {
        return err
    }
    data, _ := ioutil.ReadAll(r.Body)
    r.Body.Close()
    cr := new(CreateResponse)
    if err := from_JSON(string(data), cr); err != nil {
        return err
    }
    if !cr.Ok {
        return os.NewError("CouchDB returned not-OK")
    }

    return nil
}

//
// Database object + public methods
//

type Database struct {
    Host string
    Port string
    Name string
}

func (p Database) BaseURL() string {
    return fmt.Sprintf("http://%s:%s", p.Host, p.Port)
}

func (p Database) DBURL() string {
    return fmt.Sprintf("%s/%s", p.BaseURL(), p.Name)
}

// Test whether CouchDB is running (ignores Database.Name)
func (p Database) Running() bool {
    url := fmt.Sprintf("%s/%s", p.BaseURL(), "_all_dbs")
    s := url_to_string(url)
    if len(s) > 0 {
        return true
    }
    return false
}

type DatabaseInfo struct {
    Db_name string
    // other stuff too, ignore for now
}

// Test whether specified database exists in specified CouchDB instance
func (p Database) Exists() bool {
    di := new(DatabaseInfo)
    if err := from_JSON(url_to_string(p.DBURL()), di); err != nil {
        return false
    }
    if di.Db_name != p.Name {
        return false
    }
    return true
}

func NewDatabase(host, port, name string) (Database, os.Error) {
    db := Database{host, port, name}
    if !db.Running() {
        return db, os.NewError("CouchDB not running")
    }
    if !db.Exists() {
        if err := db.create_database(); err != nil {
            return db, err
        }
    }
    return db, nil
}

type InsertResponse struct {
    Ok  bool
    Id  string
    Rev string
}

// Inserts document to CouchDB, returning id and rev on success.
func (p Database) Insert(d interface{}) (string, string, os.Error) {
    json_str, err := to_JSON(d)
    if err != nil {
        return "", "", err
    }
    json_str = temp_hack_go_to_json(json_str)

    r, err := http.Post(p.DBURL(), "application/json", bytes.NewBufferString(json_str))
    if err != nil {
        return "", "", err
    }

    b, err := ioutil.ReadAll(r.Body)
    r.Body.Close()
    if err != nil {
        return "", "", err
    }

    ir := new(InsertResponse)
    if err := from_JSON(string(b), ir); err != nil {
        return "", "", err
    }

    if !ir.Ok {
        return "", "", os.NewError("CouchDB returned not-OK")
    }

    return ir.Id, ir.Rev, nil
}

// Unmarshals the document matching id to the given interface, returning rev.
func (p Database) Retrieve(id string, d interface{}) (string, os.Error) {
    if len(id) <= 0 {
        return "", os.NewError("no id specified")
    }

    json_str := url_to_string(fmt.Sprintf("%s/%s", p.DBURL(), id))
    json_str = temp_hack_json_to_go(json_str)
    _, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }

    return rev, from_JSON(json_str, d)
}

// Edits the given document, which must specify both Id and Rev fields, and
// returns the new revision.
func (p Database) Edit(d interface{}) (string, os.Error) {
    _, rev, err := p.Insert(d)
    return rev, err
}

// Deletes document given by id and rev.
func (p Database) Delete(id, rev string) os.Error {
    // Set up request
    var req http.Request
    req.Method = "DELETE"
    req.ProtoMajor = 1
    req.ProtoMinor = 1
    req.Close = true
    req.Header = map[string]string{
        "Content-Type": "application/json",
        "If-Match":     rev,
    }
    req.TransferEncoding = []string{"chunked"}
    req.URL, _ = http.ParseURL(fmt.Sprintf("%s/%s", p.DBURL(), id))

    // Make connection
    conn, err := net.Dial("tcp", "", fmt.Sprintf("%s:%s", p.Host, p.Port))
    if err != nil {
        return err
    }
    http_conn := http.NewClientConn(conn, nil)
    defer http_conn.Close()
    if err := http_conn.Write(&req); err != nil {
        return err
    }

    // Read response
    r, err := http_conn.Read()
    if r == nil {
        return os.NewError("no response")
    }
    if err != nil {
        return err
    }
    data, _ := ioutil.ReadAll(r.Body)
    r.Body.Close()
    ir := new(InsertResponse)
    if err := from_JSON(string(data), ir); err != nil {
        return err
    }
    if !ir.Ok {
        return os.NewError("CouchDB returned not-OK")
    }

    return nil
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view/key combo.
// view should be eg. "_design/my_foo/_view/my_bar"
func (p Database) QueryByView(view, key string) ([]string, os.Error) {
    if len(view) <= 0 || len(key) <= 0 {
        return make([]string, 0), os.NewError("empty view or key")
    }

    parameters := fmt.Sprintf(`key="%s"`, http.URLEscape(key))
    full_url := fmt.Sprintf("%s/%s?%s", p.DBURL(), view, parameters)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0), err
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids, nil
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "lumberjack"
  "time"
  proto "code.google.com/p/goprotobuf/proto"
)

func emit(events []*lumberjack.FileEvent) {
  envelope := &lumberjack.EventEnvelope{Events: events}
  data, _ := proto.Marshal(envelope)
  fmt.Printf("Flushing !!! %d events: %d encoded\n", len(events), len(data))
} /* emit */

func main() {
  // TODO(sissel): support flags for setting... stuff
  event_chan := make(chan *lumberjack.FileEvent, 32)
  emitter_chan := make(chan *lumberjack.EventEnvelope, 1)

  // The basic model of execution:
  // - prospector: finds files in paths/globs to harvest, starts harvesters
  // - harvester: reads a file, sends events to the spooler
  // - spooler: buffers events until ready to flush to the emitter
  // - emitter: writes to the network, notifies registrar
  // - registrar: records positions of files read
  // Finally, prospector uses the registrar information, on restart, to
  // determine where in each file to resume a harvester.

  // TODO(sissel): need a prospector scan for files and launch harvesters

  // Example dummy harvester
  h := lumberjack.Harvester{Path: "/var/log/messages"}
  go h.Harvest(event_chan)
  go h.Harvest(event_chan)

  var window_size uint64 = 1024 // Make this a flag
  var idle_timeout time.Duration = 1 * time.Second // Make this a flag

  // harvester -> spooler
  go lumberjack.Spooler(event_chan, emitter_chan, window_size, idle_timeout)

  // spooler -> publisher
  for x := range emitter_chan {
    // got a bunch of events, ship them out.
    fmt.Printf("Spooler gave me %d events\n", len(x.Events))
  }

  // emitter should send acknowledgements to the registrar
  // registrar records last acknowledged positions in all files.
} /* main */
<commit_msg>- s/emitter/publisher/<commit_after>package main

import (
  "fmt"
  "lumberjack"
  "os"
  "time"
  proto "code.google.com/p/goprotobuf/proto"
)

func emit(events []*lumberjack.FileEvent) {
  envelope := &lumberjack.EventEnvelope{Events: events}
  data, _ := proto.Marshal(envelope)
  fmt.Printf("Flushing !!! %d events: %d encoded\n", len(events), len(data))
} /* emit */

func main() {
  // TODO(sissel): support flags for setting... stuff
  event_chan := make(chan *lumberjack.FileEvent, 32)
  publisher_chan := make(chan *lumberjack.EventEnvelope, 1)

  // The basic model of execution:
  // - prospector: finds files in paths/globs to harvest, starts harvesters
  // - harvester: reads a file, sends events to the spooler
  // - spooler: buffers events until ready to flush to the publisher
  // - publisher: writes to the network, notifies registrar
  // - registrar: records positions of files read
  // Finally, prospector uses the registrar information, on restart, to
  // determine where in each file to resume a harvester.

  // TODO(sissel): need a prospector scan for files and launch harvesters

  // Prospect the globs/paths given on the command line and launch harvesters
  go lumberjack.Prospect(os.Args[1:], event_chan)

  var window_size uint64 = 1024 // Make this a flag
  var idle_timeout time.Duration = 1 * time.Second // Make this a flag

  // Harvesters dump events into the spooler.
  go lumberjack.Spooler(event_chan, publisher_chan, window_size, idle_timeout)

  // Spooler flushes periodically to the publisher
  for x := range publisher_chan {
    // got a bunch of events, ship them out.
    fmt.Printf("Spooler gave me %d events\n", len(x.Events))
    for _, event := range x.Events {
      fmt.Println(event)
    }
  }

  // TODO(sissel): publisher should send state to the registrar
  // TODO(sissel): registrar records last acknowledged positions in all files.
} /* main */
<|endoftext|>