<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"fmt"
	"container/list"
	zmq "github.com/alecthomas/gozmq"
)

type InitParams struct {
	Store *EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.Close()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) Close() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	nbrOfChanges int
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems) {
	a, b := zmq.Poll(items, -1)
	notifier <- zmqPollResult{a, b}
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
func loopServer(estore *EventStore, evpubsock, frontend zmq.Socket) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan [][]byte)
	go asyncPoll(pollchan, toPoll)
	for {
		select {
		case <- pollchan:
			if toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				go handleRequest(respchan, estore, msg)
			}
			go asyncPoll(pollchan, toPoll)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				fmt.Println(err)
			}
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan StoredEvent, evpub zmq.Socket) {
	msg := make([][]byte, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			// TODO: Use logger
			fmt.Println(err)
		}
	}
}

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan [][]byte, estore *EventStore, msg [][]byte) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	// TODO: Possibly wrap ZeroMQ router frames into a Type before
	// calling this method. That would yield a nicer API without
	// nitty gritty ZeroMQ details.
	resptemplate := list.New()
	emptyFrame := []byte("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.([]byte), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.([]byte))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			errstr := "Wrong number of frames for PUBLISH."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := UnstoredEvent{
				Stream: estream.([]byte),
				Data: data.([]byte),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				// TODO: Migrate to logging system
				sErr := err.Error()
				fmt.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack([]byte("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack([]byte("PUBLISHED"))
				response.PushBack([]byte(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			errstr := "Wrong number of frames for QUERY."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack([]byte("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estreamprefix := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			events := make(chan StoredEvent)
			req := QueryRequest{
				Stream: estreamprefix.([]byte),
				FromId: fromid.([]byte),
				ToId: toid.([]byte),
			}
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)

				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack([]byte("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		errstr := "Unknown request type."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack([]byte("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) [][]byte {
	frames := make([][]byte, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.([]byte)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<commit_msg>Introducing types zMsg and zFrame<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the server loop. Deals with incoming requests and delegates
// them to the event store.
package server

import (
	"bytes"
	"errors"
	"fmt"
	"container/list"
	zmq "github.com/alecthomas/gozmq"
)

type InitParams struct {
	Store *EventStore
	CommandSocketZPath *string
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.Close()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) Close() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Run() {
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	nbrOfChanges int
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems) {
	a, b := zmq.Poll(items, -1)
	notifier <- zmqPollResult{a, b}
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
func loopServer(estore *EventStore, evpubsock, frontend zmq.Socket) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)
	go asyncPoll(pollchan, toPoll)
	for {
		select {
		case <- pollchan:
			if toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				fmt.Println(err)
			}
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		event := <-toPublish

		msg[0] = event.Stream
		msg[1] = event.Id
		msg[2] = event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			// TODO: Use logger
			fmt.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	// TODO: Possibly wrap ZeroMQ router frames into a Type before
	// calling this method. That would yield a nicer API without
	// nitty gritty ZeroMQ details.
	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			errstr := "Wrong number of frames for PUBLISH."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := UnstoredEvent{
				Stream: estream.(StreamName),
				Data: data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				// TODO: Migrate to logging system
				sErr := err.Error()
				fmt.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			errstr := "Wrong number of frames for QUERY."
			// TODO: Migrate to logging system
			fmt.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estreamprefix := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			events := make(chan StoredEvent)
			req := QueryRequest{
				Stream: estreamprefix.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			go estore.Query(req, events)
			for eventdata := range(events) {
				response := copyList(resptemplate)
				response.PushBack(eventdata.Stream)
				response.PushBack(eventdata.Id)
				response.PushBack(eventdata.Data)

				respchan <- listToFrames(response)
			}
			response := copyList(resptemplate)
			response.PushBack(zFrame("END"))
			respchan <- listToFrames(response)
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		errstr := "Unknown request type."
		// TODO: Migrate to logging system
		fmt.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"gopkg.in/redis.v3"
	"net/http"
)

var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"}

func worker(client *redis.Client, data *announceData) []string {
	if RedisGetBoolKeyVal(client, data.info_hash, data) {
		x := RedisGetKeyVal(client, data.info_hash, data)

		RedisSetKeyVal(client,
			concatenateKeyMember(data.info_hash, "ip"),
			createIpPortPair(data))

		return x

	} else {
		CreateNewTorrentKey(client, data.info_hash)
		return worker(client, data)
	}
}

func requestHandler(w http.ResponseWriter, req *http.Request) {
	client := OpenClient()

	data := new(announceData)
	err := data.parseAnnounceData(req)
	if err != nil {
		panic(err)
	}

	switch data.event {
	case "started":
		data.event = "started"
		data.StartedEventHandler(client)
	case "stopped":
		data.StoppedEventHandler(client)
	case "completed":
		data.CompletedEventHandler(client)
	default:
		data.event = "started"
		data.StartedEventHandler(client)
	}
	fmt.Printf("Event: %s from host %s on port %v\n", data.event, data.ip, data.port)

	if data.event == "started" || data.event == "completed" {
		worker(client, data)
		x := RedisGetKeyVal(client, data.info_hash, data)
		// TODO(ian): Move this into a seperate function.
		// TODO(ian): Remove this magic number and use data.numwant, but limit ti
		// to 30 max, as that's the bittorrent protocol suggested limit.
		if len(x) >= 30 {
			x = x[0:30]
		} else {
			x = x[0:len(x)]
		}

		if len(x) > 0 {
			w.Header().set("Content-Type", "text/plain")
			response := formatResponseData(client, x, data)

			w.Write([]byte(response))
		} else {
			failMsg := fmt.Sprintf("No peers for torrent %s\n", data.info_hash)
			w.Write([]byte(createFailureMessage(failMsg)))
		}
	}
}

func RunServer() {
	mux := http.NewServeMux()

	mux.HandleFunc("/announce", requestHandler)
	http.ListenAndServe(":3000", mux)
}

func OpenClient() *redis.Client {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})

	return client
}
<commit_msg>Update server.go<commit_after>package server

import (
	"fmt"
	"gopkg.in/redis.v3"
	"net/http"
)

var FIELDS = []string{"port", "uploaded", "downloaded", "left", "event", "compact"}

func worker(client *redis.Client, data *announceData) []string {
	if RedisGetBoolKeyVal(client, data.info_hash, data) {
		x := RedisGetKeyVal(client, data.info_hash, data)

		RedisSetKeyVal(client,
			concatenateKeyMember(data.info_hash, "ip"),
			createIpPortPair(data))

		return x

	} else {
		CreateNewTorrentKey(client, data.info_hash)
		return worker(client, data)
	}
}

func requestHandler(w http.ResponseWriter, req *http.Request) {
	client := OpenClient()

	data := new(announceData)
	err := data.parseAnnounceData(req)
	if err != nil {
		panic(err)
	}

	switch data.event {
	case "started":
		data.event = "started"
		data.StartedEventHandler(client)
	case "stopped":
		data.StoppedEventHandler(client)
	case "completed":
		data.CompletedEventHandler(client)
	default:
		data.event = "started"
		data.StartedEventHandler(client)
	}
	fmt.Printf("Event: %s from host %s on port %v\n", data.event, data.ip, data.port)

	if data.event == "started" || data.event == "completed" {
		worker(client, data)
		x := RedisGetKeyVal(client, data.info_hash, data)
		// TODO(ian): Move this into a seperate function.
		// TODO(ian): Remove this magic number and use data.numwant, but limit ti
		// to 30 max, as that's the bittorrent protocol suggested limit.
		if len(x) >= 30 {
			x = x[0:30]
		} else {
			x = x[0:len(x)]
		}

		if len(x) > 0 {
			w.Header().Set("Content-Type", "text/plain")
			response := formatResponseData(client, x, data)

			w.Write([]byte(response))
		} else {
			failMsg := fmt.Sprintf("No peers for torrent %s\n", data.info_hash)
			w.Write([]byte(createFailureMessage(failMsg)))
		}
	}
}

func RunServer() {
	mux := http.NewServeMux()

	mux.HandleFunc("/announce", requestHandler)
	http.ListenAndServe(":3000", mux)
}

func OpenClient() *redis.Client {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "",
		DB:       0,
	})

	return client
}
<|endoftext|>


<commit_before>package sortutil

import (
	"sort"
	"testing"
	"time"
)

const (
	day = 24 * time.Hour
)

type Item struct {
	Id    int64
	Name  string
	Date  time.Time
	Valid bool
}

type SortableItems []Item

func (s SortableItems) Len() int {
	return len(s)
}

func (s SortableItems) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s SortableItems) Less(i, j int) bool {
	return s[i].Id > s[j].Id
}

func names() []string {
	return []string{"A", "C", "a", "b", "d", "g", "h", "y", "z"}
}

func namesInsensitive() []string {
	return []string{"A", "a", "b", "C", "d", "g", "h", "y", "z"}
}

var now = time.Now()

func dates() []time.Time {
	return []time.Time{
		now.Add(-3 * day),
		now.Add(-2 * day),
		now.Add(-1 * day),
		now,
		now.Add(1 * day),
		now.Add(2 * day),
		now.Add(4 * day),
		now.Add(5 * day),
		now.Add(7 * day),
	}
}

func items() []Item {
	n := names()
	d := dates()
	is := []Item{
		{6, n[4], d[0], true},
		{1, n[3], d[5], true},
		{9, n[1], d[6], true},
		{3, n[8], d[2], false},
		{7, n[7], d[8], true},
		{2, n[2], d[4], false},
		{8, n[0], d[1], false},
		{5, n[5], d[7], false},
		{4, n[6], d[3], true},
	}
	return is
}

func nestedIntSlice() [][]int {
	return [][]int{
		{4, 5, 1},
		{2, 1, 7},
		{9, 3, 3},
		{1, 6, 2},
	}
}

func TestSortReverse(t *testing.T) {
	is := items()
	SortReverse(SortableItems(is))
	for i, v := range is {
		if v.Id != int64(i+1) {
			t.Errorf("is[%d].Id was not %d, but %d", i, i+1, v.Id)
		}
	}
}

// func TestSortNoGetterReverse(t *testing.T) {

func TestSortByStringFieldAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), Ascending)
	c := names()
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name was not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByStringFieldDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), Descending)
	c := names()
	Reverse(sort.StringSlice(c))
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name was not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByStringFieldCaseInsensitiveAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), CaseInsensitiveAscending)
	c := namesInsensitive()
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name was not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByStringFieldCaseInsensitiveDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), CaseInsensitiveDescending)
	c := namesInsensitive()
	Reverse(sort.StringSlice(c))
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name was not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByInt64FieldAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Id"), Ascending)
	for i, v := range is {
		if v.Id != int64(i+1) {
			t.Errorf("is[%d].Id was not %d, but %d", i, i+1, v.Id)
		}
	}
}

func TestSortByInt64FieldDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Id"), Descending)
	l := len(is)
	for i, v := range is {
		if v.Id != int64(l-i) {
			t.Errorf("is[%d].Id was not %d, but %d", i, l-i, v.Id)
		}
	}
}

func TestSortByIntIndexAscending(t *testing.T) {
	is := nestedIntSlice()
	Sort(is, IndexGetter(2), Ascending)
	if !sort.IntsAreSorted([]int{is[0][2], is[1][2], is[2][2], is[3][2]}) {
		t.Errorf("Nested int slice was not sorted by index 2 in child slices: %v", is)
	}
}

func TestSortByTimeFieldAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Date"), Ascending)
	c := dates()
	for i, v := range is {
		if !v.Date.Equal(c[i]) {
			t.Errorf("is[%d].Date was not %v, but %v", i, c[i], v.Date)
		}
	}
}

func TestSortByTimeFieldDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Date"), Descending)
	c := dates()
	l := len(is)
	for i, v := range is {
		if !v.Date.Equal(c[l-i-1]) {
			t.Errorf("is[%d].Date was not %v, but %v", i, c[l-i], v.Date)
		}
	}
}

type TestStruct struct {
	TimePtr    *time.Time
	Invalid    InvalidType
	unexported int
}

type InvalidType struct {
	Foo string
	Bar int
}

func testStructs() []TestStruct {
	return []TestStruct{
		{
			TimePtr:    &now,
			Invalid:    InvalidType{"foo", 123},
			unexported: 5,
		},
	}
}

func TestSortInvalidType(t *testing.T) {
	// Sorting an invalid type should cause a panic
	defer func() {
		if x := recover(); x == nil {
			t.Fatal("Sorting an unrecognized type didn't cause a panic")
		}
	}()
	s := testStructs()
	Sort(s, FieldGetter("Invalid"), Ascending)
}

func TestSortUnexportedType(t *testing.T) {
	// Sorting an unexported type should cause a panic
	// TODO: This should test on a field outside the package
	return // TEMP
	defer func() {
		if x := recover(); x == nil {
			t.Fatal("Sorting an unexported type didn't cause a panic")
		}
	}()
	s := testStructs()
	Sort(s, FieldGetter("unexported"), Ascending)
}

func TestSortPointerType(t *testing.T) {
	// Sorting a pointer type shouldn't cause a panic
	s := testStructs()
	Sort(s, FieldGetter("TimePtr"), Ascending)
}

func BenchmarkSortStructByInt64(b *testing.B) {
	for i := 0; i < b.N; i++ {
		is := items()
		sort.Sort(SortableItems(is))
	}
}

func BenchmarkSortReverseStructByInt64(b *testing.B) {
	for i := 0; i < b.N; i++ {
		is := items()
		SortReverse(SortableItems(is))
	}
}
<commit_msg>s/was not/is not/<commit_after>package sortutil

import (
	"sort"
	"testing"
	"time"
)

const (
	day = 24 * time.Hour
)

type Item struct {
	Id    int64
	Name  string
	Date  time.Time
	Valid bool
}

type SortableItems []Item

func (s SortableItems) Len() int {
	return len(s)
}

func (s SortableItems) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s SortableItems) Less(i, j int) bool {
	return s[i].Id > s[j].Id
}

func names() []string {
	return []string{"A", "C", "a", "b", "d", "g", "h", "y", "z"}
}

func namesInsensitive() []string {
	return []string{"A", "a", "b", "C", "d", "g", "h", "y", "z"}
}

var now = time.Now()

func dates() []time.Time {
	return []time.Time{
		now.Add(-3 * day),
		now.Add(-2 * day),
		now.Add(-1 * day),
		now,
		now.Add(1 * day),
		now.Add(2 * day),
		now.Add(4 * day),
		now.Add(5 * day),
		now.Add(7 * day),
	}
}

func items() []Item {
	n := names()
	d := dates()
	is := []Item{
		{6, n[4], d[0], true},
		{1, n[3], d[5], true},
		{9, n[1], d[6], true},
		{3, n[8], d[2], false},
		{7, n[7], d[8], true},
		{2, n[2], d[4], false},
		{8, n[0], d[1], false},
		{5, n[5], d[7], false},
		{4, n[6], d[3], true},
	}
	return is
}

func nestedIntSlice() [][]int {
	return [][]int{
		{4, 5, 1},
		{2, 1, 7},
		{9, 3, 3},
		{1, 6, 2},
	}
}

func TestSortReverse(t *testing.T) {
	is := items()
	SortReverse(SortableItems(is))
	for i, v := range is {
		if v.Id != int64(i+1) {
			t.Errorf("is[%d].Id is not %d, but %d", i, i+1, v.Id)
		}
	}
}

// func TestSortNoGetterReverse(t *testing.T) {

func TestSortByStringFieldAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), Ascending)
	c := names()
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name is not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByStringFieldDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), Descending)
	c := names()
	Reverse(sort.StringSlice(c))
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name is not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByStringFieldCaseInsensitiveAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), CaseInsensitiveAscending)
	c := namesInsensitive()
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name is not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByStringFieldCaseInsensitiveDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Name"), CaseInsensitiveDescending)
	c := namesInsensitive()
	Reverse(sort.StringSlice(c))
	for i, v := range is {
		if v.Name != c[i] {
			t.Errorf("is[%d].Name is not %s, but %s", i, c[i], v.Name)
		}
	}
}

func TestSortByInt64FieldAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Id"), Ascending)
	for i, v := range is {
		if v.Id != int64(i+1) {
			t.Errorf("is[%d].Id is not %d, but %d", i, i+1, v.Id)
		}
	}
}

func TestSortByInt64FieldDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Id"), Descending)
	l := len(is)
	for i, v := range is {
		if v.Id != int64(l-i) {
			t.Errorf("is[%d].Id is not %d, but %d", i, l-i, v.Id)
		}
	}
}

func TestSortByIntIndexAscending(t *testing.T) {
	is := nestedIntSlice()
	Sort(is, IndexGetter(2), Ascending)
	if !sort.IntsAreSorted([]int{is[0][2], is[1][2], is[2][2], is[3][2]}) {
		t.Errorf("Nested int slice is not sorted by index 2 in child slices: %v", is)
	}
}

func TestSortByTimeFieldAscending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Date"), Ascending)
	c := dates()
	for i, v := range is {
		if !v.Date.Equal(c[i]) {
			t.Errorf("is[%d].Date is not %v, but %v", i, c[i], v.Date)
		}
	}
}

func TestSortByTimeFieldDescending(t *testing.T) {
	is := items()
	Sort(is, FieldGetter("Date"), Descending)
	c := dates()
	l := len(is)
	for i, v := range is {
		if !v.Date.Equal(c[l-i-1]) {
			t.Errorf("is[%d].Date is not %v, but %v", i, c[l-i], v.Date)
		}
	}
}

type TestStruct struct {
	TimePtr    *time.Time
	Invalid    InvalidType
	unexported int
}

type InvalidType struct {
	Foo string
	Bar int
}

func testStructs() []TestStruct {
	return []TestStruct{
		{
			TimePtr:    &now,
			Invalid:    InvalidType{"foo", 123},
			unexported: 5,
		},
	}
}

func TestSortInvalidType(t *testing.T) {
	// Sorting an invalid type should cause a panic
	defer func() {
		if x := recover(); x == nil {
			t.Fatal("Sorting an unrecognized type didn't cause a panic")
		}
	}()
	s := testStructs()
	Sort(s, FieldGetter("Invalid"), Ascending)
}

func TestSortUnexportedType(t *testing.T) {
	// Sorting an unexported type should cause a panic
	// TODO: This should test on a field outside the package
	return // TEMP
	defer func() {
		if x := recover(); x == nil {
			t.Fatal("Sorting an unexported type didn't cause a panic")
		}
	}()
	s := testStructs()
	Sort(s, FieldGetter("unexported"), Ascending)
}

func TestSortPointerType(t *testing.T) {
	// Sorting a pointer type shouldn't cause a panic
	s := testStructs()
	Sort(s, FieldGetter("TimePtr"), Ascending)
}

func BenchmarkSortStructByInt64(b *testing.B) {
	for i := 0; i < b.N; i++ {
		is := items()
		sort.Sort(SortableItems(is))
	}
}

func BenchmarkSortReverseStructByInt64(b *testing.B) {
	for i := 0; i < b.N; i++ {
		is := items()
		SortReverse(SortableItems(is))
	}
}
<|endoftext|>


<commit_before>package main

import "net/http"
import "fmt"
import "html"
import "time"
import "io/ioutil"
import "strings"
import "./players"
func getPlayerOfIp(ip string) players.NetworkPlayer {
	for _, plr := range networkPlayerList {
		plrIp := plr.GetIp()
		if plrIp == ip[:strings.Index(ip, ":")] {
			return plr
		}
	}
	//for now, add a player to the list and return it
	networkPlayerList=append(networkPlayerList,players.NewNetworkPlayer("uwe",3,ip))
	return networkPlayerList[len(networkPlayerList)-1]
}
func fileServe(w http.ResponseWriter, r *http.Request) {
	b,_ := ioutil.ReadFile("client/"+html.EscapeString(r.URL.Path)[1:])
	if r.URL.Path[len(r.URL.Path)-4:] == ".css" {
		w.Header().Set("Content-Type", "text/css; charset=utf-8")
	}
	w.Write(b)
}
func eventFunc(w http.ResponseWriter, r *http.Request) {
	plr := getPlayerOfIp(r.RemoteAddr)
	for len(plr.GetDataToBeSent()) == 0 {
		time.Sleep(time.Second / 2)
	}
	_, err := w.Write(plr.GetDataToBeSent()[0])
	if err == nil {
		plr.RemoveASentData()
	}
}
func chatFunc(w http.ResponseWriter, r *http.Request) {
	chat := ""
	r.ParseForm()
	for key := range r.Form {
		chat=key
		break
	}

	chat = strings.replace(chat,"<","<",-1)
	chat = strings.replace(chat,">",">",-1)
	chat = strings.replace(chat,"&","&",-1)

	plr := getPlayerOfIp(r.RemoteAddr)
	data := []byte("CHAT"+plr.GetName()+": "+chat)
	for _, plr2 := range networkPlayerList {
		plr2.SendData(data)
	}
}
func setupServer(quitChan chan struct{}) {
	http.HandleFunc("/ordos.html",fileServe)
	http.HandleFunc("/ordos.js",fileServe)
	http.HandleFunc("/ordos.css",fileServe)
	http.HandleFunc("/event",eventFunc)
	http.HandleFunc("/chat",chatFunc)
	go http.ListenAndServe(":8081",nil)
	fmt.Println("Server loaded")
}
<commit_msg>my code no good<commit_after>package main

import "net/http"
import "fmt"
import "html"
import "time"
import "io/ioutil"
import "strings"
import "./players"
func getPlayerOfIp(ip string) players.NetworkPlayer {
	for _, plr := range networkPlayerList {
		plrIp := plr.GetIp()
		if plrIp == ip[:strings.Index(ip, ":")] {
			return plr
		}
	}
	//for now, add a player to the list and return it
	networkPlayerList=append(networkPlayerList,players.NewNetworkPlayer("uwe",3,ip))
	return networkPlayerList[len(networkPlayerList)-1]
}
func fileServe(w http.ResponseWriter, r *http.Request) {
	b,_ := ioutil.ReadFile("client/"+html.EscapeString(r.URL.Path)[1:])
	if r.URL.Path[len(r.URL.Path)-4:] == ".css" {
		w.Header().Set("Content-Type", "text/css; charset=utf-8")
	}
	w.Write(b)
}
func eventFunc(w http.ResponseWriter, r *http.Request) {
	plr := getPlayerOfIp(r.RemoteAddr)
	for len(plr.GetDataToBeSent()) == 0 {
		time.Sleep(time.Second / 2)
	}
	_, err := w.Write(plr.GetDataToBeSent()[0])
	if err == nil {
		plr.RemoveASentData()
	}
}
func chatFunc(w http.ResponseWriter, r *http.Request) {
	chat := ""
	r.ParseForm()
	for key := range r.Form {
		chat=key
		break
	}

	chat = strings.Replace(chat,"<","<",-1)
	chat = strings.Replace(chat,">",">",-1)
	chat = strings.Replace(chat,"&","&",-1)

	plr := getPlayerOfIp(r.RemoteAddr)
	data := []byte("CHAT"+plr.GetName()+": "+chat)
	for _, plr2 := range networkPlayerList {
		plr2.SendData(data)
	}
}
func setupServer(quitChan chan struct{}) {
	http.HandleFunc("/ordos.html",fileServe)
	http.HandleFunc("/ordos.js",fileServe)
	http.HandleFunc("/ordos.css",fileServe)
	http.HandleFunc("/event",eventFunc)
	http.HandleFunc("/chat",chatFunc)
	go http.ListenAndServe(":8081",nil)
	fmt.Println("Server loaded")
}
<|endoftext|>


<commit_before>package worker

import (
	"encoding/json"
	"fmt"
	"time"

	gocontext "context"

	"github.com/Jeffail/tunny"
	"github.com/bitly/go-simplejson"
	"github.com/sirupsen/logrus"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"
)

type amqpJob struct {
	conn            *amqp.Connection
	stateUpdatePool *tunny.Pool
	logWriterChan   *amqp.Channel
	delivery        amqp.Delivery
	payload         *JobPayload
	rawPayload      *simplejson.Json
	startAttributes *backend.StartAttributes
	received        time.Time
	started         time.Time
	finished        time.Time
	stateCount      uint
}

func (j *amqpJob) GoString() string {
	return fmt.Sprintf("&amqpJob{conn: %#v, delivery: %#v, payload: %#v, startAttributes: %#v}",
		j.conn, j.delivery, j.payload, j.startAttributes)
}

func (j *amqpJob) Payload() *JobPayload {
	return j.payload
}

func (j *amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j *amqpJob) StartAttributes() *backend.StartAttributes {
	return j.startAttributes
}

func (j *amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx, time.Minute)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(ctx, FinishStateErrored)
}

func (j *amqpJob) Requeue(ctx gocontext.Context) error {
	context.LoggerFromContext(ctx).WithFields(
		logrus.Fields{
			"self":       "amqp_job",
			"job_id":     j.Payload().Job.ID,
			"repository": j.Payload().Repository.Slug,
		}).Info("requeueing job")

	metrics.Mark("worker.job.requeue")

	err := j.sendStateUpdate(ctx, "job:test:reset", "reset")
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) Received(ctx gocontext.Context) error {
	j.received = time.Now()

	if j.payload.Job.QueuedAt != nil {
		metrics.TimeSince("travis.worker.job.queue_time", *j.payload.Job.QueuedAt)
	}

	return j.sendStateUpdate(ctx, "job:test:receive", "received")
}

func (j *amqpJob) Started(ctx gocontext.Context) error {
	j.started = time.Now()

	metrics.TimeSince("travis.worker.job.start_time", j.received)

	return j.sendStateUpdate(ctx, "job:test:start", "started")
}

func (j *amqpJob) Finish(ctx gocontext.Context, state FinishState) error {
	context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"state":      state,
		"self":       "amqp_job",
		"job_id":     j.Payload().Job.ID,
		"repository": j.Payload().Repository.Slug,
	}).Info("finishing job")

	j.finished = time.Now()
	if j.received.IsZero() {
		j.received = j.finished
	}

	if j.started.IsZero() {
		j.started = j.finished
	}

	metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state))
	metrics.Mark("travis.worker.job.finish")

	err := j.sendStateUpdate(ctx, "job:test:finish", string(state))
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) {
	logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second
	if logTimeout == 0 {
		logTimeout = defaultLogTimeout
	}

	return newAMQPLogWriter(ctx, j.logWriterChan, j.payload.Job.ID, logTimeout)
}

func (j *amqpJob) createStateUpdateBody(ctx gocontext.Context, state string) map[string]interface{} {
	body := map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": state,
		"meta": map[string]interface{}{
			"state_update_count": j.stateCount,
		},
	}

	if instanceID, ok := context.InstanceIDFromContext(ctx); ok {
		body["meta"].(map[string]interface{})["instance_id"] = instanceID
	}

	if j.Payload().Job.QueuedAt != nil {
		body["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}
	if !j.received.IsZero() {
		body["received_at"] = j.received.UTC().Format(time.RFC3339)
	}
	if !j.started.IsZero() {
		body["started_at"] = j.started.UTC().Format(time.RFC3339)
	}
	if !j.finished.IsZero() {
		body["finished_at"] = j.finished.UTC().Format(time.RFC3339)
	}

	return body
}

func (j *amqpJob) sendStateUpdate(ctx gocontext.Context, event, state string) error {
	err := j.stateUpdatePool.Process(&amqpStateUpdatePayload{
		job:   j,
		ctx:   ctx,
		event: event,
		state: state,
		body:  j.createStateUpdateBody(ctx, state),
	})

	if err == nil {
		return nil
	}

	return err.(error)
}

func (j *amqpJob) SetupContext(ctx gocontext.Context) gocontext.Context { return ctx }

func (j *amqpJob) Name() string { return "amqp" }

type amqpStateUpdatePayload struct {
	job   *amqpJob
	ctx   gocontext.Context
	event string
	state string
	body  map[string]interface{}
}

type amqpStateUpdateWorker struct {
	stateUpdateChan *amqp.Channel
	ctx             gocontext.Context
	cancel          gocontext.CancelFunc
}

func (w *amqpStateUpdateWorker) Process(payload interface{}) interface{} {
	p := payload.(*amqpStateUpdatePayload)
	ctx, cancel := gocontext.WithCancel(p.ctx)

	w.ctx = ctx
	w.cancel = cancel

	return w.sendStateUpdate(p)
}

func (w *amqpStateUpdateWorker) BlockUntilReady() {
}

func (w *amqpStateUpdateWorker) Interrupt() {
	w.cancel()
}

func (w *amqpStateUpdateWorker) Terminate() {
	err := w.stateUpdateChan.Close()
	if err != nil {
		logrus.WithField("err", err).Panic("could not close state update amqp channel")
	}
}

func (w *amqpStateUpdateWorker) sendStateUpdate(payload *amqpStateUpdatePayload) error {
	select {
	case <-w.ctx.Done():
		return w.ctx.Err()
	default:
	}

	payload.job.stateCount++

	bodyBytes, err := json.Marshal(payload.body)
	if err != nil {
		return err
	}

	return w.stateUpdateChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now().UTC(),
		Type:         payload.event,
		Body:         bodyBytes,
	})
}
<commit_msg>defer panic to allow for clean shutdown<commit_after>package worker

import (
	"encoding/json"
	"fmt"
	"time"

	gocontext "context"

	"github.com/Jeffail/tunny"
	"github.com/bitly/go-simplejson"
	"github.com/sirupsen/logrus"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"
)

type amqpJob struct {
	conn            *amqp.Connection
	stateUpdatePool *tunny.Pool
	logWriterChan   *amqp.Channel
	delivery        amqp.Delivery
	payload         *JobPayload
	rawPayload      *simplejson.Json
	startAttributes *backend.StartAttributes
	received        time.Time
	started         time.Time
	finished        time.Time
	stateCount      uint
}

func (j *amqpJob) GoString() string {
	return fmt.Sprintf("&amqpJob{conn: %#v, delivery: %#v, payload: %#v, startAttributes: %#v}",
		j.conn, j.delivery, j.payload, j.startAttributes)
}

func (j *amqpJob) Payload() *JobPayload {
	return j.payload
}

func (j *amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j *amqpJob) StartAttributes() *backend.StartAttributes {
	return j.startAttributes
}

func (j *amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx, time.Minute)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(ctx, FinishStateErrored)
}

func (j *amqpJob) Requeue(ctx gocontext.Context) error {
	context.LoggerFromContext(ctx).WithFields(
		logrus.Fields{
			"self":       "amqp_job",
			"job_id":     j.Payload().Job.ID,
			"repository": j.Payload().Repository.Slug,
		}).Info("requeueing job")

	metrics.Mark("worker.job.requeue")

	err := j.sendStateUpdate(ctx, "job:test:reset", "reset")
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) Received(ctx gocontext.Context) error {
	j.received = time.Now()

	if j.payload.Job.QueuedAt != nil {
		metrics.TimeSince("travis.worker.job.queue_time", *j.payload.Job.QueuedAt)
	}

	return j.sendStateUpdate(ctx, "job:test:receive", "received")
}

func (j *amqpJob) Started(ctx gocontext.Context) error {
	j.started = time.Now()

	metrics.TimeSince("travis.worker.job.start_time", j.received)

	return j.sendStateUpdate(ctx, "job:test:start", "started")
}

func (j *amqpJob) Finish(ctx gocontext.Context, state FinishState) error {
	context.LoggerFromContext(ctx).WithFields(logrus.Fields{
		"state":      state,
		"self":       "amqp_job",
		"job_id":     j.Payload().Job.ID,
		"repository": j.Payload().Repository.Slug,
	}).Info("finishing job")

	j.finished = time.Now()
	if j.received.IsZero() {
		j.received = j.finished
	}

	if j.started.IsZero() {
		j.started = j.finished
	}

	metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state))
	metrics.Mark("travis.worker.job.finish")

	err := j.sendStateUpdate(ctx, "job:test:finish", string(state))
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) {
	logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second
	if logTimeout == 0 {
		logTimeout = defaultLogTimeout
	}

	return newAMQPLogWriter(ctx, j.logWriterChan, j.payload.Job.ID, logTimeout)
}

func (j *amqpJob) createStateUpdateBody(ctx gocontext.Context, state string) map[string]interface{} {
	body := map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": state,
		"meta": map[string]interface{}{
			"state_update_count": j.stateCount,
		},
	}

	if instanceID, ok := context.InstanceIDFromContext(ctx); ok {
		body["meta"].(map[string]interface{})["instance_id"] = instanceID
	}

	if j.Payload().Job.QueuedAt != nil {
		body["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}
	if !j.received.IsZero() {
		body["received_at"] = j.received.UTC().Format(time.RFC3339)
	}
	if !j.started.IsZero() {
		body["started_at"] = j.started.UTC().Format(time.RFC3339)
	}
	if !j.finished.IsZero() {
		body["finished_at"] = j.finished.UTC().Format(time.RFC3339)
	}

	return body
}

func (j *amqpJob) sendStateUpdate(ctx gocontext.Context, event, state string) error {
	err := j.stateUpdatePool.Process(&amqpStateUpdatePayload{
		job:   j,
		ctx:   ctx,
		event: event,
		state: state,
		body:  j.createStateUpdateBody(ctx, state),
	})

	if err == nil {
		return nil
	}

	return err.(error)
}

func (j *amqpJob) SetupContext(ctx gocontext.Context) gocontext.Context { return ctx }

func (j *amqpJob) Name() string { return "amqp" }

type amqpStateUpdatePayload struct {
	job   *amqpJob
	ctx   gocontext.Context
	event string
	state string
	body  map[string]interface{}
}

type amqpStateUpdateWorker struct {
	stateUpdateChan *amqp.Channel
	ctx             gocontext.Context
	cancel          gocontext.CancelFunc
}

func (w *amqpStateUpdateWorker) Process(payload interface{}) interface{} {
	p := payload.(*amqpStateUpdatePayload)
	ctx, cancel := gocontext.WithCancel(p.ctx)

	w.ctx = ctx
	w.cancel = cancel

	return w.sendStateUpdate(p)
}

func (w *amqpStateUpdateWorker) BlockUntilReady() {
}

func (w *amqpStateUpdateWorker) Interrupt() {
	w.cancel()
}

func (w *amqpStateUpdateWorker) Terminate() {
	err := w.stateUpdateChan.Close()
	if err != nil {
		time.Sleep(time.Minute)
		logrus.WithFields(logrus.Fields{
			"self": "amqp_state_update_worker",
			"err":  err,
		}).Panic("timed out waiting for shutdown after amqp connection error")
	}
}

func (w *amqpStateUpdateWorker) sendStateUpdate(payload *amqpStateUpdatePayload) error {
	select {
	case <-w.ctx.Done():
		return w.ctx.Err()
	default:
	}

	payload.job.stateCount++

	bodyBytes, err := json.Marshal(payload.body)
	if err != nil {
		return err
	}

	return w.stateUpdateChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now().UTC(),
		Type:         payload.event,
		Body:         bodyBytes,
	})
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"net"
	"os"
	"log"
	"github.com/talbor49/HoneyBee/grammar"
)

const (
	port      = "5000"
	ip        = "0.0.0.0"
	bufferLen = 1024
)

//DatabaseConnection is an extension of the net.Conn struct, added additional required properties.
type DatabaseConnection struct {
	net.Conn
	Bucket      string
	Connections int
	Username    string
}

// StartServer starts the database server - listens at a specific port for any incoming TCP connections.
func StartServer() {
	addr := fmt.Sprintf("%s:%s", ip, port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
	log.Printf("Listening on: %s", addr)
	// Close the listener socket when the application closes.
	defer listener.Close()

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Fatalf("Error accepting message from client, %s", err)
			os.Exit(1)
		}
		// Handle connections in a new goroutine
		dbconn := DatabaseConnection{conn, "", 0, ""}
		go handleConnection(dbconn)
	}
}

func handleConnection(conn DatabaseConnection) {
	// authenticate and process further requests
	defer conn.Close()

	var rawRequest []byte
	buff := make([]byte, bufferLen)

	for {
		reqLen, err := conn.Read(buff)
		if err != nil {
			log.Printf("Error reading buffer. %s", err)
			return
		}
		rawRequest = buff[:reqLen]
		if len(rawRequest) > 0 && rawRequest[0] == grammar.QUIT_REQUEST {
			break
		}
		go HandleRequest(rawRequest, &conn)
	}
	log.Println("Closed connection")
}
<commit_msg>Recieving port from environment variables<commit_after>package server

import (
	"fmt"
	"net"
	"os"
	"log"
	"github.com/talbor49/HoneyBee/grammar"
)

const (
	port      = os.Getenv("PORT")
	ip        = "0.0.0.0"
	bufferLen = 1024
)

//DatabaseConnection is an extension of the net.Conn struct, added additional required properties.
type DatabaseConnection struct {
	net.Conn
	Bucket      string
	Connections int
	Username    string
}

// StartServer starts the database server - listens at a specific port for any incoming TCP connections.
func StartServer() {
	addr := fmt.Sprintf("%s:%s", ip, port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
	log.Printf("Listening on: %s", addr)
	// Close the listener socket when the application closes.
	defer listener.Close()

	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Fatalf("Error accepting message from client, %s", err)
			os.Exit(1)
		}
		// Handle connections in a new goroutine
		dbconn := DatabaseConnection{conn, "", 0, ""}
		go handleConnection(dbconn)
	}
}

func handleConnection(conn DatabaseConnection) {
	// authenticate and process further requests
	defer conn.Close()

	var rawRequest []byte
	buff := make([]byte, bufferLen)

	for {
		reqLen, err := conn.Read(buff)
		if err != nil {
			log.Printf("Error reading buffer. %s", err)
			return
		}
		rawRequest = buff[:reqLen]
		if len(rawRequest) > 0 && rawRequest[0] == grammar.QUIT_REQUEST {
			break
		}
		go HandleRequest(rawRequest, &conn)
	}
	log.Println("Closed connection")
}
<|endoftext|>


<commit_before>// Copyright 2016 Jacob Taylor jacob.taylor@gmail.com
// License: Apache2
package main

import (
    "fmt"
    "net"
    "../utils"
    "bufio"
    "encoding/binary"
    "time"
    "os"
    "bytes"
    "io"
    "io/ioutil"
    "log"
)

const nbd_folder = "/sample_disks/"

var characters_per_line = 100
var newline = 0
var line_number = 0

func send_export_list_item(output *bufio.Writer, export_name string) {
    data := make([]byte, 1024)
    length := len(export_name)
    offset := 0

    // length of export name
    binary.BigEndian.PutUint32(data[offset:], uint32(length))  // length of string
    offset += 4

    // export name
    copy(data[offset:], export_name)
    offset += length

    reply_type := uint32(2)     // reply_type: NBD_REP_SERVER
    send_message(output, reply_type, uint32(offset), data)
}

func send_ack(output *bufio.Writer) {
    send_message(output, utils.NBD_COMMAND_ACK, 0, nil)
}

func export_name(output *bufio.Writer, conn net.Conn, payload_size int, payload []byte, pad_with_zeros bool) {
    fmt.Printf("have request to bind to: %s\n", string(payload[:payload_size]))

    defer conn.Close()

    var filename bytes.Buffer
    current_directory, err := os.Getwd()
    utils.ErrorCheck(err)
    filename.WriteString(current_directory)
    filename.WriteString(nbd_folder)
    filename.Write(payload[:payload_size])

    fmt.Printf("Opening file: %s\n", filename.String())

    // attempt to open the file read only
    file, err := os.OpenFile(filename.String(), os.O_RDWR, 0644)

    utils.ErrorCheck(err)
    if err != nil {
        return
    }

    buffer := make([]byte, 256)
    offset := 0

    fs, err := file.Stat()
    file_size := uint64(fs.Size())

    binary.BigEndian.PutUint64(buffer[offset:], file_size)  // size
    offset += 8

    binary.BigEndian.PutUint16(buffer[offset:], 1)  // flags
    offset += 2

    if pad_with_zeros {
        offset += 124               // pad with 124 zeroes
    }

    _, err = output.Write(buffer[:offset])
    //data_out, err := output.Write(buffer[:offset])

    output.Flush()
    utils.ErrorCheck(err)
    //fmt.Printf("Wrote %d chars: %v\n", data_out, buffer[:offset])

    buffer = make([]byte, 512*1024)
    conn_reader := bufio.NewReader(conn)
    abort := false
    for {
        offset := 0
        waiting_for := 28       // wait for at least the minimum payload size
        //fmt.Printf("Sitting at top of export loop.  offset: %d, waiting_for: %d\n", offset, waiting_for)
// Duplicate
        for offset < waiting_for {
            length, err := conn_reader.Read(buffer[offset:waiting_for])
            offset += length
            utils.ErrorCheck(err)
            if err == io.EOF {
                abort = true
                break
            }
            //utils.LogData("Reading instruction\n", offset, buffer)
            if offset < waiting_for {
                time.Sleep(5 * time.Millisecond)
            }
        }
// Duplicate
        if abort {
            fmt.Printf("Abort detected, escaping processing loop\n")
            break
        }

        //fmt.Printf("We read the buffer %v\n", buffer[:waiting_for])

        //magic := binary.BigEndian.Uint32(buffer)
        command := binary.BigEndian.Uint32(buffer[4:8])
        //handle := binary.BigEndian.Uint64(buffer[8:16])
        from := binary.BigEndian.Uint64(buffer[16:24])
        length := binary.BigEndian.Uint32(buffer[24:28])

        newline += 1;
        if newline % characters_per_line == 0 {
            line_number++
            fmt.Printf("\n%3d: ", line_number)
            newline -= characters_per_line

        }

        switch command {
        case utils.NBD_COMMAND_READ:
            //fmt.Printf("We have a request to read. handle: %v, from: %v, length: %v\n", handle, from, length)
            //fmt.Printf("Read Resquest    Offset:%x length: %v     Handle %X\n", from, length, handle)
            fmt.Printf(".")

            _, err = file.ReadAt(buffer[16:16+length], int64(from))
            utils.ErrorCheck(err)

            binary.BigEndian.PutUint32(buffer[:4], utils.NBD_REPLY_MAGIC)
            binary.BigEndian.PutUint32(buffer[4:8], 0)                      // error bits

            //utils.LogData("About to reply with", int(16+length), buffer)

            conn.Write(buffer[:16+length])

            continue
        case utils.NBD_COMMAND_WRITE:
            //fmt.Printf("We have a request to write. handle: %v, from: %v, length: %v\n", handle, from, length)
            fmt.Printf("W")

            waiting_for += int(length)                   // wait for the additional payload

// Duplicate
            for offset < waiting_for {
                length, err := conn_reader.Read(buffer[offset:waiting_for])
                offset += length
                utils.ErrorCheck(err)
                if err == io.EOF {
                    abort = true
                    break
                }
                //utils.LogData("Reading write data\n", offset, buffer)
                if offset < waiting_for {
                    time.Sleep(5 * time.Millisecond)
                }
            }
// Duplicate

            _, err = file.WriteAt(buffer[28:28+length], int64(from))
            utils.ErrorCheck(err)

            file.Sync()

            // let them know we are done
            binary.BigEndian.PutUint32(buffer[:4], utils.NBD_REPLY_MAGIC)
            binary.BigEndian.PutUint32(buffer[4:8], 0)                      // error bits

            //utils.LogData("About to reply with", int(16), buffer)
            conn.Write(buffer[:16])

            continue

        case utils.NBD_COMMAND_DISCONNECT:
            fmt.Printf("D")

            //fmt.Printf("We have received a request to disconnect\n%d\n", data_out)
            // close the file and return

            file.Sync()
            return
        }
    }
}

func send_export_list(output *bufio.Writer) {
    current_directory, err := os.Getwd()
    files, err := ioutil.ReadDir(current_directory + nbd_folder)
    if err != nil {
        log.Fatal(err)
    }
    for _, file := range files {
        send_export_list_item(output, file.Name())
    }

    send_ack(output)
}

func send_message(output *bufio.Writer, reply_type uint32, length uint32, data []byte ) {
    endian := binary.BigEndian
    buffer := make([]byte, 1024)
    offset := 0

    endian.PutUint64(buffer[offset:], utils.NBD_SERVER_SEND_REPLY_MAGIC)
    offset += 8

    endian.PutUint32(buffer[offset:], uint32(3))  // not sure what this is....
    offset += 4

    endian.PutUint32(buffer[offset:], reply_type)  // reply_type: NBD_REP_SERVER
    offset += 4

    endian.PutUint32(buffer[offset:], length)  // length of package
    offset += 4

    if data != nil {
        copy(buffer[offset:], data[0:length])
        offset += int(length)
    }

    data_to_send := buffer[:offset]
    output.Write(data_to_send)
    output.Flush()

    utils.LogData("Just sent:", offset, data_to_send)
}

func main() {

    if len(os.Args) <  3 {
        panic("missing arguments:  (ipaddress) (portnumber)")
        return
    }

    listener, err := net.Listen("tcp", os.Args[1] + ":" + os.Args[2])
    utils.ErrorCheck(err)

    fmt.Printf("Hello World, we have %v\n", listener)
    reply_magic := make([]byte, 4)
    binary.BigEndian.PutUint32(reply_magic, utils.NBD_REPLY_MAGIC)

    for {
        conn, err := listener.Accept()
        utils.ErrorCheck(err)

        fmt.Printf("We have a new connection from: %s\n", conn.RemoteAddr())
        output := bufio.NewWriter(conn)

        output.WriteString("NBDMAGIC")      // init password
        output.WriteString("IHAVEOPT")      // Magic
        output.Write([]byte{0, 3})          // Flags (3 = supports list)
        //output.Write([]byte{0, 0})

        output.Flush()

        // Fetch the data until we get the initial options
        data := make([]byte, 1024)
        offset := 0
        waiting_for := 16       // wait for at least the minimum payload size

        packet_count := 0
        for offset < waiting_for {
            length, err := conn.Read(data[offset:])
            if length > 0 {
                packet_count += 1
            }
            offset += length
            utils.ErrorCheck(err)
            //utils.LogData("Reading instruction", offset, data)
            if offset < waiting_for {
                time.Sleep(5 * time.Millisecond)
            }
            // If we are requesting an export, make sure we have the length of the data for the export name.
            if offset > 15 && binary.BigEndian.Uint32(data[12:]) == utils.NBD_COMMAND_EXPORT_NAME {
                waiting_for = 20
            }
        }

        fmt.Printf("%d packets processed to get %d bytes\n", packet_count, offset)
        utils.LogData("Received from client", offset, data)
        options := binary.BigEndian.Uint32(data[:4])
        command := binary.BigEndian.Uint32(data[12:])
        payload_size := int(binary.BigEndian.Uint32(data[16:]))
        fmt.Printf("Options are: %v\n", options)
        if (options & utils.NBD_FLAG_FIXED_NEW_STYLE) == utils.NBD_FLAG_FIXED_NEW_STYLE {
            fmt.Printf("Fixed New Style option requested\n")
        }
        pad_with_zeros := true
        if (options & utils.NBD_FLAG_NO_ZEROES) == utils.NBD_FLAG_NO_ZEROES {
            pad_with_zeros = false
            fmt.Printf("No Zero Padding option requested\n")
        }

        fmt.Sprintf("command is: %d\npayload_size is: %d\n", command, payload_size)
        waiting_for += int(payload_size)
        for offset < waiting_for {
            length, err := conn.Read(data[offset:])
            offset += length
            utils.ErrorCheck(err)
            utils.LogData("Reading instruction", offset, data)
            if offset < waiting_for {
                time.Sleep(5 * time.Millisecond)
            }
        }
        payload := make([]byte, payload_size)

        if payload_size > 0{
            copy(payload, data[20:])
        }

        utils.LogData("Payload is:", payload_size, payload)
        fmt.Printf("command is: %v\n", command)

        // At this point, we have the command, payload size, and payload.
        switch command {
        case utils.NBD_COMMAND_LIST:
            send_export_list(output)
            conn.Close()
            break
        case utils.NBD_COMMAND_EXPORT_NAME:
            go export_name(output, conn, payload_size, payload, pad_with_zeros)
            break
        }
    }

}
<commit_msg>increasing buffer size, making line numbers be the number of requests processed, adding a debugging statement<commit_after>// Copyright 2016 Jacob Taylor jacob.taylor@gmail.com
// License: Apache2
package main

import (
    "fmt"
    "net"
    "../utils"
    "bufio"
    "encoding/binary"
    "time"
    "os"
    "bytes"
    "io"
    "io/ioutil"
    "log"
)

const nbd_folder = "/sample_disks/"

var characters_per_line = 100
var newline = 0
var line_number = 0

func send_export_list_item(output *bufio.Writer, export_name string) {
    data := make([]byte, 1024)
    length := len(export_name)
    offset := 0

    // length of export name
    binary.BigEndian.PutUint32(data[offset:], uint32(length))  // length of string
    offset += 4

    // export name
    copy(data[offset:], export_name)
    offset += length

    reply_type := uint32(2)     // reply_type: NBD_REP_SERVER
    send_message(output, reply_type, uint32(offset), data)
}

func send_ack(output *bufio.Writer) {
    send_message(output, utils.NBD_COMMAND_ACK, 0, nil)
}

func export_name(output *bufio.Writer, conn net.Conn, payload_size int, payload []byte, pad_with_zeros bool) {
    fmt.Printf("have request to bind to: %s\n", string(payload[:payload_size]))

    defer conn.Close()

    var filename bytes.Buffer
    current_directory, err := os.Getwd()
    utils.ErrorCheck(err)
    filename.WriteString(current_directory)
    filename.WriteString(nbd_folder)
    filename.Write(payload[:payload_size])

    fmt.Printf("Opening file: %s\n", filename.String())

    // attempt to open the file read only
    file, err := os.OpenFile(filename.String(), os.O_RDWR, 0644)

    utils.ErrorCheck(err)
    if err != nil {
        return
    }

    buffer := make([]byte, 256)
    offset := 0

    fs, err := file.Stat()
    file_size := uint64(fs.Size())

    binary.BigEndian.PutUint64(buffer[offset:], file_size)  // size
    offset += 8

    binary.BigEndian.PutUint16(buffer[offset:], 1)  // flags
    offset += 2

    if pad_with_zeros {
        offset += 124               // pad with 124 zeroes
    }

    _, err = output.Write(buffer[:offset])
    //data_out, err := output.Write(buffer[:offset])

    output.Flush()
    utils.ErrorCheck(err)
    //fmt.Printf("Wrote %d chars: %v\n", data_out, buffer[:offset])

    buffer = make([]byte, 2048*1024)    // set the buffer to 2mb
    conn_reader := bufio.NewReader(conn)
    abort := false
    for {
        offset := 0
        waiting_for := 28       // wait for at least the minimum payload size
        //fmt.Printf("Sitting at top of export loop.  offset: %d, waiting_for: %d\n", offset, waiting_for)
// Duplicate
        for offset < waiting_for {
            length, err := conn_reader.Read(buffer[offset:waiting_for])
            offset += length
            utils.ErrorCheck(err)
            if err == io.EOF {
                abort = true
                break
            }
            //utils.LogData("Reading instruction\n", offset, buffer)
            if offset < waiting_for {
                time.Sleep(5 * time.Millisecond)
            }
        }
// Duplicate
        if abort {
            fmt.Printf("Abort detected, escaping processing loop\n")
            break
        }

        //fmt.Printf("We read the buffer %v\n", buffer[:waiting_for])

        //magic := binary.BigEndian.Uint32(buffer)
        command := binary.BigEndian.Uint32(buffer[4:8])
        //handle := binary.BigEndian.Uint64(buffer[8:16])
        from := binary.BigEndian.Uint64(buffer[16:24])
        length := binary.BigEndian.Uint32(buffer[24:28])

        newline += 1;
        if newline % characters_per_line == 0 {
            line_number++
            fmt.Printf("\n%5d: ", line_number * 100)
            newline -= characters_per_line
        }

        switch command {
        case utils.NBD_COMMAND_READ:
            //fmt.Printf("We have a request to read. handle: %v, from: %v, length: %v\n", handle, from, length)
            //fmt.Printf("Read Resquest    Offset:%x length: %v     Handle %X\n", from, length, handle)
            fmt.Printf(".")

            // working on diagnosing qemu connections from localhost to mount to os x nbd
            //fmt.Printf("len(buffer) %d, length: %d, from %d\n", len(buffer), length, int64(from))

            _, err = file.ReadAt(buffer[16:16+length], int64(from))
            utils.ErrorCheck(err)

            binary.BigEndian.PutUint32(buffer[:4], utils.NBD_REPLY_MAGIC)
            binary.BigEndian.PutUint32(buffer[4:8], 0)                      // error bits

            //utils.LogData("About to reply with", int(16+length), buffer)
            conn.Write(buffer[:16+length])

            continue
        case utils.NBD_COMMAND_WRITE:
            //fmt.Printf("We have a request to write. handle: %v, from: %v, length: %v\n", handle, from, length)
            fmt.Printf("W")

            waiting_for += int(length)                   // wait for the additional payload

// Duplicate
            for offset < waiting_for {
                length, err := conn_reader.Read(buffer[offset:waiting_for])
                offset += length
                utils.ErrorCheck(err)
                if err == io.EOF {
                    abort = true
                    break
                }
                //utils.LogData("Reading write data\n", offset, buffer)
                if offset < waiting_for {
                    time.Sleep(5 * time.Millisecond)
                }
            }
// Duplicate

            _, err = file.WriteAt(buffer[28:28+length], int64(from))
            utils.ErrorCheck(err)

            file.Sync()

            // let them know we are done
            binary.BigEndian.PutUint32(buffer[:4], utils.NBD_REPLY_MAGIC)
            binary.BigEndian.PutUint32(buffer[4:8], 0)                      // error bits

            //utils.LogData("About to reply with", int(16), buffer)
            conn.Write(buffer[:16])

            continue

        case utils.NBD_COMMAND_DISCONNECT:
            fmt.Printf("D")

            //fmt.Printf("We have received a request to disconnect\n%d\n", data_out)
            // close the file and return

            file.Sync()
            return
        }
    }
}

func send_export_list(output *bufio.Writer) {
    current_directory, err := os.Getwd()
    files, err := ioutil.ReadDir(current_directory + nbd_folder)
    if err != nil {
        log.Fatal(err)
    }
    for _, file := range files {
        send_export_list_item(output, file.Name())
    }

    send_ack(output)
}

func send_message(output *bufio.Writer, reply_type uint32, length uint32, data []byte ) {
    endian := binary.BigEndian
    buffer := make([]byte, 1024)
    offset := 0

    endian.PutUint64(buffer[offset:], utils.NBD_SERVER_SEND_REPLY_MAGIC)
    offset += 8

    endian.PutUint32(buffer[offset:], uint32(3))  // not sure what this is....
    offset += 4

    endian.PutUint32(buffer[offset:], reply_type)  // reply_type: NBD_REP_SERVER
    offset += 4

    endian.PutUint32(buffer[offset:], length)  // length of package
    offset += 4

    if data != nil {
        copy(buffer[offset:], data[0:length])
        offset += int(length)
    }

    data_to_send := buffer[:offset]
    output.Write(data_to_send)
    output.Flush()

    utils.LogData("Just sent:", offset, data_to_send)
}

func main() {

    if len(os.Args) <  3 {
        panic("missing arguments:  (ipaddress) (portnumber)")
        return
    }

    listener, err := net.Listen("tcp", os.Args[1] + ":" + os.Args[2])
    utils.ErrorCheck(err)

    fmt.Printf("Hello World, we have %v\n", listener)
    reply_magic := make([]byte, 4)
    binary.BigEndian.PutUint32(reply_magic, utils.NBD_REPLY_MAGIC)

    for {
        conn, err := listener.Accept()
        utils.ErrorCheck(err)

        fmt.Printf("We have a new connection from: %s\n", conn.RemoteAddr())
        output := bufio.NewWriter(conn)

        output.WriteString("NBDMAGIC")      // init password
        output.WriteString("IHAVEOPT")      // Magic
        //fmt.printf("arg ")
        //output.Write([]byte{0, byte(os.Args[3][1])})
        //output.Write([]byte{0, 3})          // Ubuntu
        output.Write([]byte{0, 0})        // Qemu

        output.Flush()

        // Fetch the data until we get the initial options
        data := make([]byte, 1024)
        offset := 0
        waiting_for := 16       // wait for at least the minimum payload size

        packet_count := 0
        for offset < waiting_for {
            length, err := conn.Read(data[offset:])
            if length > 0 {
                packet_count += 1
            }
            offset += length
            utils.ErrorCheck(err)
            //utils.LogData("Reading instruction", offset, data)
            if offset < waiting_for {
                time.Sleep(5 * time.Millisecond)
            }
            // If we are requesting an export, make sure we have the length of the data for the export name.
            if offset > 15 && binary.BigEndian.Uint32(data[12:]) == utils.NBD_COMMAND_EXPORT_NAME {
                waiting_for = 20
            }
        }

        fmt.Printf("%d packets processed to get %d bytes\n", packet_count, offset)
        utils.LogData("Received from client", offset, data)
        options := binary.BigEndian.Uint32(data[:4])
        command := binary.BigEndian.Uint32(data[12:])
        payload_size := int(binary.BigEndian.Uint32(data[16:]))
        fmt.Printf("Options are: %v\n", options)
        if (options & utils.NBD_FLAG_FIXED_NEW_STYLE) == utils.NBD_FLAG_FIXED_NEW_STYLE {
            fmt.Printf("Fixed New Style option requested\n")
        }
        pad_with_zeros := true
        if (options & utils.NBD_FLAG_NO_ZEROES) == utils.NBD_FLAG_NO_ZEROES {
            pad_with_zeros = false
            fmt.Printf("No Zero Padding option requested\n")
        }

        fmt.Sprintf("command is: %d\npayload_size is: %d\n", command, payload_size)
        waiting_for += int(payload_size)
        for offset < waiting_for {
            length, err := conn.Read(data[offset:])
            offset += length
            utils.ErrorCheck(err)
            utils.LogData("Reading instruction", offset, data)
            if offset < waiting_for {
                time.Sleep(5 * time.Millisecond)
            }
        }
        payload := make([]byte, payload_size)

        if payload_size > 0{
            copy(payload, data[20:])
        }

        utils.LogData("Payload is:", payload_size, payload)
        fmt.Printf("command is: %v\n", command)

        // At this point, we have the command, payload size, and payload.
        switch command {
        case utils.NBD_COMMAND_LIST:
            send_export_list(output)
            conn.Close()
            break
        case utils.NBD_COMMAND_EXPORT_NAME:
            go export_name(output, conn, payload_size, payload, pad_with_zeros)
            break
        }
    }

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/haklop/bazooka/commons/mongo"
)

const (
	BazookaEnvSCMKeyfile = "BZK_SCM_KEYFILE"
	BazookaEnvHome       = "BZK_HOME"
	BazookaEnvDockerSock = "BZK_DOCKERSOCK"
	BazookaEnvMongoAddr  = "MONGO_PORT_27017_TCP_ADDR"
	BazookaEnvMongoPort  = "MONGO_PORT_27017_TCP_PORT"

	DockerSock     = "/var/run/docker.sock"
	DockerEndpoint = "unix://" + DockerSock
	BazookaHome    = "/bazooka"
)

type errorResponse struct {
	Code    int    `json:"error_code"`
	Message string `json:"error_msg"`
}

func (e errorResponse) Error() string {
	return fmt.Sprintf("%d: %s", e.Code, e.Message)
}

type context struct {
	Connector      *mongo.MongoConnector
	DockerEndpoint string
	Env            map[string]string
}

func writeError(err error, res http.ResponseWriter) {
	res.WriteHeader(500)
	json.NewEncoder(res).Encode(&errorResponse{
		Code:    500,
		Message: err.Error(),
	})
}

type bodyFunc func(interface{})

type response struct {
	Code    int
	Payload interface{}
	Headers map[string]string
}

func ok(payload interface{}) (*response, error) {
	return &response{
		Code:    200,
		Payload: payload,
	}, nil
}

func created(payload interface{}, location string) (*response, error) {
	return &response{
		201,
		payload,
		map[string]string{"Location": location},
	}, nil
}
func accepted(payload interface{}, location string) (*response, error) {
	return &response{
		202,
		payload,
		map[string]string{"Location": location},
	}, nil
}
func badRequest(msg string) (*response, error) {
	return nil, &errorResponse{400, msg}
}

func notFound(msg string) (*response, error) {
	return nil, &errorResponse{404, msg}
}

func conflict(msg string) (*response, error) {
	return nil, &errorResponse{409, msg}
}

func mkHandler(f func(map[string]string, bodyFunc) (*response, error)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		bf := func(b interface{}) {
			defer r.Body.Close()
			decoder := json.NewDecoder(r.Body)
			if err := decoder.Decode(b); err != nil {
				panic(errorResponse{400, "Unable to decode your json : " + err.Error()})
			}
		}

		encoder := json.NewEncoder(w)

		dispatchError := func(err error) {
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			switch e := err.(type) {
			case errorResponse:
				w.WriteHeader(e.Code)
				encoder.Encode(e)
			case *errorResponse:
				w.WriteHeader(e.Code)
				encoder.Encode(e)
			default:
				writeError(e, w)
			}
		}

		defer func() {
			if r := recover(); r != nil {
				switch rt := r.(type) {
				case error:
					dispatchError(rt)
				default:
					writeError(fmt.Errorf("Caught a panic: %v", r), w)
				}
			}
		}()

		rb, err := f(mux.Vars(r), bf)

		if err != nil {
			dispatchError(err)
			return
		}

		if rb != nil {
			for k, v := range rb.Headers {
				w.Header().Set(k, v)
			}

			if rb.Payload != nil {
				w.Header().Set("Content-Type", "application/json; charset=utf-8")
				w.WriteHeader(rb.Code)
				encoder.Encode(&rb.Payload)
			} else {
				w.WriteHeader(rb.Code)
			}
		}

	}
}
<commit_msg>unauthorize method<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/haklop/bazooka/commons/mongo"
)

const (
	BazookaEnvSCMKeyfile = "BZK_SCM_KEYFILE"
	BazookaEnvHome       = "BZK_HOME"
	BazookaEnvDockerSock = "BZK_DOCKERSOCK"
	BazookaEnvMongoAddr  = "MONGO_PORT_27017_TCP_ADDR"
	BazookaEnvMongoPort  = "MONGO_PORT_27017_TCP_PORT"

	DockerSock     = "/var/run/docker.sock"
	DockerEndpoint = "unix://" + DockerSock
	BazookaHome    = "/bazooka"
)

type errorResponse struct {
	Code    int    `json:"error_code"`
	Message string `json:"error_msg"`
}

func (e errorResponse) Error() string {
	return fmt.Sprintf("%d: %s", e.Code, e.Message)
}

type context struct {
	Connector      *mongo.MongoConnector
	DockerEndpoint string
	Env            map[string]string
}

func writeError(err error, res http.ResponseWriter) {
	res.WriteHeader(500)
	json.NewEncoder(res).Encode(&errorResponse{
		Code:    500,
		Message: err.Error(),
	})
}

type bodyFunc func(interface{})

type response struct {
	Code    int
	Payload interface{}
	Headers map[string]string
}

func ok(payload interface{}) (*response, error) {
	return &response{
		Code:    200,
		Payload: payload,
	}, nil
}

func created(payload interface{}, location string) (*response, error) {
	return &response{
		201,
		payload,
		map[string]string{"Location": location},
	}, nil
}
func accepted(payload interface{}, location string) (*response, error) {
	return &response{
		202,
		payload,
		map[string]string{"Location": location},
	}, nil
}
func badRequest(msg string) (*response, error) {
	return nil, &errorResponse{400, msg}
}

func notFound(msg string) (*response, error) {
	return nil, &errorResponse{404, msg}
}

func conflict(msg string) (*response, error) {
	return nil, &errorResponse{409, msg}
}

func unauthorized() (*response, error) {
	return nil, &errorResponse{401, "Unauthorized"}
}

func mkHandler(f func(map[string]string, bodyFunc) (*response, error)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {

		bf := func(b interface{}) {
			defer r.Body.Close()
			decoder := json.NewDecoder(r.Body)
			if err := decoder.Decode(b); err != nil {
				panic(errorResponse{400, "Unable to decode your json : " + err.Error()})
			}
		}

		encoder := json.NewEncoder(w)

		dispatchError := func(err error) {
			w.Header().Set("Content-Type", "application/json; charset=utf-8")
			switch e := err.(type) {
			case errorResponse:
				w.WriteHeader(e.Code)
				encoder.Encode(e)
			case *errorResponse:
				w.WriteHeader(e.Code)
				encoder.Encode(e)
			default:
				writeError(e, w)
			}
		}

		defer func() {
			if r := recover(); r != nil {
				switch rt := r.(type) {
				case error:
					dispatchError(rt)
				default:
					writeError(fmt.Errorf("Caught a panic: %v", r), w)
				}
			}
		}()

		rb, err := f(mux.Vars(r), bf)

		if err != nil {
			dispatchError(err)
			return
		}

		if rb != nil {
			for k, v := range rb.Headers {
				w.Header().Set(k, v)
			}

			if rb.Payload != nil {
				w.Header().Set("Content-Type", "application/json; charset=utf-8")
				w.WriteHeader(rb.Code)
				encoder.Encode(&rb.Payload)
			} else {
				w.WriteHeader(rb.Code)
			}
		}

	}
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/golang/gddo/httputil"
	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/ohsu-comp-bio/funnel/config"
	"github.com/ohsu-comp-bio/funnel/events"
	"github.com/ohsu-comp-bio/funnel/logger"
	pbs "github.com/ohsu-comp-bio/funnel/proto/scheduler"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"github.com/ohsu-comp-bio/funnel/webdash"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"net"
	"net/http"
)

// Server represents a Funnel server. The server handles
// RPC traffic via gRPC, HTTP traffic for the TES API,
// and also serves the web dashboard.
type Server struct {
	RPCAddress             string
	HTTPPort               string
	Password               string
	TaskServiceServer      tes.TaskServiceServer
	EventServiceServer     events.EventServiceServer
	SchedulerServiceServer pbs.SchedulerServiceServer
	DisableHTTPCache       bool
	DialOptions            []grpc.DialOption
	Log                    *logger.Logger
}

// DefaultServer returns a new server instance.
func DefaultServer(db Database, conf config.Server) *Server {
	return &Server{
		RPCAddress:             ":" + conf.RPCPort,
		HTTPPort:               conf.HTTPPort,
		Password:               conf.Password,
		TaskServiceServer:      db,
		EventServiceServer:     db,
		SchedulerServiceServer: db,
		DisableHTTPCache:       conf.DisableHTTPCache,
		DialOptions: []grpc.DialOption{
			grpc.WithInsecure(),
		},
	}
}

// Return a new interceptor function that logs all requests at the Debug level
func newDebugInterceptor(log *logger.Logger) grpc.UnaryServerInterceptor {
	// Return a function that is the interceptor.
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
		handler grpc.UnaryHandler) (interface{}, error) {
		log.Debug(
			"received: "+info.FullMethod,
			"request", req,
		)
		resp, err := handler(ctx, req)
		log.Debug(
			"responding: "+info.FullMethod,
			"resp", resp,
			"err", err,
		)
		return resp, err
	}
}

// Serve starts the server and does not block. This will open TCP ports
// for both RPC and HTTP.
func (s *Server) Serve(pctx context.Context) error {
	ctx, cancel := context.WithCancel(pctx)
	defer cancel()

	// Open TCP connection for RPC
	lis, err := net.Listen("tcp", s.RPCAddress)
	if err != nil {
		return err
	}

	grpcServer := grpc.NewServer(
		grpc.UnaryInterceptor(
			grpc_middleware.ChainUnaryServer(
				// API auth check.
				newAuthInterceptor(s.Password),
				newDebugInterceptor(s.Log),
			),
		),
	)

	// Set up HTTP proxy of gRPC API
	mux := http.NewServeMux()
	grpcMux := runtime.NewServeMux()
	runtime.OtherErrorHandler = s.handleError

	dashmux := http.NewServeMux()
	dashmux.Handle("/", webdash.RootHandler())
	dashfs := webdash.FileServer()
	mux.Handle("/favicon.ico", dashfs)
	mux.Handle("/static/", http.StripPrefix("/static/", dashfs))

	mux.HandleFunc("/", func(resp http.ResponseWriter, req *http.Request) {

		switch negotiate(req) {
		case "html":
			// HTML was requested (by the browser)
			dashmux.ServeHTTP(resp, req)
		default:
			// Set "cache-control: no-store" to disable response caching.
			// Without this, some servers (e.g. GCE) will cache a response from ListTasks, GetTask, etc.
			// which results in confusion about the stale data.
			if s.DisableHTTPCache {
				resp.Header().Set("Cache-Control", "no-store")
			}
			grpcMux.ServeHTTP(resp, req)
		}
	})

	// Register TES service
	if s.TaskServiceServer != nil {
		tes.RegisterTaskServiceServer(grpcServer, s.TaskServiceServer)
		err := tes.RegisterTaskServiceHandlerFromEndpoint(
			ctx, grpcMux, s.RPCAddress, s.DialOptions,
		)
		if err != nil {
			return err
		}
	}

	// Register Events service
	if s.EventServiceServer != nil {
		events.RegisterEventServiceServer(grpcServer, s.EventServiceServer)
	}

	// Register Scheduler RPC service
	if s.SchedulerServiceServer != nil {
		pbs.RegisterSchedulerServiceServer(grpcServer, s.SchedulerServiceServer)
		err := pbs.RegisterSchedulerServiceHandlerFromEndpoint(
			ctx, grpcMux, s.RPCAddress, s.DialOptions,
		)
		if err != nil {
			return err
		}
	}

	httpServer := &http.Server{
		Addr:    ":" + s.HTTPPort,
		Handler: mux,
	}

	var srverr error
	go func() {
		srverr = grpcServer.Serve(lis)
		cancel()
	}()

	go func() {
		srverr = httpServer.ListenAndServe()
		cancel()
	}()

	s.Log.Info("Server listening",
		"httpPort", s.HTTPPort, "rpcAddress", s.RPCAddress,
	)

	<-ctx.Done()
	grpcServer.GracefulStop()
	httpServer.Shutdown(context.TODO())

	return srverr
}

// handleError handles errors in the HTTP stack, logging errors, stack traces,
// and returning an HTTP error code.
func (s *Server) handleError(w http.ResponseWriter, req *http.Request, err string, code int) {
	s.Log.Error("HTTP handler error", "error", err, "url", req.URL)
	http.Error(w, err, code)
}

// negotiate determines the response type based on request headers and parameters.
// Returns either "html" or "json".
func negotiate(req *http.Request) string {
	// Allow overriding the type from a URL parameter.
	// /v1/tasks?json will force a JSON response.
	q := req.URL.Query()
	if _, html := q["html"]; html {
		return "html"
	}
	if _, json := q["json"]; json {
		return "json"
	}
	// Content negotiation means that both the dashboard's HTML and the API's JSON
	// may be served at the same path.
	// In Go 1.10 we'll be able to move to a core library for this,
	// https://github.com/golang/go/issues/19307
	switch httputil.NegotiateContentType(req, []string{"text/*", "text/html"}, "text/*") {
	case "text/html":
		return "html"
	default:
		return "json"
	}
}
<commit_msg>server: indents for JSON responses<commit_after>package server

import (
	"github.com/golang/gddo/httputil"
	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/ohsu-comp-bio/funnel/config"
	"github.com/ohsu-comp-bio/funnel/events"
	"github.com/ohsu-comp-bio/funnel/logger"
	pbs "github.com/ohsu-comp-bio/funnel/proto/scheduler"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"github.com/ohsu-comp-bio/funnel/webdash"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"net"
	"net/http"
)

// Server represents a Funnel server. The server handles
// RPC traffic via gRPC, HTTP traffic for the TES API,
// and also serves the web dashboard.
type Server struct {
	RPCAddress             string
	HTTPPort               string
	Password               string
	TaskServiceServer      tes.TaskServiceServer
	EventServiceServer     events.EventServiceServer
	SchedulerServiceServer pbs.SchedulerServiceServer
	DisableHTTPCache       bool
	DialOptions            []grpc.DialOption
	Log                    *logger.Logger
}

// DefaultServer returns a new server instance.
func DefaultServer(db Database, conf config.Server) *Server {
	return &Server{
		RPCAddress:             ":" + conf.RPCPort,
		HTTPPort:               conf.HTTPPort,
		Password:               conf.Password,
		TaskServiceServer:      db,
		EventServiceServer:     db,
		SchedulerServiceServer: db,
		DisableHTTPCache:       conf.DisableHTTPCache,
		DialOptions: []grpc.DialOption{
			grpc.WithInsecure(),
		},
	}
}

// Return a new interceptor function that logs all requests at the Debug level
func newDebugInterceptor(log *logger.Logger) grpc.UnaryServerInterceptor {
	// Return a function that is the interceptor.
	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
		handler grpc.UnaryHandler) (interface{}, error) {
		log.Debug(
			"received: "+info.FullMethod,
			"request", req,
		)
		resp, err := handler(ctx, req)
		log.Debug(
			"responding: "+info.FullMethod,
			"resp", resp,
			"err", err,
		)
		return resp, err
	}
}

// Serve starts the server and does not block. This will open TCP ports
// for both RPC and HTTP.
func (s *Server) Serve(pctx context.Context) error {
	ctx, cancel := context.WithCancel(pctx)
	defer cancel()

	// Open TCP connection for RPC
	lis, err := net.Listen("tcp", s.RPCAddress)
	if err != nil {
		return err
	}

	grpcServer := grpc.NewServer(
		grpc.UnaryInterceptor(
			grpc_middleware.ChainUnaryServer(
				// API auth check.
				newAuthInterceptor(s.Password),
				newDebugInterceptor(s.Log),
			),
		),
	)

	// Set up HTTP proxy of gRPC API
	mux := http.NewServeMux()
	mar := &runtime.JSONPb{
		Indent: "   ",
	}
	grpcMux := runtime.NewServeMux(runtime.WithMarshalerOption("*/*", mar))
	runtime.OtherErrorHandler = s.handleError

	dashmux := http.NewServeMux()
	dashmux.Handle("/", webdash.RootHandler())
	dashfs := webdash.FileServer()
	mux.Handle("/favicon.ico", dashfs)
	mux.Handle("/static/", http.StripPrefix("/static/", dashfs))

	mux.HandleFunc("/", func(resp http.ResponseWriter, req *http.Request) {

		switch negotiate(req) {
		case "html":
			// HTML was requested (by the browser)
			dashmux.ServeHTTP(resp, req)
		default:
			// Set "cache-control: no-store" to disable response caching.
			// Without this, some servers (e.g. GCE) will cache a response from ListTasks, GetTask, etc.
			// which results in confusion about the stale data.
			if s.DisableHTTPCache {
				resp.Header().Set("Cache-Control", "no-store")
			}
			grpcMux.ServeHTTP(resp, req)
		}
	})

	// Register TES service
	if s.TaskServiceServer != nil {
		tes.RegisterTaskServiceServer(grpcServer, s.TaskServiceServer)
		err := tes.RegisterTaskServiceHandlerFromEndpoint(
			ctx, grpcMux, s.RPCAddress, s.DialOptions,
		)
		if err != nil {
			return err
		}
	}

	// Register Events service
	if s.EventServiceServer != nil {
		events.RegisterEventServiceServer(grpcServer, s.EventServiceServer)
	}

	// Register Scheduler RPC service
	if s.SchedulerServiceServer != nil {
		pbs.RegisterSchedulerServiceServer(grpcServer, s.SchedulerServiceServer)
		err := pbs.RegisterSchedulerServiceHandlerFromEndpoint(
			ctx, grpcMux, s.RPCAddress, s.DialOptions,
		)
		if err != nil {
			return err
		}
	}

	httpServer := &http.Server{
		Addr:    ":" + s.HTTPPort,
		Handler: mux,
	}

	var srverr error
	go func() {
		srverr = grpcServer.Serve(lis)
		cancel()
	}()

	go func() {
		srverr = httpServer.ListenAndServe()
		cancel()
	}()

	s.Log.Info("Server listening",
		"httpPort", s.HTTPPort, "rpcAddress", s.RPCAddress,
	)

	<-ctx.Done()
	grpcServer.GracefulStop()
	httpServer.Shutdown(context.TODO())

	return srverr
}

// handleError handles errors in the HTTP stack, logging errors, stack traces,
// and returning an HTTP error code.
func (s *Server) handleError(w http.ResponseWriter, req *http.Request, err string, code int) {
	s.Log.Error("HTTP handler error", "error", err, "url", req.URL)
	http.Error(w, err, code)
}

// negotiate determines the response type based on request headers and parameters.
// Returns either "html" or "json".
func negotiate(req *http.Request) string {
	// Allow overriding the type from a URL parameter.
	// /v1/tasks?json will force a JSON response.
	q := req.URL.Query()
	if _, html := q["html"]; html {
		return "html"
	}
	if _, json := q["json"]; json {
		return "json"
	}
	// Content negotiation means that both the dashboard's HTML and the API's JSON
	// may be served at the same path.
	// In Go 1.10 we'll be able to move to a core library for this,
	// https://github.com/golang/go/issues/19307
	switch httputil.NegotiateContentType(req, []string{"text/*", "text/html"}, "text/*") {
	case "text/html":
		return "html"
	default:
		return "json"
	}
}
<|endoftext|>


<commit_before>package micro

import (
	"context"
	"sync"
	"testing"

	"github.com/micro/go-micro/registry/mock"
	proto "github.com/micro/go-micro/server/debug/proto"
)

func TestFunction(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)

	// cancellation context
	ctx, cancel := context.WithCancel(context.Background())

	// create service
	fn := NewFunction(
		Name("test.function"),
		Context(ctx),
		Registry(mock.NewRegistry()),
		AfterStart(func() error {
			wg.Done()
			return nil
		}),
	)

	// we can't test fn.Init as it parses the command line
	// fn.Init()

	go func() {
		// wait for start
		wg.Wait()

		// test call debug
		req := fn.Client().NewRequest(
			"test.function",
			"Debug.Health",
			new(proto.HealthRequest),
		)

		rsp := new(proto.HealthResponse)

		err := fn.Client().Call(context.TODO(), req, rsp)
		if err != nil {
			t.Fatal(err)
		}

		if rsp.Status != "ok" {
			t.Fatalf("function response: %s", rsp.Status)
		}

		cancel()
	}()

	// run service
	if err := fn.Run(); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>context cancellation is not required<commit_after>package micro

import (
	"context"
	"sync"
	"testing"

	"github.com/micro/go-micro/registry/mock"
	proto "github.com/micro/go-micro/server/debug/proto"
)

func TestFunction(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)

	// create service
	fn := NewFunction(
		Name("test.function"),
		Registry(mock.NewRegistry()),
		AfterStart(func() error {
			wg.Done()
			return nil
		}),
	)

	// we can't test fn.Init as it parses the command line
	// fn.Init()

	go func() {
		// wait for start
		wg.Wait()

		// test call debug
		req := fn.Client().NewRequest(
			"test.function",
			"Debug.Health",
			new(proto.HealthRequest),
		)

		rsp := new(proto.HealthResponse)

		err := fn.Client().Call(context.TODO(), req, rsp)
		if err != nil {
			t.Fatal(err)
		}

		if rsp.Status != "ok" {
			t.Fatalf("function response: %s", rsp.Status)
		}
	}()

	// run service
	if err := fn.Run(); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>