<commit_before>package game

import (
	"context"
	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
	"github.com/verath/archipelago/lib/common"
	"github.com/verath/archipelago/lib/game/model"
	"sync"
	"time"
)

const defaultTickInterval time.Duration = (time.Second / 2)

// The gameLoop is what updates the game model instance that it is
// associated with. The updates are performed in ticks. Each tick
// applies all actions that has been added since the last tick
// sequentially on the model. For each such actions, zero or more
// events are created. Those events are dispatched as the last stage
// of the tick.
//
// Notice that any reads or writes (outside of actions) on the model
// once the gameLoop is started is not safe.
type gameLoop struct {
	logEntry *logrus.Entry
	// Duration between each tick
	tickInterval time.Duration
	// The game instance on which actions are to be applied
	game *model.Game

	gameOverMu sync.Mutex
	// Flag for if the game has completed
	gameOver bool

	eventHandlerMu sync.Mutex
	// A handler to handle events produced when applying actions
	// to the game instance.
	eventHandler eventHandler
	// A WaitGroup for events being handled by the registered event
	// handler.
	handleEventWG sync.WaitGroup

	actionsMu sync.Mutex
	// A slice of actions to be applied on the next tick
	actions []model.Action
}

// A handler for game events produced from applying actions.
type eventHandler interface {
	// handleEvent handles an event produced. This method will be called on a
	// separate go routine and must block until the even has been handled, or
	// the context is cancelled.
	handleEvent(ctx context.Context, event model.Event)
}

func newGameLoop(log *logrus.Logger, game *model.Game) (*gameLoop, error) {
	logEntry := common.ModuleLogEntryWithID(log, "gameLoop")

	return &gameLoop{
		logEntry:     logEntry,
		tickInterval: defaultTickInterval,
		game:         game,
		actions:      make([]model.Action, 0),
	}, nil
}

// Sets the handler for game events.
func (gl *gameLoop) SetEventHandler(eventHandler eventHandler) {
	gl.eventHandlerMu.Lock()
	gl.eventHandler = eventHandler
	gl.eventHandlerMu.Unlock()
}

// Adds an action to be processed in the next tick.
func (gl *gameLoop) AddAction(action model.Action) {
	gl.actionsMu.Lock()
	gl.actions = append(gl.actions, action)
	gl.actionsMu.Unlock()
}

// Runs the game loop. Run blocks until the context is cancelled, an
// error occurs, or the game is finished.
func (gl *gameLoop) Run(ctx context.Context) error {
	gl.logEntry.Debug("Starting")
	defer gl.logEntry.Debug("Stopped")
	err := gl.tickLoop(ctx)
	// Before returning control, wait for calls to eventHandler to finish
	gl.handleEventWG.Wait()
	return errors.Wrap(err, "error while running tickLoop")
}

// Sets the game over flag, returning an error if it is already set
func (gl *gameLoop) setGameOver() error {
	gl.gameOverMu.Lock()
	defer gl.gameOverMu.Unlock()
	if gl.gameOver {
		return errors.New("Game is already over")
	}
	gl.gameOver = true
	return nil
}

// Checks if the game over flag has been set
func (gl *gameLoop) isGameOver() bool {
	gl.gameOverMu.Lock()
	defer gl.gameOverMu.Unlock()
	return gl.gameOver
}

// Performs a "tick" each tickInterval. The tick is what updates the game, by applying
// actions that has been added since the last tick. This method blocks until the game
// is over, on an error occurs.
func (gl *gameLoop) tickLoop(ctx context.Context) error {
	tickInterval := gl.tickInterval
	ticker := time.NewTicker(tickInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
			if err := gl.tick(ctx, tickInterval); err != nil {
				return errors.Wrap(err, "Error when performing tick")
			}
		}
		// Check for game over after each tick, and stop the loop if game is over
		if gl.isGameOver() {
			return nil
		}
	}
}

// Perform a tick on the game; Applies all queued actions on the game
// sequentially, making it safe for the applied actions to modify the
// game-state. An additional TickAction is always performed as the
// last actions during a tick.
func (gl *gameLoop) tick(ctx context.Context, delta time.Duration) error {
	// Obtain a slice of the actions added since the last tick.
	acts := gl.getActions()

	// Add a tick actions as the last action to our local actions slice.
	tickAction := &model.ActionTick{Delta: delta}
	acts = append(acts, tickAction)

	for _, act := range acts {
		if err := gl.applyAction(ctx, act); err != nil {
			return err
		}
		// Check for game over after each action applied, and
		// stop processing actions if the game is over
		if gl.isGameOver() {
			return nil
		}
	}
	return nil
}

// Swaps the current slice of actions with a new empty slice, returning
// the previous actions.
func (gl *gameLoop) getActions() []model.Action {
	gl.actionsMu.Lock()
	defer gl.actionsMu.Unlock()
	acts := gl.actions

	// We slowly shrink the initial capacity of the actions here, so
	// that one tick with an abnormal number of actions doesn't result
	// in every new actions slice being allocated that same large size.
	newLen := len(acts) / 2
	gl.actions = make([]model.Action, 0, newLen)
	return acts
}

// Applies a single action to the game, and handles each event this action
// produced sequentially.
func (gl *gameLoop) applyAction(ctx context.Context, act model.Action) error {
	evts, err := act.Apply(gl.game)
	if err != nil {
		err = gl.handleActionError(ctx, err)
		return errors.Wrap(err, "Unable to handle action error")
	}
	if err := gl.handleEvents(ctx, evts); err != nil {
		return errors.Wrap(err, "Error handling events")
	}
	return nil
}

// Handles an error returned from applying an action. Non-fatal errors
// are logged and ignored. Fatal action errors results in a game over event
// being sent. All but non-fatal action errors are returned to the caller.
func (gl *gameLoop) handleActionError(ctx context.Context, err error) error {
	switch err := err.(type) {
	case model.ActionError:
		if !err.IsFatal() {
			gl.logEntry.WithError(err).Warn("Ignoring non-fatal ActionError")
			return nil
		}

		gl.logEntry.WithError(err).Debug("handle fatal ActionError")
		// Send a game over event with the opponent as winner
		var winner *model.Player
		if err.Player() != nil {
			winner = gl.game.Opponent(err.Player().ID())
		}
		gl.handleEvent(ctx, model.NewEventGameOver(winner))
	default:
		gl.logEntry.WithError(err).Debug("handle generic error")
	}
	return err
}

// handleEvent handles a single event by forwarding it to the registered
// eventHandler on a new go-routine.
func (gl *gameLoop) handleEvent(ctx context.Context, evt model.Event) {
	gl.eventHandlerMu.Lock()
	handler := gl.eventHandler
	gl.eventHandlerMu.Unlock()
	if handler == nil {
		gl.logEntry.Warn("handleEvent called, but no eventHandler was registered")
		return
	}
	gl.handleEventWG.Add(1)
	go func() {
		defer gl.handleEventWG.Done()
		handler.handleEvent(ctx, evt)
	}()
}

// handleEvents handles each event produced by delegating to the event handler.
// If an event representing the game being over is encountered, then all further
// events are discarded and the gameLoop is set to game over state.
func (gl *gameLoop) handleEvents(ctx context.Context, evts []model.Event) error {
	for _, evt := range evts {
		gl.handleEvent(ctx, evt)
		// Stop processing events if we sent a game over event
		if model.IsGameOverEvent(evt) {
			return gl.setGameOver()
		}
	}
	return nil
}
<commit_msg>Increase server tick rate (2hz -> 3hz)<commit_after>package game

import (
	"context"
	"github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
	"github.com/verath/archipelago/lib/common"
	"github.com/verath/archipelago/lib/game/model"
	"sync"
	"time"
)

const defaultTickInterval time.Duration = time.Second / 3

// The gameLoop is what updates the game model instance that it is
// associated with. The updates are performed in ticks. Each tick
// applies all actions that has been added since the last tick
// sequentially on the model. For each such actions, zero or more
// events are created. Those events are dispatched as the last stage
// of the tick.
//
// Notice that any reads or writes (outside of actions) on the model
// once the gameLoop is started is not safe.
type gameLoop struct {
	logEntry *logrus.Entry
	// Duration between each tick
	tickInterval time.Duration
	// The game instance on which actions are to be applied
	game *model.Game

	gameOverMu sync.Mutex
	// Flag for if the game has completed
	gameOver bool

	eventHandlerMu sync.Mutex
	// A handler to handle events produced when applying actions
	// to the game instance.
	eventHandler eventHandler
	// A WaitGroup for events being handled by the registered event
	// handler.
	handleEventWG sync.WaitGroup

	actionsMu sync.Mutex
	// A slice of actions to be applied on the next tick
	actions []model.Action
}

// A handler for game events produced from applying actions.
type eventHandler interface {
	// handleEvent handles an event produced. This method will be called on a
	// separate go routine and must block until the even has been handled, or
	// the context is cancelled.
	handleEvent(ctx context.Context, event model.Event)
}

func newGameLoop(log *logrus.Logger, game *model.Game) (*gameLoop, error) {
	logEntry := common.ModuleLogEntryWithID(log, "gameLoop")

	return &gameLoop{
		logEntry:     logEntry,
		tickInterval: defaultTickInterval,
		game:         game,
		actions:      make([]model.Action, 0),
	}, nil
}

// Sets the handler for game events.
func (gl *gameLoop) SetEventHandler(eventHandler eventHandler) {
	gl.eventHandlerMu.Lock()
	gl.eventHandler = eventHandler
	gl.eventHandlerMu.Unlock()
}

// Adds an action to be processed in the next tick.
func (gl *gameLoop) AddAction(action model.Action) {
	gl.actionsMu.Lock()
	gl.actions = append(gl.actions, action)
	gl.actionsMu.Unlock()
}

// Runs the game loop. Run blocks until the context is cancelled, an
// error occurs, or the game is finished.
func (gl *gameLoop) Run(ctx context.Context) error {
	gl.logEntry.Debug("Starting")
	defer gl.logEntry.Debug("Stopped")
	err := gl.tickLoop(ctx)
	// Before returning control, wait for calls to eventHandler to finish
	gl.handleEventWG.Wait()
	return errors.Wrap(err, "error while running tickLoop")
}

// Sets the game over flag, returning an error if it is already set
func (gl *gameLoop) setGameOver() error {
	gl.gameOverMu.Lock()
	defer gl.gameOverMu.Unlock()
	if gl.gameOver {
		return errors.New("Game is already over")
	}
	gl.gameOver = true
	return nil
}

// Checks if the game over flag has been set
func (gl *gameLoop) isGameOver() bool {
	gl.gameOverMu.Lock()
	defer gl.gameOverMu.Unlock()
	return gl.gameOver
}

// Performs a "tick" each tickInterval. The tick is what updates the game, by applying
// actions that has been added since the last tick. This method blocks until the game
// is over, on an error occurs.
func (gl *gameLoop) tickLoop(ctx context.Context) error {
	tickInterval := gl.tickInterval
	ticker := time.NewTicker(tickInterval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
			if err := gl.tick(ctx, tickInterval); err != nil {
				return errors.Wrap(err, "Error when performing tick")
			}
		}
		// Check for game over after each tick, and stop the loop if game is over
		if gl.isGameOver() {
			return nil
		}
	}
}

// Perform a tick on the game; Applies all queued actions on the game
// sequentially, making it safe for the applied actions to modify the
// game-state. An additional TickAction is always performed as the
// last actions during a tick.
func (gl *gameLoop) tick(ctx context.Context, delta time.Duration) error {
	// Obtain a slice of the actions added since the last tick.
	acts := gl.getActions()

	// Add a tick actions as the last action to our local actions slice.
	tickAction := &model.ActionTick{Delta: delta}
	acts = append(acts, tickAction)

	for _, act := range acts {
		if err := gl.applyAction(ctx, act); err != nil {
			return err
		}
		// Check for game over after each action applied, and
		// stop processing actions if the game is over
		if gl.isGameOver() {
			return nil
		}
	}
	return nil
}

// Swaps the current slice of actions with a new empty slice, returning
// the previous actions.
func (gl *gameLoop) getActions() []model.Action {
	gl.actionsMu.Lock()
	defer gl.actionsMu.Unlock()
	acts := gl.actions

	// We slowly shrink the initial capacity of the actions here, so
	// that one tick with an abnormal number of actions doesn't result
	// in every new actions slice being allocated that same large size.
	newLen := len(acts) / 2
	gl.actions = make([]model.Action, 0, newLen)
	return acts
}

// Applies a single action to the game, and handles each event this action
// produced sequentially.
func (gl *gameLoop) applyAction(ctx context.Context, act model.Action) error {
	evts, err := act.Apply(gl.game)
	if err != nil {
		err = gl.handleActionError(ctx, err)
		return errors.Wrap(err, "Unable to handle action error")
	}
	if err := gl.handleEvents(ctx, evts); err != nil {
		return errors.Wrap(err, "Error handling events")
	}
	return nil
}

// Handles an error returned from applying an action. Non-fatal errors
// are logged and ignored. Fatal action errors results in a game over event
// being sent. All but non-fatal action errors are returned to the caller.
func (gl *gameLoop) handleActionError(ctx context.Context, err error) error {
	switch err := err.(type) {
	case model.ActionError:
		if !err.IsFatal() {
			gl.logEntry.WithError(err).Warn("Ignoring non-fatal ActionError")
			return nil
		}

		gl.logEntry.WithError(err).Debug("handle fatal ActionError")
		// Send a game over event with the opponent as winner
		var winner *model.Player
		if err.Player() != nil {
			winner = gl.game.Opponent(err.Player().ID())
		}
		gl.handleEvent(ctx, model.NewEventGameOver(winner))
	default:
		gl.logEntry.WithError(err).Debug("handle generic error")
	}
	return err
}

// handleEvent handles a single event by forwarding it to the registered
// eventHandler on a new go-routine.
func (gl *gameLoop) handleEvent(ctx context.Context, evt model.Event) {
	gl.eventHandlerMu.Lock()
	handler := gl.eventHandler
	gl.eventHandlerMu.Unlock()
	if handler == nil {
		gl.logEntry.Warn("handleEvent called, but no eventHandler was registered")
		return
	}
	gl.handleEventWG.Add(1)
	go func() {
		defer gl.handleEventWG.Done()
		handler.handleEvent(ctx, evt)
	}()
}

// handleEvents handles each event produced by delegating to the event handler.
// If an event representing the game being over is encountered, then all further
// events are discarded and the gameLoop is set to game over state.
func (gl *gameLoop) handleEvents(ctx context.Context, evts []model.Event) error {
	for _, evt := range evts {
		gl.handleEvent(ctx, evt)
		// Stop processing events if we sent a game over event
		if model.IsGameOverEvent(evt) {
			return gl.setGameOver()
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package billing

// Person represents a bmbilling person
type Person struct {
	ID int `json:"id,omitempty"`
	// Username is the name this person uses to log in to our services.
	Username    string `json:"username"`
	Email       string `json:"email"`
	BackupEmail string `json:"email_backup,omitempty"`

	// only set in the creation request
	Password string `json:"password"`

	FirstName   string `json:"firstname"`
	LastName    string `json:"surname"`
	Address     string `json:"address"`
	City        string `json:"city"`
	StateCounty string `json:"statecounty,omitempty"`
	Postcode    string `json:"postcode"`
	Country     string `json:"country"`
	Phone       string `json:"phone"`
	MobilePhone string `json:"phonemobile,omitempty"`

	Organization         string `json:"organization,omitempty"`
	OrganizationDivision string `json:"division,omitempty"`
	VATNumber            string `json:"vatnumber,omitempty"`
}

func (p Person) IsValid() bool {
	return p.Username != ""
}
<commit_msg>Add documentation comment to billing.Person.IsValid<commit_after>package billing

// Person represents a bmbilling person
type Person struct {
	ID int `json:"id,omitempty"`
	// Username is the name this person uses to log in to our services.
	Username    string `json:"username"`
	Email       string `json:"email"`
	BackupEmail string `json:"email_backup,omitempty"`

	// only set in the creation request
	Password string `json:"password"`

	FirstName   string `json:"firstname"`
	LastName    string `json:"surname"`
	Address     string `json:"address"`
	City        string `json:"city"`
	StateCounty string `json:"statecounty,omitempty"`
	Postcode    string `json:"postcode"`
	Country     string `json:"country"`
	Phone       string `json:"phone"`
	MobilePhone string `json:"phonemobile,omitempty"`

	Organization         string `json:"organization,omitempty"`
	OrganizationDivision string `json:"division,omitempty"`
	VATNumber            string `json:"vatnumber,omitempty"`
}

// IsValid returns true if the Person is valid, false otherwise.
func (p Person) IsValid() bool {
	return p.Username != ""
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"sync"
	"testing"
	"encoding/json"
)

// Test the google cloud messaging service.
func TestGcm(t *testing.T) {

	mockResponse := testCreateGoogleCloudMsgResponse(100, 1, 0, 0)
	testAddGoogleCloudMsgResponseResult(mockResponse, "someMessageId", nadaStr, nadaStr)

	data, err := json.Marshal(mockResponse)
	if err != nil { t.Errorf("TestGcm json encode mock response broken - err: %v", err); return }

	httpClient := NewHttpRequestClientMock()
	httpClient.(*HttpRequestClientMock).AddMock("https://android.googleapis.com/gcm/send", &HttpRequestClientMockResponse{
		HttpStatusCode: 200,
		Data: data,
	})

	requestChannel := make(chan interface{})
	responseChannel := make(chan interface{})

	svc := NewGoogleCloudMessagingSvc("gcm", httpClient, requestChannel, responseChannel)

	kernel, err := baseTestStartKernel("gcmTest", func(kernel *Kernel) {
		kernel.AddComponentWithStartStopMethods("GoogleCloudMessagingSvc", svc, "Start", "Stop")
	})

	if err != nil { t.Errorf("TestGcm start kernel is broken: %v", err); return }

	msgSendCount := 100000
	msgReceivedCount := 0

	var waitGroup sync.WaitGroup

	go func() {
		waitGroup.Add(1)
		defer waitGroup.Done()
		for {
			msg := <- responseChannel
			if msg == nil { t.Errorf("TestGcm is broken - response message is nil") }
			msgReceivedCount++
			if msgReceivedCount == msgSendCount { return }
		}
	}()

	for idx := 0; idx < msgSendCount; idx++ {
		requestChannel <- &GoogleCloudMsg{
			RegistrationIds: []string { "someRegistrationId" },
			CollapseKey: "someCollapseKey",
			DelayWhileIdle: true,
			TimeToLive: 300,
			RestrictedPackageName: "somePackageName",
			DryRun: false,
			Data: map[string]interface{} { "someKey": "someValue" },
		}
	}

	waitGroup.Wait()

	close(requestChannel)

	if err := kernel.Stop(); err != nil { t.Errorf("TestGcm stop kernel is broken:", err) }
}

func testCreateGoogleCloudMsgResponse(multicastId, success, failure, canonicalIds float64) *GoogleCloudMsgResponse {
	return &GoogleCloudMsgResponse {
		MulticastId: multicastId,
		Success: success,
		Failure: failure,
		CanonicalIds: canonicalIds,
	}
}

func testAddGoogleCloudMsgResponseResult(response *GoogleCloudMsgResponse, messageId, registrationId, err string) {
	response.Results = append(response.Results, &GoogleCloudMsgResponseResult{
		MessageId: messageId,
		RegistrationId: registrationId,
		Error: err,
	})
}

<commit_msg>reduced test msg count<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"sync"
	"testing"
	"encoding/json"
)

// Test the google cloud messaging service.
func TestGcm(t *testing.T) {

	mockResponse := testCreateGoogleCloudMsgResponse(100, 1, 0, 0)
	testAddGoogleCloudMsgResponseResult(mockResponse, "someMessageId", nadaStr, nadaStr)

	data, err := json.Marshal(mockResponse)
	if err != nil { t.Errorf("TestGcm json encode mock response broken - err: %v", err); return }

	httpClient := NewHttpRequestClientMock()
	httpClient.(*HttpRequestClientMock).AddMock("https://android.googleapis.com/gcm/send", &HttpRequestClientMockResponse{
		HttpStatusCode: 200,
		Data: data,
	})

	requestChannel := make(chan interface{})
	responseChannel := make(chan interface{})

	svc := NewGoogleCloudMessagingSvc("gcm", httpClient, requestChannel, responseChannel)

	kernel, err := baseTestStartKernel("gcmTest", func(kernel *Kernel) {
		kernel.AddComponentWithStartStopMethods("GoogleCloudMessagingSvc", svc, "Start", "Stop")
	})

	if err != nil { t.Errorf("TestGcm start kernel is broken: %v", err); return }

	msgSendCount := 10000
	msgReceivedCount := 0

	var waitGroup sync.WaitGroup

	go func() {
		waitGroup.Add(1)
		defer waitGroup.Done()
		for {
			msg := <- responseChannel
			if msg == nil { t.Errorf("TestGcm is broken - response message is nil") }
			msgReceivedCount++
			if msgReceivedCount == msgSendCount { return }
		}
	}()

	for idx := 0; idx < msgSendCount; idx++ {
		requestChannel <- &GoogleCloudMsg{
			RegistrationIds: []string { "someRegistrationId" },
			CollapseKey: "someCollapseKey",
			DelayWhileIdle: true,
			TimeToLive: 300,
			RestrictedPackageName: "somePackageName",
			DryRun: false,
			Data: map[string]interface{} { "someKey": "someValue" },
		}
	}

	waitGroup.Wait()

	close(requestChannel)

	if err := kernel.Stop(); err != nil { t.Errorf("TestGcm stop kernel is broken:", err) }
}

func testCreateGoogleCloudMsgResponse(multicastId, success, failure, canonicalIds float64) *GoogleCloudMsgResponse {
	return &GoogleCloudMsgResponse {
		MulticastId: multicastId,
		Success: success,
		Failure: failure,
		CanonicalIds: canonicalIds,
	}
}

func testAddGoogleCloudMsgResponseResult(response *GoogleCloudMsgResponse, messageId, registrationId, err string) {
	response.Results = append(response.Results, &GoogleCloudMsgResponseResult{
		MessageId: messageId,
		RegistrationId: registrationId,
		Error: err,
	})
}

<|endoftext|>


<commit_before>package kbdita

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"github.com/bradfitz/slice"

	"github.com/raintreeinc/knowledgebase/kb"
	"github.com/raintreeinc/knowledgebase/kbserver"

	"github.com/raintreeinc/knowledgebase/ditaconv"
	"github.com/raintreeinc/knowledgebase/ditaconv/xmlconv"
)

var _ *kb.Page
var _ kbserver.System = &System{}

type System struct {
	name    string
	ditamap string
	server  *kbserver.Server

	store atomic.Value
}

func New(name, ditamap string, server *kbserver.Server) *System {
	sys := &System{
		name:    name,
		ditamap: ditamap,
		server:  server,
	}
	sys.init()
	return sys
}

func (sys *System) Name() string { return sys.name }

func (sys *System) init() {
	sys.store.Store(newstore())
	go sys.monitor()
}

func (sys *System) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	store := sys.store.Load().(*store)
	path := strings.TrimPrefix(r.URL.Path, "/")
	slug := kb.Slugify(path)
	if data, ok := store.raw[slug]; ok {
		w.Write(data)
		return
	}

	name := kb.Slugify(sys.name)
	switch slug {
	case name + ":conversion-errors":
	case name + ":all-pages":
		page := &kb.Page{
			Slug:     name + ":all-pages",
			Title:    "All Pages",
			Modified: time.Now(),
		}

		content := "
"
		for _, slug := range store.slugs {
			page := store.pages[slug]
			content += fmt.Sprintf("
%s
", slug, page.Title)
		}
		content += "
"

		page.Story.Append(kb.HTML(content))
		kbserver.WriteJSON(w, r, page)
		return
	}
	http.NotFound(w, r)
}

func (sys *System) reload() {
	start := time.Now()
	sys.store.Store(load(sys.name, sys.ditamap))
	log.Println("DITA reloaded (", time.Since(start), ")")
}

func (sys *System) monitor() {
	modified := time.Now()
	sys.reload()
	for range time.Tick(1 * time.Second) {
		filepath.Walk(filepath.Dir(sys.ditamap),
			func(_ string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.ModTime().After(modified) {
					modified = time.Now()
					sys.reload()
					return errors.New("stop iterate")
				}
				return nil
			})
	}
}

func newstore() *store {
	return &store{
		pages: make(map[kb.Slug]*kb.Page),
		raw:   make(map[kb.Slug][]byte),
	}
}

type store struct {
	pages map[kb.Slug]*kb.Page
	raw   map[kb.Slug][]byte
	slugs []kb.Slug

	errLoad    []error
	errMapping []error
	errConvert []convertError
}

type convertError struct {
	slug   kb.Slug
	fatal  error
	errors []error
}

func load(prefix, ditamap string) *store {
	store := newstore()

	index, errs := ditaconv.LoadIndex(ditamap)
	store.errLoad = errs

	mapping, errs := ditaconv.CreateMapping(index)
	store.errMapping = errs

	for topic, slug := range mapping.ByTopic {
		ownerslug := kb.Slugify(prefix+":") + slug
		mapping.ByTopic[topic] = ownerslug
		delete(mapping.BySlug, slug)
		mapping.BySlug[ownerslug] = topic
	}

	mapping.Rules.Merge(RaintreeDITA())
	for slug, topic := range mapping.BySlug {
		page, fatal, errs := mapping.Convert(topic)
		if fatal != nil {
			store.errConvert = append(store.errConvert, convertError{slug: slug, fatal: fatal})
		} else if len(errs) > 0 {
			store.errConvert = append(store.errConvert, convertError{slug: slug, errors: errs})
		}

		data, err := json.Marshal(page)
		if err != nil {
			log.Println(err)
		}

		store.pages[slug] = page
		store.raw[slug] = data
		store.slugs = append(store.slugs, slug)
	}

	slice.Sort(store.slugs, func(i, j int) bool {
		return store.slugs[i] < store.slugs[j]
	})

	return store
}

func RaintreeDITA() *xmlconv.Rules {
	return &xmlconv.Rules{
		Translate: map[string]string{
			"keystroke": "span",
			"secright":  "span",

			// faq
			"faq":          "dl",
			"faq-question": "dt",
			"faq-answer":   "dd",

			//UI items
			"ui-item-list": "dl",

			"ui-item-name":        "dt",
			"ui-item-description": "dd",

			// setup options
			"setup-options": "dl",

			"setup-option-name":        "dt",
			"setup-option-description": "dd",

			"settingdesc": "div",
			"settingname": "h3",
		},
		Remove: map[string]bool{
			"settinghead": true,
		},
		Unwrap: map[string]bool{
			"ui-item":      true,
			"faq-item":     true,
			"setup-option": true,

			"settings": true,
			"setting":  true,
		},
		Callback: map[string]xmlconv.Callback{
			"settingdefault": func(enc xmlconv.Encoder, dec *xml.Decoder, start *xml.StartElement) error {
				val, _ := xmlconv.Text(dec, start)
				if val != "" {
					err := enc.WriteRaw("
Default value: " + val + "
")
					if err != nil {
						return err
					}
				}
				return nil
			},
			"settinglevels": func(enc xmlconv.Encoder, dec *xml.Decoder, start *xml.StartElement) error {
				err := enc.WriteRaw("
Levels where it can be defined:
")
				if err != nil {
					return err
				}
				if err := enc.Rules().ConvertChildren(enc, dec, start); err != nil {
					return err
				}
				return nil
			},
			"settingsample": func(enc xmlconv.Encoder, dec *xml.Decoder, start *xml.StartElement) error {
				err := enc.WriteRaw("
Example:
")
				if err != nil {
					return err
				}
				if err := enc.Rules().ConvertChildren(enc, dec, start); err != nil {
					return err
				}
				return nil
			},
		},
	}
}
<commit_msg>Add dita:errors page.<commit_after>package kbdita

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"time"

	"github.com/bradfitz/slice"

	"github.com/raintreeinc/knowledgebase/kb"
	"github.com/raintreeinc/knowledgebase/kbserver"

	"github.com/raintreeinc/knowledgebase/ditaconv"
	"github.com/raintreeinc/knowledgebase/ditaconv/xmlconv"
)

var _ *kb.Page
var _ kbserver.System = &System{}

type System struct {
	name    string
	ditamap string
	server  *kbserver.Server

	store atomic.Value
}

func New(name, ditamap string, server *kbserver.Server) *System {
	sys := &System{
		name:    name,
		ditamap: ditamap,
		server:  server,
	}
	sys.init()
	return sys
}

func (sys *System) Name() string { return sys.name }

func (sys *System) init() {
	sys.store.Store(newstore())
	go sys.monitor()
}

func (sys *System) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	store := sys.store.Load().(*store)
	path := strings.TrimPrefix(r.URL.Path, "/")
	slug := kb.Slugify(path)
	if data, ok := store.raw[slug]; ok {
		w.Write(data)
		return
	}

	name := kb.Slugify(sys.name)
	switch slug {
	case name + ":errors":
		page := &kb.Page{}
		page.Slug = name + ":errors"
		page.Title = "Errors"
		page.Modified = time.Now()

		page.Story.Append(kb.HTML("
Loading
"))
		for _, err := range store.errLoad {
			page.Story.Append(kb.Paragraph(err.Error()))
		}

		page.Story.Append(kb.HTML("
Mapping
"))
		for _, err := range store.errMapping {
			page.Story.Append(kb.Paragraph(err.Error()))
		}

		page.Story.Append(kb.HTML("
Converting
"))
		for _, errs := range store.errConvert {
			text := "
[" + string(errs.slug) + "]
"
			for _, err := range errs.errors {
				text += "
" + err.Error() + "
"
			}
			page.Story.Append(kb.HTML(text))
		}
		kbserver.WriteJSON(w, r, page)
		return
	case name + ":all-pages":
		page := &kb.Page{
			Slug:     name + ":all-pages",
			Title:    "All Pages",
			Modified: time.Now(),
		}

		content := "
"
		for _, slug := range store.slugs {
			page := store.pages[slug]
			content += fmt.Sprintf("
%s
", slug, page.Title)
		}
		content += "
"

		page.Story.Append(kb.HTML(content))
		kbserver.WriteJSON(w, r, page)
		return
	}
	http.NotFound(w, r)
}

func (sys *System) reload() {
	start := time.Now()
	sys.store.Store(load(sys.name, sys.ditamap))
	log.Println("DITA reloaded (", time.Since(start), ")")
}

func (sys *System) monitor() {
	modified := time.Now()
	sys.reload()
	for range time.Tick(1 * time.Second) {
		filepath.Walk(filepath.Dir(sys.ditamap),
			func(_ string, info os.FileInfo, err error) error {
				if err != nil {
					return err
				}
				if info.ModTime().After(modified) {
					modified = time.Now()
					sys.reload()
					return errors.New("stop iterate")
				}
				return nil
			})
	}
}

func newstore() *store {
	return &store{
		pages: make(map[kb.Slug]*kb.Page),
		raw:   make(map[kb.Slug][]byte),
	}
}

type store struct {
	pages map[kb.Slug]*kb.Page
	raw   map[kb.Slug][]byte
	slugs []kb.Slug

	errLoad    []error
	errMapping []error
	errConvert []convertError
}

type convertError struct {
	slug   kb.Slug
	fatal  error
	errors []error
}

func load(prefix, ditamap string) *store {
	store := newstore()

	index, errs := ditaconv.LoadIndex(ditamap)
	store.errLoad = errs

	mapping, errs := ditaconv.CreateMapping(index)
	store.errMapping = errs

	for topic, slug := range mapping.ByTopic {
		ownerslug := kb.Slugify(prefix+":") + slug
		mapping.ByTopic[topic] = ownerslug
		delete(mapping.BySlug, slug)
		mapping.BySlug[ownerslug] = topic
	}

	mapping.Rules.Merge(RaintreeDITA())
	for slug, topic := range mapping.BySlug {
		page, fatal, errs := mapping.Convert(topic)
		if fatal != nil {
			store.errConvert = append(store.errConvert, convertError{slug: slug, fatal: fatal})
		} else if len(errs) > 0 {
			store.errConvert = append(store.errConvert, convertError{slug: slug, errors: errs})
		}

		data, err := json.Marshal(page)
		if err != nil {
			log.Println(err)
		}

		store.pages[slug] = page
		store.raw[slug] = data
		store.slugs = append(store.slugs, slug)
	}

	slice.Sort(store.slugs, func(i, j int) bool {
		return store.slugs[i] < store.slugs[j]
	})

	return store
}

func RaintreeDITA() *xmlconv.Rules {
	return &xmlconv.Rules{
		Translate: map[string]string{
			"keystroke": "span",
			"secright":  "span",

			// faq
			"faq":          "dl",
			"faq-question": "dt",
			"faq-answer":   "dd",

			//UI items
			"ui-item-list": "dl",

			"ui-item-name":        "dt",
			"ui-item-description": "dd",

			// setup options
			"setup-options": "dl",

			"setup-option-name":        "dt",
			"setup-option-description": "dd",

			"settingdesc": "div",
			"settingname": "h3",
		},
		Remove: map[string]bool{
			"settinghead": true,
		},
		Unwrap: map[string]bool{
			"ui-item":      true,
			"faq-item":     true,
			"setup-option": true,

			"settings": true,
			"setting":  true,
		},
		Callback: map[string]xmlconv.Callback{
			"settingdefault": func(enc xmlconv.Encoder, dec *xml.Decoder, start *xml.StartElement) error {
				val, _ := xmlconv.Text(dec, start)
				if val != "" {
					err := enc.WriteRaw("
Default value: " + val + "
")
					if err != nil {
						return err
					}
				}
				return nil
			},
			"settinglevels": func(enc xmlconv.Encoder, dec *xml.Decoder, start *xml.StartElement) error {
				err := enc.WriteRaw("
Levels where it can be defined:
")
				if err != nil {
					return err
				}
				if err := enc.Rules().ConvertChildren(enc, dec, start); err != nil {
					return err
				}
				return nil
			},
			"settingsample": func(enc xmlconv.Encoder, dec *xml.Decoder, start *xml.StartElement) error {
				err := enc.WriteRaw("
Example:
")
				if err != nil {
					return err
				}
				if err := enc.Rules().ConvertChildren(enc, dec, start); err != nil {
					return err
				}
				return nil
			},
		},
	}
}
<|endoftext|>


<commit_before>// Keybase file system

package main

import (
	"flag"
	_ "fmt"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"

	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/kbfs/libkbfs"
)

func GetUI() libkb.UI {
	ui := &libkbfs.UI{}
	ui.Configure()
	return ui
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write memory profile to file")
var local = flag.Bool("local", false,
	"use a fake local user DB instead of Keybase")
var localUser = flag.String("localuser", "strib",
	"fake local user (only valid when local=true)")
var client = flag.Bool("client", false, "use keybase daemon")
var debug = flag.Bool("debug", false, "Print FUSE debug messages")

func main() {
	flag.Parse()
	if len(flag.Args()) < 1 {
		log.Fatal("Usage:\n  kbfs MOUNTPOINT")
	}

	var cpuProfFile *os.File
	if *cpuprofile != "" {
		var err error
		cpuProfFile, err = os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(cpuProfFile)
		defer cpuProfFile.Close()
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt, os.Kill)
	go func() {
		_ = <-sigchan
		if *cpuprofile != "" {
			pprof.StopCPUProfile()
		}

		if *memprofile != "" {
			f, err := os.Create(*memprofile)
			if err != nil {
				log.Fatal(err)
			}
			pprof.WriteHeapProfile(f)
			f.Close()
		}
		os.Exit(1)
	}()

	config := libkbfs.NewConfigLocal()

	libkb.G.Init()
	libkb.G.ConfigureConfig()
	libkb.G.ConfigureLogging()
	libkb.G.ConfigureCaches()
	libkb.G.ConfigureMerkleClient()
	libkb.G.SetUI(GetUI())

	if *local {
		var localUid libkb.UID
		switch {
		case *localUser == "strib":
			localUid = libkb.UID{1}
		case *localUser == "max":
			localUid = libkb.UID{2}
		case *localUser == "chris":
			localUid = libkb.UID{3}
		}
		k := libkbfs.NewKBPKILocal(localUid, []*libkbfs.LocalUser{
			&libkbfs.LocalUser{"strib", libkb.UID{1}, []string{"github:strib"}},
			&libkbfs.LocalUser{"max", libkb.UID{2}, []string{"twitter:maxtaco"}},
			&libkbfs.LocalUser{
				"chris", libkb.UID{3}, []string{"twitter:malgorithms"}},
		})
		config.SetKBPKI(k)
	} else if *client {
		libkb.G.ConfigureSocketInfo()
		k := libkbfs.NewKBPKIClient()
		config.SetKBPKI(k)
	} else {
		libkb.G.ConfigureAPI()
		if err := libkb.G.LoginState.SessionLoad(); err != nil {
			log.Fatalf("Couldn't load session: %v\n", err)
		}
		if err := libkb.G.LoginState.AssertLoggedIn(); err != nil {
			log.Fatalf("Couldn't check session: %v\n", err)
		}
	}

	root := NewFuseRoot(config)

	server, _, err := nodefs.MountRoot(flag.Arg(0), root, nil)
	if err != nil {
		log.Fatalf("Mount fail: %v\n", err)
	}

	if *debug {
		server.SetDebug(true)
	}
	server.Serve()
}
<commit_msg>kbfsfuse: Use a better LoginState method to load the session<commit_after>// Keybase file system

package main

import (
	"flag"
	_ "fmt"
	"log"
	"os"
	"os/signal"
	"runtime/pprof"

	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/keybase/client/go/libkb"
	"github.com/keybase/kbfs/libkbfs"
)

func GetUI() libkb.UI {
	ui := &libkbfs.UI{}
	ui.Configure()
	return ui
}

var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
var memprofile = flag.String("memprofile", "", "write memory profile to file")
var local = flag.Bool("local", false,
	"use a fake local user DB instead of Keybase")
var localUser = flag.String("localuser", "strib",
	"fake local user (only valid when local=true)")
var client = flag.Bool("client", false, "use keybase daemon")
var debug = flag.Bool("debug", false, "Print FUSE debug messages")

func main() {
	flag.Parse()
	if len(flag.Args()) < 1 {
		log.Fatal("Usage:\n  kbfs MOUNTPOINT")
	}

	var cpuProfFile *os.File
	if *cpuprofile != "" {
		var err error
		cpuProfFile, err = os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(cpuProfFile)
		defer cpuProfFile.Close()
	}

	sigchan := make(chan os.Signal, 1)
	signal.Notify(sigchan, os.Interrupt, os.Kill)
	go func() {
		_ = <-sigchan
		if *cpuprofile != "" {
			pprof.StopCPUProfile()
		}

		if *memprofile != "" {
			f, err := os.Create(*memprofile)
			if err != nil {
				log.Fatal(err)
			}
			pprof.WriteHeapProfile(f)
			f.Close()
		}
		os.Exit(1)
	}()

	config := libkbfs.NewConfigLocal()

	libkb.G.Init()
	libkb.G.ConfigureConfig()
	libkb.G.ConfigureLogging()
	libkb.G.ConfigureCaches()
	libkb.G.ConfigureMerkleClient()
	libkb.G.SetUI(GetUI())

	if *local {
		var localUid libkb.UID
		switch {
		case *localUser == "strib":
			localUid = libkb.UID{1}
		case *localUser == "max":
			localUid = libkb.UID{2}
		case *localUser == "chris":
			localUid = libkb.UID{3}
		}
		k := libkbfs.NewKBPKILocal(localUid, []*libkbfs.LocalUser{
			&libkbfs.LocalUser{"strib", libkb.UID{1}, []string{"github:strib"}},
			&libkbfs.LocalUser{"max", libkb.UID{2}, []string{"twitter:maxtaco"}},
			&libkbfs.LocalUser{
				"chris", libkb.UID{3}, []string{"twitter:malgorithms"}},
		})
		config.SetKBPKI(k)
	} else if *client {
		libkb.G.ConfigureSocketInfo()
		k := libkbfs.NewKBPKIClient()
		config.SetKBPKI(k)
	} else {
		libkb.G.ConfigureAPI()
		if ok, err := libkb.G.LoginState.IsLoggedInLoad(); !ok || err != nil {
			log.Fatalf("Couldn't load session: %v\n", err)
		}
	}

	root := NewFuseRoot(config)

	server, _, err := nodefs.MountRoot(flag.Arg(0), root, nil)
	if err != nil {
		log.Fatalf("Mount fail: %v\n", err)
	}

	if *debug {
		server.SetDebug(true)
	}
	server.Serve()
}
<|endoftext|>


<commit_before>package network

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/verath/archipelago/lib/common"
	"sync"
	"sync/atomic"
	"time"
)

var (
	ErrClientDisconnected = errors.New("Client has disconnected")
)

const (
	// Max number of messages buffered in the readQueue. If this
	// number is exceeded, the readPump will start dropping messages.
	clientReadBufferSize = 32

	// Max time the client will wait for the underlying connection
	// to cleanly shutdown before force closing.
	clientShutdownWait = 1 * time.Second
)

// A Client represents a network peer to which it is possible to send
// and receive messages. The client handles encoding and decoding messages
// from/to envelopes, and makes sure only a single read and a single write
// is simultaneously performed on the underlying connection.
type Client interface {
	// Starts the client. Start must be called before any other methods
	// on the client. Blocks until the client is started.
	Start()

	// Stops the client, disconnecting the underlying connection.
	// Disconnect blocks until the client is fully stopped. Calling
	// Disconnect on an already disconnected Client is a no-op.
	Disconnect()

	// DisconnectCh is a channel closed when the client is disconnected.
	// A disconnected client will not successfully perform any reads or
	// writes.
	DisconnectCh() <-chan struct{}

	// Writes data to the client, provided as an envelope. Blocks until the
	// message is successfully written to the client, or the context is
	// cancelled.
	WriteEnvelope(ctx context.Context, envelope *envelope) error

	// Reads data from the client, returned as an envelope. Read blocks until
	// the read is successful, or the context is cancelled. If ReadEnvelope
	// returns ErrClientDisconnected, then any future reads will also return
	// the same error.
	ReadEnvelope(ctx context.Context) (ReceivedEnvelope, error)
}

// clientImpl is an implementation of the Client interface.
type clientImpl struct {
	logEntry *logrus.Entry
	// The underlying connection used for the client.
	conn connection
	// Buffered queue of messages that has been read from the connection.
	readQueue chan *receivedEnvelopeImpl
	// Queue of writes to be made on the connection.
	writeQueue chan *writeRequest
	// Channel closed when the client should disconnect.
	disconnectCh chan struct{}
	// Flag for if the client has been started.
	started int32
	// Wait group used to wait for the read and write pump to start.
	startWG sync.WaitGroup
	// Wait group used to wait for the read and write pump to finish.
	shutdownWG sync.WaitGroup
	// Lock around the disconnectCh so that it is only closed once.
	disconnectOnce sync.Once
}

// A struct encapsulating a message to be written and a channel
// for returning the result of the write operation.
type writeRequest struct {
	envelope *envelope
	// A channel used to return the result of the write. Sending
	// a message to this channel must not block.
	resultCh chan<- error
}

// Creates a new Client, communicating on the provided connection
func NewClient(log *logrus.Logger, conn connection) (Client, error) {
	return &clientImpl{
		logEntry:     common.ModuleLogEntryWithID(log, "network/client"),
		conn:         conn,
		readQueue:    make(chan *receivedEnvelopeImpl, clientReadBufferSize),
		writeQueue:   make(chan *writeRequest),
		disconnectCh: make(chan struct{}),
	}, nil
}

func (c *clientImpl) Start() {
	if !c.setStarted() {
		c.logEntry.Warn("Start called when already started")
		return
	}
	c.startWG.Add(2)
	c.shutdownWG.Add(2)
	go c.readPump()
	go c.writePump()
	c.startWG.Wait()
	c.logEntry.Debug("Started")
}

func (c *clientImpl) Disconnect() {
	c.disconnect()
	c.shutdownWG.Wait()
	c.logEntry.Debug("Stopped")
}

func (c *clientImpl) DisconnectCh() <-chan struct{} {
	return c.disconnectCh
}

func (c *clientImpl) WriteEnvelope(ctx context.Context, envelope *envelope) error {
	resultCh := make(chan error)
	req := &writeRequest{envelope, resultCh}
	select {
	case c.writeQueue <- req:
		// TODO(2017-01-08): select on ctx.Done() here too? If we do,
		// make resultCh buffered.
		return <-resultCh
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (c *clientImpl) ReadEnvelope(ctx context.Context) (ReceivedEnvelope, error) {
	select {
	case env, ok := <-c.readQueue:
		if !ok {
			return nil, ErrClientDisconnected
		}
		return env, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// Sets the started flag, returning true if it had not been set previously.
func (c *clientImpl) setStarted() bool {
	return atomic.CompareAndSwapInt32(&c.started, 0, 1)
}

// Disconnects the client by closing the disconnectCh, as well as the
// underlying connection. Does not block until the read and write
// pumps have finished.
func (c *clientImpl) disconnect() {
	c.disconnectOnce.Do(func() {
		close(c.disconnectCh)
		// By closing the underlying connection here, we force the
		// read and write pumps to get unblocked. We first attempt a
		// clean shutdown, if it doesn't succeed within the shutdown
		// wait timeout, we force-close the connection instead.
		ctx, cancel := context.WithTimeout(context.Background(), clientShutdownWait)
		defer cancel()
		if err := c.conn.Shutdown(ctx); err != nil {
			c.logEntry.WithError(err).Debug("Failed clean shutdown, closing.")
			c.conn.Close()
		}
		c.logEntry.Debug("Disconnected")
	})
}

// Encodes and writes an envelope to the connection
func (c *clientImpl) writeEnvelope(env *envelope) error {
	msg, err := json.Marshal(env)
	if err != nil {
		return err
	}
	return c.conn.WriteMessage(msg)
}

// The write pump takes write requests from the write queue and
// writes them to the connection.
func (c *clientImpl) writePump() {
	c.startWG.Done()
	defer c.shutdownWG.Done()
	// If we ever get an error when writing, then all following
	// writes will fail. Instead of attempting to write, we store
	// and return that same error to each write request.
	var writeErr error
	for {
		select {
		case req := <-c.writeQueue:
			if writeErr == nil {
				writeErr = c.writeEnvelope(req.envelope)
			}
			req.resultCh <- writeErr
		case <-c.disconnectCh:
			return
		}
	}
}

// Reads a message from the connection and decodes it as an envelope.
func (c *clientImpl) readEnvelope() (*receivedEnvelopeImpl, error) {
	msg, err := c.conn.ReadMessage()
	if err != nil {
		return nil, fmt.Errorf("Error reading message from conn: %v", err)
	}
	recvEnv := &receivedEnvelopeImpl{}
	return recvEnv, json.Unmarshal(msg, recvEnv)
}

// The read pump reads messages from the connection and posts them on the
// read queue. If a read fails, the read pump will disconnect the client.
// If the read queue is full, new messages will be dropped.
func (c *clientImpl) readPump() {
	c.startWG.Done()
	defer func() {
		close(c.readQueue)
		c.shutdownWG.Done()
	}()
	for {
		select {
		default:
			env, err := c.readEnvelope()
			if err != nil {
				c.logEntry.WithError(err).Debug("Error reading from connection")
				c.disconnect()
				return
			}
			// Try to add the envelope to the readQueue. If the queue is full we
			// have to drop the message so we can continue reading, otherwise we
			// cannot detect connection errors.
			select {
			case c.readQueue <- env:
			default:
				c.logEntry.Warn("readQueue full, dropping message")
			}
		case <-c.disconnectCh:
			return
		}
	}
}
<commit_msg>Listen for disconnect when writing envelopes<commit_after>package network

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/verath/archipelago/lib/common"
	"sync"
	"sync/atomic"
	"time"
)

var (
	ErrClientDisconnected = errors.New("Client has disconnected")
)

const (
	// Max number of messages buffered in the readQueue. If this
	// number is exceeded, the readPump will start dropping messages.
	clientReadBufferSize = 32

	// Max time the client will wait for the underlying connection
	// to cleanly shutdown before force closing.
	clientShutdownWait = 1 * time.Second
)

// A Client represents a network peer to which it is possible to send
// and receive messages. The client handles encoding and decoding messages
// from/to envelopes, and makes sure only a single read and a single write
// is simultaneously performed on the underlying connection.
type Client interface {
	// Starts the client. Start must be called before any other methods
	// on the client. Blocks until the client is started.
	Start()

	// Stops the client, disconnecting the underlying connection.
	// Disconnect blocks until the client is fully stopped. Calling
	// Disconnect on an already disconnected Client is a no-op.
	Disconnect()

	// DisconnectCh is a channel closed when the client is disconnected.
	// A disconnected client will not successfully perform any reads or
	// writes.
	DisconnectCh() <-chan struct{}

	// Writes data to the client, provided as an envelope. Blocks until the
	// message is successfully written to the client, or the context is
	// cancelled.
	WriteEnvelope(ctx context.Context, envelope *envelope) error

	// Reads data from the client, returned as an envelope. Read blocks until
	// the read is successful, or the context is cancelled. If ReadEnvelope
	// returns ErrClientDisconnected, then any future reads will also return
	// the same error.
	ReadEnvelope(ctx context.Context) (ReceivedEnvelope, error)
}

// clientImpl is an implementation of the Client interface.
type clientImpl struct {
	logEntry *logrus.Entry
	// The underlying connection used for the client.
	conn connection
	// Buffered queue of messages that has been read from the connection.
	readQueue chan *receivedEnvelopeImpl
	// Queue of writes to be made on the connection.
	writeQueue chan *writeRequest
	// Channel closed when the client should disconnect.
	disconnectCh chan struct{}
	// Flag for if the client has been started.
	started int32
	// Wait group used to wait for the read and write pump to start.
	startWG sync.WaitGroup
	// Wait group used to wait for the read and write pump to finish.
	shutdownWG sync.WaitGroup
	// Lock around the disconnectCh so that it is only closed once.
	disconnectOnce sync.Once
}

// A struct encapsulating a message to be written and a channel
// for returning the result of the write operation.
type writeRequest struct {
	envelope *envelope
	// A channel used to return the result of the write. Sending
	// a message to this channel must not block.
	resultCh chan<- error
}

// Creates a new Client, communicating on the provided connection
func NewClient(log *logrus.Logger, conn connection) (Client, error) {
	return &clientImpl{
		logEntry:     common.ModuleLogEntryWithID(log, "network/client"),
		conn:         conn,
		readQueue:    make(chan *receivedEnvelopeImpl, clientReadBufferSize),
		writeQueue:   make(chan *writeRequest),
		disconnectCh: make(chan struct{}),
	}, nil
}

func (c *clientImpl) Start() {
	if !c.setStarted() {
		c.logEntry.Warn("Start called when already started")
		return
	}
	c.startWG.Add(2)
	c.shutdownWG.Add(2)
	go c.readPump()
	go c.writePump()
	c.startWG.Wait()
	c.logEntry.Debug("Started")
}

func (c *clientImpl) Disconnect() {
	c.disconnect()
	c.shutdownWG.Wait()
	c.logEntry.Debug("Stopped")
}

func (c *clientImpl) DisconnectCh() <-chan struct{} {
	return c.disconnectCh
}

func (c *clientImpl) WriteEnvelope(ctx context.Context, envelope *envelope) error {
	resultCh := make(chan error)
	req := &writeRequest{envelope, resultCh}
	select {
	case c.writeQueue <- req:
		// TODO(2017-01-08): select on ctx.Done() here too? If we do,
		// make resultCh buffered.
		return <-resultCh
	case <-c.disconnectCh:
		return ErrClientDisconnected
	case <-ctx.Done():
		return ctx.Err()
	}
}

func (c *clientImpl) ReadEnvelope(ctx context.Context) (ReceivedEnvelope, error) {
	select {
	case env, ok := <-c.readQueue:
		if !ok {
			return nil, ErrClientDisconnected
		}
		return env, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// Sets the started flag, returning true if it had not been set previously.
func (c *clientImpl) setStarted() bool {
	return atomic.CompareAndSwapInt32(&c.started, 0, 1)
}

// Disconnects the client by closing the disconnectCh, as well as the
// underlying connection. Does not block until the read and write
// pumps have finished.
func (c *clientImpl) disconnect() {
	c.disconnectOnce.Do(func() {
		close(c.disconnectCh)
		// By closing the underlying connection here, we force the
		// read and write pumps to get unblocked. We first attempt a
		// clean shutdown, if it doesn't succeed within the shutdown
		// wait timeout, we force-close the connection instead.
		ctx, cancel := context.WithTimeout(context.Background(), clientShutdownWait)
		defer cancel()
		if err := c.conn.Shutdown(ctx); err != nil {
			c.logEntry.WithError(err).Debug("Failed clean shutdown, closing.")
			c.conn.Close()
		}
		c.logEntry.Debug("Disconnected")
	})
}

// Encodes and writes an envelope to the connection
func (c *clientImpl) writeEnvelope(env *envelope) error {
	msg, err := json.Marshal(env)
	if err != nil {
		return err
	}
	return c.conn.WriteMessage(msg)
}

// The write pump takes write requests from the write queue and
// writes them to the connection.
func (c *clientImpl) writePump() {
	c.startWG.Done()
	defer c.shutdownWG.Done()
	// If we ever get an error when writing, then all following
	// writes will fail. Instead of attempting to write, we store
	// and return that same error to each write request.
	var writeErr error
	for {
		select {
		case req := <-c.writeQueue:
			if writeErr == nil {
				writeErr = c.writeEnvelope(req.envelope)
			}
			req.resultCh <- writeErr
		case <-c.disconnectCh:
			return
		}
	}
}

// Reads a message from the connection and decodes it as an envelope.
func (c *clientImpl) readEnvelope() (*receivedEnvelopeImpl, error) {
	msg, err := c.conn.ReadMessage()
	if err != nil {
		return nil, fmt.Errorf("Error reading message from conn: %v", err)
	}
	recvEnv := &receivedEnvelopeImpl{}
	return recvEnv, json.Unmarshal(msg, recvEnv)
}

// The read pump reads messages from the connection and posts them on the
// read queue. If a read fails, the read pump will disconnect the client.
// If the read queue is full, new messages will be dropped.
func (c *clientImpl) readPump() {
	c.startWG.Done()
	defer func() {
		close(c.readQueue)
		c.shutdownWG.Done()
	}()
	for {
		select {
		default:
			env, err := c.readEnvelope()
			if err != nil {
				c.logEntry.WithError(err).Debug("Error reading from connection")
				c.disconnect()
				return
			}
			// Try to add the envelope to the readQueue. If the queue is full we
			// have to drop the message so we can continue reading, otherwise we
			// cannot detect connection errors.
			select {
			case c.readQueue <- env:
			default:
				c.logEntry.Warn("readQueue full, dropping message")
			}
		case <-c.disconnectCh:
			return
		}
	}
}
<|endoftext|>


<commit_before>/*

	{ "execute": "guest-exec", "arguments": {
		"command": string
		}
	}

	{ "execute": "guest-exec", "arguments": {
		"path": string,
		"arg": string,
		"env": string,
		"input": string,
		"capture-output": bool
		}
	}

*/
package main // import "github.com/vtolstov/qemu-ga"
<commit_msg>fix<commit_after>/*

	{ "execute": "guest-exec", "arguments": {
		"command": string
		}
	}

	{ "execute": "guest-exec", "arguments": {
		"path": string,
		"arg": string,
		"env": string,
		"input": string,
		"capture-output": bool
		}
	}

*/
package main // import "github.com/vtolstov/qemu-ga"
<|endoftext|>


<commit_before>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"database/sql"
	"errors"
	"fmt"
	"math"
	"strings"
	"time"
)

// EventStatus enumerates the query parameter values for filtering by
// event state.
type EventStatus uint8

const (
	// EventStatusAll does not filter events.
	EventStatusAll EventStatus = iota
	// EventStatusApplied selects only those events that have been successfully applied.
	EventStatusApplied
	// EventStatusPending selects only those events that are pending application.
	EventStatusPending
)

// DocEventID is the type of unique document event identifiers.
type DocEventID int64

// DocEvent represents a user action performed on a document in the
// system.
//
// Together with documents and nodes, events are central to the
// workflow engine in `flow`.  Events cause documents to transition
// from one state to another, usually in response to user actions.  It
// is possible for system events to cause state transitions, as well.
type DocEvent struct {
	ID      DocEventID  `json:"ID"`        // Unique ID of this event
	DocType DocTypeID   `json:"DocType"`   // Document type of the document to which this event is to be applied
	DocID   DocumentID  `json:"DocID"`     // Document to which this event is to be applied
	State   DocStateID  `json:"DocState"`  // Current state of the document must equal this
	Action  DocActionID `json:"DocAction"` // Action performed by the user
	Group   GroupID     `json:"Group"`     // Group (singleton) who caused this action
	Text    string      `json:"Text"`      // Comment or other content
	Ctime   time.Time   `json:"Ctime"`     // Time at which the event occurred
	Status  EventStatus `json:"Status"`    // Status of this event
}

// StatusInDB answers the status of this event.
func (e *DocEvent) StatusInDB() (EventStatus, error) {
	var dstatus string
	row := db.QueryRow("SELECT status FROM wf_docevents WHERE id = ?", e.ID)
	err := row.Scan(&dstatus)
	if err != nil {
		return 0, err
	}
	switch dstatus {
	case "A":
		e.Status = EventStatusApplied

	case "P":
		e.Status = EventStatusPending

	default:
		return 0, fmt.Errorf("unknown event status : %s", dstatus)
	}

	return e.Status, nil
}

// Unexported type, only for convenience methods.
type _DocEvents struct{}

// DocEvents exposes a resource-like interface to document events.
var DocEvents _DocEvents

// DocEventsNewInput holds information needed to create a new document
// event in the system.
type DocEventsNewInput struct {
	DocTypeID          // Type of the document; required
	DocumentID         // Unique identifier of the document; required
	DocStateID         // Document must be in this state for this event to be applied; required
	DocActionID        // Action performed by `Group`; required
	GroupID            // Group (user) who performed the action that raised this event; required
	Text        string // Any comments or notes
}

// New creates and initialises an event that transforms the document
// that it refers to.
func (_DocEvents) New(otx *sql.Tx, input *DocEventsNewInput) (DocEventID, error) {
	if input.DocumentID <= 0 {
		return 0, errors.New("document ID should be a positive integer")
	}
	if input.Text == "" {
		return 0, errors.New("please add comments or notes")
	}

	var tx *sql.Tx
	if otx == nil {
		tx, err := db.Begin()
		if err != nil {
			return 0, err
		}
		defer tx.Rollback()
	} else {
		tx = otx
	}

	q := `
	INSERT INTO wf_docevents(doctype_id, doc_id, docstate_id, docaction_id, group_id, data, ctime, status)
	VALUES(?, ?, ?, ?, ?, ?, NOW(), 'P')
	`
	res, err := tx.Exec(q, input.DocTypeID, input.DocumentID, input.DocStateID, input.DocActionID, input.GroupID, input.Text)
	if err != nil {
		return 0, err
	}
	var id int64
	id, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	if otx == nil {
		err = tx.Commit()
		if err != nil {
			return 0, err
		}
	}

	return DocEventID(id), nil
}

// DocEventsListInput specifies a set of filter conditions to narrow
// down document listings.
type DocEventsListInput struct {
	DocTypeID                   // Events on documents of this type are listed
	AccessContextID             // Access context from within which to list
	GroupID                     // List events created by this (singleton) group
	DocStateID                  // List events acting on this state
	CtimeStarting   time.Time   // List events created after this time
	CtimeBefore     time.Time   // List events created before this time
	Status          EventStatus // List events that are in this state of application
}

// List answers a subset of document events, based on the input
// specification.
//
// `status` should be one of `all`, `applied` and `pending`.
//
// Result set begins with ID >= `offset`, and has not more than
// `limit` elements.  A value of `0` for `offset` fetches from the
// beginning, while a value of `0` for `limit` fetches until the end.
func (_DocEvents) List(input *DocEventsListInput, offset, limit int64) ([]*DocEvent, error) {
	if offset < 0 || limit < 0 {
		return nil, errors.New("offset and limit must be non-negative integers")
	}
	if limit == 0 {
		limit = math.MaxInt64
	}

	// Base query.

	q := `
	SELECT de.id, de.doctype_id, de.doc_id, de.docstate_id, de.docaction_id, de.group_id, de.data, de.ctime, de.status
	FROM wf_docevents de
	`

	// Process input specification.

	where := []string{}
	args := []interface{}{}

	if input.AccessContextID > 0 {
		tbl := DocTypes.docStorName(input.DocTypeID)
		q += `JOIN ` + tbl + ` docs ON docs.id = de.doc_id
		`
		where = append(where, `docs.ac_id = ?`)
		args = append(args, input.AccessContextID)
	}

	switch input.Status {
	case EventStatusAll:
		// Intentionally left blank

	case EventStatusApplied:
		where = append(where, `status = 'A'`)

	case EventStatusPending:
		where = append(where, `status = 'P'`)

	default:
		return nil, fmt.Errorf("unknown event status specified in filter : %d", input.Status)
	}

	if input.GroupID > 0 {
		where = append(where, `de.group_id = ?`)
		args = append(args, input.GroupID)
	}

	if input.DocStateID > 0 {
		where = append(where, `de.docstate_id = ?`)
		args = append(args, input.DocStateID)
	}

	if !input.CtimeStarting.IsZero() {
		where = append(where, `de.ctime >= ?`)
		args = append(args, input.CtimeStarting)
	}

	if !input.CtimeBefore.IsZero() {
		where = append(where, `de.ctime < ?`)
		args = append(args, input.CtimeBefore)
	}

	if len(where) > 0 {
		q += ` WHERE ` + strings.Join(where, ` AND `)
	}

	q += `
	ORDER BY de.id
	LIMIT ? OFFSET ?
	`
	args = append(args, limit, offset)
	rows, err := db.Query(q, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var text sql.NullString
	var dstatus string
	ary := make([]*DocEvent, 0, 10)
	for rows.Next() {
		var elem DocEvent
		err = rows.Scan(&elem.ID, &elem.DocType, &elem.DocID, &elem.State, &elem.Action, &elem.Group, &text, &elem.Ctime, &dstatus)
		if err != nil {
			return nil, err
		}
		if text.Valid {
			elem.Text = text.String
		}
		switch dstatus {
		case "A":
			elem.Status = EventStatusApplied

		case "P":
			elem.Status = EventStatusPending

		default:
			return nil, fmt.Errorf("unknown event status : %s", dstatus)
		}
		ary = append(ary, &elem)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	return ary, nil
}

// Get retrieves a document event from the database, using the given
// event ID.
func (_DocEvents) Get(eid DocEventID) (*DocEvent, error) {
	if eid <= 0 {
		return nil, errors.New("event ID should be a positive integer")
	}

	var text sql.NullString
	var dstatus string
	var elem DocEvent
	q := `
	SELECT id, doctype_id, doc_id, docstate_id, docaction_id, group_id, data, ctime, status
	FROM wf_docevents
	WHERE id = ?
	`
	row := db.QueryRow(q, eid)
	err := row.Scan(&elem.ID, &elem.DocType, &elem.DocID, &elem.State, &elem.Action, &elem.Group, &text, &elem.Ctime, &dstatus)
	if err != nil {
		return nil, err
	}
	if text.Valid {
		elem.Text = text.String
	}
	switch dstatus {
	case "A":
		elem.Status = EventStatusApplied

	case "P":
		elem.Status = EventStatusPending

	default:
		return nil, fmt.Errorf("unknown event status : %s", dstatus)
	}

	return &elem, nil
}
<commit_msg>Provide a filter on document events by document type<commit_after>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"database/sql"
	"errors"
	"fmt"
	"math"
	"strings"
	"time"
)

// EventStatus enumerates the query parameter values for filtering by
// event state.
type EventStatus uint8

const (
	// EventStatusAll does not filter events.
	EventStatusAll EventStatus = iota
	// EventStatusApplied selects only those events that have been successfully applied.
	EventStatusApplied
	// EventStatusPending selects only those events that are pending application.
	EventStatusPending
)

// DocEventID is the type of unique document event identifiers.
type DocEventID int64

// DocEvent represents a user action performed on a document in the
// system.
//
// Together with documents and nodes, events are central to the
// workflow engine in `flow`.  Events cause documents to transition
// from one state to another, usually in response to user actions.  It
// is possible for system events to cause state transitions, as well.
type DocEvent struct {
	ID      DocEventID  `json:"ID"`        // Unique ID of this event
	DocType DocTypeID   `json:"DocType"`   // Document type of the document to which this event is to be applied
	DocID   DocumentID  `json:"DocID"`     // Document to which this event is to be applied
	State   DocStateID  `json:"DocState"`  // Current state of the document must equal this
	Action  DocActionID `json:"DocAction"` // Action performed by the user
	Group   GroupID     `json:"Group"`     // Group (singleton) who caused this action
	Text    string      `json:"Text"`      // Comment or other content
	Ctime   time.Time   `json:"Ctime"`     // Time at which the event occurred
	Status  EventStatus `json:"Status"`    // Status of this event
}

// StatusInDB answers the status of this event.
func (e *DocEvent) StatusInDB() (EventStatus, error) {
	var dstatus string
	row := db.QueryRow("SELECT status FROM wf_docevents WHERE id = ?", e.ID)
	err := row.Scan(&dstatus)
	if err != nil {
		return 0, err
	}
	switch dstatus {
	case "A":
		e.Status = EventStatusApplied

	case "P":
		e.Status = EventStatusPending

	default:
		return 0, fmt.Errorf("unknown event status : %s", dstatus)
	}

	return e.Status, nil
}

// Unexported type, only for convenience methods.
type _DocEvents struct{}

// DocEvents exposes a resource-like interface to document events.
var DocEvents _DocEvents

// DocEventsNewInput holds information needed to create a new document
// event in the system.
type DocEventsNewInput struct {
	DocTypeID          // Type of the document; required
	DocumentID         // Unique identifier of the document; required
	DocStateID         // Document must be in this state for this event to be applied; required
	DocActionID        // Action performed by `Group`; required
	GroupID            // Group (user) who performed the action that raised this event; required
	Text        string // Any comments or notes
}

// New creates and initialises an event that transforms the document
// that it refers to.
func (_DocEvents) New(otx *sql.Tx, input *DocEventsNewInput) (DocEventID, error) {
	if input.DocumentID <= 0 {
		return 0, errors.New("document ID should be a positive integer")
	}
	if input.Text == "" {
		return 0, errors.New("please add comments or notes")
	}

	var tx *sql.Tx
	if otx == nil {
		tx, err := db.Begin()
		if err != nil {
			return 0, err
		}
		defer tx.Rollback()
	} else {
		tx = otx
	}

	q := `
	INSERT INTO wf_docevents(doctype_id, doc_id, docstate_id, docaction_id, group_id, data, ctime, status)
	VALUES(?, ?, ?, ?, ?, ?, NOW(), 'P')
	`
	res, err := tx.Exec(q, input.DocTypeID, input.DocumentID, input.DocStateID, input.DocActionID, input.GroupID, input.Text)
	if err != nil {
		return 0, err
	}
	var id int64
	id, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	if otx == nil {
		err = tx.Commit()
		if err != nil {
			return 0, err
		}
	}

	return DocEventID(id), nil
}

// DocEventsListInput specifies a set of filter conditions to narrow
// down document listings.
type DocEventsListInput struct {
	DocTypeID                   // Events on documents of this type are listed
	AccessContextID             // Access context from within which to list
	GroupID                     // List events created by this (singleton) group
	DocStateID                  // List events acting on this state
	CtimeStarting   time.Time   // List events created after this time
	CtimeBefore     time.Time   // List events created before this time
	Status          EventStatus // List events that are in this state of application
}

// List answers a subset of document events, based on the input
// specification.
//
// `status` should be one of `all`, `applied` and `pending`.
//
// Result set begins with ID >= `offset`, and has not more than
// `limit` elements.  A value of `0` for `offset` fetches from the
// beginning, while a value of `0` for `limit` fetches until the end.
func (_DocEvents) List(input *DocEventsListInput, offset, limit int64) ([]*DocEvent, error) {
	if offset < 0 || limit < 0 {
		return nil, errors.New("offset and limit must be non-negative integers")
	}
	if limit == 0 {
		limit = math.MaxInt64
	}

	// Base query.

	q := `
	SELECT de.id, de.doctype_id, de.doc_id, de.docstate_id, de.docaction_id, de.group_id, de.data, de.ctime, de.status
	FROM wf_docevents de
	`

	// Process input specification.

	where := []string{}
	args := []interface{}{}

	if input.DocTypeID > 0 {
		where = append(where, `de.doctype_id = ?`)
		args = append(args, input.DocTypeID)
	}

	if input.AccessContextID > 0 {
		tbl := DocTypes.docStorName(input.DocTypeID)
		q += `JOIN ` + tbl + ` docs ON docs.id = de.doc_id
		`
		where = append(where, `docs.ac_id = ?`)
		args = append(args, input.AccessContextID)
	}

	switch input.Status {
	case EventStatusAll:
		// Intentionally left blank

	case EventStatusApplied:
		where = append(where, `status = 'A'`)

	case EventStatusPending:
		where = append(where, `status = 'P'`)

	default:
		return nil, fmt.Errorf("unknown event status specified in filter : %d", input.Status)
	}

	if input.GroupID > 0 {
		where = append(where, `de.group_id = ?`)
		args = append(args, input.GroupID)
	}

	if input.DocStateID > 0 {
		where = append(where, `de.docstate_id = ?`)
		args = append(args, input.DocStateID)
	}

	if !input.CtimeStarting.IsZero() {
		where = append(where, `de.ctime >= ?`)
		args = append(args, input.CtimeStarting)
	}

	if !input.CtimeBefore.IsZero() {
		where = append(where, `de.ctime < ?`)
		args = append(args, input.CtimeBefore)
	}

	if len(where) > 0 {
		q += ` WHERE ` + strings.Join(where, ` AND `)
	}

	q += `
	ORDER BY de.id
	LIMIT ? OFFSET ?
	`
	args = append(args, limit, offset)
	rows, err := db.Query(q, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var text sql.NullString
	var dstatus string
	ary := make([]*DocEvent, 0, 10)
	for rows.Next() {
		var elem DocEvent
		err = rows.Scan(&elem.ID, &elem.DocType, &elem.DocID, &elem.State, &elem.Action, &elem.Group, &text, &elem.Ctime, &dstatus)
		if err != nil {
			return nil, err
		}
		if text.Valid {
			elem.Text = text.String
		}
		switch dstatus {
		case "A":
			elem.Status = EventStatusApplied

		case "P":
			elem.Status = EventStatusPending

		default:
			return nil, fmt.Errorf("unknown event status : %s", dstatus)
		}
		ary = append(ary, &elem)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	return ary, nil
}

// Get retrieves a document event from the database, using the given
// event ID.
func (_DocEvents) Get(eid DocEventID) (*DocEvent, error) {
	if eid <= 0 {
		return nil, errors.New("event ID should be a positive integer")
	}

	var text sql.NullString
	var dstatus string
	var elem DocEvent
	q := `
	SELECT id, doctype_id, doc_id, docstate_id, docaction_id, group_id, data, ctime, status
	FROM wf_docevents
	WHERE id = ?
	`
	row := db.QueryRow(q, eid)
	err := row.Scan(&elem.ID, &elem.DocType, &elem.DocID, &elem.State, &elem.Action, &elem.Group, &text, &elem.Ctime, &dstatus)
	if err != nil {
		return nil, err
	}
	if text.Valid {
		elem.Text = text.String
	}
	switch dstatus {
	case "A":
		elem.Status = EventStatusApplied

	case "P":
		elem.Status = EventStatusPending

	default:
		return nil, fmt.Errorf("unknown event status : %s", dstatus)
	}

	return &elem, nil
}
<|endoftext|>


<commit_before>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    decoder.go
//: details: decodes netflow version 9 packets
//: author:  Mehrdad Arshad Rad
//: date:    04/10/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package netflow9

import (
	"fmt"
	"io"
	"net"

	"github.com/VerizonDigital/vflow/ipfix"
	"github.com/VerizonDigital/vflow/reader"
)

// PacketHeader represents Netflow v9  packet header
type PacketHeader struct {
	Version   uint16 // Version of Flow Record format exported in this packet
	Count     uint16 // The total number of records in the Export Packet
	SysUpTime uint32 // Time in milliseconds since this device was first booted
	UNIXSecs  uint32 // Time in seconds since 0000 UTC 197
	SeqNum    uint32 // Incremental sequence counter of all Export Packets
	SrcID     uint32 // A 32-bit value that identifies the Exporter
}

// SetHeader represents netflow v9 data flowset id and length
type SetHeader struct {
	FlowSetID uint16 // FlowSet ID value 0:: template, 1:: options template, 255< :: data
	Length    uint16 // Total length of this FlowSet
}

// TemplateHeader represents netflow v9 data template id and field count
type TemplateHeader struct {
	TemplateID     uint16 // Template ID
	FieldCount     uint16 // Number of fields in this Template Record
	OptionLen      uint16 // The length in bytes of any Scope field definition (Option)
	OptionScopeLen uint16 // The length in bytes of any options field definitions (Option)
}

// TemplateFieldSpecifier represents field properties
type TemplateFieldSpecifier struct {
	ElementID uint16
	Length    uint16
}

// TemplateRecord represents template fields
type TemplateRecord struct {
	TemplateID           uint16
	FieldCount           uint16
	FieldSpecifiers      []TemplateFieldSpecifier
	ScopeFieldCount      uint16
	ScopeFieldSpecifiers []TemplateFieldSpecifier
}

// DecodedField represents a decoded field
type DecodedField struct {
	ID    uint16
	Value interface{}
}

// Decoder represents Netflow payload and remote address
type Decoder struct {
	raddr  net.IP
	reader *reader.Reader
}

// Message represents Netflow decoded data
type Message struct {
	AgentID  string
	Header   PacketHeader
	DataSets [][]DecodedField
}

//   The Packet Header format is specified as:
//
//    0                   1                   2                   3
//    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |       Version Number          |            Count              |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                           sysUpTime                           |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                           UNIX Secs                           |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                       Sequence Number                         |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                        Source ID                              |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *PacketHeader) unmarshal(r *reader.Reader) error {
	var err error

	if h.Version, err = r.Uint16(); err != nil {
		return err
	}

	if h.Count, err = r.Uint16(); err != nil {
		return err
	}

	if h.SysUpTime, err = r.Uint32(); err != nil {
		return err
	}

	if h.UNIXSecs, err = r.Uint32(); err != nil {
		return err
	}

	if h.SeqNum, err = r.Uint32(); err != nil {
		return err
	}

	if h.SrcID, err = r.Uint32(); err != nil {
		return err
	}

	return nil
}

func (h *PacketHeader) validate() error {
	if h.Version != 9 {
		return fmt.Errorf("invalid netflow version (%d)", h.Version)
	}

	// TODO: needs more validation

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        FlowSet ID             |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *SetHeader) unmarshal(r *reader.Reader) error {
	var err error

	if h.FlowSetID, err = r.Uint16(); err != nil {
		return err
	}

	if h.Length, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Template ID            |         Field Count           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (t *TemplateHeader) unmarshal(r *reader.Reader) error {
	var err error

	if t.TemplateID, err = r.Uint16(); err != nil {
		return err
	}

	if t.FieldCount, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

func (t *TemplateHeader) unmarshalOpts(r *reader.Reader) error {
	var err error

	if t.TemplateID, err = r.Uint16(); err != nil {
		return err
	}

	if t.OptionScopeLen, err = r.Uint16(); err != nil {
		return err
	}

	if t.OptionLen, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type             |         Field Length          |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (f *TemplateFieldSpecifier) unmarshal(r *reader.Reader) error {
	var err error

	if f.ElementID, err = r.Uint16(); err != nil {
		return err
	}

	if f.Length, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |      Template ID 256          |         Field Count           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type 1           |         Field Length 1        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type 2           |         Field Length 2        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |             ...               |              ...              |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type N           |         Field Length N        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (tr *TemplateRecord) unmarshal(r *reader.Reader) {
	var (
		th = TemplateHeader{}
		tf = TemplateFieldSpecifier{}
	)

	th.unmarshal(r)
	tr.TemplateID = th.TemplateID
	tr.FieldCount = th.FieldCount

	for i := th.FieldCount; i > 0; i-- {
		tf.unmarshal(r)
		tr.FieldSpecifiers = append(tr.FieldSpecifiers, tf)
	}
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |       FlowSet ID = 1          |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |         Template ID           |      Option Scope Length      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Option Length          |       Scope 1 Field Type      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Scope 1 Field Length      |               ...             |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Scope N Field Length      |      Option 1 Field Type      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Option 1 Field Length     |             ...               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Option M Field Length     |           Padding             |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (tr *TemplateRecord) unmarshalOpts(r *reader.Reader) {
	var (
		th = TemplateHeader{}
		tf = TemplateFieldSpecifier{}
	)

	th.unmarshalOpts(r)
	tr.TemplateID = th.TemplateID

	for i := th.OptionScopeLen / 4; i > 0; i-- {
		tf.unmarshal(r)
		tr.ScopeFieldSpecifiers = append(tr.FieldSpecifiers, tf)
	}

	for i := th.OptionLen / 4; i > 0; i-- {
		tf.unmarshal(r)
		tr.FieldSpecifiers = append(tr.FieldSpecifiers, tf)
	}

}

func decodeData(r *reader.Reader, tr TemplateRecord) []DecodedField {
	var (
		fields []DecodedField
		b      []byte
	)

	for i := 0; i < len(tr.FieldSpecifiers); i++ {
		b, _ = r.Read(int(tr.FieldSpecifiers[i].Length))
		m := ipfix.InfoModel[ipfix.ElementKey{
			0,
			tr.FieldSpecifiers[i].ElementID,
		}]
		fields = append(fields, DecodedField{
			ID:    m.FieldID,
			Value: ipfix.Interpret(b, m.Type),
		})
	}

	return fields
}

// NewDecoder constructs a decoder
func NewDecoder(raddr net.IP, b []byte) *Decoder {
	return &Decoder{raddr, reader.NewReader(b)}
}

// Decode decodes the Netflow raw data
func (d *Decoder) Decode(mem MemCache) (*Message, error) {
	var (
		msg = new(Message)
		err error
	)

	// Netflow Message Header decoding
	if err = msg.Header.unmarshal(d.reader); err != nil {
		return nil, err
	}
	// Netflow Message Header validation
	if err = msg.Header.validate(); err != nil {
		return nil, err
	}

	// Add source IP address as Agent ID
	msg.AgentID = d.raddr.String()

	for d.reader.Len() > 4 {

		setHeader := new(SetHeader)
		setHeader.unmarshal(d.reader)

		if setHeader.Length < 4 {
			return nil, io.ErrUnexpectedEOF
		}

		switch {
		case setHeader.FlowSetID == 2:
			// Template set
			tr := TemplateRecord{}
			tr.unmarshal(d.reader)
			mem.insert(tr.TemplateID, d.raddr, tr)
		case setHeader.FlowSetID == 3:
			// Option set
			tr := TemplateRecord{}
			tr.unmarshalOpts(d.reader)
			mem.insert(tr.TemplateID, d.raddr, tr)
		case setHeader.FlowSetID >= 4 && setHeader.FlowSetID <= 255:
			// Reserved
		default:
			// data
			//TODO
		}
	}

	return msg, nil
}
<commit_msg>decode netflow v9 data<commit_after>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    decoder.go
//: details: decodes netflow version 9 packets
//: author:  Mehrdad Arshad Rad
//: date:    04/10/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package netflow9

import (
	"fmt"
	"io"
	"net"

	"github.com/VerizonDigital/vflow/ipfix"
	"github.com/VerizonDigital/vflow/reader"
)

// PacketHeader represents Netflow v9  packet header
type PacketHeader struct {
	Version   uint16 // Version of Flow Record format exported in this packet
	Count     uint16 // The total number of records in the Export Packet
	SysUpTime uint32 // Time in milliseconds since this device was first booted
	UNIXSecs  uint32 // Time in seconds since 0000 UTC 197
	SeqNum    uint32 // Incremental sequence counter of all Export Packets
	SrcID     uint32 // A 32-bit value that identifies the Exporter
}

// SetHeader represents netflow v9 data flowset id and length
type SetHeader struct {
	FlowSetID uint16 // FlowSet ID value 0:: template, 1:: options template, 255< :: data
	Length    uint16 // Total length of this FlowSet
}

// TemplateHeader represents netflow v9 data template id and field count
type TemplateHeader struct {
	TemplateID     uint16 // Template ID
	FieldCount     uint16 // Number of fields in this Template Record
	OptionLen      uint16 // The length in bytes of any Scope field definition (Option)
	OptionScopeLen uint16 // The length in bytes of any options field definitions (Option)
}

// TemplateFieldSpecifier represents field properties
type TemplateFieldSpecifier struct {
	ElementID uint16
	Length    uint16
}

// TemplateRecord represents template fields
type TemplateRecord struct {
	TemplateID           uint16
	FieldCount           uint16
	FieldSpecifiers      []TemplateFieldSpecifier
	ScopeFieldCount      uint16
	ScopeFieldSpecifiers []TemplateFieldSpecifier
}

// DecodedField represents a decoded field
type DecodedField struct {
	ID    uint16
	Value interface{}
}

// Decoder represents Netflow payload and remote address
type Decoder struct {
	raddr  net.IP
	reader *reader.Reader
}

// Message represents Netflow decoded data
type Message struct {
	AgentID  string
	Header   PacketHeader
	DataSets [][]DecodedField
}

//   The Packet Header format is specified as:
//
//    0                   1                   2                   3
//    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |       Version Number          |            Count              |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                           sysUpTime                           |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                           UNIX Secs                           |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                       Sequence Number                         |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
//   |                        Source ID                              |
//   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *PacketHeader) unmarshal(r *reader.Reader) error {
	var err error

	if h.Version, err = r.Uint16(); err != nil {
		return err
	}

	if h.Count, err = r.Uint16(); err != nil {
		return err
	}

	if h.SysUpTime, err = r.Uint32(); err != nil {
		return err
	}

	if h.UNIXSecs, err = r.Uint32(); err != nil {
		return err
	}

	if h.SeqNum, err = r.Uint32(); err != nil {
		return err
	}

	if h.SrcID, err = r.Uint32(); err != nil {
		return err
	}

	return nil
}

func (h *PacketHeader) validate() error {
	if h.Version != 9 {
		return fmt.Errorf("invalid netflow version (%d)", h.Version)
	}

	// TODO: needs more validation

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        FlowSet ID             |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *SetHeader) unmarshal(r *reader.Reader) error {
	var err error

	if h.FlowSetID, err = r.Uint16(); err != nil {
		return err
	}

	if h.Length, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Template ID            |         Field Count           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (t *TemplateHeader) unmarshal(r *reader.Reader) error {
	var err error

	if t.TemplateID, err = r.Uint16(); err != nil {
		return err
	}

	if t.FieldCount, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

func (t *TemplateHeader) unmarshalOpts(r *reader.Reader) error {
	var err error

	if t.TemplateID, err = r.Uint16(); err != nil {
		return err
	}

	if t.OptionScopeLen, err = r.Uint16(); err != nil {
		return err
	}

	if t.OptionLen, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type             |         Field Length          |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (f *TemplateFieldSpecifier) unmarshal(r *reader.Reader) error {
	var err error

	if f.ElementID, err = r.Uint16(); err != nil {
		return err
	}

	if f.Length, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |      Template ID 256          |         Field Count           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type 1           |         Field Length 1        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type 2           |         Field Length 2        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |             ...               |              ...              |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Field Type N           |         Field Length N        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (tr *TemplateRecord) unmarshal(r *reader.Reader) {
	var (
		th = TemplateHeader{}
		tf = TemplateFieldSpecifier{}
	)

	th.unmarshal(r)
	tr.TemplateID = th.TemplateID
	tr.FieldCount = th.FieldCount

	for i := th.FieldCount; i > 0; i-- {
		tf.unmarshal(r)
		tr.FieldSpecifiers = append(tr.FieldSpecifiers, tf)
	}
}

// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |       FlowSet ID = 1          |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |         Template ID           |      Option Scope Length      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |        Option Length          |       Scope 1 Field Type      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Scope 1 Field Length      |               ...             |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Scope N Field Length      |      Option 1 Field Type      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Option 1 Field Length     |             ...               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |     Option M Field Length     |           Padding             |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (tr *TemplateRecord) unmarshalOpts(r *reader.Reader) {
	var (
		th = TemplateHeader{}
		tf = TemplateFieldSpecifier{}
	)

	th.unmarshalOpts(r)
	tr.TemplateID = th.TemplateID

	for i := th.OptionScopeLen / 4; i > 0; i-- {
		tf.unmarshal(r)
		tr.ScopeFieldSpecifiers = append(tr.FieldSpecifiers, tf)
	}

	for i := th.OptionLen / 4; i > 0; i-- {
		tf.unmarshal(r)
		tr.FieldSpecifiers = append(tr.FieldSpecifiers, tf)
	}

}

func decodeData(r *reader.Reader, tr TemplateRecord) []DecodedField {
	var (
		fields []DecodedField
		b      []byte
	)

	for i := 0; i < len(tr.FieldSpecifiers); i++ {
		b, _ = r.Read(int(tr.FieldSpecifiers[i].Length))
		m := ipfix.InfoModel[ipfix.ElementKey{
			0,
			tr.FieldSpecifiers[i].ElementID,
		}]
		fields = append(fields, DecodedField{
			ID:    m.FieldID,
			Value: ipfix.Interpret(b, m.Type),
		})
	}

	return fields
}

// NewDecoder constructs a decoder
func NewDecoder(raddr net.IP, b []byte) *Decoder {
	return &Decoder{raddr, reader.NewReader(b)}
}

// Decode decodes the Netflow raw data
func (d *Decoder) Decode(mem MemCache) (*Message, error) {
	var (
		nextSet int
		msg     = new(Message)
		err     error
	)

	// Netflow Message Header decoding
	if err = msg.Header.unmarshal(d.reader); err != nil {
		return nil, err
	}
	// Netflow Message Header validation
	if err = msg.Header.validate(); err != nil {
		return nil, err
	}

	// Add source IP address as Agent ID
	msg.AgentID = d.raddr.String()

	for d.reader.Len() > 4 {

		setHeader := new(SetHeader)
		setHeader.unmarshal(d.reader)

		if setHeader.Length < 4 {
			return nil, io.ErrUnexpectedEOF
		}

		switch {
		case setHeader.FlowSetID == 2:
			// Template set
			tr := TemplateRecord{}
			tr.unmarshal(d.reader)
			mem.insert(tr.TemplateID, d.raddr, tr)
		case setHeader.FlowSetID == 3:
			// Option set
			tr := TemplateRecord{}
			tr.unmarshalOpts(d.reader)
			mem.insert(tr.TemplateID, d.raddr, tr)
		case setHeader.FlowSetID >= 4 && setHeader.FlowSetID <= 255:
			// Reserved
		default:
			// data
			tr, ok := mem.retrieve(setHeader.FlowSetID, d.raddr)
			if !ok {
				return msg, fmt.Errorf("%s unknown template id# %d",
					d.raddr.String(),
					setHeader.FlowSetID,
				)
			}

			// data records
			nextSet = d.reader.Len() - int(setHeader.Length) + 4
			for d.reader.Len() > nextSet {
				data := decodeData(d.reader, tr)
				msg.DataSets = append(msg.DataSets, data)
			}
		}
	}

	return msg, nil
}
<|endoftext|>


<commit_before>package shell

import (
	"fmt"
	"os"
	"os/signal"

	api "github.com/ipfs/go-ipfs-api"
	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
	embedded "github.com/noffle/ipfs-embedded-shell"
	context "golang.org/x/net/context"
)

func NewShell() (Shell, error) {
	myShell, err := getApiShell()
	if err == nil {
		// fmt.Println("got an api shell!")
		return myShell, nil
	}

	myShell, err = getEmbeddedShell()
	if err == nil {
		// fmt.Println("got an embedded shell!")
		return myShell, nil
	}

	return nil, err
}

func getApiShell() (Shell, error) {
	apiShell := api.NewShell("http://127.0.0.1:5001")
	_, _, err := apiShell.Version()
	if err != nil {
		return nil, err
	}

	return apiShell, nil
}

func getEmbeddedShell() (Shell, error) {
	ctx, cancel := context.WithCancel(context.Background())

	// Cancel the ipfs node context if the process gets interrupted or killed.
	// TODO(noffle): is this needed?
	go func() {
		interrupts := make(chan os.Signal, 1)
		signal.Notify(interrupts, os.Interrupt, os.Kill)
		<-interrupts
		cancel()
	}()

	shell, err := tryLocal(ctx)
	if err == nil {
		return shell, nil
	}

	node, err := embedded.NewTmpDirNode(ctx)
	if err != nil {
		return nil, err
	}

	return embedded.NewShell(node), nil
}

func tryLocal(ctx context.Context) (Shell, error) {
	repoPath, err := getRepoPath()
	if err != nil {
		return nil, err
	}

	node, err := embedded.NewDefaultNodeWithFSRepo(ctx, repoPath)
	if err != nil {
		return nil, fmt.Errorf("couldn't get embedded shell: %s", err)
	}

	return embedded.NewShell(node), nil
}

func getRepoPath() (string, error) {
	repoPath, err := fsrepo.BestKnownPath()
	if err != nil {
		return "", err
	}
	return repoPath, nil
}
<commit_msg>Expose NewApiShell and NewEmbeddedShell.<commit_after>package shell

import (
	"fmt"
	"os"
	"os/signal"

	api "github.com/ipfs/go-ipfs-api"
	fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
	embedded "github.com/noffle/ipfs-embedded-shell"
	context "golang.org/x/net/context"
)

func NewShell() (Shell, error) {
	myShell, err := NewApiShell()
	if err == nil {
		// fmt.Println("got an api shell!")
		return myShell, nil
	}

	myShell, err = NewEmbeddedShell()
	if err == nil {
		// fmt.Println("got an embedded shell!")
		return myShell, nil
	}

	return nil, err
}

func NewApiShell() (Shell, error) {
	apiShell := api.NewShell("http://127.0.0.1:5001")
	_, _, err := apiShell.Version()
	if err != nil {
		return nil, err
	}

	return apiShell, nil
}

func NewEmbeddedShell() (Shell, error) {
	ctx, cancel := context.WithCancel(context.Background())

	// Cancel the ipfs node context if the process gets interrupted or killed.
	// TODO(noffle): is this needed?
	go func() {
		interrupts := make(chan os.Signal, 1)
		signal.Notify(interrupts, os.Interrupt, os.Kill)
		<-interrupts
		cancel()
	}()

	shell, err := tryLocal(ctx)
	if err == nil {
		return shell, nil
	}

	node, err := embedded.NewTmpDirNode(ctx)
	if err != nil {
		return nil, err
	}

	return embedded.NewShell(node), nil
}

func tryLocal(ctx context.Context) (Shell, error) {
	repoPath, err := getRepoPath()
	if err != nil {
		return nil, err
	}

	node, err := embedded.NewDefaultNodeWithFSRepo(ctx, repoPath)
	if err != nil {
		return nil, fmt.Errorf("couldn't get embedded shell: %s", err)
	}

	return embedded.NewShell(node), nil
}

func getRepoPath() (string, error) {
	repoPath, err := fsrepo.BestKnownPath()
	if err != nil {
		return "", err
	}
	return repoPath, nil
}
<|endoftext|>