<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/gorilla/websocket"
	"github.com/martini-contrib/sessions"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

const CLEAR string = "\033[H\033[2J"
const RESET string = "\0033\0143"

const MAX_CONSOLE int = 10000

const (
	WSTerm  = 1
	WSClick = 2
)

type Config struct {
	Secret  string
	Address string
}

type LockingWebsockets struct {
	sync.RWMutex
	byId        map[int64]*websocket.Conn
	consoleToId map[int64][]int64
	currentId   int64
}

var consoleBuffers map[int64]string

var websockets *LockingWebsockets

func (c *LockingWebsockets) deleteWebsocket(id int64) {
	c.Lock()
	defer c.Unlock()
	delete(c.byId, id)
}

func (c *LockingWebsockets) addWebsocket(ws *websocket.Conn) int64 {
	c.Lock()
	defer c.Unlock()
	c.currentId += 1
	retid := c.currentId
	c.byId[retid] = ws
	return retid
}

func main() {
	consoleBuffers = make(map[int64]string)
	websockets = &LockingWebsockets{
		byId:        make(map[int64]*websocket.Conn),
		consoleToId: make(map[int64][]int64),
		currentId:   0,
	}
	consoleReadChannel = make(chan ConsoleChunk)
	go consoleDispatch()
	vzcontrol := ConnectVZControl()
	defer vzcontrol.Close()

	file, _ := os.Open("game_server.cfg")
	decoder := json.NewDecoder(file)
	config := Config{}
	decoder.Decode(&config)

	m := martini.Classic()
	store := sessions.NewCookieStore([]byte(config.Secret))
	m.Use(sessions.Sessions("session", store))

	generating := false
	gr := NewGraph()
	m.Get("/reset/:secret", func(w http.ResponseWriter, r *http.Request, params martini.Params, session sessions.Session) string {
		if params["secret"] != config.Secret {
			return ""
		}
		err := vzcontrol.Reset()
		if err != nil {
			return err.Error()
		}
		generating = false
		gr = NewGraph()
		return "Done"
	})

	m.Get("/gen", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		if generating {
			return "Already generating"
		}
		generating = true
		maxNodes := 5
		maxEdges := 5
		startNodeId := 100

		startNode := Node{Id: NodeId(startNodeId)}
		gr.AddNode(startNode)
		err := vzcontrol.ContainerCreate(int64(startNode.Id))
		err = vzcontrol.ConsoleStart(int64(startNode.Id))

		nodes := make([]Node, 0)
		nodes = append(nodes, startNode)

		steps := 1
		for len(nodes) != 0 && steps < maxNodes {
			node, nodes := nodes[len(nodes)-1], nodes[:len(nodes)-1]

			numEdges := random(1, maxEdges)
			for i := 1; i <= numEdges; i++ {
				targetNode := Node{Id: NodeId(i*steps + startNodeId)}
				if gr.AddNode(targetNode) {
					err = vzcontrol.ContainerCreate(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Container Create: %d, %d, %d\n%s", targetNode.Id, i*steps, numEdges, err.Error())
					}
					err = vzcontrol.ConsoleStart(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Console Start: %d\n%s", targetNode.Id, err.Error())
					}
					nodes = append(nodes, targetNode)
					edgeid := int64(i * steps)
					if gr.AddEdge(Edge{Id: EdgeId(edgeid), Head: node.Id, Tail: targetNode.Id}) {
						err = vzcontrol.NetworkCreate(edgeid)
						if err != nil {
							return fmt.Sprintf("Network Create: %d\n%s", edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(node.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Node: %d, %d\n%s", node.Id, edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(targetNode.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Target: %d, %d\n%s", targetNode.Id, edgeid, err.Error())
						}
					}
				}

			}
			steps += 1
		}
		return gr.String()
	})

	m.Get("/graph", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		output, err := json.Marshal(gr)
		if err != nil {
			return err.Error()
		}
		return string(output)
	})

	m.Get("/ws", func(w http.ResponseWriter, r *http.Request, session sessions.Session) {
		var currentVm int64 = -1
		ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
		if _, ok := err.(websocket.HandshakeError); ok {
			http.Error(w, "Not a websocket handshake", 400)
			log.Println(err)
			return
		} else if err != nil {
			log.Println(err)
			return
		}
		defer ws.Close()
		websocketId := websockets.addWebsocket(ws)
		defer websockets.deleteWebsocket(websocketId)
		ws.WriteMessage(websocket.TextMessage, []byte("Welcome to ginux!\r\n"))
		for {
			_, message, err := ws.ReadMessage()
			if err != nil {
				log.Println(err)
				return
			} else {
				msgType := message[0]
				msgData := message[1:len(message)]
				switch msgType {
				case WSTerm:
					if currentVm != -1 {
						vzcontrol.ConsoleWrite(currentVm, msgData)
					}
				case WSClick:
					prevVm := currentVm
					tmp, _ := strconv.Atoi(string(msgData))
					currentVm = int64(tmp)
					websockets.Lock()
					if prevVm != -1 {
						for index, wsId := range websockets.consoleToId[prevVm] {
							if wsId == websocketId {
								websockets.consoleToId[prevVm] = append(websockets.consoleToId[prevVm][:index], websockets.consoleToId[prevVm][index+1:]...)
							}
						}
					}
					websockets.consoleToId[currentVm] = append(websockets.consoleToId[currentVm], websocketId)
					websockets.Unlock()
					ws.WriteMessage(websocket.TextMessage, []byte(RESET))
					//ws.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("Selected Container %d\r\n", currentVm)))
					ws.WriteMessage(websocket.TextMessage, []byte(consoleBuffers[currentVm]))
				}
			}
		}
	})
	log.Println("Game Server started on", config.Address)
	log.Fatal(http.ListenAndServe(config.Address, m))
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func consoleDispatch() {
	for chunk := range consoleReadChannel {
		if len(consoleBuffers[chunk.Id]) > MAX_CONSOLE {
			consoleBuffers[chunk.Id] = consoleBuffers[chunk.Id][len(string(chunk.Data)):] + string(chunk.Data)
		} else {
			consoleBuffers[chunk.Id] += string(chunk.Data)
		}
		websockets.RLock()
		for _, wsId := range websockets.consoleToId[chunk.Id] {
			if socket, ok := websockets.byId[wsId]; ok {
				socket.WriteMessage(websocket.TextMessage, chunk.Data)
			}
		}
		websockets.RUnlock()
	}
}
<commit_msg>Changed reset command<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/gorilla/websocket"
	"github.com/martini-contrib/sessions"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

const CLEAR string = "\033[H\033[2J"
const RESET string = "\033[!p\033[?3;4l\033[4l\033>"

const MAX_CONSOLE int = 10000

const (
	WSTerm  = 1
	WSClick = 2
)

type Config struct {
	Secret  string
	Address string
}

type LockingWebsockets struct {
	sync.RWMutex
	byId        map[int64]*websocket.Conn
	consoleToId map[int64][]int64
	currentId   int64
}

var consoleBuffers map[int64]string

var websockets *LockingWebsockets

func (c *LockingWebsockets) deleteWebsocket(id int64) {
	c.Lock()
	defer c.Unlock()
	delete(c.byId, id)
}

func (c *LockingWebsockets) addWebsocket(ws *websocket.Conn) int64 {
	c.Lock()
	defer c.Unlock()
	c.currentId += 1
	retid := c.currentId
	c.byId[retid] = ws
	return retid
}

func main() {
	consoleBuffers = make(map[int64]string)
	websockets = &LockingWebsockets{
		byId:        make(map[int64]*websocket.Conn),
		consoleToId: make(map[int64][]int64),
		currentId:   0,
	}
	consoleReadChannel = make(chan ConsoleChunk)
	go consoleDispatch()
	vzcontrol := ConnectVZControl()
	defer vzcontrol.Close()

	file, _ := os.Open("game_server.cfg")
	decoder := json.NewDecoder(file)
	config := Config{}
	decoder.Decode(&config)

	m := martini.Classic()
	store := sessions.NewCookieStore([]byte(config.Secret))
	m.Use(sessions.Sessions("session", store))

	generating := false
	gr := NewGraph()
	m.Get("/reset/:secret", func(w http.ResponseWriter, r *http.Request, params martini.Params, session sessions.Session) string {
		if params["secret"] != config.Secret {
			return ""
		}
		err := vzcontrol.Reset()
		if err != nil {
			return err.Error()
		}
		generating = false
		gr = NewGraph()
		return "Done"
	})

	m.Get("/gen", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		if generating {
			return "Already generating"
		}
		generating = true
		maxNodes := 5
		maxEdges := 5
		startNodeId := 100

		startNode := Node{Id: NodeId(startNodeId)}
		gr.AddNode(startNode)
		err := vzcontrol.ContainerCreate(int64(startNode.Id))
		err = vzcontrol.ConsoleStart(int64(startNode.Id))

		nodes := make([]Node, 0)
		nodes = append(nodes, startNode)

		steps := 1
		for len(nodes) != 0 && steps < maxNodes {
			node, nodes := nodes[len(nodes)-1], nodes[:len(nodes)-1]

			numEdges := random(1, maxEdges)
			for i := 1; i <= numEdges; i++ {
				targetNode := Node{Id: NodeId(i*steps + startNodeId)}
				if gr.AddNode(targetNode) {
					err = vzcontrol.ContainerCreate(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Container Create: %d, %d, %d\n%s", targetNode.Id, i*steps, numEdges, err.Error())
					}
					err = vzcontrol.ConsoleStart(int64(targetNode.Id))
					if err != nil {
						return fmt.Sprintf("Console Start: %d\n%s", targetNode.Id, err.Error())
					}
					nodes = append(nodes, targetNode)
					edgeid := int64(i * steps)
					if gr.AddEdge(Edge{Id: EdgeId(edgeid), Head: node.Id, Tail: targetNode.Id}) {
						err = vzcontrol.NetworkCreate(edgeid)
						if err != nil {
							return fmt.Sprintf("Network Create: %d\n%s", edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(node.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Node: %d, %d\n%s", node.Id, edgeid, err.Error())
						}
						err = vzcontrol.NetworkAdd(int64(targetNode.Id), edgeid)
						if err != nil {
							return fmt.Sprintf("Network Add Target: %d, %d\n%s", targetNode.Id, edgeid, err.Error())
						}
					}
				}

			}
			steps += 1
		}
		return gr.String()
	})

	m.Get("/graph", func(w http.ResponseWriter, r *http.Request, session sessions.Session) string {
		output, err := json.Marshal(gr)
		if err != nil {
			return err.Error()
		}
		return string(output)
	})

	m.Get("/ws", func(w http.ResponseWriter, r *http.Request, session sessions.Session) {
		var currentVm int64 = -1
		ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
		if _, ok := err.(websocket.HandshakeError); ok {
			http.Error(w, "Not a websocket handshake", 400)
			log.Println(err)
			return
		} else if err != nil {
			log.Println(err)
			return
		}
		defer ws.Close()
		websocketId := websockets.addWebsocket(ws)
		defer websockets.deleteWebsocket(websocketId)
		ws.WriteMessage(websocket.TextMessage, []byte("Welcome to ginux!\r\n"))
		for {
			_, message, err := ws.ReadMessage()
			if err != nil {
				log.Println(err)
				return
			} else {
				msgType := message[0]
				msgData := message[1:len(message)]
				switch msgType {
				case WSTerm:
					if currentVm != -1 {
						vzcontrol.ConsoleWrite(currentVm, msgData)
					}
				case WSClick:
					prevVm := currentVm
					tmp, _ := strconv.Atoi(string(msgData))
					currentVm = int64(tmp)
					websockets.Lock()
					if prevVm != -1 {
						for index, wsId := range websockets.consoleToId[prevVm] {
							if wsId == websocketId {
								websockets.consoleToId[prevVm] = append(websockets.consoleToId[prevVm][:index], websockets.consoleToId[prevVm][index+1:]...)
							}
						}
					}
					websockets.consoleToId[currentVm] = append(websockets.consoleToId[currentVm], websocketId)
					websockets.Unlock()
					ws.WriteMessage(websocket.TextMessage, []byte(RESET))
					//ws.WriteMessage(websocket.TextMessage, []byte(fmt.Sprintf("Selected Container %d\r\n", currentVm)))
					ws.WriteMessage(websocket.TextMessage, []byte(consoleBuffers[currentVm]))
				}
			}
		}
	})
	log.Println("Game Server started on", config.Address)
	log.Fatal(http.ListenAndServe(config.Address, m))
}

func random(min, max int) int {
	rand.Seed(time.Now().Unix())
	return rand.Intn(max-min) + min
}

func consoleDispatch() {
	for chunk := range consoleReadChannel {
		if len(consoleBuffers[chunk.Id]) > MAX_CONSOLE {
			consoleBuffers[chunk.Id] = consoleBuffers[chunk.Id][len(string(chunk.Data)):] + string(chunk.Data)
		} else {
			consoleBuffers[chunk.Id] += string(chunk.Data)
		}
		websockets.RLock()
		for _, wsId := range websockets.consoleToId[chunk.Id] {
			if socket, ok := websockets.byId[wsId]; ok {
				socket.WriteMessage(websocket.TextMessage, chunk.Data)
			}
		}
		websockets.RUnlock()
	}
}
<|endoftext|>


<commit_before>// Package qstat provides an interface to GridEngine's job and queue status facilities
package qstat

import (
	"encoding/xml"
	"errors"
	"github.com/kisielk/gorge/util"
	"os/exec"
	"path"
	"strings"
)

// Resource represents a GridEngine resource request
// See man 5 sge_complex for a more detailed description of the fields
type Resource struct {
	Name        string  `xml:"CE_name"`         // The name of the complex resource
	ValType     int     `xml:"CE_valtype"`      // The type of value
	StringVal   string  `xml:"CE_stringval"`    // The value as a string
	DoubleVal   float64 `xml:"CE_doubleval"`    // The value as a double
	RelOp       int     `xml:"CE_relop"`        // The relation operator used to compare the value
	Consumable  bool    `xml:"CE_consumable"`   // True if the resource is a consumable resourece
	Dominant    bool    `xml:"CE_dominant"`     // ?
	PJDoubleVal float64 `xml:"CE_pj_doubleval"` // ?
	PJDominant  bool    `xml:"CE_pj_dominant"`  // ?
	Requestable bool    `xml:"CE_requestable"`  // True if the resource is a requestable resource
	Tagged      bool    `xml:"CE_tagged"`       // ?
}

// MailAddress represents an email address
type MailAddress struct {
	User string `xml:"MR_user"`
	Host string `xml:"MR_host"`
}

// String implements the Stringer interface
func (a MailAddress) String() string {
	return a.User + "@" + a.Host
}

// EnvVar represents a job environment variable
type EnvVar struct {
	Variable string `xml:"VA_variable"` // The name of the variable
	Value    string `xml:"VA_value"`    // The value of the variable
}

type PathList struct {
	Path        string `xml:"PN_path"`
	Host        string `xml:"PN_host"`
	FileHost    string `xml:"PN_file_host"`
	FileStaging bool   `xml:"PN_file_staging"`
}

// TaskIDRange represents a range of job array task identifiers
type TaskIDRange struct {
	Min  int `xml:"RN_min"`  // The minimum task ID
	Max  int `xml:"RN_max"`  // The maximum task ID
	Step int `xml:"RN_step"` // The ID step size between tasks
}

type JATMessage struct {
	Type    int    `xml:"QIM_type"`
	Message string `xml:"QIM_message"`
}

type Messages struct {
	Messages       []SMEMessage `xml:"SME_message_list>element"`
	GlobalMessages []SMEMessage `xml:"SME_global_message_list>element"`
}

type SMEMessage struct {
	JobNumbers []int  `xml:"MES_job_number_list>ulong_sublist>ULNG_value"`
	Number     int    `xml:"MES_message_number"`
	Message    string `xml:"MES_message"`
}

type Task struct {
	Status      int        `xml:"JAT_status"`
	TaskNumber  int        `xml:"JAT_task_number"`
	MessageList JATMessage `xml:"JAT_message_list>ulong_sublist"`
}

type JobInfo struct {
	JobNumber            int           `xml:"JB_job_number"`
	AdvanceReservation   int           `xml:"JB_ar"`
	ExecFile             string        `xml:"JB_exec_file"`
	SubmissionTime       int           `xml:"JB_submission_time"`
	Owner                string        `xml:"JB_owner"`
	Uid                  int           `xml:"JB_uid"`
	Group                string        `xml:"JB_group"`
	Gid                  int           `xml:"JB_gid"`
	Account              string        `xml:"JB_account"`
	MergeStdErr          bool          `xml:"JB_merge_stderr"`
	MailList             []MailAddress `xml:"JB_mail_list>element"`
	Project              string        `xml:"JB_project"`
	Notify               bool          `xml:"JB_notify"`
	JobName              string        `xml:"JB_job_name"`
	StdoutPathList       []PathList    `xml:"JB_stdout_path_list>path_list"`
	AltStdoutPathList    []PathList    `xml:"JB_stdout_path_list>stdout_path_list"` // Alternate stdout path list
	JobShare             int           `xml:"JB_jobshare"`
	HardResourceList     []Resource    `xml:"JB_hard_resource_list>qstat_l_requests"`
	EnvList              []EnvVar      `xml:"JB_env_list>job_sublist"`
	JobArgs              []string      `xml:"JB_job_args>element>ST_name"`
	ScriptFile           string        `xml:"JB_script_file"`
	JobArrayTasks        []Task        `xml:"JB_ja_tasks>ulong_sublist"`
	Cwd                  string        `xml:"JB_cwd"`
	StderrPathList       []PathList    `xml:"JB_stderr_path_list>path_list"`
	AltStderrPathList    []PathList    `xml:"JB_stderr_path_list>stderr_path_list"` // Alternate stderr path list
	JIDRequestList       []int         `xml:"JB_jid_request_list>element>JRE_job_name"`
	JIDSuccessorList     []int         `xml:"JB_jid_successor_list>ulong_sublist>JRE_job_number"`
	Deadline             bool          `xml:"JB_deadline"`
	ExecutionTime        int           `xml:"JB_execution_time"`
	CheckpointAttr       int           `xml:"JB_checkpoint_attr"`
	CheckpointInterval   int           `xml:"JB_checkpoint_interval"`
	Reserve              bool          `xml:"JB_reserve"`
	MailOptions          int           `xml:"JB_mail_options"`
	Priority             int           `xml:"JB_priority"`
	Restart              int           `xml:"JB_restart"`
	Verify               bool          `xml:"JB_verify"`
	ScriptSize           int           `xml:"JB_script_size"`
	VerifySuitableQueues bool          `xml:"JB_verify_suitable_queues"`
	SoftWallClockGMT     int           `xml:"JB_soft_wallclock_gmt"`
	HardWallClockGMT     int           `xml:"JB_hard_wallclock_gmt"`
	OverrideTickets      int           `xml:"JB_override_tickets"`
	Version              int           `xml:"JB_version"`
	JobArray             TaskIDRange   `xml:"JB_ja_structure>task_id_range"`
	Type                 int           `xml:"JB_type"`
}

// Type DetailedJobInfo represents the job information returned by qstat -j
type DetailedJobInfo struct {
	Jobs     []JobInfo `xml:"djob_info>element"`
	Messages Messages  `xml:"messages>element"`
}

type QueueJob struct {
	JobNumber          int     `xml:"JB_job_number"`
	NormalizedPriority float32 `xml:"JAT_prio"`
	POSIXPriority      int     `xml:"JB_priority"`
	Name               string  `xml:"JB_name"`
	Owner              string  `xml:"JB_owner"`
	Project            string  `xml:"JB_project"`
	Department         string  `xml:"JB_department"`
	State              string  `xml:"state"`
	StartTime          string  `xml:"JAT_start_time"`
	CpuUsage           float64 `xml:"cpu_usage"`
	MemUsage           float64 `xml:"mem_usage"`
	IOUsage            float64 `xml:"io_usage"`
	Tickets            int     `xml:"tickets"`
	OverrideTickets    int     `xml:"otickets"`
	FairshareTickets   int     `xml:"ftickets"`
	ShareTreeTickets   int     `xml:"stickets"`
	QueueName          string  `xml:"queue_name"`
	Slots              int     `xml:"slots"`
	Tasks              string  `xml:"tasks"`
}

// DeletionState returns true if the job is in the (d)eletion state
func (j QueueJob) DeletionState() bool {
	return strings.Contains(j.State, "d")
}

// ErrorState returns true if the job is in the (E)rror state
func (j QueueJob) ErrorState() bool {
	return strings.Contains(j.State, "E")
}

// HoldState returns true if the job is in the (h)old state
func (j QueueJob) HoldState() bool {
	return strings.Contains(j.State, "h")
}

// RunningState returns true if the job is in the (r)unning state
func (j QueueJob) RunningState() bool {
	return strings.Contains(j.State, "r")
}

// RestartedState returns true if the job is in the (R)estarted state
func (j QueueJob) RestartedState() bool {
	return strings.Contains(j.State, "R")
}

// SuspendedState returns true if the job is in the (s)uspended state
func (j QueueJob) SuspendedState() bool {
	return strings.Contains(j.State, "s")
}

// QueueSuspendedState returns true if the job is in the queue (S)uspended state
func (j QueueJob) QueueSuspendedState() bool {
	return strings.Contains(j.State, "S")
}

// TransferringState returns true if the job is in the (t)ransferring state
func (j QueueJob) TransferringState() bool {
	return strings.Contains(j.State, "t")
}

// ThresholdState returns true if the job is in the (T)hreshold state
func (j QueueJob) ThresholdState() bool {
	return strings.Contains(j.State, "T")
}

// WaitingState returns true if the job is in the (w)aiting state
func (j QueueJob) WaitingState() bool {
	return strings.Contains(j.State, "w")
}

// QueuedState returns true if the job is in the (q)ueued state
func (j QueueJob) QueuedState() bool {
	return strings.Contains(j.State, "q")
}

type Queue struct {
	Name          string     `xml:"name"`
	QType         string     `xml:"qtype"`
	SlotsUsed     int        `xml:"slots_used"`
	SlotsReserved int        `xml:"slots_resv"`
	SlotsTotal    int        `xml:"slots_total"`
	Arch          string     `xml:"arch"`
	Joblist       []QueueJob `xml:"job_list"`
}

type QueueInfo struct {
	QueuedJobs  []QueueJob `xml:"queue_info>job_list"` // A list of jobs currently assigned to queues, eg: executing
	PendingJobs []QueueJob `xml:"job_info>job_list"`   // A list of jobs that are not yet executing in any queue
}

// Function absPaths converts the paths of a list of PathList structs in to absolute paths of root if they are not already absolute.
func absPaths(root string, ps []PathList) []PathList {
	var paths []PathList
	for _, p := range ps {
		if !path.IsAbs(p.Path) {
			p.Path = path.Join(root, p.Path)
		}
		paths = append(paths, p)
	}
	return paths
}

func (i *JobInfo) StdoutPaths() []PathList {
	var paths []PathList
	paths = append(paths, absPaths(i.Cwd, i.StdoutPathList)...)
	paths = append(paths, absPaths(i.Cwd, i.AltStdoutPathList)...)
	return paths
}

func (i *JobInfo) StderrPaths() []PathList {
	var paths []PathList
	if !i.MergeStdErr {
		paths = append(paths, absPaths(i.Cwd, i.StderrPathList)...)
		paths = append(paths, absPaths(i.Cwd, i.AltStderrPathList)...)
	}
	return paths
}

func (i *JobInfo) Command() string {
	return i.ScriptFile + " " + strings.Join(i.JobArgs, " ")
}

// GetDetailedJobInfo returns a DetailedJobInfo structure contianing all jobs matching the provided pattern.
// The pattern should match the type wc_job_list as defined in man 1 sge_types
func GetDetailedJobInfo(pattern string) (q *DetailedJobInfo, err error) {
	cmd := exec.Command("qstat", "-j", pattern, "-xml")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err = cmd.Start(); err != nil {
		return nil, err
	}

	d := xml.NewDecoder(util.NewValidUTF8Reader(stdout))
	d.Strict = false
	if err = d.Decode(&q); err != nil {
		// Qstat just produces unparseable XML instead of doing real error reporting. Hurrah.
		if err.Error() == "XML syntax error on line 3: expected element name after <" {
			return nil, errors.New("Unknown job: " + pattern)
		}
	}

	if err = cmd.Wait(); err != nil {
		return nil, err
	}

	return q, nil
}

// GetQueueInfo returns a QueueInfo reflecting the current state of the GridEngine queue.
// The argument u can be used to limit the results to a particular user.
// If u is the string "*" then results are returned for all users.
// If u is the empty string then results are returned for the current user.
func GetQueueInfo(u string) (q *QueueInfo, err error) {
	args := []string{"-xml", "-pri", "-ext"}

	if u != "" {
		args = append(args, "-u", "*")
	}

	cmd := exec.Command("qstat", args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err = cmd.Start(); err != nil {
		return nil, err
	}
	d := xml.NewDecoder(util.NewValidUTF8Reader(stdout))
	d.Strict = false

	if err = d.Decode(&q); err != nil {
		return nil, err
	}

	if err = cmd.Wait(); err != nil {
		return nil, err
	}

	return q, nil
}
<commit_msg>Correctly restrict GetQueueInfo to the supplied user<commit_after>// Package qstat provides an interface to GridEngine's job and queue status facilities
package qstat

import (
	"encoding/xml"
	"errors"
	"github.com/kisielk/gorge/util"
	"os/exec"
	"path"
	"strings"
)

// Resource represents a GridEngine resource request
// See man 5 sge_complex for a more detailed description of the fields
type Resource struct {
	Name        string  `xml:"CE_name"`         // The name of the complex resource
	ValType     int     `xml:"CE_valtype"`      // The type of value
	StringVal   string  `xml:"CE_stringval"`    // The value as a string
	DoubleVal   float64 `xml:"CE_doubleval"`    // The value as a double
	RelOp       int     `xml:"CE_relop"`        // The relation operator used to compare the value
	Consumable  bool    `xml:"CE_consumable"`   // True if the resource is a consumable resourece
	Dominant    bool    `xml:"CE_dominant"`     // ?
	PJDoubleVal float64 `xml:"CE_pj_doubleval"` // ?
	PJDominant  bool    `xml:"CE_pj_dominant"`  // ?
	Requestable bool    `xml:"CE_requestable"`  // True if the resource is a requestable resource
	Tagged      bool    `xml:"CE_tagged"`       // ?
}

// MailAddress represents an email address
type MailAddress struct {
	User string `xml:"MR_user"`
	Host string `xml:"MR_host"`
}

// String implements the Stringer interface
func (a MailAddress) String() string {
	return a.User + "@" + a.Host
}

// EnvVar represents a job environment variable
type EnvVar struct {
	Variable string `xml:"VA_variable"` // The name of the variable
	Value    string `xml:"VA_value"`    // The value of the variable
}

type PathList struct {
	Path        string `xml:"PN_path"`
	Host        string `xml:"PN_host"`
	FileHost    string `xml:"PN_file_host"`
	FileStaging bool   `xml:"PN_file_staging"`
}

// TaskIDRange represents a range of job array task identifiers
type TaskIDRange struct {
	Min  int `xml:"RN_min"`  // The minimum task ID
	Max  int `xml:"RN_max"`  // The maximum task ID
	Step int `xml:"RN_step"` // The ID step size between tasks
}

type JATMessage struct {
	Type    int    `xml:"QIM_type"`
	Message string `xml:"QIM_message"`
}

type Messages struct {
	Messages       []SMEMessage `xml:"SME_message_list>element"`
	GlobalMessages []SMEMessage `xml:"SME_global_message_list>element"`
}

type SMEMessage struct {
	JobNumbers []int  `xml:"MES_job_number_list>ulong_sublist>ULNG_value"`
	Number     int    `xml:"MES_message_number"`
	Message    string `xml:"MES_message"`
}

type Task struct {
	Status      int        `xml:"JAT_status"`
	TaskNumber  int        `xml:"JAT_task_number"`
	MessageList JATMessage `xml:"JAT_message_list>ulong_sublist"`
}

type JobInfo struct {
	JobNumber            int           `xml:"JB_job_number"`
	AdvanceReservation   int           `xml:"JB_ar"`
	ExecFile             string        `xml:"JB_exec_file"`
	SubmissionTime       int           `xml:"JB_submission_time"`
	Owner                string        `xml:"JB_owner"`
	Uid                  int           `xml:"JB_uid"`
	Group                string        `xml:"JB_group"`
	Gid                  int           `xml:"JB_gid"`
	Account              string        `xml:"JB_account"`
	MergeStdErr          bool          `xml:"JB_merge_stderr"`
	MailList             []MailAddress `xml:"JB_mail_list>element"`
	Project              string        `xml:"JB_project"`
	Notify               bool          `xml:"JB_notify"`
	JobName              string        `xml:"JB_job_name"`
	StdoutPathList       []PathList    `xml:"JB_stdout_path_list>path_list"`
	AltStdoutPathList    []PathList    `xml:"JB_stdout_path_list>stdout_path_list"` // Alternate stdout path list
	JobShare             int           `xml:"JB_jobshare"`
	HardResourceList     []Resource    `xml:"JB_hard_resource_list>qstat_l_requests"`
	EnvList              []EnvVar      `xml:"JB_env_list>job_sublist"`
	JobArgs              []string      `xml:"JB_job_args>element>ST_name"`
	ScriptFile           string        `xml:"JB_script_file"`
	JobArrayTasks        []Task        `xml:"JB_ja_tasks>ulong_sublist"`
	Cwd                  string        `xml:"JB_cwd"`
	StderrPathList       []PathList    `xml:"JB_stderr_path_list>path_list"`
	AltStderrPathList    []PathList    `xml:"JB_stderr_path_list>stderr_path_list"` // Alternate stderr path list
	JIDRequestList       []int         `xml:"JB_jid_request_list>element>JRE_job_name"`
	JIDSuccessorList     []int         `xml:"JB_jid_successor_list>ulong_sublist>JRE_job_number"`
	Deadline             bool          `xml:"JB_deadline"`
	ExecutionTime        int           `xml:"JB_execution_time"`
	CheckpointAttr       int           `xml:"JB_checkpoint_attr"`
	CheckpointInterval   int           `xml:"JB_checkpoint_interval"`
	Reserve              bool          `xml:"JB_reserve"`
	MailOptions          int           `xml:"JB_mail_options"`
	Priority             int           `xml:"JB_priority"`
	Restart              int           `xml:"JB_restart"`
	Verify               bool          `xml:"JB_verify"`
	ScriptSize           int           `xml:"JB_script_size"`
	VerifySuitableQueues bool          `xml:"JB_verify_suitable_queues"`
	SoftWallClockGMT     int           `xml:"JB_soft_wallclock_gmt"`
	HardWallClockGMT     int           `xml:"JB_hard_wallclock_gmt"`
	OverrideTickets      int           `xml:"JB_override_tickets"`
	Version              int           `xml:"JB_version"`
	JobArray             TaskIDRange   `xml:"JB_ja_structure>task_id_range"`
	Type                 int           `xml:"JB_type"`
}

// Type DetailedJobInfo represents the job information returned by qstat -j
type DetailedJobInfo struct {
	Jobs     []JobInfo `xml:"djob_info>element"`
	Messages Messages  `xml:"messages>element"`
}

type QueueJob struct {
	JobNumber          int     `xml:"JB_job_number"`
	NormalizedPriority float32 `xml:"JAT_prio"`
	POSIXPriority      int     `xml:"JB_priority"`
	Name               string  `xml:"JB_name"`
	Owner              string  `xml:"JB_owner"`
	Project            string  `xml:"JB_project"`
	Department         string  `xml:"JB_department"`
	State              string  `xml:"state"`
	StartTime          string  `xml:"JAT_start_time"`
	CpuUsage           float64 `xml:"cpu_usage"`
	MemUsage           float64 `xml:"mem_usage"`
	IOUsage            float64 `xml:"io_usage"`
	Tickets            int     `xml:"tickets"`
	OverrideTickets    int     `xml:"otickets"`
	FairshareTickets   int     `xml:"ftickets"`
	ShareTreeTickets   int     `xml:"stickets"`
	QueueName          string  `xml:"queue_name"`
	Slots              int     `xml:"slots"`
	Tasks              string  `xml:"tasks"`
}

// DeletionState returns true if the job is in the (d)eletion state
func (j QueueJob) DeletionState() bool {
	return strings.Contains(j.State, "d")
}

// ErrorState returns true if the job is in the (E)rror state
func (j QueueJob) ErrorState() bool {
	return strings.Contains(j.State, "E")
}

// HoldState returns true if the job is in the (h)old state
func (j QueueJob) HoldState() bool {
	return strings.Contains(j.State, "h")
}

// RunningState returns true if the job is in the (r)unning state
func (j QueueJob) RunningState() bool {
	return strings.Contains(j.State, "r")
}

// RestartedState returns true if the job is in the (R)estarted state
func (j QueueJob) RestartedState() bool {
	return strings.Contains(j.State, "R")
}

// SuspendedState returns true if the job is in the (s)uspended state
func (j QueueJob) SuspendedState() bool {
	return strings.Contains(j.State, "s")
}

// QueueSuspendedState returns true if the job is in the queue (S)uspended state
func (j QueueJob) QueueSuspendedState() bool {
	return strings.Contains(j.State, "S")
}

// TransferringState returns true if the job is in the (t)ransferring state
func (j QueueJob) TransferringState() bool {
	return strings.Contains(j.State, "t")
}

// ThresholdState returns true if the job is in the (T)hreshold state
func (j QueueJob) ThresholdState() bool {
	return strings.Contains(j.State, "T")
}

// WaitingState returns true if the job is in the (w)aiting state
func (j QueueJob) WaitingState() bool {
	return strings.Contains(j.State, "w")
}

// QueuedState returns true if the job is in the (q)ueued state
func (j QueueJob) QueuedState() bool {
	return strings.Contains(j.State, "q")
}

type Queue struct {
	Name          string     `xml:"name"`
	QType         string     `xml:"qtype"`
	SlotsUsed     int        `xml:"slots_used"`
	SlotsReserved int        `xml:"slots_resv"`
	SlotsTotal    int        `xml:"slots_total"`
	Arch          string     `xml:"arch"`
	Joblist       []QueueJob `xml:"job_list"`
}

type QueueInfo struct {
	QueuedJobs  []QueueJob `xml:"queue_info>job_list"` // A list of jobs currently assigned to queues, eg: executing
	PendingJobs []QueueJob `xml:"job_info>job_list"`   // A list of jobs that are not yet executing in any queue
}

// Function absPaths converts the paths of a list of PathList structs in to absolute paths of root if they are not already absolute.
func absPaths(root string, ps []PathList) []PathList {
	var paths []PathList
	for _, p := range ps {
		if !path.IsAbs(p.Path) {
			p.Path = path.Join(root, p.Path)
		}
		paths = append(paths, p)
	}
	return paths
}

func (i *JobInfo) StdoutPaths() []PathList {
	var paths []PathList
	paths = append(paths, absPaths(i.Cwd, i.StdoutPathList)...)
	paths = append(paths, absPaths(i.Cwd, i.AltStdoutPathList)...)
	return paths
}

func (i *JobInfo) StderrPaths() []PathList {
	var paths []PathList
	if !i.MergeStdErr {
		paths = append(paths, absPaths(i.Cwd, i.StderrPathList)...)
		paths = append(paths, absPaths(i.Cwd, i.AltStderrPathList)...)
	}
	return paths
}

func (i *JobInfo) Command() string {
	return i.ScriptFile + " " + strings.Join(i.JobArgs, " ")
}

// GetDetailedJobInfo returns a DetailedJobInfo structure contianing all jobs matching the provided pattern.
// The pattern should match the type wc_job_list as defined in man 1 sge_types
func GetDetailedJobInfo(pattern string) (q *DetailedJobInfo, err error) {
	cmd := exec.Command("qstat", "-j", pattern, "-xml")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err = cmd.Start(); err != nil {
		return nil, err
	}

	d := xml.NewDecoder(util.NewValidUTF8Reader(stdout))
	d.Strict = false
	if err = d.Decode(&q); err != nil {
		// Qstat just produces unparseable XML instead of doing real error reporting. Hurrah.
		if err.Error() == "XML syntax error on line 3: expected element name after <" {
			return nil, errors.New("Unknown job: " + pattern)
		}
	}

	if err = cmd.Wait(); err != nil {
		return nil, err
	}

	return q, nil
}

// GetQueueInfo returns a QueueInfo reflecting the current state of the GridEngine queue.
// The argument u can be used to limit the results to a particular user.
// If u is the string "*" then results are returned for all users.
// If u is the empty string then results are returned for the current user.
func GetQueueInfo(u string) (q *QueueInfo, err error) {
	args := []string{"-xml", "-pri", "-ext"}

	if u != "" {
		args = append(args, "-u", u)
	}

	cmd := exec.Command("qstat", args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}

	if err = cmd.Start(); err != nil {
		return nil, err
	}
	d := xml.NewDecoder(util.NewValidUTF8Reader(stdout))
	d.Strict = false

	if err = d.Decode(&q); err != nil {
		return nil, err
	}

	if err = cmd.Wait(); err != nil {
		return nil, err
	}

	return q, nil
}
<|endoftext|>


<commit_before>package weavedns

import (
	"github.com/miekg/dns"
	"log"
	"net"
	"testing"
	"time"
)

var (
	test_addr1 = "10.0.2.1/24"
)

func sendQuery(name string, querytype uint16) error {
	m := new(dns.Msg)
	m.SetQuestion(name, querytype)
	m.RecursionDesired = false
	buf, err := m.Pack()
	if err != nil {
		return err
	}
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return err
	}
	_, err = conn.WriteTo(buf, ipv4Addr)
	return err
}

func TestServerSimpleQuery(t *testing.T) {
	log.Println("TestServerSimpleQuery starting")
	var zone = new(ZoneDb)
	docker_ip := net.ParseIP("9.8.7.6")
	weave_ip, subnet, _ := net.ParseCIDR(test_addr1)
	zone.AddRecord("test.weave.", docker_ip, weave_ip, subnet)

	mdnsServer, err := NewMDNSServer(zone)
	if err != nil {
		t.Fatal(err)
	}
	err = mdnsServer.Start(nil)
	if err != nil {
		t.Fatal(err)
	}

	var received_addr net.IP

	// Implement a minimal listener for responses
	multicast, err := LinkLocalMulticastListener(nil)
	if err != nil {
		t.Fatal(err)
	}

	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
		// Only handle responses here
		if len(r.Answer) > 0 {
			for _, answer := range r.Answer {
				switch rr := answer.(type) {
				case *dns.A:
					received_addr = rr.A
				}
			}
		}
	}

	server := &dns.Server{Listener: nil, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
	go server.ActivateAndServe()

	sendQuery("test.weave.", dns.TypeA)

	time.Sleep(time.Second)

	if !received_addr.Equal(weave_ip) {
		t.Log("Unexpected result for test.weave", received_addr)
		t.Fail()
	}
}
<commit_msg>Shut down listener when test finished<commit_after>package weavedns

import (
	"github.com/miekg/dns"
	"log"
	"net"
	"testing"
	"time"
)

var (
	test_addr1 = "10.0.2.1/24"
)

func sendQuery(name string, querytype uint16) error {
	m := new(dns.Msg)
	m.SetQuestion(name, querytype)
	m.RecursionDesired = false
	buf, err := m.Pack()
	if err != nil {
		return err
	}
	conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return err
	}
	_, err = conn.WriteTo(buf, ipv4Addr)
	return err
}

func TestServerSimpleQuery(t *testing.T) {
	log.Println("TestServerSimpleQuery starting")
	var zone = new(ZoneDb)
	docker_ip := net.ParseIP("9.8.7.6")
	weave_ip, subnet, _ := net.ParseCIDR(test_addr1)
	zone.AddRecord("test.weave.", docker_ip, weave_ip, subnet)

	mdnsServer, err := NewMDNSServer(zone)
	if err != nil {
		t.Fatal(err)
	}
	err = mdnsServer.Start(nil)
	if err != nil {
		t.Fatal(err)
	}

	var received_addr net.IP

	// Implement a minimal listener for responses
	multicast, err := LinkLocalMulticastListener(nil)
	if err != nil {
		t.Fatal(err)
	}

	handleMDNS := func(w dns.ResponseWriter, r *dns.Msg) {
		// Only handle responses here
		if len(r.Answer) > 0 {
			for _, answer := range r.Answer {
				switch rr := answer.(type) {
				case *dns.A:
					received_addr = rr.A
				}
			}
		}
	}

	server := &dns.Server{Listener: nil, PacketConn: multicast, Handler: dns.HandlerFunc(handleMDNS)}
	go server.ActivateAndServe()
	defer server.Shutdown()

	sendQuery("test.weave.", dns.TypeA)

	time.Sleep(time.Second)

	if !received_addr.Equal(weave_ip) {
		t.Log("Unexpected result for test.weave", received_addr)
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package webhook

import (
	"encoding/json"
	"time"
)

type oldW struct {
	// Configuration for message delivery
	Config struct {
		// The URL to deliver messages to.
		URL string `json:"url"`

		// The content-type to set the messages to (unless specified by WRP).
		ContentType string `json:"content_type"`

		// The secret to use for the SHA1 HMAC.
		// Optional, set to "" to disable behavior.
		Secret string `json:"secret,omitempty"`
	} `json:"config"`

	// The list of regular expressions to match event type against.
	Events []string `json:"events"`

	// Matcher type contains values to match against the metadata.
	Matcher struct {
		// The list of regular expressions to match device id type against.
		DeviceId []string `json:"device_id"`
	} `json:"matcher,omitempty"`

	// The specified duration for this hook to live
	Duration int64 `json:"duration"`

	// The absolute time when this hook is to be disabled
	Until int64 `json:"until"`

	// The address that performed the registration
	Address string `json:"registered_from_address"`
}

func convertOldHooksToNewHooks(body []byte) (hooks []W, err error) {
	var oldHooks []oldW
	err = json.Unmarshal(body, &oldHooks)
	if err != nil {
		return
	}

	for _, oldHook range oldHooks {
		var tempHook W
		tempHook.Config.URL = oldHook.Config.URL
		tempHook.Config.ContentType = oldHook.Config.ContentType
		tempHook.Config.Secret = oldHook.Config.Secret
		tempHook.Events = oldHook.Events
		tempHook.Matcher = oldHook.Matcher
		tempHook.Duration = time.Duration(oldHook.Duration) * time.Second
		tempHook.Until = time.Unix(oldHook.Until, 0)
		tempHook.Address = oldHook.Address

		hooks = append(hooks, tempHook)
	}

	return
}
<commit_msg>fix syntax error<commit_after>package webhook

import (
	"encoding/json"
	"time"
)

type oldW struct {
	// Configuration for message delivery
	Config struct {
		// The URL to deliver messages to.
		URL string `json:"url"`

		// The content-type to set the messages to (unless specified by WRP).
		ContentType string `json:"content_type"`

		// The secret to use for the SHA1 HMAC.
		// Optional, set to "" to disable behavior.
		Secret string `json:"secret,omitempty"`
	} `json:"config"`

	// The list of regular expressions to match event type against.
	Events []string `json:"events"`

	// Matcher type contains values to match against the metadata.
	Matcher struct {
		// The list of regular expressions to match device id type against.
		DeviceId []string `json:"device_id"`
	} `json:"matcher,omitempty"`

	// The specified duration for this hook to live
	Duration int64 `json:"duration"`

	// The absolute time when this hook is to be disabled
	Until int64 `json:"until"`

	// The address that performed the registration
	Address string `json:"registered_from_address"`
}

func convertOldHooksToNewHooks(body []byte) (hooks []W, err error) {
	var oldHooks []oldW
	err = json.Unmarshal(body, &oldHooks)
	if err != nil {
		return
	}

	for _, oldHook := range oldHooks {
		var tempHook W
		tempHook.Config.URL = oldHook.Config.URL
		tempHook.Config.ContentType = oldHook.Config.ContentType
		tempHook.Config.Secret = oldHook.Config.Secret
		tempHook.Events = oldHook.Events
		tempHook.Matcher = oldHook.Matcher
		tempHook.Duration = time.Duration(oldHook.Duration) * time.Second
		tempHook.Until = time.Unix(oldHook.Until, 0)
		tempHook.Address = oldHook.Address

		hooks = append(hooks, tempHook)
	}

	return
}
<|endoftext|>


<commit_before>// Package queue implements a boldDB backed queue for MDM Commands.
package queue

import (
	"context"
	"fmt"

	"github.com/boltdb/bolt"
	"github.com/groob/plist"
	"github.com/pkg/errors"

	"github.com/micromdm/mdm"
	"github.com/micromdm/micromdm/command"
	"github.com/micromdm/micromdm/pubsub"
)

const (
	DeviceCommandBucket = "mdm.DeviceCommands"
)

type Store struct {
	*bolt.DB
}

func (db *Store) Next(ctx context.Context, resp mdm.Response) (*Command, error) {
	dc, err := db.DeviceCommand(resp.UDID)
	if err != nil {
		return nil, errors.Wrapf(err, "get device command from queue, udid: %s", resp.UDID)
	}

	var cmd *Command
	switch resp.Status {
	case "NotNow":
		// move down, send next
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil {
			break
		}
		dc.Commands = append(dc.Commands, *x)

	case "Acknowledged":
		// move to completed, send next
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil {
			break
		}
		dc.Completed = append(dc.Completed, *x)
	case "Error":
		// move to failed, send next
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil { // must've already bin ackd
			break
		}
		dc.Failed = append(dc.Failed, *x)

	case "CommandFormatError":
		// move to failed
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil {
			break
		}
		dc.Failed = append(dc.Failed, *x)

	case "Idle":
		// will send next command below

	default:
		return nil, fmt.Errorf("unknown response status: %s", resp.Status)
	}

	// pop the first command from the queue and add it to the end.
	cmd, dc.Commands = popFirst(dc.Commands)
	if cmd != nil {
		dc.Commands = append(dc.Commands, *cmd)
	}

	if cmd.UUID == resp.CommandUUID && resp.Status == "NotNow" {
		// This command was just handled by NotNow, ignore.
		cmd = nil
	}

	if err := db.Save(dc); err != nil {
		return nil, err
	}

	return cmd, nil
}

func popFirst(all []Command) (*Command, []Command) {
	if len(all) == 0 {
		return nil, all
	}
	first := all[0]
	all = append(all[:0], all[1:]...)
	return &first, all
}

func cut(all []Command, uuid string) (*Command, []Command) {
	for i, cmd := range all {
		if cmd.UUID == uuid {
			all = append(all[:i], all[i+1:]...)
			return &cmd, all
		}
	}
	return nil, all
}

func NewQueue(db *bolt.DB, sub pubsub.Subscriber) (*Store, error) {
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(DeviceCommandBucket))
		return err
	})
	if err != nil {
		return nil, errors.Wrapf(err, "creating %s bucket", DeviceCommandBucket)
	}
	datastore := &Store{DB: db}
	if err := datastore.pollCommands(sub); err != nil {
		return nil, err
	}
	return datastore, nil
}

func (db *Store) Save(cmd *DeviceCommand) error {
	tx, err := db.DB.Begin(true)
	if err != nil {
		return errors.Wrap(err, "begin transaction")
	}
	bkt := tx.Bucket([]byte(DeviceCommandBucket))
	if bkt == nil {
		return fmt.Errorf("bucket %q not found!", DeviceCommandBucket)
	}
	devproto, err := MarshalDeviceCommand(cmd)
	if err != nil {
		return errors.Wrap(err, "marshalling DeviceCommand")
	}
	key := []byte(cmd.DeviceUDID)
	if err := bkt.Put(key, devproto); err != nil {
		return errors.Wrap(err, "put DeviceCommand to boltdb")
	}
	return tx.Commit()
}

func (db *Store) DeviceCommand(udid string) (*DeviceCommand, error) {
	var dev DeviceCommand
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(DeviceCommandBucket))
		v := b.Get([]byte(udid))
		if v == nil {
			return &notFound{"DeviceCommand", fmt.Sprintf("udid %s", udid)}
		}
		return UnmarshalDeviceCommand(v, &dev)
	})
	if err != nil {
		return nil, err
	}
	return &dev, nil
}

type notFound struct {
	ResourceType string
	Message      string
}

func (e *notFound) Error() string {
	return fmt.Sprintf("not found: %s %s", e.ResourceType, e.Message)
}

func (db *Store) pollCommands(sub pubsub.Subscriber) error {
	commandEvents, err := sub.Subscribe("command-queue", command.CommandTopic)
	if err != nil {
		return errors.Wrapf(err,
			"subscribing push to %s topic", command.CommandTopic)
	}
	go func() {
		for {
			select {
			case event := <-commandEvents:
				var ev command.Event
				if err := command.UnmarshalEvent(event.Message, &ev); err != nil {
					fmt.Println(err)
					continue
				}

				cmd := new(DeviceCommand)
				cmd.DeviceUDID = ev.DeviceUDID
				byUDID, err := db.DeviceCommand(ev.DeviceUDID)
				if err == nil && byUDID != nil {
					cmd = byUDID
				}
				newPayload, err := plist.Marshal(&ev.Payload)
				if err != nil {
					fmt.Println(err)
					continue
				}
				newCmd := Command{
					UUID:    ev.Payload.CommandUUID,
					Payload: newPayload,
				}
				cmd.Commands = append(cmd.Commands, newCmd)
				if err := db.Save(cmd); err != nil {
					fmt.Println(err)
					continue
				}
				fmt.Printf("queued event for device: %s\n", ev.DeviceUDID)
			}
		}
	}()

	return nil
}

func isNotFound(err error) bool {
	if _, ok := err.(*notFound); ok {
		return true
	}
	return false
}
<commit_msg>Fix panic where popFirst() can return nil (but we were immediately using it)<commit_after>// Package queue implements a boldDB backed queue for MDM Commands.
package queue

import (
	"context"
	"fmt"

	"github.com/boltdb/bolt"
	"github.com/groob/plist"
	"github.com/pkg/errors"

	"github.com/micromdm/mdm"
	"github.com/micromdm/micromdm/command"
	"github.com/micromdm/micromdm/pubsub"
)

const (
	DeviceCommandBucket = "mdm.DeviceCommands"
)

type Store struct {
	*bolt.DB
}

func (db *Store) Next(ctx context.Context, resp mdm.Response) (*Command, error) {
	dc, err := db.DeviceCommand(resp.UDID)
	if err != nil {
		return nil, errors.Wrapf(err, "get device command from queue, udid: %s", resp.UDID)
	}

	var cmd *Command
	switch resp.Status {
	case "NotNow":
		// move down, send next
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil {
			break
		}
		dc.Commands = append(dc.Commands, *x)

	case "Acknowledged":
		// move to completed, send next
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil {
			break
		}
		dc.Completed = append(dc.Completed, *x)
	case "Error":
		// move to failed, send next
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil { // must've already bin ackd
			break
		}
		dc.Failed = append(dc.Failed, *x)

	case "CommandFormatError":
		// move to failed
		x, a := cut(dc.Commands, resp.CommandUUID)
		dc.Commands = a
		if x == nil {
			break
		}
		dc.Failed = append(dc.Failed, *x)

	case "Idle":
		// will send next command below

	default:
		return nil, fmt.Errorf("unknown response status: %s", resp.Status)
	}

	// pop the first command from the queue and add it to the end.
	cmd, dc.Commands = popFirst(dc.Commands)
	if cmd != nil {
		dc.Commands = append(dc.Commands, *cmd)

		if cmd.UUID == resp.CommandUUID && resp.Status == "NotNow" {
			// This command was just handled by NotNow, ignore.
			cmd = nil
		}
	}

	if err := db.Save(dc); err != nil {
		return nil, err
	}

	return cmd, nil
}

func popFirst(all []Command) (*Command, []Command) {
	if len(all) == 0 {
		return nil, all
	}
	first := all[0]
	all = append(all[:0], all[1:]...)
	return &first, all
}

func cut(all []Command, uuid string) (*Command, []Command) {
	for i, cmd := range all {
		if cmd.UUID == uuid {
			all = append(all[:i], all[i+1:]...)
			return &cmd, all
		}
	}
	return nil, all
}

func NewQueue(db *bolt.DB, sub pubsub.Subscriber) (*Store, error) {
	err := db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists([]byte(DeviceCommandBucket))
		return err
	})
	if err != nil {
		return nil, errors.Wrapf(err, "creating %s bucket", DeviceCommandBucket)
	}
	datastore := &Store{DB: db}
	if err := datastore.pollCommands(sub); err != nil {
		return nil, err
	}
	return datastore, nil
}

func (db *Store) Save(cmd *DeviceCommand) error {
	tx, err := db.DB.Begin(true)
	if err != nil {
		return errors.Wrap(err, "begin transaction")
	}
	bkt := tx.Bucket([]byte(DeviceCommandBucket))
	if bkt == nil {
		return fmt.Errorf("bucket %q not found!", DeviceCommandBucket)
	}
	devproto, err := MarshalDeviceCommand(cmd)
	if err != nil {
		return errors.Wrap(err, "marshalling DeviceCommand")
	}
	key := []byte(cmd.DeviceUDID)
	if err := bkt.Put(key, devproto); err != nil {
		return errors.Wrap(err, "put DeviceCommand to boltdb")
	}
	return tx.Commit()
}

func (db *Store) DeviceCommand(udid string) (*DeviceCommand, error) {
	var dev DeviceCommand
	err := db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(DeviceCommandBucket))
		v := b.Get([]byte(udid))
		if v == nil {
			return &notFound{"DeviceCommand", fmt.Sprintf("udid %s", udid)}
		}
		return UnmarshalDeviceCommand(v, &dev)
	})
	if err != nil {
		return nil, err
	}
	return &dev, nil
}

type notFound struct {
	ResourceType string
	Message      string
}

func (e *notFound) Error() string {
	return fmt.Sprintf("not found: %s %s", e.ResourceType, e.Message)
}

func (db *Store) pollCommands(sub pubsub.Subscriber) error {
	commandEvents, err := sub.Subscribe("command-queue", command.CommandTopic)
	if err != nil {
		return errors.Wrapf(err,
			"subscribing push to %s topic", command.CommandTopic)
	}
	go func() {
		for {
			select {
			case event := <-commandEvents:
				var ev command.Event
				if err := command.UnmarshalEvent(event.Message, &ev); err != nil {
					fmt.Println(err)
					continue
				}

				cmd := new(DeviceCommand)
				cmd.DeviceUDID = ev.DeviceUDID
				byUDID, err := db.DeviceCommand(ev.DeviceUDID)
				if err == nil && byUDID != nil {
					cmd = byUDID
				}
				newPayload, err := plist.Marshal(&ev.Payload)
				if err != nil {
					fmt.Println(err)
					continue
				}
				newCmd := Command{
					UUID:    ev.Payload.CommandUUID,
					Payload: newPayload,
				}
				cmd.Commands = append(cmd.Commands, newCmd)
				if err := db.Save(cmd); err != nil {
					fmt.Println(err)
					continue
				}
				fmt.Printf("queued event for device: %s\n", ev.DeviceUDID)
			}
		}
	}()

	return nil
}

func isNotFound(err error) bool {
	if _, ok := err.(*notFound); ok {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"regexp"
	"time"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/retry"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
)

// generateXUnitTestSuite generates an xUnit test suite that
// encapsulates the given input.
func generateXUnitTestSuite(ctx *tool.Context, failure *xunit.Failure, pkg string, duration time.Duration) *xunit.TestSuite {
	// Generate an xUnit test suite describing the result.
	s := xunit.TestSuite{Name: pkg}
	c := xunit.TestCase{
		Classname: pkg,
		Name:      "Test",
		Time:      fmt.Sprintf("%.2f", duration.Seconds()),
	}
	if failure != nil {
		fmt.Fprintf(ctx.Stdout(), "%s ... failed\n%v\n", pkg, failure.Data)
		c.Failures = append(c.Failures, *failure)
		s.Failures++
	} else {
		fmt.Fprintf(ctx.Stdout(), "%s ... ok\n", pkg)
	}
	s.Tests++
	s.Cases = append(s.Cases, c)
	return &s
}

// testSingleProdService test the given production service.
func testSingleProdService(ctx *tool.Context, vroot, principalDir string, service prodService) *xunit.TestSuite {
	bin := filepath.Join(vroot, "release", "go", "bin", "vrpc")
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Stdout = &out
	opts.Stderr = &out
	start := time.Now()
	args := []string{}
	if principalDir != "" {
		args = append(args, "--v23.credentials", principalDir)
	}
	args = append(args, "signature", "--show-reserved")
	if principalDir == "" {
		args = append(args, "--insecure")
	}
	args = append(args, service.objectName)
	if err := ctx.Run().TimedCommandWithOpts(test.DefaultTimeout, opts, bin, args...); err != nil {
		return generateXUnitTestSuite(ctx, &xunit.Failure{Message: "vrpc", Data: out.String()}, service.name, time.Now().Sub(start))
	}
	if !service.regexp.Match(out.Bytes()) {
		fmt.Fprintf(ctx.Stderr(), "couldn't match regexp %q in output:\n%v\n", service.regexp, out.String())
		return generateXUnitTestSuite(ctx, &xunit.Failure{Message: "vrpc", Data: "mismatching signature"}, service.name, time.Now().Sub(start))
	}
	return generateXUnitTestSuite(ctx, nil, service.name, time.Now().Sub(start))
}

type prodService struct {
	name       string         // Name to use for the test description
	objectName string         // Object name of the service to connect to
	regexp     *regexp.Regexp // Regexp that should match the signature output
}

// vanadiumProdServicesTest runs a test of vanadium production services.
func vanadiumProdServicesTest(ctx *tool.Context, testName string, opts ...Opt) (_ *test.Result, e error) {
	// Initialize the test.
	// Need the new-stype base profile since many web tests will build
	// go apps that need it.
	cleanup, err := initTest(ctx, testName, []string{"base"})
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	vroot, err := project.JiriRoot()
	if err != nil {
		return nil, err
	}

	// Install the vrpc tool.
	if err := ctx.Run().Command("jiri", "go", "install", "v.io/x/ref/cmd/vrpc"); err != nil {
		return nil, internalTestError{err, "Install VRPC"}
	}
	// Install the principal tool.
	if err := ctx.Run().Command("jiri", "go", "install", "v.io/x/ref/cmd/principal"); err != nil {
		return nil, internalTestError{err, "Install Principal"}
	}
	tmpdir, err := ctx.Run().TempDir("", "prod-services-test")
	if err != nil {
		return nil, internalTestError{err, "Create temporary directory"}
	}
	defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpdir) }, &e)

	blessingRoot, namespaceRoot := getServiceOpts(opts)
	allPassed, suites := true, []xunit.TestSuite{}

	// Fetch the "root" blessing that all services are blessed by.
	suite, pubkey, blessingNames := testIdentityProviderHTTP(ctx, blessingRoot)
	suites = append(suites, *suite)

	if suite.Failures == 0 {
		// Setup a principal that will be used by testAllProdServices and will
		// recognize the blessings of the prod services.
		principalDir, err := setupPrincipal(ctx, vroot, tmpdir, pubkey, blessingNames)
		if err != nil {
			return nil, err
		}
		for _, suite := range testAllProdServices(ctx, vroot, principalDir, namespaceRoot) {
			allPassed = allPassed && (suite.Failures == 0)
			suites = append(suites, *suite)
		}
	}

	// Create the xUnit report.
	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}
	for _, suite := range suites {
		if suite.Failures > 0 {
			// At least one test failed:
			return &test.Result{Status: test.Failed}, nil
		}
	}
	return &test.Result{Status: test.Passed}, nil
}

func testAllProdServices(ctx *tool.Context, vroot, principalDir, namespaceRoot string) []*xunit.TestSuite {
	services := []prodService{
		prodService{
			name:       "mounttable",
			objectName: namespaceRoot,
			regexp:     regexp.MustCompile(`MountTable[[:space:]]+interface`),
		},
		prodService{
			name:       "application repository",
			objectName: namespaceRoot + "/applications",
			regexp:     regexp.MustCompile(`Application[[:space:]]+interface`),
		},
		prodService{
			name:       "binary repository",
			objectName: namespaceRoot + "/binaries",
			regexp:     regexp.MustCompile(`Binary[[:space:]]+interface`),
		},
		prodService{
			name:       "macaroon service",
			objectName: namespaceRoot + "/identity/dev.v.io/u/macaroon",
			regexp:     regexp.MustCompile(`MacaroonBlesser[[:space:]]+interface`),
		},
		prodService{
			name:       "google identity service",
			objectName: namespaceRoot + "/identity/dev.v.io/u/google",
			regexp:     regexp.MustCompile(`OAuthBlesser[[:space:]]+interface`),
		},
		prodService{
			objectName: namespaceRoot + "/identity/dev.v.io/u/discharger",
			name:       "binary discharger",
			regexp:     regexp.MustCompile(`Discharger[[:space:]]+interface`),
		},
		prodService{
			objectName: namespaceRoot + "/proxy-mon/__debug",
			name:       "proxy service",
			// We just check that the returned signature has the __Reserved interface since
			// proxy-mon doesn't implement any other services.
			regexp: regexp.MustCompile(`__Reserved[[:space:]]+interface`),
		},
	}

	var suites []*xunit.TestSuite
	for _, service := range services {
		suites = append(suites, testSingleProdService(ctx, vroot, principalDir, service))
	}
	return suites
}

// testIdentityProviderHTTP tests that the identity provider's HTTP server is
// up and running and also fetches the set of blessing names that the provider
// claims to be authoritative on and the public key (encoded) used by that
// identity provider to sign certificates for blessings.
//
// PARANOIA ALERT:
// This function is subject to man-in-the-middle attacks because it does not
// verify the TLS certificates presented by the server. This does open the
// door for an attack where a parallel universe of services could be setup
// and fool this production services test into thinking all services are
// up and running when they may not be.
//
// The attacker in this case will have to be able to mess with the routing
// tables on the machine running this test, or the network routes of routers
// used by the machine, or mess up DNS entries.
func testIdentityProviderHTTP(ctx *tool.Context, blessingRoot string) (suite *xunit.TestSuite, publickey string, blessingNames []string) {
	url := fmt.Sprintf("https://%s/auth/blessing-root", blessingRoot)
	var response struct {
		Names     []string `json:"names"`
		PublicKey string   `json:"publicKey"`
	}
	var resp *http.Response
	var err error
	var start time.Time
	fn := func() error {
		start = time.Now()
		resp, err = http.Get(url)
		return err
	}
	if err = retry.Function(ctx, fn); err == nil {
		defer resp.Body.Close()
		err = json.NewDecoder(resp.Body).Decode(&response)
	}
	var failure *xunit.Failure
	if err != nil {
		failure = &xunit.Failure{Message: "identityd HTTP", Data: err.Error()}
	}
	return generateXUnitTestSuite(ctx, failure, url, time.Now().Sub(start)), response.PublicKey, response.Names
}

func setupPrincipal(ctx *tool.Context, vroot, tmpdir, pubkey string, blessingNames []string) (string, error) {
	dir := filepath.Join(tmpdir, "credentials")
	bin := filepath.Join(vroot, "release", "go", "bin", "principal")
	if err := ctx.Run().TimedCommand(test.DefaultTimeout, bin, "create", dir, "prod-services-tester"); err != nil {
		fmt.Fprintf(ctx.Stderr(), "principal create failed: %v\n", err)
		return "", err
	}
	for _, name := range blessingNames {
		if err := ctx.Run().TimedCommand(test.DefaultTimeout, bin, "--v23.credentials", dir, "recognize", name, pubkey); err != nil {
			fmt.Fprintf(ctx.Stderr(), "principal recognize %v %v failed: %v\n", name, pubkey, err)
			return "", err
		}
	}
	return dir, nil
}

// getServiceOpts extracts blessing root and namespace root from the
// given Opts.
func getServiceOpts(opts []Opt) (string, string) {
	blessingRoot := "dev.v.io"
	namespaceRoot := "/ns.dev.v.io:8101"
	for _, opt := range opts {
		switch v := opt.(type) {
		case BlessingsRootOpt:
			blessingRoot = string(v)
		case NamespaceRootOpt:
			namespaceRoot = string(v)
		}
	}
	return blessingRoot, namespaceRoot
}
<commit_msg>jiri-test/internal/test/prod.go: print more info about failed commands<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"path/filepath"
	"regexp"
	"time"

	"v.io/jiri/collect"
	"v.io/jiri/project"
	"v.io/jiri/retry"
	"v.io/jiri/tool"
	"v.io/x/devtools/internal/test"
	"v.io/x/devtools/internal/xunit"
)

// generateXUnitTestSuite generates an xUnit test suite that
// encapsulates the given input.
func generateXUnitTestSuite(ctx *tool.Context, failure *xunit.Failure, pkg string, duration time.Duration) *xunit.TestSuite {
	// Generate an xUnit test suite describing the result.
	s := xunit.TestSuite{Name: pkg}
	c := xunit.TestCase{
		Classname: pkg,
		Name:      "Test",
		Time:      fmt.Sprintf("%.2f", duration.Seconds()),
	}
	if failure != nil {
		fmt.Fprintf(ctx.Stdout(), "%s ... failed\n%v\n", pkg, failure.Data)
		c.Failures = append(c.Failures, *failure)
		s.Failures++
	} else {
		fmt.Fprintf(ctx.Stdout(), "%s ... ok\n", pkg)
	}
	s.Tests++
	s.Cases = append(s.Cases, c)
	return &s
}

// testSingleProdService test the given production service.
func testSingleProdService(ctx *tool.Context, vroot, principalDir string, service prodService) *xunit.TestSuite {
	bin := filepath.Join(vroot, "release", "go", "bin", "vrpc")
	var out bytes.Buffer
	opts := ctx.Run().Opts()
	opts.Verbose = true
	opts.Stdout = &out
	opts.Stderr = &out
	start := time.Now()
	args := []string{}
	if principalDir != "" {
		args = append(args, "--v23.credentials", principalDir)
	}
	args = append(args, "signature", "--show-reserved")
	if principalDir == "" {
		args = append(args, "--insecure")
	}
	args = append(args, service.objectName)
	if err := ctx.Run().TimedCommandWithOpts(test.DefaultTimeout, opts, bin, args...); err != nil {
		fmt.Fprintf(ctx.Stderr(), "Failed running %q: %v. Output:\n%v\n", append([]string{bin}, args...), err, out.String())
		return generateXUnitTestSuite(ctx, &xunit.Failure{Message: "vrpc", Data: out.String()}, service.name, time.Now().Sub(start))
	}
	if !service.regexp.Match(out.Bytes()) {
		fmt.Fprintf(ctx.Stderr(), "couldn't match regexp %q in output:\n%v\n", service.regexp, out.String())
		return generateXUnitTestSuite(ctx, &xunit.Failure{Message: "vrpc", Data: "mismatching signature"}, service.name, time.Now().Sub(start))
	}
	return generateXUnitTestSuite(ctx, nil, service.name, time.Now().Sub(start))
}

type prodService struct {
	name       string         // Name to use for the test description
	objectName string         // Object name of the service to connect to
	regexp     *regexp.Regexp // Regexp that should match the signature output
}

// vanadiumProdServicesTest runs a test of vanadium production services.
func vanadiumProdServicesTest(ctx *tool.Context, testName string, opts ...Opt) (_ *test.Result, e error) {
	// Initialize the test.
	// Need the new-stype base profile since many web tests will build
	// go apps that need it.
	cleanup, err := initTest(ctx, testName, []string{"base"})
	if err != nil {
		return nil, internalTestError{err, "Init"}
	}
	defer collect.Error(func() error { return cleanup() }, &e)

	vroot, err := project.JiriRoot()
	if err != nil {
		return nil, err
	}

	// Install the vrpc tool.
	if err := ctx.Run().Command("jiri", "go", "install", "v.io/x/ref/cmd/vrpc"); err != nil {
		return nil, internalTestError{err, "Install VRPC"}
	}
	// Install the principal tool.
	if err := ctx.Run().Command("jiri", "go", "install", "v.io/x/ref/cmd/principal"); err != nil {
		return nil, internalTestError{err, "Install Principal"}
	}
	tmpdir, err := ctx.Run().TempDir("", "prod-services-test")
	if err != nil {
		return nil, internalTestError{err, "Create temporary directory"}
	}
	defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpdir) }, &e)

	blessingRoot, namespaceRoot := getServiceOpts(opts)
	allPassed, suites := true, []xunit.TestSuite{}

	// Fetch the "root" blessing that all services are blessed by.
	suite, pubkey, blessingNames := testIdentityProviderHTTP(ctx, blessingRoot)
	suites = append(suites, *suite)

	if suite.Failures == 0 {
		// Setup a principal that will be used by testAllProdServices and will
		// recognize the blessings of the prod services.
		principalDir, err := setupPrincipal(ctx, vroot, tmpdir, pubkey, blessingNames)
		if err != nil {
			return nil, err
		}
		for _, suite := range testAllProdServices(ctx, vroot, principalDir, namespaceRoot) {
			allPassed = allPassed && (suite.Failures == 0)
			suites = append(suites, *suite)
		}
	}

	// Create the xUnit report.
	if err := xunit.CreateReport(ctx, testName, suites); err != nil {
		return nil, err
	}
	for _, suite := range suites {
		if suite.Failures > 0 {
			// At least one test failed:
			return &test.Result{Status: test.Failed}, nil
		}
	}
	return &test.Result{Status: test.Passed}, nil
}

func testAllProdServices(ctx *tool.Context, vroot, principalDir, namespaceRoot string) []*xunit.TestSuite {
	services := []prodService{
		prodService{
			name:       "mounttable",
			objectName: namespaceRoot,
			regexp:     regexp.MustCompile(`MountTable[[:space:]]+interface`),
		},
		prodService{
			name:       "application repository",
			objectName: namespaceRoot + "/applications",
			regexp:     regexp.MustCompile(`Application[[:space:]]+interface`),
		},
		prodService{
			name:       "binary repository",
			objectName: namespaceRoot + "/binaries",
			regexp:     regexp.MustCompile(`Binary[[:space:]]+interface`),
		},
		prodService{
			name:       "macaroon service",
			objectName: namespaceRoot + "/identity/dev.v.io/u/macaroon",
			regexp:     regexp.MustCompile(`MacaroonBlesser[[:space:]]+interface`),
		},
		prodService{
			name:       "google identity service",
			objectName: namespaceRoot + "/identity/dev.v.io/u/google",
			regexp:     regexp.MustCompile(`OAuthBlesser[[:space:]]+interface`),
		},
		prodService{
			objectName: namespaceRoot + "/identity/dev.v.io/u/discharger",
			name:       "binary discharger",
			regexp:     regexp.MustCompile(`Discharger[[:space:]]+interface`),
		},
		prodService{
			objectName: namespaceRoot + "/proxy-mon/__debug",
			name:       "proxy service",
			// We just check that the returned signature has the __Reserved interface since
			// proxy-mon doesn't implement any other services.
			regexp: regexp.MustCompile(`__Reserved[[:space:]]+interface`),
		},
	}

	var suites []*xunit.TestSuite
	for _, service := range services {
		suites = append(suites, testSingleProdService(ctx, vroot, principalDir, service))
	}
	return suites
}

// testIdentityProviderHTTP tests that the identity provider's HTTP server is
// up and running and also fetches the set of blessing names that the provider
// claims to be authoritative on and the public key (encoded) used by that
// identity provider to sign certificates for blessings.
//
// PARANOIA ALERT:
// This function is subject to man-in-the-middle attacks because it does not
// verify the TLS certificates presented by the server. This does open the
// door for an attack where a parallel universe of services could be setup
// and fool this production services test into thinking all services are
// up and running when they may not be.
//
// The attacker in this case will have to be able to mess with the routing
// tables on the machine running this test, or the network routes of routers
// used by the machine, or mess up DNS entries.
func testIdentityProviderHTTP(ctx *tool.Context, blessingRoot string) (suite *xunit.TestSuite, publickey string, blessingNames []string) {
	url := fmt.Sprintf("https://%s/auth/blessing-root", blessingRoot)
	var response struct {
		Names     []string `json:"names"`
		PublicKey string   `json:"publicKey"`
	}
	var resp *http.Response
	var err error
	var start time.Time
	fn := func() error {
		start = time.Now()
		resp, err = http.Get(url)
		return err
	}
	if err = retry.Function(ctx, fn); err == nil {
		defer resp.Body.Close()
		err = json.NewDecoder(resp.Body).Decode(&response)
	}
	var failure *xunit.Failure
	if err != nil {
		failure = &xunit.Failure{Message: "identityd HTTP", Data: err.Error()}
	}
	return generateXUnitTestSuite(ctx, failure, url, time.Now().Sub(start)), response.PublicKey, response.Names
}

func setupPrincipal(ctx *tool.Context, vroot, tmpdir, pubkey string, blessingNames []string) (string, error) {
	dir := filepath.Join(tmpdir, "credentials")
	bin := filepath.Join(vroot, "release", "go", "bin", "principal")
	if err := ctx.Run().TimedCommand(test.DefaultTimeout, bin, "create", dir, "prod-services-tester"); err != nil {
		fmt.Fprintf(ctx.Stderr(), "principal create failed: %v\n", err)
		return "", err
	}
	for _, name := range blessingNames {
		if err := ctx.Run().TimedCommand(test.DefaultTimeout, bin, "--v23.credentials", dir, "recognize", name, pubkey); err != nil {
			fmt.Fprintf(ctx.Stderr(), "principal recognize %v %v failed: %v\n", name, pubkey, err)
			return "", err
		}
	}
	return dir, nil
}

// getServiceOpts extracts blessing root and namespace root from the
// given Opts.
func getServiceOpts(opts []Opt) (string, string) {
	blessingRoot := "dev.v.io"
	namespaceRoot := "/ns.dev.v.io:8101"
	for _, opt := range opts {
		switch v := opt.(type) {
		case BlessingsRootOpt:
			blessingRoot = string(v)
		case NamespaceRootOpt:
			namespaceRoot = string(v)
		}
	}
	return blessingRoot, namespaceRoot
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 Marco Peereboom <marco@peereboom.us>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package queueb

import (
	"fmt"
	"sync"
	"testing"
)

const (
	Audio   = "audio"
	Network = "network"
)

var (
	q *Queueb
)

func TestQueueb(t *testing.T) {
	var err error
	q, err = New("myqueueb", 10)
	if err != nil {
		t.Error(err)
		return
	}

	err = q.Register("subsystem", 10)
	if err != nil {
		t.Error(err)
		return
	}

	// should fail
	err = q.Register("subsystem", 10)
	if err == nil {
		t.Error("duplicate QueuebChannelPair not detected")
		return
	}

	q.Unregister("subsystem")
	if q.Len() != 0 {
		t.Error("invalid QueuebChannelPair count")
		return
	}

	// should fail
	q.Unregister("subsystem")
	if err == nil {
		t.Error("QueuebChannelPair should not exist")
		return
	}
}

func TestQueuebMessage(t *testing.T) {
	err := q.Register(Audio, 10)
	if err != nil {
		t.Error(err)
		return
	}

	err = q.Register(Network, 20)
	if err != nil {
		t.Error(err)
		return
	}

	err = q.Send(Audio, []string{Network}, "Hello world!")
	if err != nil {
		t.Error(err)
		return
	}

	m, err := q.Receive(Network)
	if err != nil {
		t.Error(err)
		return
	}
	if m.Error() != nil {
		t.Error("unexpected error message")
		return
	}

	if m.Message.(string) != "Hello world!" {
		t.Error("invalid message")
		return
	}

	// make sure we don't crash
	q.Unregister(Audio)
	err = q.Send(Audio, []string{Network}, "Hello world!")
	// should fail
	if err == nil {
		t.Error("queue should have been deleted")
		return
	}
	_, err = q.Receive(Audio)
	// should fail
	if err == nil {
		t.Error("receive should have failed")
		return
	}

	err = q.Send(Network, []string{Audio}, "Hello world!")
	if err != nil {
		t.Error("source queue should exist")
		return
	}
	m, err = q.Receive(Network)
	// get error
	if err != nil {
		t.Error("receive error failed")
		return
	}

	// should fail
	if m.Error() == nil {
		t.Error("not an error message")
		return
	}

	mm, ok := m.Message.(*QueuebMessageError)
	if !ok {
		t.Error("could not type assert m")
		return
	}
	t.Log(mm.Error)
}

func TestQueuebPrioQueueSamePrio(t *testing.T) {
	err := q.Register(Audio, 10)
	if err != nil {
		t.Error(err)
		return
	}

	for i := 0; i < 15; i++ {
		err := q.Send(Audio, []string{Network}, fmt.Sprintf("%v", i))
		if err != nil {
			t.Error(err)
			return
		}
	}
	for i := 0; i < 15; i++ {
		m, err := q.Receive(Network)
		if err != nil {
			t.Error(err)
			return
		}
		if m.Error() != nil {
			t.Error("unexpected error message")
			return
		}
		t.Log("got %v expected %v", m.Message, fmt.Sprintf("%v", i))
		if m.Message != fmt.Sprintf("%v", i) {
			t.Error("out of order", m.Message, fmt.Sprintf("%v", i))
			return
		}
	}
}

func TestQueuebPrioQueueDifferentPrio(t *testing.T) {
	for i := 0; i < 15; i++ {
		err := q.Send(Audio, []string{Network}, fmt.Sprintf("%v", i))
		if err != nil {
			t.Error(err)
			return
		}
	}
	for i := 15; i < 30; i++ {
		err := q.Send(Network, []string{Audio}, fmt.Sprintf("%v", i))
		if err != nil {
			t.Error(err)
			return
		}
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < 15; i++ {
			m, err := q.Receive(Network)
			if err != nil {
				t.Error(err)
				return
			}
			if m.Error() != nil {
				t.Error("unexpected error message")
				return
			}
			t.Log("got %v expected %v", m.Message, fmt.Sprintf("%v", i))
			if m.Message != fmt.Sprintf("%v", i) {
				t.Error("out of order", m.Message, fmt.Sprintf("%v", i))
				return
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 15; i < 30; i++ {
			m, err := q.Receive(Audio)
			if err != nil {
				t.Error(err)
				return
			}
			if m.Error() != nil {
				t.Error("unexpected error message")
				return
			}
			t.Log("got %v expected %v", m.Message, fmt.Sprintf("%v", i))
			if m.Message != fmt.Sprintf("%v", i) {
				t.Error("out of order", m.Message, fmt.Sprintf("%v", i))
				return
			}
		}
	}()
	wg.Wait()
}
<commit_msg>add benchmark to meassure roundtrip<commit_after>/*
 * Copyright (c) 2014 Marco Peereboom <marco@peereboom.us>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package queueb

import (
	"fmt"
	"sync"
	"testing"
	"time"
)

const (
	Audio   = "audio"
	Network = "network"
)

var (
	q *Queueb
)

func TestQueueb(t *testing.T) {
	var err error
	q, err = New("myqueueb", 10)
	if err != nil {
		t.Error(err)
		return
	}

	err = q.Register("subsystem", 10)
	if err != nil {
		t.Error(err)
		return
	}

	// should fail
	err = q.Register("subsystem", 10)
	if err == nil {
		t.Error("duplicate QueuebChannelPair not detected")
		return
	}

	q.Unregister("subsystem")
	if q.Len() != 0 {
		t.Error("invalid QueuebChannelPair count")
		return
	}

	// should fail
	q.Unregister("subsystem")
	if err == nil {
		t.Error("QueuebChannelPair should not exist")
		return
	}
}

func TestQueuebMessage(t *testing.T) {
	err := q.Register(Audio, 10)
	if err != nil {
		t.Error(err)
		return
	}

	err = q.Register(Network, 20)
	if err != nil {
		t.Error(err)
		return
	}

	err = q.Send(Audio, []string{Network}, "Hello world!")
	if err != nil {
		t.Error(err)
		return
	}

	m, err := q.Receive(Network)
	if err != nil {
		t.Error(err)
		return
	}
	if m.Error() != nil {
		t.Error("unexpected error message")
		return
	}

	if m.Message.(string) != "Hello world!" {
		t.Error("invalid message")
		return
	}

	// make sure we don't crash
	q.Unregister(Audio)
	err = q.Send(Audio, []string{Network}, "Hello world!")
	// should fail
	if err == nil {
		t.Error("queue should have been deleted")
		return
	}
	_, err = q.Receive(Audio)
	// should fail
	if err == nil {
		t.Error("receive should have failed")
		return
	}

	err = q.Send(Network, []string{Audio}, "Hello world!")
	if err != nil {
		t.Error("source queue should exist")
		return
	}
	m, err = q.Receive(Network)
	// get error
	if err != nil {
		t.Error("receive error failed")
		return
	}

	// should fail
	if m.Error() == nil {
		t.Error("not an error message")
		return
	}

	mm, ok := m.Message.(*QueuebMessageError)
	if !ok {
		t.Error("could not type assert m")
		return
	}
	t.Log(mm.Error)
}

func TestQueuebPrioQueueSamePrio(t *testing.T) {
	err := q.Register(Audio, 10)
	if err != nil {
		t.Error(err)
		return
	}

	for i := 0; i < 15; i++ {
		err := q.Send(Audio, []string{Network}, fmt.Sprintf("%v", i))
		if err != nil {
			t.Error(err)
			return
		}
	}
	for i := 0; i < 15; i++ {
		m, err := q.Receive(Network)
		if err != nil {
			t.Error(err)
			return
		}
		if m.Error() != nil {
			t.Error("unexpected error message")
			return
		}
		t.Log("got %v expected %v", m.Message, fmt.Sprintf("%v", i))
		if m.Message != fmt.Sprintf("%v", i) {
			t.Error("out of order", m.Message, fmt.Sprintf("%v", i))
			return
		}
	}
}

func TestQueuebPrioQueueDifferentPrio(t *testing.T) {
	for i := 0; i < 15; i++ {
		err := q.Send(Audio, []string{Network}, fmt.Sprintf("%v", i))
		if err != nil {
			t.Error(err)
			return
		}
	}
	for i := 15; i < 30; i++ {
		err := q.Send(Network, []string{Audio}, fmt.Sprintf("%v", i))
		if err != nil {
			t.Error(err)
			return
		}
	}

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 0; i < 15; i++ {
			m, err := q.Receive(Network)
			if err != nil {
				t.Error(err)
				return
			}
			if m.Error() != nil {
				t.Error("unexpected error message")
				return
			}
			t.Log("got %v expected %v", m.Message, fmt.Sprintf("%v", i))
			if m.Message != fmt.Sprintf("%v", i) {
				t.Error("out of order", m.Message, fmt.Sprintf("%v", i))
				return
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		for i := 15; i < 30; i++ {
			m, err := q.Receive(Audio)
			if err != nil {
				t.Error(err)
				return
			}
			if m.Error() != nil {
				t.Error("unexpected error message")
				return
			}
			t.Log("got %v expected %v", m.Message, fmt.Sprintf("%v", i))
			if m.Message != fmt.Sprintf("%v", i) {
				t.Error("out of order", m.Message, fmt.Sprintf("%v", i))
				return
			}
		}
	}()
	wg.Wait()
}

func BenchmarkRoundTrip(b *testing.B) {
	count := 10000
	start := time.Now()
	for i := 0; i < count; i++ {
		err := q.Send(Audio, []string{Network}, "a")
		if err != nil {
			b.Error(err)
			return
		}
		_, err = q.Receive(Network)
		if err != nil {
			b.Error(err)
			return
		}
	}
	end := time.Now()
	b.Logf("Roundtrip: %v", end.Sub(start)/10000)
}
<|endoftext|>


<commit_before>package indicators

import (
	"errors"
	"github.com/thetruetrade/gotrade"
)

// A Minus Directional Indicator (MinusDi), no storage, for use in other indicators
type MinusDiWithoutStorage struct {
	*baseIndicator
	*baseFloatBounds

	// private variables
	valueAvailableAction ValueAvailableActionFloat
	periodCounter        int
	previousHigh         float64
	previousLow          float64
	previousMinusDM      float64
	previousTrueRange    float64
	currentTrueRange     float64
	trueRange            *TrueRange
	timePeriod           int
}

// NewMinusDiWithoutStorage creates a Minus Directional Indicator (MinusDi) without storage
func NewMinusDiWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *MinusDiWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := 1
	if timePeriod > 1 {
		lookback = timePeriod
	}
	ind := MinusDiWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseFloatBounds:      newBaseFloatBounds(),
		periodCounter:        -1,
		previousMinusDM:      0.0,
		previousTrueRange:    0.0,
		currentTrueRange:     0.0,
		valueAvailableAction: valueAvailableAction,
		timePeriod:           timePeriod,
	}

	ind.trueRange, err = NewTrueRange()

	ind.trueRange.valueAvailableAction = func(dataItem float64, streamBarIndex int) {
		ind.currentTrueRange = dataItem
	}

	return &ind, nil
}

// A Minus Directional Indicator (MinusDi)
type MinusDi struct {
	*MinusDiWithoutStorage

	// public variables
	Data []float64
}

// NewMinusDi creates a Minus Directional Indicator (MinusDi) for online usage
func NewMinusDi(timePeriod int) (indicator *MinusDi, err error) {
	ind := MinusDi{}
	ind.MinusDiWithoutStorage, err = NewMinusDiWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultMinusDi creates a Minus Directional Indicator (MinusDi) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultMinusDi() (indicator *MinusDi, err error) {
	timePeriod := 14
	return NewMinusDi(timePeriod)
}

// NewMinusDiWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage
func NewMinusDiWithSrcLen(sourceLength uint, timePeriod int) (indicator *MinusDi, err error) {
	ind, err := NewMinusDi(timePeriod)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultMinusDiWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage with default parameters
func NewDefaultMinusDiWithSrcLen(sourceLength uint) (indicator *MinusDi, err error) {
	ind, err := NewDefaultMinusDi()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewMinusDiForStream creates a Minus Directional Indicator (MinusDi) for online usage with a source data stream
func NewMinusDiForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDi, err error) {
	ind, err := NewMinusDi(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDiForStream creates a Minus Directional Indicator (MinusDi) for online usage with a source data stream
func NewDefaultMinusDiForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDi, err error) {
	ind, err := NewDefaultMinusDi()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewMinusDiForStreamWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage with a source data stream
func NewMinusDiForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDi, err error) {
	ind, err := NewMinusDiWithSrcLen(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDiForStreamWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage with a source data stream
func NewDefaultMinusDiForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDi, err error) {
	ind, err := NewDefaultMinusDiWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *MinusDiWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {

	// forward to the true range indicator first using previous data
	ind.trueRange.ReceiveDOHLCVTick(tickData, streamBarIndex)

	ind.periodCounter += 1
	high := tickData.H()
	low := tickData.L()
	diffP := high - ind.previousHigh
	diffM := ind.previousLow - low

	if ind.lookbackPeriod == 1 {
		if ind.periodCounter > 0 {

			// forward to the true range indicator first using previous data
			ind.trueRange.ReceiveDOHLCVTick(tickData, streamBarIndex)

			var result float64
			if (diffM > 0) && (diffP < diffM) && ind.currentTrueRange != 0.0 {
				result = diffM / ind.currentTrueRange
			} else {
				result = 0
			}

			// increment the number of results this indicator can be expected to return
			ind.dataLength += 1

			if ind.validFromBar == -1 {
				// set the streamBarIndex from which this indicator returns valid results
				ind.validFromBar = streamBarIndex
			}

			// update the maximum result value
			if result > ind.maxValue {
				ind.maxValue = result
			}

			// update the minimum result value
			if result < ind.minValue {
				ind.minValue = result
			}

			// notify of a new result value though the value available action
			ind.valueAvailableAction(result, streamBarIndex)
		}
	} else {
		if ind.periodCounter > 0 {
			if ind.periodCounter < ind.timePeriod {
				if (diffM > 0) && (diffP < diffM) {
					ind.previousMinusDM += diffM
				}
				ind.previousTrueRange += ind.currentTrueRange
			} else {
				var result float64
				ind.previousTrueRange = ind.previousTrueRange - (ind.previousTrueRange / float64(ind.timePeriod)) + ind.currentTrueRange
				if (diffM > 0) && (diffP < diffM) {
					ind.previousMinusDM = ind.previousMinusDM - (ind.previousMinusDM / float64(ind.timePeriod)) + diffM
				} else {
					ind.previousMinusDM = ind.previousMinusDM - (ind.previousMinusDM / float64(ind.timePeriod))
				}

				if ind.previousTrueRange != 0.0 {
					result = float64(100.0) * ind.previousMinusDM / ind.previousTrueRange
				} else {
					result = 0.0
				}

				// increment the number of results this indicator can be expected to return
				ind.dataLength += 1

				if ind.validFromBar == -1 {
					// set the streamBarIndex from which this indicator returns valid results
					ind.validFromBar = streamBarIndex
				}

				// update the maximum result value
				if result > ind.maxValue {
					ind.maxValue = result
				}

				// update the minimum result value
				if result < ind.minValue {
					ind.minValue = result
				}

				// notify of a new result value though the value available action
				ind.valueAvailableAction(result, streamBarIndex)
			}
		}
	}

	ind.previousHigh = high
	ind.previousLow = low
}
<commit_msg>#76 Remove duplication - minusdi<commit_after>package indicators

import (
	"errors"
	"github.com/thetruetrade/gotrade"
)

// A Minus Directional Indicator (MinusDi), no storage, for use in other indicators
type MinusDiWithoutStorage struct {
	*baseIndicatorWithFloatBounds

	// private variables
	periodCounter     int
	previousHigh      float64
	previousLow       float64
	previousMinusDM   float64
	previousTrueRange float64
	currentTrueRange  float64
	trueRange         *TrueRange
	timePeriod        int
}

// NewMinusDiWithoutStorage creates a Minus Directional Indicator (MinusDi) without storage
func NewMinusDiWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *MinusDiWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 1
	if timePeriod < 1 {
		return nil, errors.New("timePeriod is less than the minimum (1)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := 1
	if timePeriod > 1 {
		lookback = timePeriod
	}
	ind := MinusDiWithoutStorage{
		baseIndicatorWithFloatBounds: newBaseIndicatorWithFloatBounds(lookback, valueAvailableAction),
		periodCounter:                -1,
		previousMinusDM:              0.0,
		previousTrueRange:            0.0,
		currentTrueRange:             0.0,
		timePeriod:                   timePeriod,
	}

	ind.trueRange, err = NewTrueRange()

	ind.trueRange.valueAvailableAction = func(dataItem float64, streamBarIndex int) {
		ind.currentTrueRange = dataItem
	}

	return &ind, nil
}

// A Minus Directional Indicator (MinusDi)
type MinusDi struct {
	*MinusDiWithoutStorage

	// public variables
	Data []float64
}

// NewMinusDi creates a Minus Directional Indicator (MinusDi) for online usage
func NewMinusDi(timePeriod int) (indicator *MinusDi, err error) {
	ind := MinusDi{}
	ind.MinusDiWithoutStorage, err = NewMinusDiWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultMinusDi creates a Minus Directional Indicator (MinusDi) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultMinusDi() (indicator *MinusDi, err error) {
	timePeriod := 14
	return NewMinusDi(timePeriod)
}

// NewMinusDiWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage
func NewMinusDiWithSrcLen(sourceLength uint, timePeriod int) (indicator *MinusDi, err error) {
	ind, err := NewMinusDi(timePeriod)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultMinusDiWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage with default parameters
func NewDefaultMinusDiWithSrcLen(sourceLength uint) (indicator *MinusDi, err error) {
	ind, err := NewDefaultMinusDi()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewMinusDiForStream creates a Minus Directional Indicator (MinusDi) for online usage with a source data stream
func NewMinusDiForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDi, err error) {
	ind, err := NewMinusDi(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDiForStream creates a Minus Directional Indicator (MinusDi) for online usage with a source data stream
func NewDefaultMinusDiForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDi, err error) {
	ind, err := NewDefaultMinusDi()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewMinusDiForStreamWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage with a source data stream
func NewMinusDiForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *MinusDi, err error) {
	ind, err := NewMinusDiWithSrcLen(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultMinusDiForStreamWithSrcLen creates a Minus Directional Indicator (MinusDi) for offline usage with a source data stream
func NewDefaultMinusDiForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *MinusDi, err error) {
	ind, err := NewDefaultMinusDiWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *MinusDiWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {

	// forward to the true range indicator first using previous data
	ind.trueRange.ReceiveDOHLCVTick(tickData, streamBarIndex)

	ind.periodCounter += 1
	high := tickData.H()
	low := tickData.L()
	diffP := high - ind.previousHigh
	diffM := ind.previousLow - low

	if ind.lookbackPeriod == 1 {
		if ind.periodCounter > 0 {

			// forward to the true range indicator first using previous data
			ind.trueRange.ReceiveDOHLCVTick(tickData, streamBarIndex)

			var result float64
			if (diffM > 0) && (diffP < diffM) && ind.currentTrueRange != 0.0 {
				result = diffM / ind.currentTrueRange
			} else {
				result = 0
			}

			ind.UpdateIndicatorWithNewValue(result, streamBarIndex)
		}
	} else {
		if ind.periodCounter > 0 {
			if ind.periodCounter < ind.timePeriod {
				if (diffM > 0) && (diffP < diffM) {
					ind.previousMinusDM += diffM
				}
				ind.previousTrueRange += ind.currentTrueRange
			} else {
				var result float64
				ind.previousTrueRange = ind.previousTrueRange - (ind.previousTrueRange / float64(ind.timePeriod)) + ind.currentTrueRange
				if (diffM > 0) && (diffP < diffM) {
					ind.previousMinusDM = ind.previousMinusDM - (ind.previousMinusDM / float64(ind.timePeriod)) + diffM
				} else {
					ind.previousMinusDM = ind.previousMinusDM - (ind.previousMinusDM / float64(ind.timePeriod))
				}

				if ind.previousTrueRange != 0.0 {
					result = float64(100.0) * ind.previousMinusDM / ind.previousTrueRange
				} else {
					result = 0.0
				}

				ind.UpdateIndicatorWithNewValue(result, streamBarIndex)
			}
		}
	}

	ind.previousHigh = high
	ind.previousLow = low
}
<|endoftext|>


<commit_before>package internal

import (
	"reflect"
	"unsafe"
)

const (
	sizeOfUint64 = 8
)

// Hash is a means of converting indices into strings.
type Hash struct {
	bytes  []byte
	header *reflect.SliceHeader
}

// NewHash creates a hash.
func NewHash(ni uint) *Hash {
	hash := &Hash{bytes: make([]byte, 0)}
	hash.header = (*reflect.SliceHeader)(unsafe.Pointer(&hash.bytes))
	hash.header.Cap = int(ni * sizeOfUint64)
	hash.header.Len = hash.header.Cap
	return hash
}

// Key converts an index into a string.
func (self *Hash) Key(index []uint64) string {
	self.header.Data = uintptr(((*reflect.SliceHeader)(unsafe.Pointer(&index))).Data)
	key := string(self.bytes)
	self.header.Data = 0
	return key
}
<commit_msg>a/internal: fix the description of Hash<commit_after>package internal

import (
	"reflect"
	"unsafe"
)

const (
	sizeOfUint64 = 8
)

// Hash is a means of creating hash keys from indices.
type Hash struct {
	bytes  []byte
	header *reflect.SliceHeader
}

// NewHash creates a hash.
func NewHash(ni uint) *Hash {
	hash := &Hash{bytes: make([]byte, 0)}
	hash.header = (*reflect.SliceHeader)(unsafe.Pointer(&hash.bytes))
	hash.header.Cap = int(ni * sizeOfUint64)
	hash.header.Len = hash.header.Cap
	return hash
}

// Key creates a hash key from an index.
func (self *Hash) Key(index []uint64) string {
	self.header.Data = uintptr(((*reflect.SliceHeader)(unsafe.Pointer(&index))).Data)
	key := string(self.bytes)
	self.header.Data = 0
	return key
}
<|endoftext|>


<commit_before><commit_msg>first commit<commit_after><|endoftext|>