<commit_before>// Queue implements a queue that will grow as needed, minimize unnecessary growth, and
// can be safely accessed from multiple routines without introducing race conditions.
// If necessary, the growth of the queue can be capped to a configurable size. When a
// maximum capacity for the queue is defined, an error will occur if the queue is full
// and an attempt is made to add another item to the queue.
//
// A queue is created with a minimum length and an optional maximum size (capacity).
// If the max size of the queue == 0, the queue will be unbounded. The growth rate of
// the queue is similar to that of a slice. When the queue grows, all items in the
// queue are shifted so that the head of the queue points to the first element in the
// queue.
//
// Before an item is enqueued, the queue is checked to see if thie new item
// will cause it to grow. If the tail == length, growth may occur. If the
// head of the queue is past a certain point in the queue, which is currently
// calculated using a percentage, the items in the queue will be shifted to start at
// the beginning of the slice, instead of growing the slice. The queue's head and tail
// will then be updated to reflect the shift.
//
// After dequeuing an item, the head position will be checked. If the queue
// io empty, head > tail, head and tail will be set to 0. This allows for
// efficient reuse of the queue without having to check to see if the queue items
// should be shifted or the queue should be grown.
//
// Once a queue grows, it will not be shrunk. This behavior may change in the future.
//
// All publicly exposed methods on the queue use locking to protect the queue from
// race conditions in situations where the queue is being accessed concurrently.
// Unexposed methods do not do any locking/unlocking since it is expected that the
// calling function has already obtained the lock and will release it as appropriate.
package queue

import (
	"fmt"
	"sync"
)

// shiftPercent is the default value for shifting the queue items to the
// front of the queue instead of growing the queue. If at least the % of
// the items have been removed from the queue, the items in the queue will
// be shifted to make room; otherwise the queue will grow
var shiftPercent = 20

// Queue represents a queue and everything needed to manage it. The preferred method
// for creating a new Queue is to use the New() func.
type Queue struct {
	mu           sync.Mutex
	items        []interface{}
	head         int // current item in queue
	tail         int // tail is the next insert point. last item is tail - 1
	length       int // the current length (cap) of the queue,
	maxCap       int // if > 0, the queue's cap cannot grow beyond this value
	shiftPercent int // the % of items that need to be removed before shifting occurs
}

// New returns an empty queue with a capacity equal to the recieved size value. If
// maxCap is > 0, the queue will not grow larger than maxCap; if it is at maxCap
// and growth is requred to enqueue an item, an error will occur.
func New(size, maxCap int) *Queue {
	return &Queue{items: make([]interface{}, size, size), length: size, maxCap: maxCap, shiftPercent: shiftPercent}
}

// Enqueue: adds an item to the queue. If adding the item requires growing
// the queue, the queue will either be shifted, to make room at the end of the queue
// or it will grow. If the queue cannot be grown, an error will be returned.
func (q *Queue) Enqueue(item interface{}) error {
	q.mu.Lock()
	defer q.mu.Unlock()
	// See if it needs to grow
	if q.tail == q.length {
		shifted := q.shift()
		// if we weren't able to make room by shifting, grow the queue/
		if !shifted {
			err := q.grow()
			if err != nil {
				return err
			}
		}
	}
	q.items[q.tail] = item
	q.tail++
	return nil
}

// Dequeue removes an item from the queue. If the removal of the item empties
// the queue, the head and tail will be set to 0.
func (q *Queue) Dequeue() interface{} {
	q.mu.Lock()
	i := q.items[q.head]
	q.head++
	if q.head > q.tail {
		q.mu.Unlock()
		q.Reset()
		return i
	}
	q.mu.Unlock()
	return i
}

// IsEmpty returns whether or not the queue is empty
func (q *Queue) IsEmpty() bool {
	q.mu.Lock()
	if q.tail == 0 || q.head > q.tail {
		q.mu.Unlock()
		return true
	}
	q.mu.Unlock()
	return false
}

// shift: if shiftPercent items have been removed from the queue, the remaining items
// in the queue will be shifted to element 0-n, where n is the number of remaining
// items in the queue. Returns whether or not a shift occurred
func (q *Queue) shift() bool {
	if q.head <= (q.length*q.shiftPercent)/100 {
		return false
	}
	copy(q.items, q.items[q.head:q.tail])
	// set the pointers to the correct position
	q.tail = q.tail - q.head
	q.head = 0
	return true
}

// grow grows the slice using an algorithm similar to growSlice(). This is a bit slower
// than relying on slice's automatic growth, but allows for capacity enforcement w/o
// growing the slice cap beyond the configured maxCap, if applicable.
//
// Since a temporary slice is created to store the current queue, all items in queue
// are automatically shifted
func (q *Queue) grow() error {
	if q.length == q.maxCap && q.maxCap > 0 {
		return fmt.Errorf("groweQueue: cannot grow beyond max capacity of %d", q.maxCap)
	}
	if q.length < 1024 {
		q.length += q.length
	} else {
		q.length += q.length / 4
	}
	// If the maxCap is set, cannot grow it beyond that
	if q.length > q.maxCap && q.maxCap > 0 {
		q.length = q.maxCap
	}
	// grow the slice
	l := q.tail - q.head
	tmp := make([]interface{}, l, l)
	copy(tmp, q.items[q.head:q.tail])
	q.items = make([]interface{}, q.length, q.length)
	copy(q.items, tmp)
	q.tail = l
	q.head = 0
	return nil
}

// Reset resets the queue; head and tail point to element 0.
func (q *Queue) Reset() {
	q.mu.Lock()
	q.head = 0
	q.tail = 0
	q.mu.Unlock()
}
<commit_msg>fix bug in IsEmpty()<commit_after>// Queue implements a queue that will grow as needed, minimize unnecessary growth, and
// can be safely accessed from multiple routines without introducing race conditions.
// If necessary, the growth of the queue can be capped to a configurable size. When a
// maximum capacity for the queue is defined, an error will occur if the queue is full
// and an attempt is made to add another item to the queue.
//
// A queue is created with a minimum length and an optional maximum size (capacity).
// If the max size of the queue == 0, the queue will be unbounded. The growth rate of
// the queue is similar to that of a slice. When the queue grows, all items in the
// queue are shifted so that the head of the queue points to the first element in the
// queue.
//
// Before an item is enqueued, the queue is checked to see if thie new item
// will cause it to grow. If the tail == length, growth may occur. If the
// head of the queue is past a certain point in the queue, which is currently
// calculated using a percentage, the items in the queue will be shifted to start at
// the beginning of the slice, instead of growing the slice. The queue's head and tail
// will then be updated to reflect the shift.
//
// After dequeuing an item, the head position will be checked. If the queue
// io empty, head > tail, head and tail will be set to 0. This allows for
// efficient reuse of the queue without having to check to see if the queue items
// should be shifted or the queue should be grown.
//
// Once a queue grows, it will not be shrunk. This behavior may change in the future.
//
// All publicly exposed methods on the queue use locking to protect the queue from
// race conditions in situations where the queue is being accessed concurrently.
// Unexposed methods do not do any locking/unlocking since it is expected that the
// calling function has already obtained the lock and will release it as appropriate.
package queue

import (
	"fmt"
	"sync"
)

// shiftPercent is the default value for shifting the queue items to the
// front of the queue instead of growing the queue. If at least the % of
// the items have been removed from the queue, the items in the queue will
// be shifted to make room; otherwise the queue will grow
var shiftPercent = 20

// Queue represents a queue and everything needed to manage it. The preferred method
// for creating a new Queue is to use the New() func.
type Queue struct {
	mu           sync.Mutex
	items        []interface{}
	head         int // current item in queue
	tail         int // tail is the next insert point. last item is tail - 1
	length       int // the current length (cap) of the queue,
	maxCap       int // if > 0, the queue's cap cannot grow beyond this value
	shiftPercent int // the % of items that need to be removed before shifting occurs
}

// New returns an empty queue with a capacity equal to the recieved size value. If
// maxCap is > 0, the queue will not grow larger than maxCap; if it is at maxCap
// and growth is requred to enqueue an item, an error will occur.
func New(size, maxCap int) *Queue {
	return &Queue{items: make([]interface{}, size, size), length: size, maxCap: maxCap, shiftPercent: shiftPercent}
}

// Enqueue: adds an item to the queue. If adding the item requires growing
// the queue, the queue will either be shifted, to make room at the end of the queue
// or it will grow. If the queue cannot be grown, an error will be returned.
func (q *Queue) Enqueue(item interface{}) error {
	q.mu.Lock()
	defer q.mu.Unlock()
	// See if it needs to grow
	if q.tail == q.length {
		shifted := q.shift()
		// if we weren't able to make room by shifting, grow the queue/
		if !shifted {
			err := q.grow()
			if err != nil {
				return err
			}
		}
	}
	q.items[q.tail] = item
	q.tail++
	return nil
}

// Dequeue removes an item from the queue. If the removal of the item empties
// the queue, the head and tail will be set to 0.
func (q *Queue) Dequeue() interface{} {
	q.mu.Lock()
	i := q.items[q.head]
	q.head++
	if q.head > q.tail {
		q.mu.Unlock()
		q.Reset()
		return i
	}
	q.mu.Unlock()
	return i
}

// IsEmpty returns whether or not the queue is empty
func (q *Queue) IsEmpty() bool {
	q.mu.Lock()
	if q.tail == 0 || q.head == q.tail {
		q.mu.Unlock()
		return true
	}
	q.mu.Unlock()
	return false
}

func (q *Queue) Tail() int {
	return q.tail
}

func (q *Queue) Head() int {
	return q.head
}

// shift: if shiftPercent items have been removed from the queue, the remaining items
// in the queue will be shifted to element 0-n, where n is the number of remaining
// items in the queue. Returns whether or not a shift occurred
func (q *Queue) shift() bool {
	if q.head <= (q.length*q.shiftPercent)/100 {
		return false
	}
	copy(q.items, q.items[q.head:q.tail])
	// set the pointers to the correct position
	q.tail = q.tail - q.head
	q.head = 0
	return true
}

// grow grows the slice using an algorithm similar to growSlice(). This is a bit slower
// than relying on slice's automatic growth, but allows for capacity enforcement w/o
// growing the slice cap beyond the configured maxCap, if applicable.
//
// Since a temporary slice is created to store the current queue, all items in queue
// are automatically shifted
func (q *Queue) grow() error {
	if q.length == q.maxCap && q.maxCap > 0 {
		return fmt.Errorf("groweQueue: cannot grow beyond max capacity of %d", q.maxCap)
	}
	if q.length < 1024 {
		q.length += q.length
	} else {
		q.length += q.length / 4
	}
	// If the maxCap is set, cannot grow it beyond that
	if q.length > q.maxCap && q.maxCap > 0 {
		q.length = q.maxCap
	}
	// grow the slice
	l := q.tail - q.head
	tmp := make([]interface{}, l, l)
	copy(tmp, q.items[q.head:q.tail])
	q.items = make([]interface{}, q.length, q.length)
	copy(q.items, tmp)
	q.tail = l
	q.head = 0
	return nil
}

// Reset resets the queue; head and tail point to element 0.
func (q *Queue) Reset() {
	q.mu.Lock()
	q.head = 0
	q.tail = 0
	q.mu.Unlock()
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Huawei Technologies Co., Ltd. All Rights Reserved.
//
//    Licensed under the Apache License, Version 2.0 (the "License"); you may
//    not use this file except in compliance with the License. You may obtain
//    a copy of the License at
//
//         http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//    License for the specific language governing permissions and limitations
//    under the License.

/*
This module implements the entry into operations of storageDock module.

*/

package discovery

import (
	"os"

	"github.com/opensds/opensds/pkg/db"
	dockHub "github.com/opensds/opensds/pkg/dock"
	api "github.com/opensds/opensds/pkg/model"
	"github.com/opensds/opensds/pkg/utils"
	. "github.com/opensds/opensds/pkg/utils/config"

	log "github.com/golang/glog"
	"github.com/satori/go.uuid"
)

type Discoverer interface {
	Init() error
	Discovery() error
	Store() error
}

type DockDiscoverer struct {
	dcks []*api.DockSpec
	pols []*api.StoragePoolSpec

	c db.Client
}

func NewDiscover() Discoverer {
	return &DockDiscoverer{
		c: db.C,
	}
}

func (dd *DockDiscoverer) Init() error {
	// Load resource from specified file
	name2Backend := map[string]BackendProperties{
		"ceph":   BackendProperties(CONF.Ceph),
		"cinder": BackendProperties(CONF.Cinder),
		"sample": BackendProperties(CONF.Sample),
	}

	host, err := os.Hostname()
	if err != nil {
		log.Error("When get os hostname:", err)
		return err
	}

	for _, v := range CONF.EnableBackends {
		b := name2Backend[v]
		if b.Name == "" {
			continue
		}

		dck := &api.DockSpec{
			BaseModel: &api.BaseModel{
				Id: uuid.NewV5(uuid.NamespaceOID, host+":"+b.DriverName).String(),
			},
			Name:        b.Name,
			Description: b.Description,
			DriverName:  b.DriverName,
			Endpoint:    CONF.OsdsDock.ApiEndpoint,
		}
		dd.dcks = append(dd.dcks, dck)
	}
	return nil
}

func (dd *DockDiscoverer) Discovery() error {
	var pols []*api.StoragePoolSpec
	var err error

	for _, dck := range dd.dcks {
		pols, err = dockHub.NewDockHub(dck.GetDriverName()).ListPools()
		if err != nil {
			log.Error("When list pools:", err)
			return err
		}

		if len(pols) == 0 {
			log.Warningf("The pool of dock %s is empty!\n", dck.GetId())
		}

		for _, pol := range pols {
			pol.DockId = dck.GetId()
		}
		dd.pols = append(dd.pols, pols...)
	}

	return err
}

func (dd *DockDiscoverer) Store() error {
	var err error

	// Store dock resources in database.
	for _, dck := range dd.dcks {
		if err = utils.ValidateData(dck, utils.S); err != nil {
			log.Error("When validate dock structure:", err)
			return err
		}

		// Call db module to create dock resource.
		if err = db.C.CreateDock(&dock); err != nil {
			log.Error("When create dock %s in db: %v\n", dock.GetId(), err)
			return err
		}
	}

	// Store pool resources in database.
	for _, pol := range dd.pols {
		if err = utils.ValidateData(pol, utils.S); err != nil {
			log.Error("When validate pool structure:", err)
			return err
		}

		// Call db module to create pool resource.
		if err = db.C.CreatePool(&pool); err != nil {
			log.Error("When create pool %s in db: %v\n", pool.GetId(), err)
			return err
		}
	}

	return err
}

func Discovery(d Discoverer) error {
	var err error

	if err = d.Init(); err != nil {
		return err
	}
	if err = d.Discovery(); err != nil {
		return err
	}
	if err = d.Store(); err != nil {
		return err
	}

	return err
}
<commit_msg>Fix some errors<commit_after>// Copyright (c) 2017 Huawei Technologies Co., Ltd. All Rights Reserved.
//
//    Licensed under the Apache License, Version 2.0 (the "License"); you may
//    not use this file except in compliance with the License. You may obtain
//    a copy of the License at
//
//         http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//    License for the specific language governing permissions and limitations
//    under the License.

/*
This module implements the entry into operations of storageDock module.

*/

package discovery

import (
	"os"

	"github.com/opensds/opensds/pkg/db"
	dockHub "github.com/opensds/opensds/pkg/dock"
	api "github.com/opensds/opensds/pkg/model"
	"github.com/opensds/opensds/pkg/utils"
	. "github.com/opensds/opensds/pkg/utils/config"

	log "github.com/golang/glog"
	"github.com/satori/go.uuid"
)

type Discoverer interface {
	Init() error
	Discovery() error
	Store() error
}

type DockDiscoverer struct {
	dcks []*api.DockSpec
	pols []*api.StoragePoolSpec

	c db.Client
}

func NewDiscover() Discoverer {
	return &DockDiscoverer{
		c: db.C,
	}
}

func (dd *DockDiscoverer) Init() error {
	// Load resource from specified file
	name2Backend := map[string]BackendProperties{
		"ceph":   BackendProperties(CONF.Ceph),
		"cinder": BackendProperties(CONF.Cinder),
		"sample": BackendProperties(CONF.Sample),
	}

	host, err := os.Hostname()
	if err != nil {
		log.Error("When get os hostname:", err)
		return err
	}

	for _, v := range CONF.EnableBackends {
		b := name2Backend[v]
		if b.Name == "" {
			continue
		}

		dck := &api.DockSpec{
			BaseModel: &api.BaseModel{
				Id: uuid.NewV5(uuid.NamespaceOID, host+":"+b.DriverName).String(),
			},
			Name:        b.Name,
			Description: b.Description,
			DriverName:  b.DriverName,
			Endpoint:    CONF.OsdsDock.ApiEndpoint,
		}
		dd.dcks = append(dd.dcks, dck)
	}
	return nil
}

func (dd *DockDiscoverer) Discovery() error {
	var pols []*api.StoragePoolSpec
	var err error

	for _, dck := range dd.dcks {
		pols, err = dockHub.NewDockHub(dck.GetDriverName()).ListPools()
		if err != nil {
			log.Error("When list pools:", err)
			return err
		}

		if len(pols) == 0 {
			log.Warningf("The pool of dock %s is empty!\n", dck.GetId())
		}

		for _, pol := range pols {
			pol.DockId = dck.GetId()
		}
		dd.pols = append(dd.pols, pols...)
	}

	return err
}

func (dd *DockDiscoverer) Store() error {
	var err error

	// Store dock resources in database.
	for _, dck := range dd.dcks {
		if err = utils.ValidateData(dck, utils.S); err != nil {
			log.Error("When validate dock structure:", err)
			return err
		}

		// Call db module to create dock resource.
		if err = db.C.CreateDock(dck); err != nil {
			log.Error("When create dock %s in db: %v\n", dck.GetId(), err)
			return err
		}
	}

	// Store pool resources in database.
	for _, pol := range dd.pols {
		if err = utils.ValidateData(pol, utils.S); err != nil {
			log.Error("When validate pool structure:", err)
			return err
		}

		// Call db module to create pool resource.
		if err = db.C.CreatePool(pol); err != nil {
			log.Error("When create pool %s in db: %v\n", pol.GetId(), err)
			return err
		}
	}

	return err
}

func Discovery(d Discoverer) error {
	var err error

	if err = d.Init(); err != nil {
		return err
	}
	if err = d.Discovery(); err != nil {
		return err
	}
	if err = d.Store(); err != nil {
		return err
	}

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"net/http"
	"time"

	"github.com/TeaMeow/KitSvc/module/logger"
	"github.com/TeaMeow/KitSvc/module/sd"
	"github.com/TeaMeow/KitSvc/router"
	"github.com/TeaMeow/KitSvc/router/middleware"
	"github.com/TeaMeow/KitSvc/shared/eventutil"
	"github.com/TeaMeow/KitSvc/shared/mqutil"
	"github.com/TeaMeow/KitSvc/shared/wsutil"
	"github.com/codegangsta/cli"
	"github.com/gin-gonic/gin"
)

var serverFlags = []cli.Flag{
	// Common flags.
	cli.StringFlag{
		EnvVar: "KITSVC_NAME",
		Name:   "name",
		Usage:  "the name of the service, exposed for service discovery.",
		Value:  "Service",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_URL",
		Name:   "url",
		Usage:  "the url of the service.",
		Value:  "http://127.0.0.1:8080",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_ADDR",
		Name:   "addr",
		Usage:  "the address of the service (with the port).",
		Value:  "127.0.0.1:8080",
	},
	cli.IntFlag{
		EnvVar: "KITSVC_PORT",
		Name:   "port",
		Usage:  "the port of the service.",
		Value:  8080,
	},
	cli.StringFlag{
		EnvVar: "KITSVC_USAGE",
		Name:   "usage",
		Usage:  "the usage of the service, exposed for service discovery.",
		Value:  "Operations about the users.",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_JWT_SECRET",
		Name:   "jwt-secret",
		Usage:  "the secert used to encode the json web token.",
		Value:  "4Rtg8BPKwixXy2ktDPxoMMAhRzmo9mmuZjvKONGPZZQSaJWNLijxR42qRgq0iBb5",
	},
	cli.IntFlag{
		EnvVar: "KITSVC_MAX_PING_COUNT",
		Name:   "max-ping-count",
		Usage:  "",
		Value:  20,
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DEBUG",
		Name:   "debug",
		Usage:  "enable the debug mode.",
	},

	// Database flags.
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_DRIVER",
		Name:   "database-driver",
		Usage:  "the driver of the database.",
		Value:  "mysql",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_NAME",
		Name:   "database-name",
		Usage:  "the name of the database.",
		Value:  "service",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_HOST",
		Name:   "database-host",
		Usage:  "the host of the database (with the port).",
		Value:  "127.0.0.1:3306",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_USER",
		Name:   "database-user",
		Usage:  "the user of the database.",
		Value:  "root",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_PASSWORD",
		Name:   "database-password",
		Usage:  "the password of the database.",
		Value:  "root",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_CHARSET",
		Name:   "database-charset",
		Usage:  "the charset of the database.",
		Value:  "utf8",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_LOC",
		Name:   "database-loc",
		Usage:  "the timezone of the database.",
		Value:  "Local",
	},
	cli.BoolFlag{
		EnvVar: "KITSVC_DATABASE_PARSE_TIME",
		Name:   "database-parse_time",
		Usage:  "parse the time.",
	},

	// NSQ flags.
	cli.StringFlag{
		EnvVar: "KITSVC_NSQ_PRODUCER",
		Name:   "nsq-producer",
		Usage:  "the address of the TCP NSQ producer (with the port).",
		Value:  "127.0.0.1:4150",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_NSQ_PRODUCER_HTTP",
		Name:   "nsq-producer-http",
		Usage:  "the address of the HTTP NSQ producer (with the port).",
		Value:  "127.0.0.1:4151",
	},
	cli.StringSliceFlag{
		EnvVar: "KITSVC_NSQ_LOOKUPDS",
		Name:   "nsq-lookupds",
		Usage:  "the address of the NSQ lookupds (with the port).",
		Value: &cli.StringSlice{
			"127.0.0.1:4161",
		},
	},

	// Event store flags.
	cli.StringFlag{
		EnvVar: "KITSVC_ES_SERVER_URL",
		Name:   "es-url",
		Usage:  "the url of the event store server.",
		Value:  "http://127.0.0.1:2113",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_ES_USERNAME",
		Name:   "es-username",
		Usage:  "the username of the event store.",
		Value:  "admin",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_ES_PASSWORD",
		Name:   "es-password",
		Usage:  "the password of the event store.",
		Value:  "changeit",
	},

	// Prometheus flags.
	cli.StringFlag{
		EnvVar: "KITSVC_PROMETHEUS_NAMESPACE",
		Name:   "prometheus-namespace",
		Usage:  "the prometheus namespace.",
		Value:  "service",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_PROMETHEUS_SUBSYSTEM",
		Name:   "prometheus-subsystem",
		Usage:  "the subsystem of the promethues.",
		Value:  "user",
	},

	// Consul flags.
	cli.StringFlag{
		EnvVar: "KITSVC_CONSUL_CHECK_INTERVAL",
		Name:   "consul-check_interval",
		Usage:  "the interval of consul health check.",
		Value:  "30s",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_CONSUL_CHECK_TIMEOUT",
		Name:   "consul-check_timeout",
		Usage:  "the timeout of consul health check.",
		Value:  "1s",
	},
	cli.StringSliceFlag{
		EnvVar: "KITSVC_CONSUL_TAGS",
		Name:   "consul-tags",
		Usage:  "the service tags for consul.",
		Value: &cli.StringSlice{
			"user",
			"micro",
		},
	},
}

// server runs the server.
func server(c *cli.Context, started chan bool) error {
	// `deployed` will be closed when the router is deployed.
	deployed := make(chan bool)
	// `replayed` will be closed after the events are all replayed.
	replayed := make(chan bool)

	// Debug mode.
	if !c.Bool("debug") {
		gin.SetMode(gin.ReleaseMode)
	}

	// Initialize the logger.
	logger.Init(c)
	// Create the Gin engine.
	g := gin.New()
	// Event handlers.
	event := eventutil.New(g)
	// Websocket handlers.
	ws := wsutil.New(g)
	// Message queue handlers.
	mq := mqutil.New(g)

	// Routes.
	router.Load(
		// Cores.
		g, event, ws, mq,
		// Middlwares.
		middleware.Config(c),
		middleware.Store(c),
		middleware.Logging(),
		middleware.Event(c, event, replayed, deployed),
		middleware.MQ(c, mq, deployed),
		middleware.Metrics(),
	)

	// Register to the service registry when the events were replayed.
	go func() {
		<-replayed

		sd.Register(c)
		// After the service is registered to the consul,
		// close the `started` channel to make it non-blocking.
		close(started)
	}()

	// Ping the server to make sure the router is working.
	go func() {
		if err := pingServer(c); err != nil {
			logger.Fatal("The router has no response, or it might took too long to start up.")
		}
		logger.Info("The router has been deployed successfully.")
		// Close the `deployed` channel to make it non-blocking.
		close(deployed)
	}()

	// Start to listening the incoming requests.
	return http.ListenAndServe(c.String("addr"), g)
}

// pingServer pings the http server to make sure the router is working.
func pingServer(c *cli.Context) error {
	for i := 0; i < c.Int("max-ping-count"); i++ {
		// Ping the server by sending a GET request to `/health`.
		resp, err := http.Get(c.String("url") + "/sd/health")
		if err == nil && resp.StatusCode == 200 {
			return nil
		}

		// Sleep for a second to continue the next ping.
		logger.Info("Waiting for the router, retry in 1 second.")
		time.Sleep(time.Second)
	}
	return errors.New("Cannot connect to the router.")
}
<commit_msg>Added usage for KITSVC_MAX_PING_COUNT<commit_after>package main

import (
	"errors"
	"net/http"
	"time"

	"github.com/TeaMeow/KitSvc/module/logger"
	"github.com/TeaMeow/KitSvc/module/sd"
	"github.com/TeaMeow/KitSvc/router"
	"github.com/TeaMeow/KitSvc/router/middleware"
	"github.com/TeaMeow/KitSvc/shared/eventutil"
	"github.com/TeaMeow/KitSvc/shared/mqutil"
	"github.com/TeaMeow/KitSvc/shared/wsutil"
	"github.com/codegangsta/cli"
	"github.com/gin-gonic/gin"
)

var serverFlags = []cli.Flag{
	// Common flags.
	cli.StringFlag{
		EnvVar: "KITSVC_NAME",
		Name:   "name",
		Usage:  "the name of the service, exposed for service discovery.",
		Value:  "Service",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_URL",
		Name:   "url",
		Usage:  "the url of the service.",
		Value:  "http://127.0.0.1:8080",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_ADDR",
		Name:   "addr",
		Usage:  "the address of the service (with the port).",
		Value:  "127.0.0.1:8080",
	},
	cli.IntFlag{
		EnvVar: "KITSVC_PORT",
		Name:   "port",
		Usage:  "the port of the service.",
		Value:  8080,
	},
	cli.StringFlag{
		EnvVar: "KITSVC_USAGE",
		Name:   "usage",
		Usage:  "the usage of the service, exposed for service discovery.",
		Value:  "Operations about the users.",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_JWT_SECRET",
		Name:   "jwt-secret",
		Usage:  "the secert used to encode the json web token.",
		Value:  "4Rtg8BPKwixXy2ktDPxoMMAhRzmo9mmuZjvKONGPZZQSaJWNLijxR42qRgq0iBb5",
	},
	cli.IntFlag{
		EnvVar: "KITSVC_MAX_PING_COUNT",
		Name:   "max-ping-count",
		Usage:  "the amount to ping the server before we give up.",
		Value:  20,
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DEBUG",
		Name:   "debug",
		Usage:  "enable the debug mode.",
	},

	// Database flags.
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_DRIVER",
		Name:   "database-driver",
		Usage:  "the driver of the database.",
		Value:  "mysql",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_NAME",
		Name:   "database-name",
		Usage:  "the name of the database.",
		Value:  "service",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_HOST",
		Name:   "database-host",
		Usage:  "the host of the database (with the port).",
		Value:  "127.0.0.1:3306",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_USER",
		Name:   "database-user",
		Usage:  "the user of the database.",
		Value:  "root",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_PASSWORD",
		Name:   "database-password",
		Usage:  "the password of the database.",
		Value:  "root",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_CHARSET",
		Name:   "database-charset",
		Usage:  "the charset of the database.",
		Value:  "utf8",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_DATABASE_LOC",
		Name:   "database-loc",
		Usage:  "the timezone of the database.",
		Value:  "Local",
	},
	cli.BoolFlag{
		EnvVar: "KITSVC_DATABASE_PARSE_TIME",
		Name:   "database-parse_time",
		Usage:  "parse the time.",
	},

	// NSQ flags.
	cli.StringFlag{
		EnvVar: "KITSVC_NSQ_PRODUCER",
		Name:   "nsq-producer",
		Usage:  "the address of the TCP NSQ producer (with the port).",
		Value:  "127.0.0.1:4150",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_NSQ_PRODUCER_HTTP",
		Name:   "nsq-producer-http",
		Usage:  "the address of the HTTP NSQ producer (with the port).",
		Value:  "127.0.0.1:4151",
	},
	cli.StringSliceFlag{
		EnvVar: "KITSVC_NSQ_LOOKUPDS",
		Name:   "nsq-lookupds",
		Usage:  "the address of the NSQ lookupds (with the port).",
		Value: &cli.StringSlice{
			"127.0.0.1:4161",
		},
	},

	// Event store flags.
	cli.StringFlag{
		EnvVar: "KITSVC_ES_SERVER_URL",
		Name:   "es-url",
		Usage:  "the url of the event store server.",
		Value:  "http://127.0.0.1:2113",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_ES_USERNAME",
		Name:   "es-username",
		Usage:  "the username of the event store.",
		Value:  "admin",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_ES_PASSWORD",
		Name:   "es-password",
		Usage:  "the password of the event store.",
		Value:  "changeit",
	},

	// Prometheus flags.
	cli.StringFlag{
		EnvVar: "KITSVC_PROMETHEUS_NAMESPACE",
		Name:   "prometheus-namespace",
		Usage:  "the prometheus namespace.",
		Value:  "service",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_PROMETHEUS_SUBSYSTEM",
		Name:   "prometheus-subsystem",
		Usage:  "the subsystem of the promethues.",
		Value:  "user",
	},

	// Consul flags.
	cli.StringFlag{
		EnvVar: "KITSVC_CONSUL_CHECK_INTERVAL",
		Name:   "consul-check_interval",
		Usage:  "the interval of consul health check.",
		Value:  "30s",
	},
	cli.StringFlag{
		EnvVar: "KITSVC_CONSUL_CHECK_TIMEOUT",
		Name:   "consul-check_timeout",
		Usage:  "the timeout of consul health check.",
		Value:  "1s",
	},
	cli.StringSliceFlag{
		EnvVar: "KITSVC_CONSUL_TAGS",
		Name:   "consul-tags",
		Usage:  "the service tags for consul.",
		Value: &cli.StringSlice{
			"user",
			"micro",
		},
	},
}

// server runs the server.
func server(c *cli.Context, started chan bool) error {
	// `deployed` will be closed when the router is deployed.
	deployed := make(chan bool)
	// `replayed` will be closed after the events are all replayed.
	replayed := make(chan bool)

	// Debug mode.
	if !c.Bool("debug") {
		gin.SetMode(gin.ReleaseMode)
	}

	// Initialize the logger.
	logger.Init(c)
	// Create the Gin engine.
	g := gin.New()
	// Event handlers.
	event := eventutil.New(g)
	// Websocket handlers.
	ws := wsutil.New(g)
	// Message queue handlers.
	mq := mqutil.New(g)

	// Routes.
	router.Load(
		// Cores.
		g, event, ws, mq,
		// Middlwares.
		middleware.Config(c),
		middleware.Store(c),
		middleware.Logging(),
		middleware.Event(c, event, replayed, deployed),
		middleware.MQ(c, mq, deployed),
		middleware.Metrics(),
	)

	// Register to the service registry when the events were replayed.
	go func() {
		<-replayed

		sd.Register(c)
		// After the service is registered to the consul,
		// close the `started` channel to make it non-blocking.
		close(started)
	}()

	// Ping the server to make sure the router is working.
	go func() {
		if err := pingServer(c); err != nil {
			logger.Fatal("The router has no response, or it might took too long to start up.")
		}
		logger.Info("The router has been deployed successfully.")
		// Close the `deployed` channel to make it non-blocking.
		close(deployed)
	}()

	// Start to listening the incoming requests.
	return http.ListenAndServe(c.String("addr"), g)
}

// pingServer pings the http server to make sure the router is working.
func pingServer(c *cli.Context) error {
	for i := 0; i < c.Int("max-ping-count"); i++ {
		// Ping the server by sending a GET request to `/health`.
		resp, err := http.Get(c.String("url") + "/sd/health")
		if err == nil && resp.StatusCode == 200 {
			return nil
		}

		// Sleep for a second to continue the next ping.
		logger.Info("Waiting for the router, retry in 1 second.")
		time.Sleep(time.Second)
	}
	return errors.New("Cannot connect to the router.")
}
<|endoftext|>


<commit_before>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the ZeroMQ server loop. Deals with incoming requests and
// delegates them to the event store. Also publishes newly stored events
// using a PUB socket.
//
// See README file for an up-to-date documentation of the ZeroMQ wire
// format.
package server

import (
	"bytes"
	"errors"
	"log"
	"container/list"
	"time"
	"sync"
	zmq "github.com/alecthomas/gozmq"
	"github.com/JensRantil/gorewind/eventstore"
)

// StartParams are parameters required for starting the server.
type InitParams struct {
	// The event store to use as backend.
	Store *eventstore.EventStore
	// The ZeroMQ path that the command receiving socket will bind
	// to.
	CommandSocketZPath *string
	// The ZeroMQ path that the event publishing socket will bind
	// to.
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context

	runningMutex sync.Mutex
	running bool
	stopChan chan bool
}

// IsRunning returns true if the server is running, false otherwise.
func (v *Server) IsRunning() bool {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	return v.running
}

// Stop stops a running server. Blocks until the server is stopped. If
// the server is not running, an error is returned.
func (v* Server) Stop() error {
	if !v.IsRunning() {
		return errors.New("Not running.")
	}

	select {
	case v.stopChan <- true:
	default:
		return errors.New("Stop already signalled.")
	}
	<-v.stopChan
	// v.running is modified by Server.Run(...)

	if v.IsRunning() {
		return errors.New("Signalled stopped, but never stopped.")
	}

	return nil
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
		running: false,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.closeZmq()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = &context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = &commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = &evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) closeZmq() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

func (v *Server) setRunningState(newState bool) {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	v.running = newState
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Start() {
	v.setRunningState(true)
	defer v.setRunningState(false)
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock, v.stopChan)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems, stop chan bool) {
	for {
		timeout := time.Duration(1)*time.Second
		count, err := zmq.Poll(items, int64(timeout))
		if count > 0 || err != nil {
			notifier <- zmqPollResult{err}
		}

		select {
		case <-stop:
			stop <- true
			return
		default:
		}
	}
}

func stopPoller(cancelChan chan bool) {
	cancelChan <- true
	<-cancelChan
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
//
// TODO: Make this a type function of `Server` to remove a lot of
// parameters.
func loopServer(estore *eventstore.EventStore, evpubsock, frontend zmq.Socket,
stop chan bool) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan eventstore.StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)

	pollCancel := make(chan bool)
	defer stopPoller(pollCancel)

	go asyncPoll(pollchan, toPoll, pollCancel)
	for {
		select {
		case res := <-pollchan:
			if res.err != nil {
				log.Print("Could not poll:", res.err)
			}
			if res.err == nil && toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll, pollCancel)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				log.Println(err)
			}
		case <- stop:
			stop <- true
			return
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan eventstore.StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		stored := <-toPublish

		msg[0] = stored.Event.Stream
		msg[1] = stored.Id
		msg[2] = stored.Event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			log.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *eventstore.EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for PUBLISH."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := eventstore.Event{
				estream.(eventstore.StreamName),
				data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for QUERY."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			req := eventstore.QueryRequest{
				Stream: estream.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			events, err := estore.Query(req)

			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				for eventdata := range(events) {
					response := copyList(resptemplate)
					response.PushBack([]byte("EVENT"))
					response.PushBack(eventdata.Id)
					response.PushBack(eventdata.Data)

					respchan <- listToFrames(response)
				}
				response := copyList(resptemplate)
				response.PushBack(zFrame("END"))
				respchan <- listToFrames(response)
			}
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		// TODO: Constantify this error message
		errstr := "Unknown request type."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<commit_msg>Patching to support latest GoZMQ<commit_after>// gorewind is an event store server written in Python that talks ZeroMQ.
// Copyright (C) 2013  Jens Rantil
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

// Contains the ZeroMQ server loop. Deals with incoming requests and
// delegates them to the event store. Also publishes newly stored events
// using a PUB socket.
//
// See README file for an up-to-date documentation of the ZeroMQ wire
// format.
package server

import (
	"bytes"
	"errors"
	"log"
	"container/list"
	"time"
	"sync"
	zmq "github.com/alecthomas/gozmq"
	"github.com/JensRantil/gorewind/eventstore"
)

// StartParams are parameters required for starting the server.
type InitParams struct {
	// The event store to use as backend.
	Store *eventstore.EventStore
	// The ZeroMQ path that the command receiving socket will bind
	// to.
	CommandSocketZPath *string
	// The ZeroMQ path that the event publishing socket will bind
	// to.
	EvPubSocketZPath *string
}

// Check all required initialization parameters are set.
func checkAllInitParamsSet(p *InitParams) error {
	if p.Store == nil {
		return errors.New("Missing param: Store")
	}
	if p.CommandSocketZPath == nil {
		return errors.New("Missing param: CommandSocketZPath")
	}
	if p.EvPubSocketZPath == nil {
		return errors.New("Missing param: EvPubSocketZPath")
	}
	return nil
}

// A server instance. Can be run.
type Server struct {
	params InitParams

	evpubsock *zmq.Socket
	commandsock *zmq.Socket
	context *zmq.Context

	runningMutex sync.Mutex
	running bool
	stopChan chan bool
}

// IsRunning returns true if the server is running, false otherwise.
func (v *Server) IsRunning() bool {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	return v.running
}

// Stop stops a running server. Blocks until the server is stopped. If
// the server is not running, an error is returned.
func (v* Server) Stop() error {
	if !v.IsRunning() {
		return errors.New("Not running.")
	}

	select {
	case v.stopChan <- true:
	default:
		return errors.New("Stop already signalled.")
	}
	<-v.stopChan
	// v.running is modified by Server.Run(...)

	if v.IsRunning() {
		return errors.New("Signalled stopped, but never stopped.")
	}

	return nil
}

// Initialize a new event store server and return a handle to it. The
// event store is not started. It's up to the caller to execute Run()
// on the server handle.
func New(params *InitParams) (*Server, error) {
	if params == nil {
		return nil, errors.New("Missing init params")
	}
	if err := checkAllInitParamsSet(params); err != nil {
		return nil, err
	}

	server := Server{
		params: *params,
		running: false,
	}

	var allOkay *bool = new(bool)
	*allOkay = false
	defer func() {
		if (!*allOkay) {
			server.closeZmq()
		}
	}()

	context, err := zmq.NewContext()
	if err != nil {
		return nil, err
	}
	server.context = context

	commandsock, err := context.NewSocket(zmq.ROUTER)
	if err != nil {
		return nil, err
	}
	server.commandsock = commandsock
	err = commandsock.Bind(*params.CommandSocketZPath)
	if err != nil {
		return nil, err
	}

	evpubsock, err := context.NewSocket(zmq.PUB)
	if err != nil {
		return nil, err
	}
	server.evpubsock = evpubsock
	if binderr := evpubsock.Bind(*params.EvPubSocketZPath); binderr != nil {
		return nil, err
	}

	*allOkay = true

	return &server, nil
}

func (v *Server) closeZmq() {
	(*v.evpubsock).Close()
	v.evpubsock = nil
	(*v.commandsock).Close()
	v.commandsock = nil
	(*v.context).Close()
	v.context = nil
}

func (v *Server) setRunningState(newState bool) {
	v.runningMutex.Lock()
	defer v.runningMutex.Unlock()
	v.running = newState
}

// Runs the server that distributes requests to workers.
// Panics on error since it is an essential piece of code required to
// run the application correctly.
func (v *Server) Start() {
	v.setRunningState(true)
	defer v.setRunningState(false)
	loopServer((*v).params.Store, *(*v).evpubsock, *(*v).commandsock, v.stopChan)
}

// The result of an asynchronous zmq.Poll call.
type zmqPollResult struct {
	err error
}

// Polls a bunch of ZeroMQ sockets and notifies the result through a
// channel. This makes it possible to combine ZeroMQ polling with Go's
// own built-in channels.
func asyncPoll(notifier chan zmqPollResult, items zmq.PollItems, stop chan bool) {
	for {
		timeout := time.Duration(1)*time.Second
		count, err := zmq.Poll(items, timeout)
		if count > 0 || err != nil {
			notifier <- zmqPollResult{err}
		}

		select {
		case <-stop:
			stop <- true
			return
		default:
		}
	}
}

func stopPoller(cancelChan chan bool) {
	cancelChan <- true
	<-cancelChan
}

// The core ZeroMQ messaging loop. Handles requests and responses
// asynchronously using the router socket. Every request is delegated to
// a goroutine for maximum concurrency.
//
// `gozmq` does currently not support copy-free messages/frames. This
// means that every message passing through this function needs to be
// copied in-memory. If this becomes a bottleneck in the future,
// multiple router sockets can be hooked to this final router to scale
// message copying.
//
// TODO: Make this a type function of `Server` to remove a lot of
// parameters.
func loopServer(estore *eventstore.EventStore, evpubsock, frontend zmq.Socket,
stop chan bool) {
	toPoll := zmq.PollItems{
		zmq.PollItem{Socket: &frontend, zmq.Events: zmq.POLLIN},
	}

	pubchan := make(chan eventstore.StoredEvent)
	estore.RegisterPublishedEventsChannel(pubchan)
	go publishAllSavedEvents(pubchan, evpubsock)

	pollchan := make(chan zmqPollResult)
	respchan := make(chan zMsg)

	pollCancel := make(chan bool)
	defer stopPoller(pollCancel)

	go asyncPoll(pollchan, toPoll, pollCancel)
	for {
		select {
		case res := <-pollchan:
			if res.err != nil {
				log.Print("Could not poll:", res.err)
			}
			if res.err == nil && toPoll[0].REvents&zmq.POLLIN != 0 {
				msg, _ := toPoll[0].Socket.RecvMultipart(0)
				zmsg := zMsg(msg)
				go handleRequest(respchan, estore, zmsg)
			}
			go asyncPoll(pollchan, toPoll, pollCancel)
		case frames := <-respchan:
			if err := frontend.SendMultipart(frames, 0); err != nil {
				log.Println(err)
			}
		case <- stop:
			stop <- true
			return
		}
	}
}

// Publishes stored events to event listeners.
//
// Pops previously stored messages off a channel and published them to a
// ZeroMQ socket.
func publishAllSavedEvents(toPublish chan eventstore.StoredEvent, evpub zmq.Socket) {
	msg := make(zMsg, 3)
	for {
		stored := <-toPublish

		msg[0] = stored.Event.Stream
		msg[1] = stored.Id
		msg[2] = stored.Event.Data

		if err := evpub.SendMultipart(msg, 0); err != nil {
			log.Println(err)
		}
	}
}

// A single frame in a ZeroMQ message.
type zFrame []byte

// A ZeroMQ message.
//
// I wish it could have been `[]zFrame`, but that would make conversion
// from `[][]byte` pretty messy[1].
//
// [1] http://stackoverflow.com/a/15650327/260805
type zMsg [][]byte

// Handles a single ZeroMQ RES/REQ loop synchronously.
//
// The full request message stored in `msg` and the full ZeroMQ response
// is pushed to `respchan`. The function does not return any error
// because it is expected to be called asynchronously as a goroutine.
func handleRequest(respchan chan zMsg, estore *eventstore.EventStore, msg zMsg) {

	// TODO: Rename to 'framelist'
	parts := list.New()
	for _, msgpart := range msg {
		parts.PushBack(msgpart)
	}

	resptemplate := list.New()
	emptyFrame := zFrame("")
	for true {
		resptemplate.PushBack(parts.Remove(parts.Front()))

		if bytes.Equal(parts.Front().Value.(zFrame), emptyFrame) {
			break
		}
	}

	if parts.Len() == 0 {
		errstr := "Incoming command was empty. Ignoring it."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
		return
	}

	command := string(parts.Front().Value.(zFrame))
	switch command {
	case "PUBLISH":
		parts.Remove(parts.Front())
		if parts.Len() != 2 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for PUBLISH."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			data := parts.Remove(parts.Front())
			newevent := eventstore.Event{
				estream.(eventstore.StreamName),
				data.(zFrame),
			}
			newId, err := estore.Add(newevent)
			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				// the event was added
				response := copyList(resptemplate)
				response.PushBack(zFrame("PUBLISHED"))
				response.PushBack(zFrame(newId))
				respchan <- listToFrames(response)
			}
		}
	case "QUERY":
		parts.Remove(parts.Front())
		if parts.Len() != 3 {
			// TODO: Constantify this error message
			errstr := "Wrong number of frames for QUERY."
			log.Println(errstr)
			response := copyList(resptemplate)
			response.PushBack(zFrame("ERROR " + errstr))
			respchan <- listToFrames(response)
		} else {
			estream := parts.Remove(parts.Front())
			fromid := parts.Remove(parts.Front())
			toid := parts.Remove(parts.Front())

			req := eventstore.QueryRequest{
				Stream: estream.(zFrame),
				FromId: fromid.(zFrame),
				ToId: toid.(zFrame),
			}
			events, err := estore.Query(req)

			if err != nil {
				sErr := err.Error()
				log.Println(sErr)

				response := copyList(resptemplate)
				response.PushBack(zFrame("ERROR " + sErr))
				respchan <- listToFrames(response)
			} else {
				for eventdata := range(events) {
					response := copyList(resptemplate)
					response.PushBack([]byte("EVENT"))
					response.PushBack(eventdata.Id)
					response.PushBack(eventdata.Data)

					respchan <- listToFrames(response)
				}
				response := copyList(resptemplate)
				response.PushBack(zFrame("END"))
				respchan <- listToFrames(response)
			}
		}
	default:
		// TODO: Move these error strings out as constants of
		//       this package.

		// TODO: Move the chunk of code below into a separate
		// function and reuse for similar piece of code above.
		// TODO: Constantify this error message
		errstr := "Unknown request type."
		log.Println(errstr)
		response := copyList(resptemplate)
		response.PushBack(zFrame("ERROR " + errstr))
		respchan <- listToFrames(response)
	}
}

// Convert a doubly linked list of message frames to a slice of message
// fram
func listToFrames(l *list.List) zMsg {
	frames := make(zMsg, l.Len())
	i := 0
	for e := l.Front(); e != nil; e = e.Next() {
		frames[i] = e.Value.(zFrame)
	}
	return frames
}

// Helper function for copying a doubly linked list.
func copyList(l *list.List) *list.List {
	replica := list.New()
	replica.PushBackList(l)
	return replica
}

<|endoftext|>


<commit_before>package strategy

import (
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

// TODO: at some point, the strategy needs to plug in here
// to help decide how to sort tasks (on add) and how to select
// tasks (on getnext). For now, we are assuming a dumb/nice strategy.
type taskQueue struct {
	tasks   []*Task
	taskmap map[string]*Task
}

func newTaskQueue() *taskQueue {
	return &taskQueue{
		taskmap: make(map[string]*Task),
	}
}

type Task struct {
	Key           u.Key
	Target        peer.Peer
	theirPriority int
}

// Push currently adds a new task to the end of the list
// TODO: make this into a priority queue
func (tl *taskQueue) Push(block u.Key, priority int, to peer.Peer) {
	if task, ok := tl.taskmap[taskKey(to, block)]; ok {
		// TODO: when priority queue is implemented,
		//       rearrange this Task
		task.theirPriority = priority
		return
	}
	task := &Task{
		Key:           block,
		Target:        to,
		theirPriority: priority,
	}
	tl.tasks = append(tl.tasks, task)
	tl.taskmap[taskKey(to, block)] = task
}

// Pop 'pops' the next task to be performed. Returns nil no task exists.
func (tl *taskQueue) Pop() *Task {
	var out *Task
	for len(tl.tasks) > 0 {
		// TODO: instead of zero, use exponential distribution
		//       it will help reduce the chance of receiving
		//		 the same block from multiple peers
		out = tl.tasks[0]
		tl.tasks = tl.tasks[1:]
		delete(tl.taskmap, taskKey(out.Target, out.Key))
		// Filter out blocks that have been cancelled
		if out.theirPriority >= 0 {
			break
		}
	}

	return out
}

// Remove lazily removes a task from the queue
func (tl *taskQueue) Remove(k u.Key, p peer.Peer) {
	t, ok := tl.taskmap[taskKey(p, k)]
	if ok {
		t.theirPriority = -1
	}
}

// taskKey returns a key that uniquely identifies a task.
func taskKey(p peer.Peer, k u.Key) string {
	return string(p.Key() + k)
}
<commit_msg>privatize Task<commit_after>package strategy

import (
	peer "github.com/jbenet/go-ipfs/peer"
	u "github.com/jbenet/go-ipfs/util"
)

// TODO: at some point, the strategy needs to plug in here
// to help decide how to sort tasks (on add) and how to select
// tasks (on getnext). For now, we are assuming a dumb/nice strategy.
type taskQueue struct {
	tasks   []*task
	taskmap map[string]*task
}

func newTaskQueue() *taskQueue {
	return &taskQueue{
		taskmap: make(map[string]*task),
	}
}

type task struct {
	Key           u.Key
	Target        peer.Peer
	theirPriority int
}

// Push currently adds a new task to the end of the list
// TODO: make this into a priority queue
func (tl *taskQueue) Push(block u.Key, priority int, to peer.Peer) {
	if task, ok := tl.taskmap[taskKey(to, block)]; ok {
		// TODO: when priority queue is implemented,
		//       rearrange this task
		task.theirPriority = priority
		return
	}
	task := &task{
		Key:           block,
		Target:        to,
		theirPriority: priority,
	}
	tl.tasks = append(tl.tasks, task)
	tl.taskmap[taskKey(to, block)] = task
}

// Pop 'pops' the next task to be performed. Returns nil no task exists.
func (tl *taskQueue) Pop() *task {
	var out *task
	for len(tl.tasks) > 0 {
		// TODO: instead of zero, use exponential distribution
		//       it will help reduce the chance of receiving
		//		 the same block from multiple peers
		out = tl.tasks[0]
		tl.tasks = tl.tasks[1:]
		delete(tl.taskmap, taskKey(out.Target, out.Key))
		// Filter out blocks that have been cancelled
		if out.theirPriority >= 0 {
			break
		}
	}

	return out
}

// Remove lazily removes a task from the queue
func (tl *taskQueue) Remove(k u.Key, p peer.Peer) {
	t, ok := tl.taskmap[taskKey(p, k)]
	if ok {
		t.theirPriority = -1
	}
}

// taskKey returns a key that uniquely identifies a task.
func taskKey(p peer.Peer, k u.Key) string {
	return string(p.Key() + k)
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package watchdog is responsible for monitoring the sentry for tasks that may
// potentially be stuck or looping inderterminally causing hard to debug hungs in
// the untrusted app.
//
// It works by periodically querying all tasks to check whether they are in user
// mode (RunUser), kernel mode (RunSys), or blocked in the kernel (OffCPU). Tasks
// that have been running in kernel mode for a long time in the same syscall
// without blocking are considered stuck and are reported.
//
// When a stuck task is detected, the watchdog can take one of the following actions:
//		1. LogWarning: Logs a warning message followed by a stack dump of all goroutines.
//			 If a tasks continues to be stuck, the message will repeat every minute, unless
//			 a new stuck task is detected
//		2. Panic: same as above, followed by panic()
//
package watchdog

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
	"gvisor.googlesource.com/gvisor/pkg/log"
	"gvisor.googlesource.com/gvisor/pkg/metric"
	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
	ktime "gvisor.googlesource.com/gvisor/pkg/sentry/kernel/time"
)

// DefaultTimeout is a resonable timeout value for most applications.
const DefaultTimeout = 3 * time.Minute

// descheduleThreshold is the amount of time scheduling needs to be off before the entire wait period
// is discounted from task's last update time. It's set high enough that small scheduling delays won't
// trigger it.
const descheduleThreshold = 1 * time.Second

var stuckTasks = metric.MustCreateNewUint64Metric("/watchdog/stuck_tasks_detected", true /* sync */, "Cumulative count of stuck tasks detected")

// Amount of time to wait before dumping the stack to the log again when the same task(s) remains stuck.
var stackDumpSameTaskPeriod = time.Minute

// Action defines what action to take when a stuck task is detected.
type Action int

const (
	// LogWarning logs warning message followed by stack trace.
	LogWarning Action = iota
	// Panic will do the same logging as LogWarning and panic().
	Panic
)

// String returns Action's string representation.
func (a Action) String() string {
	switch a {
	case LogWarning:
		return "LogWarning"
	case Panic:
		return "Panic"
	default:
		panic(fmt.Sprintf("Invalid action: %d", a))
	}
}

// Watchdog is the main watchdog class. It controls a goroutine that periodically
// analyses all tasks and reports if any of them appear to be stuck.
type Watchdog struct {
	// period indicates how often to check all tasks. It's calculated based on
	// 'taskTimeout'.
	period time.Duration

	// taskTimeout is the amount of time to allow a task to execute the same syscall
	// without blocking before it's declared stuck.
	taskTimeout time.Duration

	// timeoutAction indicates what action to take when a stuck tasks is detected.
	timeoutAction Action

	// k is where the tasks come from.
	k *kernel.Kernel

	// stop is used to notify to watchdog should stop.
	stop chan struct{}

	// done is used to notify when the watchdog has stopped.
	done chan struct{}

	// offenders map contains all tasks that are currently stuck.
	offenders map[*kernel.Task]*offender

	// lastStackDump tracks the last time a stack dump was generated to prevent
	// spamming the log.
	lastStackDump time.Time

	// lastRun is set to the last time the watchdog executed a monitoring loop.
	lastRun ktime.Time

	// mu protects the fields below.
	mu sync.Mutex

	// started is true if the watchdog has been started before.
	started bool
}

type offender struct {
	lastUpdateTime ktime.Time
}

// New creates a new watchdog.
func New(k *kernel.Kernel, taskTimeout time.Duration, a Action) *Watchdog {
	// 4 is arbitrary, just don't want to prolong 'taskTimeout' too much.
	period := taskTimeout / 4
	return &Watchdog{
		k:             k,
		period:        period,
		taskTimeout:   taskTimeout,
		timeoutAction: a,
		offenders:     make(map[*kernel.Task]*offender),
		stop:          make(chan struct{}),
		done:          make(chan struct{}),
	}
}

// Start starts the watchdog.
func (w *Watchdog) Start() {
	if w.taskTimeout == 0 {
		log.Infof("Watchdog disabled")
		return
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	if w.started {
		return
	}

	w.lastRun = w.k.MonotonicClock().Now()

	log.Infof("Starting watchdog, period: %v, timeout: %v, action: %v", w.period, w.taskTimeout, w.timeoutAction)
	go w.loop() // S/R-SAFE: watchdog is stopped during save and restarted after restore.
	w.started = true
}

// Stop requests the watchdog to stop and wait for it.
func (w *Watchdog) Stop() {
	if w.taskTimeout == 0 {
		return
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.started {
		return
	}
	log.Infof("Stopping watchdog")
	w.stop <- struct{}{}
	<-w.done
	w.started = false
	log.Infof("Watchdog stopped")
}

// loop is the main watchdog routine. It only returns when 'Stop()' is called.
func (w *Watchdog) loop() {
	// Loop until someone stops it.
	for {
		select {
		case <-w.stop:
			w.done <- struct{}{}
			return
		case <-time.After(w.period):
			w.runTurn()
		}
	}
}

// runTurn runs a single pass over all tasks and reports anything it finds.
func (w *Watchdog) runTurn() {
	tasks := w.k.TaskSet().Root.Tasks()

	newOffenders := make(map[*kernel.Task]*offender)
	newTaskFound := false
	now := ktime.FromNanoseconds(int64(w.k.CPUClockNow() * uint64(linux.ClockTick)))

	// The process may be running with low CPU limit making tasks appear stuck because
	// are starved of CPU cycles. An estimate is that Tasks could have been starved
	// since the last time the watchdog run. If the watchdog detects that scheduling
	// is off, it will discount the entire duration since last run from 'lastUpdateTime'.
	discount := time.Duration(0)
	if now.Sub(w.lastRun.Add(w.period)) > descheduleThreshold {
		discount = now.Sub(w.lastRun)
	}
	w.lastRun = now

	log.Infof("Watchdog starting loop, tasks: %d, discount: %v", len(tasks), discount)
	for _, t := range tasks {
		tsched := t.TaskGoroutineSchedInfo()

		// An offender is a task running inside the kernel for longer than the specified timeout.
		if tsched.State == kernel.TaskGoroutineRunningSys {
			lastUpdateTime := ktime.FromNanoseconds(int64(tsched.Timestamp * uint64(linux.ClockTick)))
			elapsed := now.Sub(lastUpdateTime) - discount
			if elapsed > w.taskTimeout {
				tc, ok := w.offenders[t]
				if !ok {
					// New stuck task detected.
					//
					// TODO: Tasks blocked doing IO may be considered stuck in kernel.
					tc = &offender{lastUpdateTime: lastUpdateTime}
					stuckTasks.Increment()
					newTaskFound = true
				}
				newOffenders[t] = tc
			}
		}
	}
	if len(newOffenders) > 0 {
		w.report(newOffenders, newTaskFound, now)
	}

	// Remember which tasks have been reported.
	w.offenders = newOffenders
}

// report takes appropriate action when a stuck task is detected.
func (w *Watchdog) report(offenders map[*kernel.Task]*offender, newTaskFound bool, now ktime.Time) {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("Sentry detected %d stuck task(s):\n", len(offenders)))
	for t, o := range offenders {
		tid := w.k.TaskSet().Root.IDOfTask(t)
		buf.WriteString(fmt.Sprintf("\tTask tid: %v (%#x), entered RunSys state %v ago.\n", tid, uint64(tid), now.Sub(o.lastUpdateTime)))
	}
	buf.WriteString("Search for '(*Task).run(0x..., 0x<tid>)' in the stack dump to find the offending goroutine")

	switch w.timeoutAction {
	case LogWarning:
		// Dump stack only if a new task is detected or if it sometime has passed since
		// the last time a stack dump was generated.
		if !newTaskFound && time.Since(w.lastStackDump) < stackDumpSameTaskPeriod {
			buf.WriteString("\n...[stack dump skipped]...")
			log.Warningf(buf.String())
		} else {
			log.TracebackAll(buf.String())
			w.lastStackDump = time.Now()
		}

	case Panic:
		// Panic will skip over running tasks, which is likely the culprit here. So manually
		// dump all stacks before panic'ing.
		log.TracebackAll(buf.String())

		// Attempt to flush metrics, timeout and move on in case metrics are stuck as well.
		metricsEmitted := make(chan struct{}, 1)
		go func() { // S/R-SAFE: watchdog is stopped during save and restarted after restore.
			// Flush metrics before killing process.
			metric.EmitMetricUpdate()
			metricsEmitted <- struct{}{}
		}()
		select {
		case <-metricsEmitted:
		case <-time.After(1 * time.Second):
		}
		panic("Sentry detected stuck task(s). See stack trace and message above for more details")
	}
}
<commit_msg>Dumps stacks if watchdog thread is stuck<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package watchdog is responsible for monitoring the sentry for tasks that may
// potentially be stuck or looping inderterminally causing hard to debug hungs in
// the untrusted app.
//
// It works by periodically querying all tasks to check whether they are in user
// mode (RunUser), kernel mode (RunSys), or blocked in the kernel (OffCPU). Tasks
// that have been running in kernel mode for a long time in the same syscall
// without blocking are considered stuck and are reported.
//
// When a stuck task is detected, the watchdog can take one of the following actions:
//		1. LogWarning: Logs a warning message followed by a stack dump of all goroutines.
//			 If a tasks continues to be stuck, the message will repeat every minute, unless
//			 a new stuck task is detected
//		2. Panic: same as above, followed by panic()
//
package watchdog

import (
	"bytes"
	"fmt"
	"sync"
	"time"

	"gvisor.googlesource.com/gvisor/pkg/abi/linux"
	"gvisor.googlesource.com/gvisor/pkg/log"
	"gvisor.googlesource.com/gvisor/pkg/metric"
	"gvisor.googlesource.com/gvisor/pkg/sentry/kernel"
	ktime "gvisor.googlesource.com/gvisor/pkg/sentry/kernel/time"
)

// DefaultTimeout is a resonable timeout value for most applications.
const DefaultTimeout = 3 * time.Minute

// descheduleThreshold is the amount of time scheduling needs to be off before the entire wait period
// is discounted from task's last update time. It's set high enough that small scheduling delays won't
// trigger it.
const descheduleThreshold = 1 * time.Second

var stuckTasks = metric.MustCreateNewUint64Metric("/watchdog/stuck_tasks_detected", true /* sync */, "Cumulative count of stuck tasks detected")

// Amount of time to wait before dumping the stack to the log again when the same task(s) remains stuck.
var stackDumpSameTaskPeriod = time.Minute

// Action defines what action to take when a stuck task is detected.
type Action int

const (
	// LogWarning logs warning message followed by stack trace.
	LogWarning Action = iota
	// Panic will do the same logging as LogWarning and panic().
	Panic
)

// String returns Action's string representation.
func (a Action) String() string {
	switch a {
	case LogWarning:
		return "LogWarning"
	case Panic:
		return "Panic"
	default:
		panic(fmt.Sprintf("Invalid action: %d", a))
	}
}

// Watchdog is the main watchdog class. It controls a goroutine that periodically
// analyses all tasks and reports if any of them appear to be stuck.
type Watchdog struct {
	// period indicates how often to check all tasks. It's calculated based on
	// 'taskTimeout'.
	period time.Duration

	// taskTimeout is the amount of time to allow a task to execute the same syscall
	// without blocking before it's declared stuck.
	taskTimeout time.Duration

	// timeoutAction indicates what action to take when a stuck tasks is detected.
	timeoutAction Action

	// k is where the tasks come from.
	k *kernel.Kernel

	// stop is used to notify to watchdog should stop.
	stop chan struct{}

	// done is used to notify when the watchdog has stopped.
	done chan struct{}

	// offenders map contains all tasks that are currently stuck.
	offenders map[*kernel.Task]*offender

	// lastStackDump tracks the last time a stack dump was generated to prevent
	// spamming the log.
	lastStackDump time.Time

	// lastRun is set to the last time the watchdog executed a monitoring loop.
	lastRun ktime.Time

	// mu protects the fields below.
	mu sync.Mutex

	// started is true if the watchdog has been started before.
	started bool
}

type offender struct {
	lastUpdateTime ktime.Time
}

// New creates a new watchdog.
func New(k *kernel.Kernel, taskTimeout time.Duration, a Action) *Watchdog {
	// 4 is arbitrary, just don't want to prolong 'taskTimeout' too much.
	period := taskTimeout / 4
	return &Watchdog{
		k:             k,
		period:        period,
		taskTimeout:   taskTimeout,
		timeoutAction: a,
		offenders:     make(map[*kernel.Task]*offender),
		stop:          make(chan struct{}),
		done:          make(chan struct{}),
	}
}

// Start starts the watchdog.
func (w *Watchdog) Start() {
	if w.taskTimeout == 0 {
		log.Infof("Watchdog disabled")
		return
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	if w.started {
		return
	}

	w.lastRun = w.k.MonotonicClock().Now()

	log.Infof("Starting watchdog, period: %v, timeout: %v, action: %v", w.period, w.taskTimeout, w.timeoutAction)
	go w.loop() // S/R-SAFE: watchdog is stopped during save and restarted after restore.
	w.started = true
}

// Stop requests the watchdog to stop and wait for it.
func (w *Watchdog) Stop() {
	if w.taskTimeout == 0 {
		return
	}

	w.mu.Lock()
	defer w.mu.Unlock()
	if !w.started {
		return
	}
	log.Infof("Stopping watchdog")
	w.stop <- struct{}{}
	<-w.done
	w.started = false
	log.Infof("Watchdog stopped")
}

// loop is the main watchdog routine. It only returns when 'Stop()' is called.
func (w *Watchdog) loop() {
	// Loop until someone stops it.
	for {
		select {
		case <-w.stop:
			w.done <- struct{}{}
			return
		case <-time.After(w.period):
			w.runTurn()
		}
	}
}

// runTurn runs a single pass over all tasks and reports anything it finds.
func (w *Watchdog) runTurn() {
	// Someone needs to watch the watchdog. The call below can get stuck if there
	// is a deadlock affecting root's PID namespace mutex. Run it in a goroutine
	// and report if it takes too long to return.
	var tasks []*kernel.Task
	done := make(chan struct{})
	go func() { // S/R-SAFE: watchdog is stopped and restarted during S/R.
		tasks = w.k.TaskSet().Root.Tasks()
		close(done)
	}()

	select {
	case <-done:
	case <-time.After(w.taskTimeout):
		// Report if the watchdog is not making progress.
		// No one is wathching the watchdog watcher though.
		w.reportStuckWatchdog()
		<-done
	}

	newOffenders := make(map[*kernel.Task]*offender)
	newTaskFound := false
	now := ktime.FromNanoseconds(int64(w.k.CPUClockNow() * uint64(linux.ClockTick)))

	// The process may be running with low CPU limit making tasks appear stuck because
	// are starved of CPU cycles. An estimate is that Tasks could have been starved
	// since the last time the watchdog run. If the watchdog detects that scheduling
	// is off, it will discount the entire duration since last run from 'lastUpdateTime'.
	discount := time.Duration(0)
	if now.Sub(w.lastRun.Add(w.period)) > descheduleThreshold {
		discount = now.Sub(w.lastRun)
	}
	w.lastRun = now

	log.Infof("Watchdog starting loop, tasks: %d, discount: %v", len(tasks), discount)
	for _, t := range tasks {
		tsched := t.TaskGoroutineSchedInfo()

		// An offender is a task running inside the kernel for longer than the specified timeout.
		if tsched.State == kernel.TaskGoroutineRunningSys {
			lastUpdateTime := ktime.FromNanoseconds(int64(tsched.Timestamp * uint64(linux.ClockTick)))
			elapsed := now.Sub(lastUpdateTime) - discount
			if elapsed > w.taskTimeout {
				tc, ok := w.offenders[t]
				if !ok {
					// New stuck task detected.
					//
					// TODO: Tasks blocked doing IO may be considered stuck in kernel.
					tc = &offender{lastUpdateTime: lastUpdateTime}
					stuckTasks.Increment()
					newTaskFound = true
				}
				newOffenders[t] = tc
			}
		}
	}
	if len(newOffenders) > 0 {
		w.report(newOffenders, newTaskFound, now)
	}

	// Remember which tasks have been reported.
	w.offenders = newOffenders
}

// report takes appropriate action when a stuck task is detected.
func (w *Watchdog) report(offenders map[*kernel.Task]*offender, newTaskFound bool, now ktime.Time) {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("Sentry detected %d stuck task(s):\n", len(offenders)))
	for t, o := range offenders {
		tid := w.k.TaskSet().Root.IDOfTask(t)
		buf.WriteString(fmt.Sprintf("\tTask tid: %v (%#x), entered RunSys state %v ago.\n", tid, uint64(tid), now.Sub(o.lastUpdateTime)))
	}
	buf.WriteString("Search for '(*Task).run(0x..., 0x<tid>)' in the stack dump to find the offending goroutine")
	w.onStuckTask(newTaskFound, &buf)
}

func (w *Watchdog) reportStuckWatchdog() {
	var buf bytes.Buffer
	buf.WriteString("Watchdog goroutine is stuck:\n")
	w.onStuckTask(true, &buf)
}

func (w *Watchdog) onStuckTask(newTaskFound bool, buf *bytes.Buffer) {
	switch w.timeoutAction {
	case LogWarning:
		// Dump stack only if a new task is detected or if it sometime has passed since
		// the last time a stack dump was generated.
		if !newTaskFound && time.Since(w.lastStackDump) < stackDumpSameTaskPeriod {
			buf.WriteString("\n...[stack dump skipped]...")
			log.Warningf(buf.String())
		} else {
			log.TracebackAll(buf.String())
			w.lastStackDump = time.Now()
		}

	case Panic:
		// Panic will skip over running tasks, which is likely the culprit here. So manually
		// dump all stacks before panic'ing.
		log.TracebackAll(buf.String())

		// Attempt to flush metrics, timeout and move on in case metrics are stuck as well.
		metricsEmitted := make(chan struct{}, 1)
		go func() { // S/R-SAFE: watchdog is stopped during save and restarted after restore.
			// Flush metrics before killing process.
			metric.EmitMetricUpdate()
			metricsEmitted <- struct{}{}
		}()
		select {
		case <-metricsEmitted:
		case <-time.After(1 * time.Second):
		}
		panic("Sentry detected stuck task(s). See stack trace and message above for more details")
	}
}
<|endoftext|>


<commit_before>package pushbullet

import (
	"encoding/json"
	"errors"
	"net/http"

	"github.com/mitsuse/bullet/pushbullet/responses"
)

// Get the devices thath can be pushed to.
func (pb *Pushbullet) GetDevices() ([]*responses.Device, error) {
	// TODO: Implement this.
	req, err := http.NewRequest("GET", ENDPOINT_DEVICES, nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(pb.token, "")

	res, err := pb.client.Do(req)
	if err != nil {
		return nil, err
	}

	// TODO: Return an error value with human friendly message.
	if res.StatusCode != 200 {
		return nil, errors.New(res.Status)
	}

	var devices *devicesResponse

	decoder := json.NewDecoder(res.Body)
	if err := decoder.Decode(&devices); err != nil {
		return nil, err
	}

	return devices.Devices, nil
}

type devicesResponse struct {
	Devices []*responses.Device `json:"devices"`
}
<commit_msg>Remove a needless comment.<commit_after>package pushbullet

import (
	"encoding/json"
	"errors"
	"net/http"

	"github.com/mitsuse/bullet/pushbullet/responses"
)

// Get the devices thath can be pushed to.
func (pb *Pushbullet) GetDevices() ([]*responses.Device, error) {
	req, err := http.NewRequest("GET", ENDPOINT_DEVICES, nil)
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(pb.token, "")

	res, err := pb.client.Do(req)
	if err != nil {
		return nil, err
	}

	// TODO: Return an error value with human friendly message.
	if res.StatusCode != 200 {
		return nil, errors.New(res.Status)
	}

	var devices *devicesResponse

	decoder := json.NewDecoder(res.Body)
	if err := decoder.Decode(&devices); err != nil {
		return nil, err
	}

	return devices.Devices, nil
}

type devicesResponse struct {
	Devices []*responses.Device `json:"devices"`
}
<|endoftext|>


<commit_before>// +build windows

package windows

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"unsafe"

	ansiterm "github.com/Azure/go-ansiterm"
	"github.com/Azure/go-ansiterm/winterm"
)

const (
	escapeSequence = ansiterm.KEY_ESC_CSI
)

// ansiReader wraps a standard input file (e.g., os.Stdin) providing ANSI sequence translation.
type ansiReader struct {
	file     *os.File
	fd       uintptr
	buffer   []byte
	cbBuffer int
	command  []byte
}

// NewAnsiReader returns an io.ReadCloser that provides VT100 terminal emulation on top of a
// Windows console input handle.
func NewAnsiReader(nFile int) io.ReadCloser {
	initLogger()
	file, fd := winterm.GetStdFile(nFile)
	return &ansiReader{
		file:    file,
		fd:      fd,
		command: make([]byte, 0, ansiterm.ANSI_MAX_CMD_LENGTH),
		buffer:  make([]byte, 0),
	}
}

// Close closes the wrapped file.
func (ar *ansiReader) Close() (err error) {
	return ar.file.Close()
}

// Fd returns the file descriptor of the wrapped file.
func (ar *ansiReader) Fd() uintptr {
	return ar.fd
}

// Read reads up to len(p) bytes of translated input events into p.
func (ar *ansiReader) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}

	// Previously read bytes exist, read as much as we can and return
	if len(ar.buffer) > 0 {
		logger.Debugf("Reading previously cached bytes")

		originalLength := len(ar.buffer)
		copiedLength := copy(p, ar.buffer)

		if copiedLength == originalLength {
			ar.buffer = make([]byte, 0, len(p))
		} else {
			ar.buffer = ar.buffer[copiedLength:]
		}

		logger.Debugf("Read from cache p[%d]: % x", copiedLength, p)
		return copiedLength, nil
	}

	// Read and translate key events
	events, err := readInputEvents(ar.fd, len(p))
	if err != nil {
		return 0, err
	} else if len(events) == 0 {
		logger.Debug("No input events detected")
		return 0, nil
	}

	keyBytes := translateKeyEvents(events, []byte(escapeSequence))

	// Save excess bytes and right-size keyBytes
	if len(keyBytes) > len(p) {
		logger.Debugf("Received %d keyBytes, only room for %d bytes", len(keyBytes), len(p))
		ar.buffer = keyBytes[len(p):]
		keyBytes = keyBytes[:len(p)]
	} else if len(keyBytes) == 0 {
		logger.Debug("No key bytes returned from the translator")
		return 0, nil
	}

	copiedLength := copy(p, keyBytes)
	if copiedLength != len(keyBytes) {
		return 0, errors.New("Unexpected copy length encountered.")
	}

	logger.Debugf("Read        p[%d]: % x", copiedLength, p)
	logger.Debugf("Read keyBytes[%d]: % x", copiedLength, keyBytes)
	return copiedLength, nil
}

// readInputEvents polls until at least one event is available.
func readInputEvents(fd uintptr, maxBytes int) ([]winterm.INPUT_RECORD, error) {
	// Determine the maximum number of records to retrieve
	// -- Cast around the type system to obtain the size of a single INPUT_RECORD.
	//    unsafe.Sizeof requires an expression vs. a type-reference; the casting
	//    tricks the type system into believing it has such an expression.
	recordSize := int(unsafe.Sizeof(*((*winterm.INPUT_RECORD)(unsafe.Pointer(&maxBytes)))))
	countRecords := maxBytes / recordSize
	if countRecords > ansiterm.MAX_INPUT_EVENTS {
		countRecords = ansiterm.MAX_INPUT_EVENTS
	}
	logger.Debugf("[windows] readInputEvents: Reading %v records (buffer size %v, record size %v)", countRecords, maxBytes, recordSize)

	// Wait for and read input events
	events := make([]winterm.INPUT_RECORD, countRecords)
	nEvents := uint32(0)
	eventsExist, err := winterm.WaitForSingleObject(fd, winterm.WAIT_INFINITE)
	if err != nil {
		return nil, err
	}

	if eventsExist {
		err = winterm.ReadConsoleInput(fd, events, &nEvents)
		if err != nil {
			return nil, err
		}
	}

	// Return a slice restricted to the number of returned records
	logger.Debugf("[windows] readInputEvents: Read %v events", nEvents)
	return events[:nEvents], nil
}

// KeyEvent Translation Helpers

var arrowKeyMapPrefix = map[uint16]string{
	winterm.VK_UP:    "%s%sA",
	winterm.VK_DOWN:  "%s%sB",
	winterm.VK_RIGHT: "%s%sC",
	winterm.VK_LEFT:  "%s%sD",
}

var keyMapPrefix = map[uint16]string{
	winterm.VK_UP:     "\x1B[%sA",
	winterm.VK_DOWN:   "\x1B[%sB",
	winterm.VK_RIGHT:  "\x1B[%sC",
	winterm.VK_LEFT:   "\x1B[%sD",
	winterm.VK_HOME:   "\x1B[1%s~", // showkey shows ^[[1
	winterm.VK_END:    "\x1B[4%s~", // showkey shows ^[[4
	winterm.VK_INSERT: "\x1B[2%s~",
	winterm.VK_DELETE: "\x1B[3%s~",
	winterm.VK_PRIOR:  "\x1B[5%s~",
	winterm.VK_NEXT:   "\x1B[6%s~",
	winterm.VK_F1:     "",
	winterm.VK_F2:     "",
	winterm.VK_F3:     "\x1B[13%s~",
	winterm.VK_F4:     "\x1B[14%s~",
	winterm.VK_F5:     "\x1B[15%s~",
	winterm.VK_F6:     "\x1B[17%s~",
	winterm.VK_F7:     "\x1B[18%s~",
	winterm.VK_F8:     "\x1B[19%s~",
	winterm.VK_F9:     "\x1B[20%s~",
	winterm.VK_F10:    "\x1B[21%s~",
	winterm.VK_F11:    "\x1B[23%s~",
	winterm.VK_F12:    "\x1B[24%s~",
}

// translateKeyEvents converts the input events into the appropriate ANSI string.
func translateKeyEvents(events []winterm.INPUT_RECORD, escapeSequence []byte) []byte {
	var buffer bytes.Buffer
	for _, event := range events {
		if event.EventType == winterm.KEY_EVENT && event.KeyEvent.KeyDown != 0 {
			buffer.WriteString(keyToString(&event.KeyEvent, escapeSequence))
		}
	}

	return buffer.Bytes()
}

// keyToString maps the given input event record to the corresponding string.
func keyToString(keyEvent *winterm.KEY_EVENT_RECORD, escapeSequence []byte) string {
	if keyEvent.UnicodeChar == 0 {
		return formatVirtualKey(keyEvent.VirtualKeyCode, keyEvent.ControlKeyState, escapeSequence)
	}

	_, alt, control := getControlKeys(keyEvent.ControlKeyState)
	if control {
		// TODO(azlinux): Implement following control sequences
		// <Ctrl>-D  Signals the end of input from the keyboard; also exits current shell.
		// <Ctrl>-H  Deletes the first character to the left of the cursor. Also called the ERASE key.
		// <Ctrl>-Q  Restarts printing after it has been stopped with <Ctrl>-s.
		// <Ctrl>-S  Suspends printing on the screen (does not stop the program).
		// <Ctrl>-U  Deletes all characters on the current line. Also called the KILL key.
		// <Ctrl>-E  Quits current command and creates a core

	}

	// <Alt>+Key generates ESC N Key
	if !control && alt {
		return ansiterm.KEY_ESC_N + strings.ToLower(string(keyEvent.UnicodeChar))
	}

	return string(keyEvent.UnicodeChar)
}

// formatVirtualKey converts a virtual key (e.g., up arrow) into the appropriate ANSI string.
func formatVirtualKey(key uint16, controlState uint32, escapeSequence []byte) string {
	shift, alt, control := getControlKeys(controlState)
	modifier := getControlKeysModifier(shift, alt, control)

	if format, ok := arrowKeyMapPrefix[key]; ok {
		return fmt.Sprintf(format, escapeSequence, modifier)
	}

	if format, ok := keyMapPrefix[key]; ok {
		return fmt.Sprintf(format, modifier)
	}

	return ""
}

// getControlKeys extracts the shift, alt, and ctrl key states.
func getControlKeys(controlState uint32) (shift, alt, control bool) {
	shift = 0 != (controlState & winterm.SHIFT_PRESSED)
	alt = 0 != (controlState & (winterm.LEFT_ALT_PRESSED | winterm.RIGHT_ALT_PRESSED))
	control = 0 != (controlState & (winterm.LEFT_CTRL_PRESSED | winterm.RIGHT_CTRL_PRESSED))
	return shift, alt, control
}

// getControlKeysModifier returns the ANSI modifier for the given combination of control keys.
func getControlKeysModifier(shift, alt, control bool) string {
	if shift && alt && control {
		return ansiterm.KEY_CONTROL_PARAM_8
	}
	if alt && control {
		return ansiterm.KEY_CONTROL_PARAM_7
	}
	if shift && control {
		return ansiterm.KEY_CONTROL_PARAM_6
	}
	if control {
		return ansiterm.KEY_CONTROL_PARAM_5
	}
	if shift && alt {
		return ansiterm.KEY_CONTROL_PARAM_4
	}
	if alt {
		return ansiterm.KEY_CONTROL_PARAM_3
	}
	if shift {
		return ansiterm.KEY_CONTROL_PARAM_2
	}
	return ""
}
<commit_msg>error strings should not be capitalized or end with punctuation<commit_after>// +build windows

package windows

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"
	"unsafe"

	ansiterm "github.com/Azure/go-ansiterm"
	"github.com/Azure/go-ansiterm/winterm"
)

const (
	escapeSequence = ansiterm.KEY_ESC_CSI
)

// ansiReader wraps a standard input file (e.g., os.Stdin) providing ANSI sequence translation.
type ansiReader struct {
	file     *os.File
	fd       uintptr
	buffer   []byte
	cbBuffer int
	command  []byte
}

// NewAnsiReader returns an io.ReadCloser that provides VT100 terminal emulation on top of a
// Windows console input handle.
func NewAnsiReader(nFile int) io.ReadCloser {
	initLogger()
	file, fd := winterm.GetStdFile(nFile)
	return &ansiReader{
		file:    file,
		fd:      fd,
		command: make([]byte, 0, ansiterm.ANSI_MAX_CMD_LENGTH),
		buffer:  make([]byte, 0),
	}
}

// Close closes the wrapped file.
func (ar *ansiReader) Close() (err error) {
	return ar.file.Close()
}

// Fd returns the file descriptor of the wrapped file.
func (ar *ansiReader) Fd() uintptr {
	return ar.fd
}

// Read reads up to len(p) bytes of translated input events into p.
func (ar *ansiReader) Read(p []byte) (int, error) {
	if len(p) == 0 {
		return 0, nil
	}

	// Previously read bytes exist, read as much as we can and return
	if len(ar.buffer) > 0 {
		logger.Debugf("Reading previously cached bytes")

		originalLength := len(ar.buffer)
		copiedLength := copy(p, ar.buffer)

		if copiedLength == originalLength {
			ar.buffer = make([]byte, 0, len(p))
		} else {
			ar.buffer = ar.buffer[copiedLength:]
		}

		logger.Debugf("Read from cache p[%d]: % x", copiedLength, p)
		return copiedLength, nil
	}

	// Read and translate key events
	events, err := readInputEvents(ar.fd, len(p))
	if err != nil {
		return 0, err
	} else if len(events) == 0 {
		logger.Debug("No input events detected")
		return 0, nil
	}

	keyBytes := translateKeyEvents(events, []byte(escapeSequence))

	// Save excess bytes and right-size keyBytes
	if len(keyBytes) > len(p) {
		logger.Debugf("Received %d keyBytes, only room for %d bytes", len(keyBytes), len(p))
		ar.buffer = keyBytes[len(p):]
		keyBytes = keyBytes[:len(p)]
	} else if len(keyBytes) == 0 {
		logger.Debug("No key bytes returned from the translator")
		return 0, nil
	}

	copiedLength := copy(p, keyBytes)
	if copiedLength != len(keyBytes) {
		return 0, errors.New("unexpected copy length encountered")
	}

	logger.Debugf("Read        p[%d]: % x", copiedLength, p)
	logger.Debugf("Read keyBytes[%d]: % x", copiedLength, keyBytes)
	return copiedLength, nil
}

// readInputEvents polls until at least one event is available.
func readInputEvents(fd uintptr, maxBytes int) ([]winterm.INPUT_RECORD, error) {
	// Determine the maximum number of records to retrieve
	// -- Cast around the type system to obtain the size of a single INPUT_RECORD.
	//    unsafe.Sizeof requires an expression vs. a type-reference; the casting
	//    tricks the type system into believing it has such an expression.
	recordSize := int(unsafe.Sizeof(*((*winterm.INPUT_RECORD)(unsafe.Pointer(&maxBytes)))))
	countRecords := maxBytes / recordSize
	if countRecords > ansiterm.MAX_INPUT_EVENTS {
		countRecords = ansiterm.MAX_INPUT_EVENTS
	}
	logger.Debugf("[windows] readInputEvents: Reading %v records (buffer size %v, record size %v)", countRecords, maxBytes, recordSize)

	// Wait for and read input events
	events := make([]winterm.INPUT_RECORD, countRecords)
	nEvents := uint32(0)
	eventsExist, err := winterm.WaitForSingleObject(fd, winterm.WAIT_INFINITE)
	if err != nil {
		return nil, err
	}

	if eventsExist {
		err = winterm.ReadConsoleInput(fd, events, &nEvents)
		if err != nil {
			return nil, err
		}
	}

	// Return a slice restricted to the number of returned records
	logger.Debugf("[windows] readInputEvents: Read %v events", nEvents)
	return events[:nEvents], nil
}

// KeyEvent Translation Helpers

var arrowKeyMapPrefix = map[uint16]string{
	winterm.VK_UP:    "%s%sA",
	winterm.VK_DOWN:  "%s%sB",
	winterm.VK_RIGHT: "%s%sC",
	winterm.VK_LEFT:  "%s%sD",
}

var keyMapPrefix = map[uint16]string{
	winterm.VK_UP:     "\x1B[%sA",
	winterm.VK_DOWN:   "\x1B[%sB",
	winterm.VK_RIGHT:  "\x1B[%sC",
	winterm.VK_LEFT:   "\x1B[%sD",
	winterm.VK_HOME:   "\x1B[1%s~", // showkey shows ^[[1
	winterm.VK_END:    "\x1B[4%s~", // showkey shows ^[[4
	winterm.VK_INSERT: "\x1B[2%s~",
	winterm.VK_DELETE: "\x1B[3%s~",
	winterm.VK_PRIOR:  "\x1B[5%s~",
	winterm.VK_NEXT:   "\x1B[6%s~",
	winterm.VK_F1:     "",
	winterm.VK_F2:     "",
	winterm.VK_F3:     "\x1B[13%s~",
	winterm.VK_F4:     "\x1B[14%s~",
	winterm.VK_F5:     "\x1B[15%s~",
	winterm.VK_F6:     "\x1B[17%s~",
	winterm.VK_F7:     "\x1B[18%s~",
	winterm.VK_F8:     "\x1B[19%s~",
	winterm.VK_F9:     "\x1B[20%s~",
	winterm.VK_F10:    "\x1B[21%s~",
	winterm.VK_F11:    "\x1B[23%s~",
	winterm.VK_F12:    "\x1B[24%s~",
}

// translateKeyEvents converts the input events into the appropriate ANSI string.
func translateKeyEvents(events []winterm.INPUT_RECORD, escapeSequence []byte) []byte {
	var buffer bytes.Buffer
	for _, event := range events {
		if event.EventType == winterm.KEY_EVENT && event.KeyEvent.KeyDown != 0 {
			buffer.WriteString(keyToString(&event.KeyEvent, escapeSequence))
		}
	}

	return buffer.Bytes()
}

// keyToString maps the given input event record to the corresponding string.
func keyToString(keyEvent *winterm.KEY_EVENT_RECORD, escapeSequence []byte) string {
	if keyEvent.UnicodeChar == 0 {
		return formatVirtualKey(keyEvent.VirtualKeyCode, keyEvent.ControlKeyState, escapeSequence)
	}

	_, alt, control := getControlKeys(keyEvent.ControlKeyState)
	if control {
		// TODO(azlinux): Implement following control sequences
		// <Ctrl>-D  Signals the end of input from the keyboard; also exits current shell.
		// <Ctrl>-H  Deletes the first character to the left of the cursor. Also called the ERASE key.
		// <Ctrl>-Q  Restarts printing after it has been stopped with <Ctrl>-s.
		// <Ctrl>-S  Suspends printing on the screen (does not stop the program).
		// <Ctrl>-U  Deletes all characters on the current line. Also called the KILL key.
		// <Ctrl>-E  Quits current command and creates a core

	}

	// <Alt>+Key generates ESC N Key
	if !control && alt {
		return ansiterm.KEY_ESC_N + strings.ToLower(string(keyEvent.UnicodeChar))
	}

	return string(keyEvent.UnicodeChar)
}

// formatVirtualKey converts a virtual key (e.g., up arrow) into the appropriate ANSI string.
func formatVirtualKey(key uint16, controlState uint32, escapeSequence []byte) string {
	shift, alt, control := getControlKeys(controlState)
	modifier := getControlKeysModifier(shift, alt, control)

	if format, ok := arrowKeyMapPrefix[key]; ok {
		return fmt.Sprintf(format, escapeSequence, modifier)
	}

	if format, ok := keyMapPrefix[key]; ok {
		return fmt.Sprintf(format, modifier)
	}

	return ""
}

// getControlKeys extracts the shift, alt, and ctrl key states.
func getControlKeys(controlState uint32) (shift, alt, control bool) {
	shift = 0 != (controlState & winterm.SHIFT_PRESSED)
	alt = 0 != (controlState & (winterm.LEFT_ALT_PRESSED | winterm.RIGHT_ALT_PRESSED))
	control = 0 != (controlState & (winterm.LEFT_CTRL_PRESSED | winterm.RIGHT_CTRL_PRESSED))
	return shift, alt, control
}

// getControlKeysModifier returns the ANSI modifier for the given combination of control keys.
func getControlKeysModifier(shift, alt, control bool) string {
	if shift && alt && control {
		return ansiterm.KEY_CONTROL_PARAM_8
	}
	if alt && control {
		return ansiterm.KEY_CONTROL_PARAM_7
	}
	if shift && control {
		return ansiterm.KEY_CONTROL_PARAM_6
	}
	if control {
		return ansiterm.KEY_CONTROL_PARAM_5
	}
	if shift && alt {
		return ansiterm.KEY_CONTROL_PARAM_4
	}
	if alt {
		return ansiterm.KEY_CONTROL_PARAM_3
	}
	if shift {
		return ansiterm.KEY_CONTROL_PARAM_2
	}
	return ""
}
<|endoftext|>


<commit_before>package server

import (
	"net/http"
	"log"
	"github.com/castiron/tandygram-backend/datastore"
	"encoding/json"
)

func Serve() {
	http.HandleFunc("/api/composites", handleComposites)
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func handleComposites(w http.ResponseWriter, r *http.Request) {
	log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
	if(r.Method == "GET") {
		handleIndex(w, r)
	} else if(r.Method == "POST") {
		handleCreate(w, r)
	} else {
		handleError(w, r)
	}
}

func handleIndex(w http.ResponseWriter, r *http.Request) {
    json, err := json.Marshal(datastore.GetAllComposites())
    if err != nil {
        handleError(w, r)
        return
    }
    w.Write(json)
}

func handleCreate(w http.ResponseWriter, r *http.Request) {
	composite, err := datastore.CreateComposite(r.Body)
	if err != nil {
		handleError(w, r)
	}
	json, err := json.Marshal(composite)
    if err != nil {
        handleError(w, r)
        return
    }
    w.Write(json)
}

func handleError(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(422)
}
<commit_msg>[F] Stab wildly at CORS issues<commit_after>package server

import (
	"net/http"
	"log"
	"github.com/castiron/tandygram-backend/datastore"
	"encoding/json"
)

func Serve() {
	http.HandleFunc("/api/composites", handleComposites)
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func handleComposites(w http.ResponseWriter, r *http.Request) {
	log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)

	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, DELETE")
	w.Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")

	// Stop here if its Preflighted OPTIONS request
	if r.Method == "OPTIONS" {
		return
	}

	if(r.Method == "GET") {
		handleIndex(w, r)
	} else if(r.Method == "POST") {
		handleCreate(w, r)
	} else {
		handleError(w, r)
	}
}

func handleIndex(w http.ResponseWriter, r *http.Request) {
    json, err := json.Marshal(datastore.GetAllComposites())
    if err != nil {
        handleError(w, r)
        return
    }
    w.Write(json)
}

func handleCreate(w http.ResponseWriter, r *http.Request) {
	composite, err := datastore.CreateComposite(r.Body)
	if err != nil {
		handleError(w, r)
	}
	json, err := json.Marshal(composite)
    if err != nil {
        handleError(w, r)
        return
    }
    w.Write(json)
}

func handleError(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(422)
}
<|endoftext|>