<commit_before>package pgx

import (
	"context"

	"github.com/jackc/pgconn"
	errors "golang.org/x/xerrors"
)

type batchItem struct {
	query     string
	arguments []interface{}
}

// Batch queries are a way of bundling multiple queries together to avoid
// unnecessary network round trips.
type Batch struct {
	items []*batchItem
}

// Queue queues a query to batch b. query can be an SQL query or the name of a prepared statement.
func (b *Batch) Queue(query string, arguments ...interface{}) {
	b.items = append(b.items, &batchItem{
		query:     query,
		arguments: arguments,
	})
}

type BatchResults interface {
	// Exec reads the results from the next query in the batch as if the query has been sent with Conn.Exec.
	Exec() (pgconn.CommandTag, error)

	// Query reads the results from the next query in the batch as if the query has been sent with Conn.Query.
	Query() (Rows, error)

	// QueryRow reads the results from the next query in the batch as if the query has been sent with Conn.QueryRow.
	QueryRow() Row

	// Close closes the batch operation. This must be called before the underlying connection can be used again. Any error
	// that occurred during a batch operation may have made it impossible to resyncronize the connection with the server.
	// In this case the underlying connection will have been closed.
	Close() error
}

type batchResults struct {
	ctx  context.Context
	conn *Conn
	mrr  *pgconn.MultiResultReader
	err  error
}

// Exec reads the results from the next query in the batch as if the query has been sent with Exec.
func (br *batchResults) Exec() (pgconn.CommandTag, error) {
	if br.err != nil {
		return nil, br.err
	}

	if !br.mrr.NextResult() {
		err := br.mrr.Close()
		if err == nil {
			err = errors.New("no result")
		}
		return nil, err
	}

	return br.mrr.ResultReader().Close()
}

// Query reads the results from the next query in the batch as if the query has been sent with Query.
func (br *batchResults) Query() (Rows, error) {
	rows := br.conn.getRows(br.ctx, "batch query", nil)

	if br.err != nil {
		rows.err = br.err
		rows.closed = true
		return rows, br.err
	}

	if !br.mrr.NextResult() {
		rows.err = br.mrr.Close()
		if rows.err == nil {
			rows.err = errors.New("no result")
		}
		rows.closed = true
		return rows, rows.err
	}

	rows.resultReader = br.mrr.ResultReader()
	return rows, nil
}

// QueryRow reads the results from the next query in the batch as if the query has been sent with QueryRow.
func (br *batchResults) QueryRow() Row {
	rows, _ := br.Query()
	return (*connRow)(rows.(*connRows))

}

// Close closes the batch operation. Any error that occurred during a batch operation may have made it impossible to
// resyncronize the connection with the server. In this case the underlying connection will have been closed.
func (br *batchResults) Close() error {
	if br.err != nil {
		return br.err
	}

	return br.mrr.Close()
}
<commit_msg>Add Len() method to *pgx.Batch<commit_after>package pgx

import (
	"context"

	"github.com/jackc/pgconn"
	errors "golang.org/x/xerrors"
)

type batchItem struct {
	query     string
	arguments []interface{}
}

// Batch queries are a way of bundling multiple queries together to avoid
// unnecessary network round trips.
type Batch struct {
	items []*batchItem
}

// Queue queues a query to batch b. query can be an SQL query or the name of a prepared statement.
func (b *Batch) Queue(query string, arguments ...interface{}) {
	b.items = append(b.items, &batchItem{
		query:     query,
		arguments: arguments,
	})
}

// Len returns number of queries that have been queued so far.
func (b *Batch) Len() int {
	return len(b.items)
}

type BatchResults interface {
	// Exec reads the results from the next query in the batch as if the query has been sent with Conn.Exec.
	Exec() (pgconn.CommandTag, error)

	// Query reads the results from the next query in the batch as if the query has been sent with Conn.Query.
	Query() (Rows, error)

	// QueryRow reads the results from the next query in the batch as if the query has been sent with Conn.QueryRow.
	QueryRow() Row

	// Close closes the batch operation. This must be called before the underlying connection can be used again. Any error
	// that occurred during a batch operation may have made it impossible to resyncronize the connection with the server.
	// In this case the underlying connection will have been closed.
	Close() error
}

type batchResults struct {
	ctx  context.Context
	conn *Conn
	mrr  *pgconn.MultiResultReader
	err  error
}

// Exec reads the results from the next query in the batch as if the query has been sent with Exec.
func (br *batchResults) Exec() (pgconn.CommandTag, error) {
	if br.err != nil {
		return nil, br.err
	}

	if !br.mrr.NextResult() {
		err := br.mrr.Close()
		if err == nil {
			err = errors.New("no result")
		}
		return nil, err
	}

	return br.mrr.ResultReader().Close()
}

// Query reads the results from the next query in the batch as if the query has been sent with Query.
func (br *batchResults) Query() (Rows, error) {
	rows := br.conn.getRows(br.ctx, "batch query", nil)

	if br.err != nil {
		rows.err = br.err
		rows.closed = true
		return rows, br.err
	}

	if !br.mrr.NextResult() {
		rows.err = br.mrr.Close()
		if rows.err == nil {
			rows.err = errors.New("no result")
		}
		rows.closed = true
		return rows, rows.err
	}

	rows.resultReader = br.mrr.ResultReader()
	return rows, nil
}

// QueryRow reads the results from the next query in the batch as if the query has been sent with QueryRow.
func (br *batchResults) QueryRow() Row {
	rows, _ := br.Query()
	return (*connRow)(rows.(*connRows))

}

// Close closes the batch operation. Any error that occurred during a batch operation may have made it impossible to
// resyncronize the connection with the server. In this case the underlying connection will have been closed.
func (br *batchResults) Close() error {
	if br.err != nil {
		return br.err
	}

	return br.mrr.Close()
}
<|endoftext|>


<commit_before>package shutdown

import (
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var (
	sysSigChan     chan os.Signal
	shutdownLogger = log.New(os.Stderr, "", 0) // log to stderr without the timestamps
)

// Destructor is a func takes no args and returns no values. It is executed as an injectable destructor
// so that the calling context could use this func to sync.Wait() or print a pretty exit message.
type Destructor func()

// SignalChan is a blank channel used to signal a shutdown
type SignalChan chan struct{}

// Watch is our signal watching func. It's worth noting that this is a
// blocking action so it should be run in a goroutine or as the last function call
// such that all the other goroutines can continue working while the application
// sits blocked here.
// For a deeper discussion of the close channel idiom: http://dave.cheney.net/2013/04/30/curious-channels
func Watch(shutdown SignalChan, destruct Destructor) {

	sysSigChan = make(chan os.Signal, 1)
	signal.Notify(sysSigChan, syscall.SIGINT)
	signal.Notify(sysSigChan, syscall.SIGTERM)

	var sig os.Signal

	select { // block until we get a signal
	case sig = <-sysSigChan:
		close(shutdown) // idiom via: http://dave.cheney.net/2013/04/30/curious-channels
	}

	shutdownLogger.Printf("\n.signal: %s; shutting down...\n", sig.String())
	destruct()
	shutdownLogger.Printf(".shutdown: program exit at %s\n", time.Now().Format(time.RFC3339))
	os.Exit(1)
}
<commit_msg>cleans up comments<commit_after>package shutdown

import (
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"
)

var (
	sysSigChan     chan os.Signal
	shutdownLogger = log.New(os.Stderr, "", 0) // log to stderr without the timestamps
)

// Destructor is a func takes no args and returns no values. It is executed as an injectable destructor
// so that the calling context could use this func to sync.Wait() or print a pretty exit message.
type Destructor func()

// SignalChan is a blank channel used to signal a shutdown
type SignalChan chan struct{}

// Watch is our signal watching func. It's worth noting that this is a
// blocking action so it should be run in a goroutine or as the last function call
// such that all the other goroutines can continue working while the application
// sits blocked here.
func Watch(shutdown SignalChan, destruct Destructor) {

	sysSigChan = make(chan os.Signal, 1)
	signal.Notify(sysSigChan, syscall.SIGINT)
	signal.Notify(sysSigChan, syscall.SIGTERM)

	var sig os.Signal

	select { // block until we get a signal
	case sig = <-sysSigChan:
		close(shutdown)
	}

	shutdownLogger.Printf("\n.signal: %s; shutting down...\n", sig.String())
	destruct()
	shutdownLogger.Printf(".shutdown: program exit at %s\n", time.Now().Format(time.RFC3339))
	os.Exit(1)
}

// For a deeper discussion of the close channel idiom: http://dave.cheney.net/2013/04/30/curious-channels
<|endoftext|>


<commit_before>package mailgun

import (
	"encoding/json"
	"fmt"
	"net"
	"strings"
	"time"
)

type EventType uint8

const (
	EventUnknown EventType = iota
	EventAccepted
	EventRejected
	EventDelivered
	EventFailed
	EventOpened
	EventClicked
	EventUnsubscribed
	EventComplained
	EventStored
	EventDropped
)

var eventTypes = []string{
	"unknown",
	"accepted",
	"rejected",
	"delivered",
	"failed",
	"opened",
	"clicked",
	"unsubscribed",
	"complained",
	"stored",
	"dropped",
}

func (et EventType) String() string {
	return eventTypes[et]
}

// MarshalText satisfies TextMarshaler
func (et EventType) MarshalText() ([]byte, error) {
	return []byte(et.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (et *EventType) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(eventTypes); i++ {
		if enum == eventTypes[i] {
			*et = EventType(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event type '%s'", enum)
}

type TimestampNano time.Time

// MarshalText satisfies JSONMarshaler
func (tn TimestampNano) MarshalJSON() ([]byte, error) {
	t := time.Time(tn)
	v := float64(t.Unix()) + float64(t.Nanosecond())/float64(time.Nanosecond)
	return json.Marshal(v)
}

// UnmarshalText satisfies JSONUnmarshaler
func (tn *TimestampNano) UnmarshalJSON(data []byte) error {
	var v float64
	err := json.Unmarshal(data, &v)
	if err == nil {
		*tn = TimestampNano(time.Unix(0, int64(v*float64(time.Second))))
	}
	return err
}

type IP net.IP

// MarshalText satisfies TextMarshaler
func (i IP) MarshalText() ([]byte, error) {
	return []byte(net.IP(i).String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (i *IP) UnmarshalText(text []byte) error {
	s := strings.Trim(string(text), "\"")
	v := net.ParseIP(s)
	if v != nil {
		*i = IP(v)
	}
	return nil
}

type Method uint8

const (
	MethodUnknown Method = iota
	MethodSMTP
	MethodHTTP
)

var methods = []string{
	"unknown",
	"smtp",
	"http",
}

func (m Method) String() string {
	return methods[m]
}

// MarshalText satisfies TextMarshaler
func (m Method) MarshalText() ([]byte, error) {
	return []byte(m.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (m *Method) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(methods); i++ {
		if enum == methods[i] {
			*m = Method(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event method '%s'", enum)
}

type EventSeverity uint8

const (
	SeverityUnknown EventSeverity = iota
	SeverityTemporary
	SeverityPermanent
	SeverityInternal
)

var severities = []string{
	"unknown",
	"temporary",
	"permanent",
	"internal",
}

func (es EventSeverity) String() string {
	return severities[es]
}

// MarshalText satisfies TextMarshaler
func (es EventSeverity) MarshalText() ([]byte, error) {
	return []byte(es.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (es *EventSeverity) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(severities); i++ {
		if enum == severities[i] {
			*es = EventSeverity(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event severity '%s'", enum)
}

type EventReason uint8

const (
	ReasonUnknown EventReason = iota
	ReasonGeneric
	ReasonBounce
	ReasonESPBlock
	ReasonSuppressBounce
	ReasonSuppressComplaint
	ReasonSuppressUnsubscribe
	ReasonOld
	ReasonHardFail
)

var eventReasons = []string{
	"unknown",
	"generic",
	"bounce",
	"espblock",
	"suppress-bounce",
	"suppress-complaint",
	"suppress-unsubscribe",
	"old",
	"hardfail",
}

func (er EventReason) String() string {
	return eventReasons[er]
}

// MarshalText satisfies TextMarshaler
func (er EventReason) MarshalText() ([]byte, error) {
	return []byte(er.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (er *EventReason) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(eventReasons); i++ {
		if enum == eventReasons[i] {
			*er = EventReason(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event reason '%s'", enum)
}

type ClientType uint

const (
	ClientUnknown ClientType = iota
	ClientMobileBrowser
	ClientBrowser
	ClientEmail
	ClientLibrary
	ClientRobot
	ClientOther
)

var clientTypes = []string{
	"unknown",
	"mobile browser",
	"browser",
	"email client",
	"library",
	"robot",
	"other",
}

func (ct ClientType) String() string {
	return clientTypes[ct]
}

// MarshalText satisfies TextMarshaler
func (ct ClientType) MarshalText() ([]byte, error) {
	return []byte(ct.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (ct *ClientType) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(clientTypes); i++ {
		if enum == clientTypes[i] {
			*ct = ClientType(i)
			return nil
		}
	}
	return fmt.Errorf("unknown client type '%s'", enum)
}

type DeviceType uint

const (
	DeviceUnknown DeviceType = iota
	DeviceMobileBrowser
	DeviceBrowser
	DeviceEmail
	DeviceOther
)

var deviceTypes = []string{
	"unknown",
	"desktop",
	"mobile",
	"tablet",
	"other",
}

func (ct DeviceType) String() string {
	return deviceTypes[ct]
}

// MarshalText satisfies TextMarshaler
func (ct DeviceType) MarshalText() ([]byte, error) {
	return []byte(ct.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (ct *DeviceType) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(deviceTypes); i++ {
		if enum == deviceTypes[i] {
			*ct = DeviceType(i)
			return nil
		}
	}
	return fmt.Errorf("unknown device type '%s'", enum)
}

type TransportMethod uint

const (
	TransportUnknown TransportMethod = iota
	TransportHTTP
	TransportSMTP
)

var transportMethods = []string{
	"unknown",
	"http",
	"smtp",
}

func (tm TransportMethod) String() string {
	return transportMethods[tm]
}

// MarshalText satisfies TextMarshaler
func (tm TransportMethod) MarshalText() ([]byte, error) {
	return []byte(tm.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (tm *TransportMethod) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(transportMethods); i++ {
		if enum == transportMethods[i] {
			*tm = TransportMethod(i)
			return nil
		}
	}
	return fmt.Errorf("unknown transport method '%s'", enum)
}
<commit_msg>Add greylisted and blacklisted reasons<commit_after>package mailgun

import (
	"encoding/json"
	"fmt"
	"net"
	"strings"
	"time"
)

type EventType uint8

const (
	EventUnknown EventType = iota
	EventAccepted
	EventRejected
	EventDelivered
	EventFailed
	EventOpened
	EventClicked
	EventUnsubscribed
	EventComplained
	EventStored
	EventDropped
)

var eventTypes = []string{
	"unknown",
	"accepted",
	"rejected",
	"delivered",
	"failed",
	"opened",
	"clicked",
	"unsubscribed",
	"complained",
	"stored",
	"dropped",
}

func (et EventType) String() string {
	return eventTypes[et]
}

// MarshalText satisfies TextMarshaler
func (et EventType) MarshalText() ([]byte, error) {
	return []byte(et.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (et *EventType) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(eventTypes); i++ {
		if enum == eventTypes[i] {
			*et = EventType(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event type '%s'", enum)
}

type TimestampNano time.Time

// MarshalText satisfies JSONMarshaler
func (tn TimestampNano) MarshalJSON() ([]byte, error) {
	t := time.Time(tn)
	v := float64(t.Unix()) + float64(t.Nanosecond())/float64(time.Nanosecond)
	return json.Marshal(v)
}

// UnmarshalText satisfies JSONUnmarshaler
func (tn *TimestampNano) UnmarshalJSON(data []byte) error {
	var v float64
	err := json.Unmarshal(data, &v)
	if err == nil {
		*tn = TimestampNano(time.Unix(0, int64(v*float64(time.Second))))
	}
	return err
}

type IP net.IP

// MarshalText satisfies TextMarshaler
func (i IP) MarshalText() ([]byte, error) {
	return []byte(net.IP(i).String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (i *IP) UnmarshalText(text []byte) error {
	s := strings.Trim(string(text), "\"")
	v := net.ParseIP(s)
	if v != nil {
		*i = IP(v)
	}
	return nil
}

type Method uint8

const (
	MethodUnknown Method = iota
	MethodSMTP
	MethodHTTP
)

var methods = []string{
	"unknown",
	"smtp",
	"http",
}

func (m Method) String() string {
	return methods[m]
}

// MarshalText satisfies TextMarshaler
func (m Method) MarshalText() ([]byte, error) {
	return []byte(m.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (m *Method) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(methods); i++ {
		if enum == methods[i] {
			*m = Method(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event method '%s'", enum)
}

type EventSeverity uint8

const (
	SeverityUnknown EventSeverity = iota
	SeverityTemporary
	SeverityPermanent
	SeverityInternal
)

var severities = []string{
	"unknown",
	"temporary",
	"permanent",
	"internal",
}

func (es EventSeverity) String() string {
	return severities[es]
}

// MarshalText satisfies TextMarshaler
func (es EventSeverity) MarshalText() ([]byte, error) {
	return []byte(es.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (es *EventSeverity) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(severities); i++ {
		if enum == severities[i] {
			*es = EventSeverity(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event severity '%s'", enum)
}

type EventReason uint8

const (
	ReasonUnknown EventReason = iota
	ReasonGeneric
	ReasonBounce
	ReasonESPBlock
	ReasonGreylisted
	ReasonBlacklisted
	ReasonSuppressBounce
	ReasonSuppressComplaint
	ReasonSuppressUnsubscribe
	ReasonOld
	ReasonHardFail
)

var eventReasons = []string{
	"unknown",
	"generic",
	"bounce",
	"espblock",
	"greylisted",
	"blacklisted",
	"suppress-bounce",
	"suppress-complaint",
	"suppress-unsubscribe",
	"old",
	"hardfail",
}

func (er EventReason) String() string {
	return eventReasons[er]
}

// MarshalText satisfies TextMarshaler
func (er EventReason) MarshalText() ([]byte, error) {
	return []byte(er.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (er *EventReason) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(eventReasons); i++ {
		if enum == eventReasons[i] {
			*er = EventReason(i)
			return nil
		}
	}
	return fmt.Errorf("unknown event reason '%s'", enum)
}

type ClientType uint

const (
	ClientUnknown ClientType = iota
	ClientMobileBrowser
	ClientBrowser
	ClientEmail
	ClientLibrary
	ClientRobot
	ClientOther
)

var clientTypes = []string{
	"unknown",
	"mobile browser",
	"browser",
	"email client",
	"library",
	"robot",
	"other",
}

func (ct ClientType) String() string {
	return clientTypes[ct]
}

// MarshalText satisfies TextMarshaler
func (ct ClientType) MarshalText() ([]byte, error) {
	return []byte(ct.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (ct *ClientType) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(clientTypes); i++ {
		if enum == clientTypes[i] {
			*ct = ClientType(i)
			return nil
		}
	}
	return fmt.Errorf("unknown client type '%s'", enum)
}

type DeviceType uint

const (
	DeviceUnknown DeviceType = iota
	DeviceMobileBrowser
	DeviceBrowser
	DeviceEmail
	DeviceOther
)

var deviceTypes = []string{
	"unknown",
	"desktop",
	"mobile",
	"tablet",
	"other",
}

func (ct DeviceType) String() string {
	return deviceTypes[ct]
}

// MarshalText satisfies TextMarshaler
func (ct DeviceType) MarshalText() ([]byte, error) {
	return []byte(ct.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (ct *DeviceType) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(deviceTypes); i++ {
		if enum == deviceTypes[i] {
			*ct = DeviceType(i)
			return nil
		}
	}
	return fmt.Errorf("unknown device type '%s'", enum)
}

type TransportMethod uint

const (
	TransportUnknown TransportMethod = iota
	TransportHTTP
	TransportSMTP
)

var transportMethods = []string{
	"unknown",
	"http",
	"smtp",
}

func (tm TransportMethod) String() string {
	return transportMethods[tm]
}

// MarshalText satisfies TextMarshaler
func (tm TransportMethod) MarshalText() ([]byte, error) {
	return []byte(tm.String()), nil
}

// UnmarshalText satisfies TextUnmarshaler
func (tm *TransportMethod) UnmarshalText(text []byte) error {
	enum := string(text)
	for i := 0; i < len(transportMethods); i++ {
		if enum == transportMethods[i] {
			*tm = TransportMethod(i)
			return nil
		}
	}
	return fmt.Errorf("unknown transport method '%s'", enum)
}
<|endoftext|>


<commit_before><commit_msg>backward compatability<commit_after><|endoftext|>


<commit_before>package main

import (
	"github.com/opslabjpl/etcdq.git/etcdq"
	"log"
	"os"
)

func main() {
	config := &etcdq.WorkerConfig{
		AwsAccessKey: os.Getenv("AWS_ACCESS_KEY"),
		AwsSecretKey: os.Getenv("AWS_SECRET_KEY"),
		AwsRegion:    os.Getenv("AWS_REGION"),
		S3Bucket:     os.Getenv("S3_BUCKET"),
		Workspace:    os.Getenv("WORKSPACE"),
		EarthkitImg:  os.Getenv("EARTHKIT_IMG"),
		DataDir:      os.Getenv("DATA_DIR"),
	}
	if config.AwsAccessKey == "" {
		log.Fatal("Missing environment variable AWS_ACCESS_KEY")
	}
	if config.AwsSecretKey == "" {
		log.Fatal("Missing environment variable AWS_SECRET_KEY")
	}
	if config.AwsRegion == "" {
		log.Fatal("Missing environment variable AWS_REGION")
	}
	if config.S3Bucket == "" {
		log.Fatal("Missing environment variable S3_BUCKET")
	}
	if config.Workspace == "" {
		log.Fatal("Missing environment variable WORKSPACE")
	}
	if config.EarthkitImg == "" {
		log.Fatal("Missing environment variable EARTHKIT_IMG")
	}
	if config.DataDir == "" {
		config.DataDir = "/tmp"
	}

	worker, err := etcdq.NewWorker(config)
	if err != nil {
		log.Printf("Error: %s", err)
		return
	}
	worker.Start()
}
<commit_msg>remove .git suffix from imports<commit_after>package main

import (
	"github.com/opslabjpl/etcdq/etcdq"
	"log"
	"os"
)

func main() {
	config := &etcdq.WorkerConfig{
		AwsAccessKey: os.Getenv("AWS_ACCESS_KEY"),
		AwsSecretKey: os.Getenv("AWS_SECRET_KEY"),
		AwsRegion:    os.Getenv("AWS_REGION"),
		S3Bucket:     os.Getenv("S3_BUCKET"),
		Workspace:    os.Getenv("WORKSPACE"),
		EarthkitImg:  os.Getenv("EARTHKIT_IMG"),
		DataDir:      os.Getenv("DATA_DIR"),
	}
	if config.AwsAccessKey == "" {
		log.Fatal("Missing environment variable AWS_ACCESS_KEY")
	}
	if config.AwsSecretKey == "" {
		log.Fatal("Missing environment variable AWS_SECRET_KEY")
	}
	if config.AwsRegion == "" {
		log.Fatal("Missing environment variable AWS_REGION")
	}
	if config.S3Bucket == "" {
		log.Fatal("Missing environment variable S3_BUCKET")
	}
	if config.Workspace == "" {
		log.Fatal("Missing environment variable WORKSPACE")
	}
	if config.EarthkitImg == "" {
		log.Fatal("Missing environment variable EARTHKIT_IMG")
	}
	if config.DataDir == "" {
		config.DataDir = "/tmp"
	}

	worker, err := etcdq.NewWorker(config)
	if err != nil {
		log.Printf("Error: %s", err)
		return
	}
	worker.Start()
}
<|endoftext|>


<commit_before>package traytor

import "fmt"

func ExampleVec3_ToZero() {
	v := NewVec3(1, 2, 3)
	fmt.Printf("%s\n", v)
	v.ToZero()
	fmt.Printf("%s\n", v)

	// Output:
	// (1, 2, 3)
	// (0, 0, 0)
	//
}
<commit_msg>Fixed vector so it's exported<commit_after>package traytor

import (
	"fmt"
)

func ExampleVec3_ToZero() {
	v := NewVec3(1, 2, 3)
	fmt.Printf("%s\n", v)
	v.ToZero()
	fmt.Printf("%s\n", v)

	// Output:
	// (1, 2, 3)
	// (0, 0, 0)
	//
}

func ExampleVec3_Length() {
	v := NewVec3(1, 2, 2)
	fmt.Printf("%.3g\n", v.Length())

	// Output:
	// 3
	//
}
<|endoftext|>


<commit_before>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    sflow.go
//: details: sflow decoding raw data samples
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package main

import (
	"bytes"
	"encoding/json"
	"net"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/VerizonDigital/vflow/packet"
	"github.com/VerizonDigital/vflow/producer"
	"github.com/VerizonDigital/vflow/sflow"
)

// SFUDPMsg represents sFlow UDP message
type SFUDPMsg struct {
	raddr *net.UDPAddr
	body  []byte
}

// SFlow represents sFlow collector
type SFlow struct {
	port    int
	addr    string
	workers int
	stop    bool
	stats   SFlowStats
}

// SFlowStats represents sflow stats
type SFlowStats struct {
	UDPQueue     int
	MessageQueue int
	UDPCount     uint64
	DecodedCount uint64
	MQErrorCount uint64
	BusyWorker   int32
}

var (
	sFlowUDPCh = make(chan SFUDPMsg, 1000)
	sFlowMQCh  = make(chan []byte, 1000)

	// sflow udp payload pool
	sFlowBuffer = &sync.Pool{
		New: func() interface{} {
			return make([]byte, opts.SFlowUDPSize)
		},
	}
)

// NewSFlow constructs sFlow collector
func NewSFlow() *SFlow {
	logger = opts.Logger

	return &SFlow{
		port:    opts.SFlowPort,
		workers: opts.SFlowWorkers,
	}
}

func (s *SFlow) run() {
	var wg sync.WaitGroup

	// exit if the sflow is disabled
	if !opts.SFlowEnabled {
		logger.Println("sflow has been disabled")
		return
	}

	hostPort := net.JoinHostPort(s.addr, strconv.Itoa(s.port))
	udpAddr, _ := net.ResolveUDPAddr("udp", hostPort)

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		logger.Fatal(err)
	}

	wg.Add(s.workers)
	atomic.AddInt32(&s.stats.BusyWorker, int32(s.workers))

	for i := 0; i < s.workers; i++ {
		go func() {
			defer wg.Done()
			s.sFlowWorker()
		}()
	}

	logger.Printf("sFlow is running (workers#: %d)", s.workers)

	go func() {
		p := producer.NewProducer(opts.MQName)

		p.MQConfigFile = opts.MQConfigFile
		p.MQErrorCount = &s.stats.MQErrorCount
		p.Logger = logger
		p.Chan = sFlowMQCh
		p.Topic = "sflow"

		if err := p.Run(); err != nil {
			logger.Fatal(err)
		}
	}()

	for !s.stop {
		b := sFlowBuffer.Get().([]byte)
		conn.SetReadDeadline(time.Now().Add(1e9))
		n, raddr, err := conn.ReadFromUDP(b)
		if err != nil {
			continue
		}
		atomic.AddUint64(&s.stats.UDPCount, 1)
		sFlowUDPCh <- SFUDPMsg{raddr, b[:n]}
	}

	wg.Wait()
}

func (s *SFlow) shutdown() {
	s.stop = true
	logger.Println("stopping sflow service gracefully ...")
	time.Sleep(1 * time.Second)
	logger.Println("vFlow has been shutdown")
	close(sFlowUDPCh)
}

func (s *SFlow) sFlowWorker() {
	var (
		filter = []uint32{sflow.DataCounterSample}
		reader *bytes.Reader
		msg    SFUDPMsg
		ok     bool
		b      []byte
	)

	for {
		atomic.AddInt32(&s.stats.BusyWorker, -1)

		if msg, ok = <-sFlowUDPCh; !ok {
			break
		}

		if opts.Verbose {
			logger.Printf("rcvd sflow data from: %s, size: %d bytes",
				msg.raddr, len(msg.body))
		}

		atomic.AddInt32(&s.stats.BusyWorker, 1)

		reader = bytes.NewReader(msg.body)
		d := sflow.NewSFDecoder(reader, filter)
		records, err := d.SFDecode()
		if err != nil || len(records) < 1 {
			sFlowBuffer.Put(msg.body[:opts.SFlowUDPSize])
			continue
		}

		decodedMsg := sflow.Message{}

		for _, data := range records {
			switch data.(type) {
			case *packet.Packet:
				decodedMsg.Packet = data.(*packet.Packet)
			case *sflow.ExtSwitchData:
				decodedMsg.ExtSWData = data.(*sflow.ExtSwitchData)
			case *sflow.FlowSample:
				decodedMsg.Sample = data.(*sflow.FlowSample)
			case *sflow.SFDatagram:
				decodedMsg.Header = data.(*sflow.SFDatagram)
			}
		}

		b, err = json.Marshal(decodedMsg)
		if err != nil {
			sFlowBuffer.Put(msg.body[:opts.SFlowUDPSize])
			logger.Println(err)
			continue
		}

		atomic.AddUint64(&s.stats.DecodedCount, 1)

		if opts.Verbose {
			logger.Println(string(b))
		}

		select {
		case sFlowMQCh <- b:
		default:
		}

		sFlowBuffer.Put(msg.body[:opts.SFlowUDPSize])
	}
}

func (s *SFlow) status() *SFlowStats {
	return &SFlowStats{
		UDPQueue:     len(sFlowUDPCh),
		MessageQueue: len(sFlowMQCh),
		UDPCount:     atomic.LoadUint64(&s.stats.UDPCount),
		DecodedCount: atomic.LoadUint64(&s.stats.DecodedCount),
		MQErrorCount: atomic.LoadUint64(&s.stats.MQErrorCount),
		BusyWorker:   atomic.LoadInt32(&s.stats.BusyWorker),
	}
}
<commit_msg>add worker pool to achieve dynamic workers<commit_after>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    sflow.go
//: details: sflow decoding raw data samples
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------
package main

import (
	"bytes"
	"encoding/json"
	"net"
	"runtime"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/VerizonDigital/vflow/packet"
	"github.com/VerizonDigital/vflow/producer"
	"github.com/VerizonDigital/vflow/sflow"
)

// SFUDPMsg represents sFlow UDP message
type SFUDPMsg struct {
	raddr *net.UDPAddr
	body  []byte
}

// SFlow represents sFlow collector
type SFlow struct {
	port    int
	addr    string
	workers int
	stop    bool
	stats   SFlowStats
	pool    chan chan struct{}
}

// SFlowStats represents sflow stats
type SFlowStats struct {
	UDPQueue     int
	MessageQueue int
	UDPCount     uint64
	DecodedCount uint64
	MQErrorCount uint64
	Workers      int32
}

var (
	sFlowUDPCh = make(chan SFUDPMsg, 1000)
	sFlowMQCh  = make(chan []byte, 1000)

	// sflow udp payload pool
	sFlowBuffer = &sync.Pool{
		New: func() interface{} {
			return make([]byte, opts.SFlowUDPSize)
		},
	}
)

// NewSFlow constructs sFlow collector
func NewSFlow() *SFlow {
	logger = opts.Logger

	return &SFlow{
		port:    opts.SFlowPort,
		workers: opts.SFlowWorkers,
		pool:    make(chan chan struct{}, 1e6),
	}
}

func (s *SFlow) run() {
	// exit if the sflow is disabled
	if !opts.SFlowEnabled {
		logger.Println("sflow has been disabled")
		return
	}

	hostPort := net.JoinHostPort(s.addr, strconv.Itoa(s.port))
	udpAddr, _ := net.ResolveUDPAddr("udp", hostPort)

	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		logger.Fatal(err)
	}

	atomic.AddInt32(&s.stats.Workers, int32(s.workers))
	for i := 0; i < s.workers; i++ {
		go func() {
			wQuit := make(chan struct{})
			s.pool <- wQuit
			s.sFlowWorker(wQuit)
		}()
	}

	logger.Printf("sFlow is running (workers#: %d)", s.workers)

	go func() {
		p := producer.NewProducer(opts.MQName)

		p.MQConfigFile = opts.MQConfigFile
		p.MQErrorCount = &s.stats.MQErrorCount
		p.Logger = logger
		p.Chan = sFlowMQCh
		p.Topic = "sflow"

		if err := p.Run(); err != nil {
			logger.Fatal(err)
		}
	}()

	go func() {
		if !opts.DynWorkers {
			logger.Println("sFlow dynamic worker disabled")
			return
		}

		s.dynWorkers()
	}()

	for !s.stop {
		b := sFlowBuffer.Get().([]byte)
		conn.SetReadDeadline(time.Now().Add(1e9))
		n, raddr, err := conn.ReadFromUDP(b)
		if err != nil {
			continue
		}

		atomic.AddUint64(&s.stats.UDPCount, 1)
		sFlowUDPCh <- SFUDPMsg{raddr, b[:n]}
	}

}

func (s *SFlow) shutdown() {
	s.stop = true
	logger.Println("stopping sflow service gracefully ...")
	time.Sleep(1 * time.Second)
	logger.Println("vFlow has been shutdown")
	close(sFlowUDPCh)
}

func (s *SFlow) sFlowWorker(wQuit chan struct{}) {
	var (
		filter = []uint32{sflow.DataCounterSample}
		reader *bytes.Reader
		msg    SFUDPMsg
		ok     bool
		b      []byte
	)

LOOP:
	for {

		select {
		case <-wQuit:
			break LOOP
		case msg, ok = <-sFlowUDPCh:
			if !ok {
				break LOOP
			}
		}

		if opts.Verbose {
			logger.Printf("rcvd sflow data from: %s, size: %d bytes",
				msg.raddr, len(msg.body))
		}

		reader = bytes.NewReader(msg.body)
		d := sflow.NewSFDecoder(reader, filter)
		records, err := d.SFDecode()
		if err != nil || len(records) < 1 {
			sFlowBuffer.Put(msg.body[:opts.SFlowUDPSize])
			continue
		}

		decodedMsg := sflow.Message{}

		for _, data := range records {
			switch data.(type) {
			case *packet.Packet:
				decodedMsg.Packet = data.(*packet.Packet)
			case *sflow.ExtSwitchData:
				decodedMsg.ExtSWData = data.(*sflow.ExtSwitchData)
			case *sflow.FlowSample:
				decodedMsg.Sample = data.(*sflow.FlowSample)
			case *sflow.SFDatagram:
				decodedMsg.Header = data.(*sflow.SFDatagram)
			}
		}

		b, err = json.Marshal(decodedMsg)
		if err != nil {
			sFlowBuffer.Put(msg.body[:opts.SFlowUDPSize])
			logger.Println(err)
			continue
		}

		atomic.AddUint64(&s.stats.DecodedCount, 1)

		if opts.Verbose {
			logger.Println(string(b))
		}

		select {
		case sFlowMQCh <- b:
		default:
		}

		sFlowBuffer.Put(msg.body[:opts.SFlowUDPSize])
	}
}

func (s *SFlow) status() *SFlowStats {
	return &SFlowStats{
		UDPQueue:     len(sFlowUDPCh),
		MessageQueue: len(sFlowMQCh),
		UDPCount:     atomic.LoadUint64(&s.stats.UDPCount),
		DecodedCount: atomic.LoadUint64(&s.stats.DecodedCount),
		MQErrorCount: atomic.LoadUint64(&s.stats.MQErrorCount),
		Workers:      atomic.LoadInt32(&s.stats.Workers),
	}
}

func (s *SFlow) dynWorkers() {
	var load, normalSeq, newWorkers, n int

	maxWorkers := int32(runtime.NumCPU() * 1e4)
	tick := time.Tick(120 * time.Second)

	for {
		<-tick
		load = 0

		for n = 0; n < 30; n++ {
			time.Sleep(1 * time.Second)
			load += len(sFlowUDPCh)
		}

		if load > 15 && s.stats.Workers < maxWorkers {

			switch {
			case load > 300:
				newWorkers = 100
			case load > 200:
				newWorkers = 60
			case load > 100:
				newWorkers = 40
			default:
				newWorkers = 30
			}

			for n = 0; n < newWorkers; n++ {
				go func() {
					atomic.AddInt32(&s.stats.Workers, 1)
					wQuit := make(chan struct{})
					s.pool <- wQuit
					s.sFlowWorker(wQuit)
				}()
			}

		}

		if load == 0 {
			normalSeq++
		} else {
			normalSeq = 0
			continue
		}

		if normalSeq > 15 {
			for n = 0; n < 10; n++ {
				if len(s.pool) > s.workers {
					atomic.AddInt32(&s.stats.Workers, -1)
					wQuit := <-s.pool
					close(wQuit)
				}
			}
			normalSeq = 0
		}
	}
}
<|endoftext|>


<commit_before>package logger

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	"github.com/oinume/lekcije/server/config"
	"github.com/uber-go/zap"
)

var (
	Access = zap.New(zap.NewJSONEncoder(zap.RFC3339Formatter("ts")), zap.Output(os.Stdout))
	App    = zap.New(zap.NewJSONEncoder(zap.RFC3339Formatter("ts")), zap.Output(os.Stderr))
)

func init() {
	if !config.IsProductionEnv() {
		App.SetLevel(zap.DebugLevel)
	}
}

func InitializeAccessLogger(writer io.Writer) {
	Access = zap.New(
		zap.NewJSONEncoder(zap.RFC3339Formatter("ts")),
		zap.Output(zap.AddSync(writer)),
	)
}

func InitializeAppLogger(writer io.Writer) {
	App = zap.New(
		zap.NewJSONEncoder(zap.RFC3339Formatter("ts")),
		zap.Output(zap.AddSync(writer)),
	)
	if !config.IsProductionEnv() {
		App.SetLevel(zap.DebugLevel)
	}
}

func NewLevel(level string) zap.Level {
	var l zap.Level
	switch strings.ToLower(level) {
	case "debug":
		l = zap.DebugLevel
	case "info":
		l = zap.InfoLevel
	case "warn":
		l = zap.WarnLevel
	case "error":
		l = zap.ErrorLevel
	case "panic":
		l = zap.PanicLevel
	case "fatal":
		l = zap.FatalLevel
	default:
		l = zap.InfoLevel
	}
	return l
}

type LoggingHTTPTransport struct {
	DumpHeaderBody bool
}

func (t *LoggingHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	var reqDump bytes.Buffer
	chunked := false
	if t.DumpHeaderBody {
		if len(req.Header) > 0 {
			fmt.Fprintln(&reqDump, "--- Request Header ---")
			for k, v := range req.Header {
				fmt.Fprintf(&reqDump, "%s: %s\n", k, strings.Join(v, ","))
			}
			chunked = len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"
			if len(req.TransferEncoding) > 0 {
				fmt.Fprintf(&reqDump, "Transfer-Encoding: %s\r\n", strings.Join(req.TransferEncoding, ","))
			}
			if req.Close {
				fmt.Fprintf(&reqDump, "Connection: close\r\n")
			}
		}
		if req.Body != nil {
			fmt.Fprintln(&reqDump, "--- Request Body ---")
			dump, _ := dumpRequestBody(req, chunked)
			fmt.Fprint(&reqDump, string(dump))
		}
	}

	start := time.Now()
	resp, err := http.DefaultTransport.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	end := time.Now()

	var out bytes.Buffer
	fmt.Fprintf(
		&out, "%s %s %d %d\n",
		req.Method, req.URL,
		resp.StatusCode, time.Duration(end.Sub(start).Nanoseconds())/time.Millisecond,
	)

	var respDump bytes.Buffer
	if t.DumpHeaderBody {
		if len(resp.Header) > 0 {
			fmt.Fprintln(&respDump, "--- Response Header ---")
			for k, v := range resp.Header {
				fmt.Fprintf(&respDump, "%s: %s\n", k, strings.Join(v, ","))
			}
		}
		save := resp.Body
		savecl := resp.ContentLength
		if resp.Body != nil {
			save, resp.Body, _ = drainBody(resp.Body)
			fmt.Fprintln(&respDump, "--- Response Body ---")
			_, _ = io.Copy(&respDump, resp.Body)
		}
		resp.Body = save
		resp.ContentLength = savecl
	}

	if s := reqDump.String(); s != "" {
		fmt.Fprintln(&out, s)
	}
	if s := respDump.String(); s != "" {
		fmt.Fprintln(&out, s)
	}

	if !config.IsProductionEnv() {
		fmt.Println(out.String())
	}

	return resp, err
}

func (t *LoggingHTTPTransport) CancelRequest(r *http.Request) {
}

func dumpRequestBody(req *http.Request, chunked bool) ([]byte, error) {
	// https://github.com/golang/go/blob/master/src/net/http/httputil/dump.go#L187 のDumpRequestを参考にしている
	if req.Body == nil {
		return []byte{}, nil
	}

	var err error
	var save io.ReadCloser
	save, req.Body, err = drainBody(req.Body)
	if err != nil {
		return nil, err
	}

	var b bytes.Buffer
	var dest io.Writer = &b
	if chunked {
		dest = httputil.NewChunkedWriter(dest)
	}
	_, err = io.Copy(dest, req.Body)
	if chunked {
		dest.(io.Closer).Close()
		io.WriteString(&b, "\r\n")
	}

	req.Body = save
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, nil, err
	}
	if err = b.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}
<commit_msg>Close response body<commit_after>package logger

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httputil"
	"os"
	"strings"
	"time"

	"github.com/oinume/lekcije/server/config"
	"github.com/uber-go/zap"
)

var (
	Access = zap.New(zap.NewJSONEncoder(zap.RFC3339Formatter("ts")), zap.Output(os.Stdout))
	App    = zap.New(zap.NewJSONEncoder(zap.RFC3339Formatter("ts")), zap.Output(os.Stderr))
)

func init() {
	if !config.IsProductionEnv() {
		App.SetLevel(zap.DebugLevel)
	}
}

func InitializeAccessLogger(writer io.Writer) {
	Access = zap.New(
		zap.NewJSONEncoder(zap.RFC3339Formatter("ts")),
		zap.Output(zap.AddSync(writer)),
	)
}

func InitializeAppLogger(writer io.Writer) {
	App = zap.New(
		zap.NewJSONEncoder(zap.RFC3339Formatter("ts")),
		zap.Output(zap.AddSync(writer)),
	)
	if !config.IsProductionEnv() {
		App.SetLevel(zap.DebugLevel)
	}
}

func NewLevel(level string) zap.Level {
	var l zap.Level
	switch strings.ToLower(level) {
	case "debug":
		l = zap.DebugLevel
	case "info":
		l = zap.InfoLevel
	case "warn":
		l = zap.WarnLevel
	case "error":
		l = zap.ErrorLevel
	case "panic":
		l = zap.PanicLevel
	case "fatal":
		l = zap.FatalLevel
	default:
		l = zap.InfoLevel
	}
	return l
}

type LoggingHTTPTransport struct {
	DumpHeaderBody bool
}

func (t *LoggingHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	var reqDump bytes.Buffer
	chunked := false
	if t.DumpHeaderBody {
		if len(req.Header) > 0 {
			fmt.Fprintln(&reqDump, "--- Request Header ---")
			for k, v := range req.Header {
				fmt.Fprintf(&reqDump, "%s: %s\n", k, strings.Join(v, ","))
			}
			chunked = len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"
			if len(req.TransferEncoding) > 0 {
				fmt.Fprintf(&reqDump, "Transfer-Encoding: %s\r\n", strings.Join(req.TransferEncoding, ","))
			}
			if req.Close {
				fmt.Fprint(&reqDump, "Connection: close\r\n")
			}
		}
		if req.Body != nil {
			fmt.Fprintln(&reqDump, "--- Request Body ---")
			dump, _ := dumpRequestBody(req, chunked)
			fmt.Fprint(&reqDump, string(dump))
		}
	}

	start := time.Now()
	resp, err := http.DefaultTransport.RoundTrip(req)
	if err != nil {
		return nil, err
	}
	end := time.Now()

	var out bytes.Buffer
	fmt.Fprintf(
		&out, "%s %s %d %d\n",
		req.Method, req.URL,
		resp.StatusCode, time.Duration(end.Sub(start).Nanoseconds())/time.Millisecond,
	)

	var respDump bytes.Buffer
	if t.DumpHeaderBody {
		if len(resp.Header) > 0 {
			fmt.Fprintln(&respDump, "--- Response Header ---")
			for k, v := range resp.Header {
				fmt.Fprintf(&respDump, "%s: %s\n", k, strings.Join(v, ","))
			}
		}
		save := resp.Body
		savecl := resp.ContentLength
		respBody := resp.Body
		defer respBody.Close()
		if resp.Body != nil {
			save, resp.Body, _ = drainBody(resp.Body)
			fmt.Fprintln(&respDump, "--- Response Body ---")
			_, _ = io.Copy(&respDump, resp.Body)
		}
		resp.Body = save
		resp.ContentLength = savecl
	}

	if s := reqDump.String(); s != "" {
		fmt.Fprintln(&out, s)
	}
	if s := respDump.String(); s != "" {
		fmt.Fprintln(&out, s)
	}

	if !config.IsProductionEnv() {
		fmt.Println(out.String())
	}

	return resp, err
}

func (t *LoggingHTTPTransport) CancelRequest(r *http.Request) {
}

func dumpRequestBody(req *http.Request, chunked bool) ([]byte, error) {
	// https://github.com/golang/go/blob/master/src/net/http/httputil/dump.go#L187 のDumpRequestを参考にしている
	if req.Body == nil {
		return []byte{}, nil
	}

	var err error
	var save io.ReadCloser
	save, req.Body, err = drainBody(req.Body)
	if err != nil {
		return nil, err
	}

	var b bytes.Buffer
	var dest io.Writer = &b
	if chunked {
		dest = httputil.NewChunkedWriter(dest)
	}
	_, err = io.Copy(dest, req.Body)
	if chunked {
		dest.(io.Closer).Close()
		io.WriteString(&b, "\r\n")
	}

	req.Body = save
	if err != nil {
		return nil, err
	}
	return b.Bytes(), nil
}

func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, nil, err
	}
	if err = b.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil
}
<|endoftext|>


<commit_before>package model

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/errors"
)

const teacherUrlBase = "http://eikaiwa.dmm.com/teacher/index/%v/"

var (
	defaultLastLessonAt = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
	idsRegexp           = regexp.MustCompile(`^[\d,]+$`)
	teacherUrlRegexp    = regexp.MustCompile(`https?://eikaiwa.dmm.com/teacher/index/([\d]+)`)
)

type Teacher struct {
	ID                uint32
	Name              string
	CountryID         uint16
	Gender            string
	Birthday          time.Time
	YearsOfExperience uint8
	FavoriteCount     uint32
	ReviewCount       uint32
	Rating            float32
	LastLessonAt      time.Time
	FetchErrorCount   uint8
	CreatedAt         time.Time
	UpdatedAt         time.Time
}

func (*Teacher) TableName() string {
	return "teacher"
}

func NewTeacher(id uint32) *Teacher {
	return &Teacher{ID: id}
}

func NewTeachersFromIDsOrURL(idsOrURL string) ([]*Teacher, error) {
	if idsRegexp.MatchString(idsOrURL) {
		ids := strings.Split(idsOrURL, ",")
		teachers := make([]*Teacher, 0, len(ids))
		for _, sid := range ids {
			if id, err := strconv.ParseUint(sid, 10, 32); err == nil {
				teachers = append(teachers, NewTeacher(uint32(id)))
			} else {
				continue
			}
		}
		return teachers, nil
	} else if group := teacherUrlRegexp.FindStringSubmatch(idsOrURL); len(group) > 0 {
		id, _ := strconv.ParseUint(group[1], 10, 32)
		return []*Teacher{NewTeacher(uint32(id))}, nil
	}
	return nil, errors.NewInvalidArgumentError(
		errors.WithMessage("Failed to parse idsOrURL"),
		errors.WithResource(errors.NewResource("teacher", "idsOrURL", idsOrURL)),
	)
}

func (t *Teacher) URL() string {
	return fmt.Sprintf(teacherUrlBase, t.ID)
}

type TeacherLessons struct {
	Teacher *Teacher
	Lessons []*Lesson
}

func NewTeacherLessons(t *Teacher, l []*Lesson) *TeacherLessons {
	return &TeacherLessons{Teacher: t, Lessons: l}
}

type TeacherService struct {
	db *gorm.DB
}

func NewTeacherService(db *gorm.DB) *TeacherService {
	return &TeacherService{db: db}
}

func (s *TeacherService) CreateOrUpdate(t *Teacher) error {
	if t.LastLessonAt.IsZero() {
		t.LastLessonAt = defaultLastLessonAt
	}
	sql := fmt.Sprintf("INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", t.TableName())
	sql += " ON DUPLICATE KEY UPDATE"
	sql += " country_id=?, gender=?, years_of_experience=?, birthday=?, favorite_count=?, review_count=?, rating=?, last_lesson_at=?"

	now := time.Now().UTC()
	values := []interface{}{
		t.ID,
		t.Name,
		t.CountryID,
		t.Gender,
		t.Birthday.Format(dbDateFormat),
		t.YearsOfExperience,
		t.FavoriteCount,
		t.ReviewCount,
		t.Rating,
		t.LastLessonAt.Format(dbDatetimeFormat),
		t.FetchErrorCount,
		now.Format(dbDatetimeFormat),
		now.Format(dbDatetimeFormat),
		// update
		t.CountryID,
		t.Gender,
		t.YearsOfExperience,
		t.Birthday.Format(dbDateFormat),
		t.FavoriteCount,
		t.ReviewCount,
		t.Rating,
		t.LastLessonAt.Format(dbDatetimeFormat),
	}
	if err := s.db.Exec(sql, values...).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to INSERT or UPDATE teacher"),
			errors.WithResource(errors.NewResource("teacher", "id", t.ID)),
		)
	}
	return nil
}

func (s *TeacherService) FindByPK(id uint32) (*Teacher, error) {
	teacher := &Teacher{}
	if err := s.db.First(teacher, &Teacher{ID: id}).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to FindByPK"),
			errors.WithResource(errors.NewResource(teacher.TableName(), "id", id)),
		)
	}
	return teacher, nil
}

func (s *TeacherService) IncrementFetchErrorCount(id uint32, value int) error {
	tableName := new(Teacher).TableName()
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = fetch_error_count + ?, updated_at = NOW() WHERE id = ?`,
		tableName,
	)
	if err := s.db.Exec(sql, value, id).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to UPDATE teacher"),
			errors.WithResource(errors.NewResource(tableName, "id", id)),
		)
	}
	return nil
}

func (s *TeacherService) ResetFetchErrorCount(id uint32) error {
	tableName := new(Teacher).TableName()
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = 0, updated_at = NOW() WHERE id = ?`,
		tableName,
	)
	if err := s.db.Exec(sql, id).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to UPDATE teacher"),
			errors.WithResource(errors.NewResource(tableName, "id", id)),
		)
	}
	return nil
}

func (s *TeacherService) FindByFetchErrorCountGt(count uint32) ([]*Teacher, error) {
	values := make([]*Teacher, 0, 3000)
	sql := fmt.Sprintf(`SELECT * FROM teacher WHERE fetch_error_count > ? ORDER BY id LIMIT 3000`)
	if result := s.db.Raw(sql, count).Scan(&values); result.Error != nil {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to FindByFetchErrorCountGt"),
		)
	}
	return values, nil
}
<commit_msg>Dont update last_lesson_at when it is default value<commit_after>package model

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/oinume/lekcije/server/errors"
)

const teacherUrlBase = "http://eikaiwa.dmm.com/teacher/index/%v/"

var (
	defaultLastLessonAt = time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
	idsRegexp           = regexp.MustCompile(`^[\d,]+$`)
	teacherUrlRegexp    = regexp.MustCompile(`https?://eikaiwa.dmm.com/teacher/index/([\d]+)`)
)

type Teacher struct {
	ID                uint32
	Name              string
	CountryID         uint16
	Gender            string
	Birthday          time.Time
	YearsOfExperience uint8
	FavoriteCount     uint32
	ReviewCount       uint32
	Rating            float32
	LastLessonAt      time.Time
	FetchErrorCount   uint8
	CreatedAt         time.Time
	UpdatedAt         time.Time
}

func (*Teacher) TableName() string {
	return "teacher"
}

func NewTeacher(id uint32) *Teacher {
	return &Teacher{ID: id}
}

func NewTeachersFromIDsOrURL(idsOrURL string) ([]*Teacher, error) {
	if idsRegexp.MatchString(idsOrURL) {
		ids := strings.Split(idsOrURL, ",")
		teachers := make([]*Teacher, 0, len(ids))
		for _, sid := range ids {
			if id, err := strconv.ParseUint(sid, 10, 32); err == nil {
				teachers = append(teachers, NewTeacher(uint32(id)))
			} else {
				continue
			}
		}
		return teachers, nil
	} else if group := teacherUrlRegexp.FindStringSubmatch(idsOrURL); len(group) > 0 {
		id, _ := strconv.ParseUint(group[1], 10, 32)
		return []*Teacher{NewTeacher(uint32(id))}, nil
	}
	return nil, errors.NewInvalidArgumentError(
		errors.WithMessage("Failed to parse idsOrURL"),
		errors.WithResource(errors.NewResource("teacher", "idsOrURL", idsOrURL)),
	)
}

func (t *Teacher) URL() string {
	return fmt.Sprintf(teacherUrlBase, t.ID)
}

type TeacherLessons struct {
	Teacher *Teacher
	Lessons []*Lesson
}

func NewTeacherLessons(t *Teacher, l []*Lesson) *TeacherLessons {
	return &TeacherLessons{Teacher: t, Lessons: l}
}

type TeacherService struct {
	db *gorm.DB
}

func NewTeacherService(db *gorm.DB) *TeacherService {
	return &TeacherService{db: db}
}

func (s *TeacherService) CreateOrUpdate(t *Teacher) error {
	sql := fmt.Sprintf("INSERT INTO %s VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", t.TableName())
	sql += " ON DUPLICATE KEY UPDATE"
	sql += " country_id=?, gender=?, years_of_experience=?, birthday=?, favorite_count=?, review_count=?, rating=?"

	if t.LastLessonAt.IsZero() {
		t.LastLessonAt = defaultLastLessonAt
	}
	now := time.Now().UTC()
	values := []interface{}{
		t.ID,
		t.Name,
		t.CountryID,
		t.Gender,
		t.Birthday.Format(dbDateFormat),
		t.YearsOfExperience,
		t.FavoriteCount,
		t.ReviewCount,
		t.Rating,
		t.LastLessonAt.Format(dbDatetimeFormat),
		t.FetchErrorCount,
		now.Format(dbDatetimeFormat),
		now.Format(dbDatetimeFormat),
		// update
		t.CountryID,
		t.Gender,
		t.YearsOfExperience,
		t.Birthday.Format(dbDateFormat),
		t.FavoriteCount,
		t.ReviewCount,
		t.Rating,
	}

	if !t.LastLessonAt.Equal(defaultLastLessonAt) {
		// UPDATE last_lesson_at only when it's NOT default value
		sql += ", last_lesson_at=?"
		values = append(values, t.LastLessonAt.Format(dbDatetimeFormat))
	}

	if err := s.db.Exec(sql, values...).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to INSERT or UPDATE teacher"),
			errors.WithResource(errors.NewResource("teacher", "id", t.ID)),
		)
	}
	return nil
}

func (s *TeacherService) FindByPK(id uint32) (*Teacher, error) {
	teacher := &Teacher{}
	if err := s.db.First(teacher, &Teacher{ID: id}).Error; err != nil {
		return nil, errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to FindByPK"),
			errors.WithResource(errors.NewResource(teacher.TableName(), "id", id)),
		)
	}
	return teacher, nil
}

func (s *TeacherService) IncrementFetchErrorCount(id uint32, value int) error {
	tableName := new(Teacher).TableName()
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = fetch_error_count + ?, updated_at = NOW() WHERE id = ?`,
		tableName,
	)
	if err := s.db.Exec(sql, value, id).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to UPDATE teacher"),
			errors.WithResource(errors.NewResource(tableName, "id", id)),
		)
	}
	return nil
}

func (s *TeacherService) ResetFetchErrorCount(id uint32) error {
	tableName := new(Teacher).TableName()
	sql := fmt.Sprintf(
		`UPDATE %s SET fetch_error_count = 0, updated_at = NOW() WHERE id = ?`,
		tableName,
	)
	if err := s.db.Exec(sql, id).Error; err != nil {
		return errors.NewInternalError(
			errors.WithError(err),
			errors.WithMessage("Failed to UPDATE teacher"),
			errors.WithResource(errors.NewResource(tableName, "id", id)),
		)
	}
	return nil
}

func (s *TeacherService) FindByFetchErrorCountGt(count uint32) ([]*Teacher, error) {
	values := make([]*Teacher, 0, 3000)
	sql := fmt.Sprintf(`SELECT * FROM teacher WHERE fetch_error_count > ? ORDER BY id LIMIT 3000`)
	if result := s.db.Raw(sql, count).Scan(&values); result.Error != nil {
		return nil, errors.NewInternalError(
			errors.WithError(result.Error),
			errors.WithMessage("Failed to FindByFetchErrorCountGt"),
		)
	}
	return values, nil
}
<|endoftext|>


<commit_before>package heap

import (
	"GoVM/chapter2-class/classpath"
	"fmt"
	"GoVM/chapter3-cf/classfile"
)

type ClassLoader struct {
	cp          *classpath.Classpath
	//是否输出加载class信息
	verboseFlag bool
	//key 是类的完全限定名称
	classMap    map[string]*Class
}

func NewClassLoader(cp *classpath.Classpath, verboseFlag bool) *ClassLoader {
	return &ClassLoader{
		cp:        cp,
		verboseFlag:        verboseFlag,
		classMap:        make(map[string]*Class),
	}
}

func (self *ClassLoader) LoadClass(name string) *Class {
	if class, ok := self.classMap[name]; ok {
		//类已经被加载过了
		return class
	}

	if name[0] == '[' {
		return self.loadArrayClass(name)
	}
	return self.loadNonArrayClass(name)
}

func (self *ClassLoader) loadNonArrayClass(name string) *Class {
	data, entry := self.readClass(name)
	class := self.defineClass(data)
	link(class)

	if self.verboseFlag {
		fmt.Printf("[Loaded %s from %s]\n", name, entry)
	}

	return class
}

/**
	加载数组类
 */
func (self *ClassLoader) loadArrayClass(name string) *Class {
	class := &Class{
		accessFlags: ACC_PUBLIC, // todo
		name:        name,
		loader:      self,
		initStarted: true,
		superClass:  self.LoadClass("java/lang/Object"),
		interfaces: []*Class{
			//数组默认实现了Cloneable和Serializable接口
			self.LoadClass("java/lang/Cloneable"),
			self.LoadClass("java/io/Serializable"),
		},
	}
	self.classMap[name] = class
	return class
}

func (self *ClassLoader) readClass(name string) ([]byte, classpath.Entry) {
	data, entry, err := self.cp.ReadClass(name)
	if err != nil {
		panic("java.lang.ClassNotFoundException: " + name)
	}
	return data, entry
}

func (self *ClassLoader) defineClass(data []byte) *Class {
	//byte转成class结构体
	class := parseClass(data)
	class.loader = self
	resolveSuperClass(class)
	resolveInterfaces(class)
	self.classMap[class.name] = class
	return class
}

func parseClass(data []byte) *Class {
	cf, err := chapter3_cf.Parse(data)
	if err != nil {
		panic("java.lang.ClassFormatError")
	}
	return newClass(cf)
}

/**
	除了object，每一个类都有且只有一个超类。需要递归调用LoadClass方法加载他的超类
 */
func resolveSuperClass(class *Class) {
	if class.name != "java/lang/Object" {
		class.superClass = class.loader.LoadClass(class.superClassName)
	}
}

func resolveInterfaces(class *Class) {
	interfaceCount := len(class.interfaceNames)
	if interfaceCount > 0 {
		class.interfaces = make([]*Class, interfaceCount)
		for i, interfaceName := range class.interfaceNames {
			class.interfaces[i] = class.loader.LoadClass(interfaceName)
		}
	}
}

func link(class *Class) {
	verify(class)
	prepare(class)
}

func verify(class *Class) {
	//todo
}

/**
	1.类是可以继承的，要递归地计算超类的实例变量
	2.long double要占两个位置
	3.静态字段和实例字段要分开编号，否则会混乱
	4.实例字段，一定要从Object开始编号。编号也要考虑long和double占两个位置的问题
 */
func prepare(class *Class) {
	calcInstanceFieldSlotIds(class)
	calcStaticFieldSlotIds(class)
	allocAndInitStaticVars(class)
}

func calcInstanceFieldSlotIds(class *Class) {
	slotId := uint(0)
	if class.superClass != nil {
		slotId = class.superClass.InstanceSlotCount
	}
	for _, field := range class.fields {
		if !field.IsStatic() {
			field.slotId = slotId
			slotId++
			if field.isLongOrDouble() {
				slotId++
			}
		}
	}
	class.InstanceSlotCount = slotId
}

func calcStaticFieldSlotIds(class *Class) {
	slotId := uint(0)
	for _, field := range class.fields {
		if field.IsStatic() {
			field.slotId = slotId
			slotId++
			if field.isLongOrDouble() {
				slotId++
			}
		}
	}
	class.staticSlotCount = slotId
}

func allocAndInitStaticVars(class *Class) {
	class.staticVars = NewSlots(class.staticSlotCount)
	for _, field := range class.fields {
		if field.IsStatic() && field.IsFinal() {
			//如果是static final 修饰的静态变量，从class文件的常量池里加载数值
			initStaticFinalVar(class, field)
		}
	}
}

func initStaticFinalVar(class *Class, field *Field) {
	vars := class.staticVars
	cp := class.constantPool
	cpIndex := field.ConstValueIndex()
	slotId := field.SlotId()

	if cpIndex > 0 {
		switch field.Descriptor() {
		case "Z", "B", "C", "S", "I" :
			val := cp.GetConstant(cpIndex).(int32)
			vars.SetInt(slotId, val)
		case "J":
			val := cp.GetConstant(cpIndex).(int64)
			vars.SetLong(slotId, val)
		case "F":
			val := cp.GetConstant(cpIndex).(float32)
			vars.SetFloat(slotId, val)
		case "D":
			val := cp.GetConstant(cpIndex).(float64)
			vars.SetDouble(slotId, val)
		case "Ljava/lang/String":
			goStr := cp.GetConstant(cpIndex).(string)
			jStr := JString(class.Loader(), goStr)
			vars.SetRef(slotId, jStr)
		}
	}
}<commit_msg>很乱。慢慢捋头绪<commit_after>package heap

import (
	"GoVM/chapter2-class/classpath"
	"fmt"
	"GoVM/chapter3-cf/classfile"
)

type ClassLoader struct {
	cp          *classpath.Classpath
	//是否输出加载class信息
	verboseFlag bool
	//key 是类的完全限定名称
	classMap    map[string]*Class
}

func NewClassLoader(cp *classpath.Classpath, verboseFlag bool) *ClassLoader {
	loader := &ClassLoader{
		cp:        cp,
		verboseFlag:        verboseFlag,
		classMap:        make(map[string]*Class),
	}
	loader.loadBasicClasses()
	return loader
}

/**
	先加载 java.lang.Class 类，这会触发 java.lang.Object等类和接口的加载。
	然后遍历classMap，给已经加载的每一个 类 关联 类对象。
 */
func (self *ClassLoader) loadBasicClasses() {
	jlClassClass := self.LoadClass("java/lang/Class")
	for _, class := range self.classMap {
		if class.jClass == nil {
			class.jClass = jlClassClass.NewObject()
			class.jClass.extra = class
		}
	}
}

func (self *ClassLoader) LoadClass(name string) *Class {
	if class, ok := self.classMap[name]; ok {
		//类已经被加载过了
		return class
	}

	var class *Class
	if name[0] == '[' {
		class = self.loadArrayClass(name)
	} else {
		class = self.loadNonArrayClass(name)
	}

	if jlClassClass, ok := self.classMap["java/lang/Class"]; ok {
		class.jClass = jlClassClass.NewObject()
		class.jClass.extra = class
	}

	return class
}

/**
	加载 所有 非数组 的类
 */
func (self *ClassLoader) loadNonArrayClass(name string) *Class {
	data, entry := self.readClass(name)
	class := self.defineClass(data)
	link(class)

	if self.verboseFlag {
		fmt.Printf("[Loaded %s from %s]\n", name, entry)
	}

	return class
}

/**
	加载数组类
 */
func (self *ClassLoader) loadArrayClass(name string) *Class {
	class := &Class{
		accessFlags: ACC_PUBLIC, // todo
		name:        name,
		loader:      self,
		initStarted: true,
		superClass:  self.LoadClass("java/lang/Object"),
		interfaces: []*Class{
			//数组默认实现了Cloneable和Serializable接口
			self.LoadClass("java/lang/Cloneable"),
			self.LoadClass("java/io/Serializable"),
		},
	}
	self.classMap[name] = class
	return class
}

func (self *ClassLoader) readClass(name string) ([]byte, classpath.Entry) {
	data, entry, err := self.cp.ReadClass(name)
	if err != nil {
		panic("java.lang.ClassNotFoundException: " + name)
	}
	return data, entry
}

func (self *ClassLoader) defineClass(data []byte) *Class {
	//byte转成class结构体
	class := parseClass(data)
	class.loader = self
	resolveSuperClass(class)
	resolveInterfaces(class)
	self.classMap[class.name] = class
	return class
}

func parseClass(data []byte) *Class {
	cf, err := chapter3_cf.Parse(data)
	if err != nil {
		panic("java.lang.ClassFormatError")
	}
	return newClass(cf)
}

/**
	除了object，每一个类都有且只有一个超类。需要递归调用LoadClass方法加载他的超类
 */
func resolveSuperClass(class *Class) {
	if class.name != "java/lang/Object" {
		class.superClass = class.loader.LoadClass(class.superClassName)
	}
}

func resolveInterfaces(class *Class) {
	interfaceCount := len(class.interfaceNames)
	if interfaceCount > 0 {
		class.interfaces = make([]*Class, interfaceCount)
		for i, interfaceName := range class.interfaceNames {
			class.interfaces[i] = class.loader.LoadClass(interfaceName)
		}
	}
}

func link(class *Class) {
	verify(class)
	prepare(class)
}

func verify(class *Class) {
	//todo
}

/**
	1.类是可以继承的，要递归地计算超类的实例变量
	2.long double要占两个位置
	3.静态字段和实例字段要分开编号，否则会混乱
	4.实例字段，一定要从Object开始编号。编号也要考虑long和double占两个位置的问题
 */
func prepare(class *Class) {
	calcInstanceFieldSlotIds(class)
	calcStaticFieldSlotIds(class)
	allocAndInitStaticVars(class)
}

func calcInstanceFieldSlotIds(class *Class) {
	slotId := uint(0)
	if class.superClass != nil {
		slotId = class.superClass.InstanceSlotCount
	}
	for _, field := range class.fields {
		if !field.IsStatic() {
			field.slotId = slotId
			slotId++
			if field.isLongOrDouble() {
				slotId++
			}
		}
	}
	class.InstanceSlotCount = slotId
}

func calcStaticFieldSlotIds(class *Class) {
	slotId := uint(0)
	for _, field := range class.fields {
		if field.IsStatic() {
			field.slotId = slotId
			slotId++
			if field.isLongOrDouble() {
				slotId++
			}
		}
	}
	class.staticSlotCount = slotId
}

func allocAndInitStaticVars(class *Class) {
	class.staticVars = NewSlots(class.staticSlotCount)
	for _, field := range class.fields {
		if field.IsStatic() && field.IsFinal() {
			//如果是static final 修饰的静态变量，从class文件的常量池里加载数值
			initStaticFinalVar(class, field)
		}
	}
}

func initStaticFinalVar(class *Class, field *Field) {
	vars := class.staticVars
	cp := class.constantPool
	cpIndex := field.ConstValueIndex()
	slotId := field.SlotId()

	if cpIndex > 0 {
		switch field.Descriptor() {
		case "Z", "B", "C", "S", "I" :
			val := cp.GetConstant(cpIndex).(int32)
			vars.SetInt(slotId, val)
		case "J":
			val := cp.GetConstant(cpIndex).(int64)
			vars.SetLong(slotId, val)
		case "F":
			val := cp.GetConstant(cpIndex).(float32)
			vars.SetFloat(slotId, val)
		case "D":
			val := cp.GetConstant(cpIndex).(float64)
			vars.SetDouble(slotId, val)
		case "Ljava/lang/String":
			goStr := cp.GetConstant(cpIndex).(string)
			jStr := JString(class.Loader(), goStr)
			vars.SetRef(slotId, jStr)
		}
	}
}<|endoftext|>


<commit_before>/**
 * Copyright 2015 @ z3q.net.
 * name : tcpserve.go
 * author : jarryliu
 * date : 2015-11-23 14:19
 * description :
 * history :
 */
package tcpserve

import (
	"errors"
	"github.com/garyburd/redigo/redis"
	"github.com/jsix/gof"
	"github.com/jsix/gof/net/nc"
	"go2o/src/app/util"
	"go2o/src/core"
	"go2o/src/core/service/dps"
	"net"
	"strconv"
	"strings"
	"time"
)

var (
	s        *nc.SocketServer
	handlers map[string]nc.CmdFunc = map[string]nc.CmdFunc{
		"PRINT": cliPrint,
		"MGET":  cliMGet,
		"PING":  cliPing,
	}
)

func ListenTcp(addr string) {
	s = nc.NewSocketServer()
	s.ReadDeadLine = time.Minute * 5
	serveLoop(s) // server loop,send some message to client
	//s.OutputOff()
	s.Listen(addr, func(conn net.Conn, b []byte) ([]byte, error) {
		cmd := string(b)
		id, ok := s.GetCli(conn)
		if !ok { // not join,auth first!
			if err := connAuth(s, conn, cmd); err != nil {
				return nil, err
			}
			return []byte("ok"), nil
		}
		if strings.HasPrefix(cmd, "MAUTH:") { //auth member
			return memberAuth(s, id, cmd[6:])
		}
		if !strings.HasPrefix(cmd, "PING") {
			s.Print("[ CLIENT][ MESSAGE] - send by %d ; %s", id.Source, cmd)
		}
		return handleCommand(id, cmd)
	})
}

// Add socket command handler
func Handle(cmd string, handler nc.CmdFunc) {
	handlers[cmd] = handler
}

// auth connection
func connAuth(s *nc.SocketServer, conn net.Conn, line string) error {
	if strings.HasPrefix(line, "AUTH:") {
		arr := strings.Split(line[5:], "#") // AUTH:API_ID#SECRET#VERSION
		if len(arr) == 3 {
			var af nc.AuthFunc = func() (int, error) {
				partnerId := dps.PartnerService.GetPartnerIdByApiId(arr[0])
				apiInfo := dps.PartnerService.GetApiInfo(partnerId)
				if apiInfo != nil && apiInfo.ApiSecret == arr[1] {
					if apiInfo.Enabled == 0 {
						return partnerId, errors.New("api has exipres")
					}
				}
				return partnerId, nil
			}

			if err := s.Auth(conn, af); err != nil {
				return err
			}

			s.Print("[ CLIENT] - Version = %s", arr[2])
			return nil
		}
	}
	return errors.New("conn reject")
}

// member auth,command like 'MAUTH:jarrysix#3234234242342342'
func memberAuth(s *nc.SocketServer, id *nc.Client, param string) ([]byte, error) {
	var err error
	arr := strings.Split(param, "#")
	if len(arr) == 2 {

		f := func() (int, error) {
			memberId, _ := strconv.Atoi(arr[0])
			authOk := util.CompareMemberApiToken(gof.CurrentApp.Storage(),
				memberId, arr[1])
			if !authOk {
				return memberId, errors.New("auth fail")
			}
			return memberId, nil
		}

		if err = s.UAuth(id.Conn, f); err == nil { //验证成功
			return []byte("ok"), nil
		}
	}
	return nil, err
}

// Handle command of client sending.
func handleCommand(ci *nc.Client, cmd string) ([]byte, error) {
	ci.LatestConnectTime = time.Now()
	i := strings.Index(cmd, ":")
	if i != -1 {
		plan := cmd[i+1:]
		if v, ok := handlers[cmd[:i]]; ok {
			return v(ci, plan)
		}
	}
	return nil, errors.New("unknown command:" + cmd)
}

func serveLoop(s *nc.SocketServer) {
	conn := core.GetRedisPool()
	go notifyMup(s, conn)
}

func notifyMup(s *nc.SocketServer, conn redis.Conn) {
	time.Sleep(time.Second * 10) // 等待监听服务
	for {
		err := mmSummaryNotify(s, conn)
		err1 := mmAccountNotify(s, conn)
		if err != nil || err1 != nil {
			time.Sleep(time.Second * 1) //阻塞,避免轮询占用CPU
		}
	}
}

// print text by client sending.
func cliPrint(id *nc.Client, params string) ([]byte, error) {
	return []byte(params), nil
}

func cliPing(id *nc.Client, plan string) ([]byte, error) {
	return []byte("PONG"), nil
}
<commit_msg>commit<commit_after>/**
 * Copyright 2015 @ z3q.net.
 * name : tcpserve.go
 * author : jarryliu
 * date : 2015-11-23 14:19
 * description :
 * history :
 */
package tcpserve

import (
	"errors"
	"github.com/garyburd/redigo/redis"
	"github.com/jsix/gof"
	"github.com/jsix/gof/net/nc"
	"go2o/src/app/util"
	"go2o/src/core"
	"go2o/src/core/service/dps"
	"net"
	"strconv"
	"strings"
	"time"
)

var (
	s        *nc.SocketServer
	handlers map[string]nc.CmdFunc = map[string]nc.CmdFunc{
		"PRINT": cliPrint,
		"MGET":  cliMGet,
		"PING":  cliPing,
	}
)

func ListenTcp(addr string) {
	s = nc.NewSocketServer()
	s.ReadDeadLine = time.Minute * 5
	serveLoop(s) // server loop,send some message to client
	//s.OutputOff()
	s.Listen(addr, func(conn net.Conn, b []byte) ([]byte, error) {
		cmd := string(b)
		id, ok := s.GetCli(conn)
		if !ok { // not join,auth first!
			if err := connAuth(s, conn, cmd); err != nil {
				return nil, err
			}
			return []byte("ok"), nil
		}
		if strings.HasPrefix(cmd, "MAUTH:") { //auth member
			return memberAuth(s, id, cmd[6:])
		}
		if !strings.HasPrefix(cmd, "PING") {
			s.Print("[ CLIENT][ MESSAGE] - send by %d ; %s", id.Source, cmd)
		}
		return handleCommand(id, cmd)
	})
}

// Add socket command handler
func Handle(cmd string, handler nc.CmdFunc) {
	handlers[cmd] = handler
}

// auth connection
func connAuth(s *nc.SocketServer, conn net.Conn, line string) error {
	if strings.HasPrefix(line, "AUTH:") {
		arr := strings.Split(line[5:], "#") // AUTH:API_ID#SECRET#VERSION
		if len(arr) == 3 {
			var af nc.AuthFunc = func() (int, error) {
				partnerId := dps.PartnerService.GetPartnerIdByApiId(arr[0])
				apiInfo := dps.PartnerService.GetApiInfo(partnerId)
				if apiInfo != nil && apiInfo.ApiSecret == arr[1] {
					if apiInfo.Enabled == 0 {
						return partnerId, errors.New("api has exipres")
					}
				}
				return partnerId, nil
			}

			if err := s.Auth(conn, af); err != nil {
				return err
			}

			s.Print("[ CLIENT] - Version = %s", arr[2])
			return nil
		}
	}
	return errors.New("conn reject")
}

// member auth,command like 'MAUTH:jarrysix#3234234242342342'
func memberAuth(s *nc.SocketServer, id *nc.Client, param string) ([]byte, error) {
	var err error
	arr := strings.Split(param, "#")
	if len(arr) == 2 {

		f := func() (int, error) {
			memberId, _ := strconv.Atoi(arr[0])
			authOk := util.CompareMemberApiToken(gof.CurrentApp.Storage(),
				memberId, arr[1])
			if !authOk {
				return memberId, errors.New("auth fail")
			}
			return memberId, nil
		}

		if err = s.UAuth(id.Conn, f); err == nil { //验证成功
			return []byte("ok"), nil
		}
	}
	return nil, err
}

// Handle command of client sending.
func handleCommand(ci *nc.Client, cmd string) ([]byte, error) {
	ci.LatestConnectTime = time.Now()
	i := strings.Index(cmd, ":")
	if i != -1 {
		plan := cmd[i+1:]
		if v, ok := handlers[cmd[:i]]; ok {
			return v(ci, plan)
		}
	}
	return nil, errors.New("unknown command:" + cmd)
}

func serveLoop(s *nc.SocketServer) {
	conn := core.GetRedisConn()
	go notifyMup(s, conn)
}

func notifyMup(s *nc.SocketServer, conn redis.Conn) {
	time.Sleep(time.Second * 10) // 等待监听服务
	for {
		err := mmSummaryNotify(s, conn)
		err1 := mmAccountNotify(s, conn)
		if err != nil || err1 != nil {
			time.Sleep(time.Second * 1) //阻塞,避免轮询占用CPU
		}
	}
}

// print text by client sending.
func cliPrint(id *nc.Client, params string) ([]byte, error) {
	return []byte(params), nil
}

func cliPing(id *nc.Client, plan string) ([]byte, error) {
	return []byte("PONG"), nil
}
<|endoftext|>


<commit_before>package testutil

import (
	"path/filepath"

	"veyron.io/tools/lib/envutil"
	"veyron.io/tools/lib/runutil"
	"veyron.io/tools/lib/util"
)

// VeyronTutorial runs the veyron tutorial examples.
//
// TODO(jregan): Merge the mdrip logic into this package.
func VeyronTutorial(ctx *util.Context, testName string) (*TestResult, error) {
	root, err := util.VeyronRoot()
	if err != nil {
		return nil, err
	}

	// Install the mdrip tool.
	opts := ctx.Run().Opts()
	env := envutil.NewSnapshotFromOS()
	env.Set("GOPATH", filepath.Join(root, "tutorial", "testing"))
	opts.Env = env.Map()
	if err := ctx.Run().CommandWithOpts(opts, "go", "install", "mdrip"); err != nil {
		return nil, err
	}

	// Run the tutorials.
	content := filepath.Join(root, "tutorial", "www", "content")
	mdrip := filepath.Join(root, "tutorial", "testing", "bin", "mdrip")
	args := []string{"--subshell", "1",
		filepath.Join(content, "docs", "installation", "index.md"),
		filepath.Join(content, "tutorials", "basics.md"),
		filepath.Join(content, "tutorials", "security.md"),
	}
	if err := ctx.Run().TimedCommand(DefaultTestTimeout, mdrip, args...); err != nil {
		if err == runutil.CommandTimedOutErr {
			return &TestResult{Status: TestTimedOut}, nil
		}
		return &TestResult{Status: TestFailed}, nil
	}
	return &TestResult{Status: TestPassed}, nil
}
<commit_msg>tools: fixing the tutorial test<commit_after>package testutil

import (
	"path/filepath"

	"veyron.io/tools/lib/envutil"
	"veyron.io/tools/lib/runutil"
	"veyron.io/tools/lib/util"
)

// VeyronTutorial runs the veyron tutorial examples.
//
// TODO(jregan): Merge the mdrip logic into this package.
func VeyronTutorial(ctx *util.Context, testName string) (*TestResult, error) {
	root, err := util.VeyronRoot()
	if err != nil {
		return nil, err
	}

	// Initialize the test.
	cleanup, err := initTest(ctx, testName, nil)
	if err != nil {
		return nil, err
	}
	defer cleanup()

	// Install the mdrip tool.
	opts := ctx.Run().Opts()
	env := envutil.NewSnapshotFromOS()
	env.Set("GOPATH", filepath.Join(root, "tutorial", "testing"))
	opts.Env = env.Map()
	if err := ctx.Run().CommandWithOpts(opts, "go", "install", "mdrip"); err != nil {
		return nil, err
	}

	// Run the tutorials.
	content := filepath.Join(root, "tutorial", "www", "content")
	mdrip := filepath.Join(root, "tutorial", "testing", "bin", "mdrip")
	args := []string{"--subshell", "1",
		filepath.Join(content, "docs", "installation", "index.md"),
		filepath.Join(content, "tutorials", "basics.md"),
		filepath.Join(content, "tutorials", "security.md"),
	}
	if err := ctx.Run().TimedCommand(DefaultTestTimeout, mdrip, args...); err != nil {
		if err == runutil.CommandTimedOutErr {
			return &TestResult{Status: TestTimedOut}, nil
		}
		return &TestResult{Status: TestFailed}, nil
	}
	return &TestResult{Status: TestPassed}, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Build the parser:
//go:generate goyacc -v y.output -o parser.go -p mtail parser.y

package vm

import (
	"io"
	"path/filepath"
	"regexp"

	"github.com/google/mtail/metrics"
)

// Options contains all the parameters that affect the behaviour of the compiler.
type Options struct {
	CompileOnly          bool // Do not start the program after compilation.
	SyslogUseCurrentYear bool // Use the current year if no year is present in the log file timestamp.
	EmitAst              bool // Print the AST after parse
	EmitAstTypes         bool // Print the AST with types after typechecking
}

// Compile compiles a program from the input into a virtual machine or a list
// of compile errors.  It takes the program's name and the metric store as
// additional arguments to build the virtual machine.
func Compile(name string, input io.Reader, o *Options) (*VM, error) {
	name = filepath.Base(name)

	ast, err := Parse(name, input)
	if err != nil {
		return nil, err
	}
	if o.EmitAst {
		u := Unparser{}
		u.Unparse(ast)
	}

	if err := Check(ast); err != nil {
		return nil, err
	}
	if o.EmitAstTypes {
		u := Unparser{}
		u.emitTypes = true
		u.Unparse(ast)
	}

	obj, err := CodeGen(name, ast)
	if err != nil {
		return nil, err
	}

	if o.CompileOnly {
		return nil, nil
	}

	vm := New(name, obj, o.SyslogUseCurrentYear)
	return vm, nil
}

// object describes a built object of data and bytecode
type object struct {
	prog []instr           // The emitted program.
	str  []string          // Static strings.
	re   []*regexp.Regexp  // Static regular expressions.
	m    []*metrics.Metric // Metrics accessible to this program.
}
<commit_msg>Dump the unparsed program and with types when asked.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Build the parser:
//go:generate goyacc -v y.output -o parser.go -p mtail parser.y

package vm

import (
	"io"
	"path/filepath"
	"regexp"

	"github.com/golang/glog"
	"github.com/google/mtail/metrics"
)

// Options contains all the parameters that affect the behaviour of the compiler.
type Options struct {
	CompileOnly          bool // Do not start the program after compilation.
	SyslogUseCurrentYear bool // Use the current year if no year is present in the log file timestamp.
	EmitAst              bool // Print the AST after parse
	EmitAstTypes         bool // Print the AST with types after typechecking
}

// Compile compiles a program from the input into a virtual machine or a list
// of compile errors.  It takes the program's name and the metric store as
// additional arguments to build the virtual machine.
func Compile(name string, input io.Reader, o *Options) (*VM, error) {
	name = filepath.Base(name)

	ast, err := Parse(name, input)
	if err != nil {
		return nil, err
	}
	if o.EmitAst {
		u := Unparser{}
		glog.Infof("%s AST:\n%s", name, u.Unparse(ast))
	}

	if err := Check(ast); err != nil {
		return nil, err
	}
	if o.EmitAstTypes {
		u := Unparser{}
		u.emitTypes = true
		glog.Infof("%s AST with Types:\n%s", name, u.Unparse(ast))
	}

	obj, err := CodeGen(name, ast)
	if err != nil {
		return nil, err
	}

	if o.CompileOnly {
		return nil, nil
	}

	vm := New(name, obj, o.SyslogUseCurrentYear)
	return vm, nil
}

// object describes a built object of data and bytecode
type object struct {
	prog []instr           // The emitted program.
	str  []string          // Static strings.
	re   []*regexp.Regexp  // Static regular expressions.
	m    []*metrics.Metric // Metrics accessible to this program.
}
<|endoftext|>


<commit_before>package voice

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/messagebird/go-rest-api"
)

const apiRoot = "https://voice.messagebird.com"

type ErrorResponse struct {
	Errors []Error
}

type Error struct {
	Code int
	Message string
}

func init() {
	// The Voice API returns errors in a format that slightly differs from other
	// APIs. Here we instruct package messagebird to use our custom
	// voice.errorReader func, which has access to voice.ErrorResponse, to
	// unmarshal those. Package messagebird must not import the voice package to
	// safeguard against import cycles, so it can not use voice.ErrorResponse
	// directly.
	messagebird.SetVoiceErrorReader(errorReader)
}

// errorReader takes a []byte representation of a Voice API JSON error and
// parses it to a voice.ErrorResponse.
func errorReader(b []byte) error {
	var er ErrorResponse
	if err := json.Unmarshal(b, &er); err != nil {
		return fmt.Errorf("encoding/json: Unmarshal: %v", err)
	}
	return er
}

func (e ErrorResponse) Error() string {
	if len(e.Errors) == 1 {
		return e.Errors[0].Error()
	}

	errStrings := make([]string, len(e.Errors))
	for i, v := range e.Errors {
		errStrings[i] = v.Error()
	}
	return strings.Join(errStrings, "; ")
}

func (e Error) Error() string {
	return fmt.Sprintf("code: %d, message: %q", e.Code, e.Message)
}
<commit_msg>Remove quick exit from Error().<commit_after>package voice

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/messagebird/go-rest-api"
)

const apiRoot = "https://voice.messagebird.com"

type ErrorResponse struct {
	Errors []Error
}

type Error struct {
	Code int
	Message string
}

func init() {
	// The Voice API returns errors in a format that slightly differs from other
	// APIs. Here we instruct package messagebird to use our custom
	// voice.errorReader func, which has access to voice.ErrorResponse, to
	// unmarshal those. Package messagebird must not import the voice package to
	// safeguard against import cycles, so it can not use voice.ErrorResponse
	// directly.
	messagebird.SetVoiceErrorReader(errorReader)
}

// errorReader takes a []byte representation of a Voice API JSON error and
// parses it to a voice.ErrorResponse.
func errorReader(b []byte) error {
	var er ErrorResponse
	if err := json.Unmarshal(b, &er); err != nil {
		return fmt.Errorf("encoding/json: Unmarshal: %v", err)
	}
	return er
}

func (e ErrorResponse) Error() string {
	errStrings := make([]string, len(e.Errors))
	for i, v := range e.Errors {
		errStrings[i] = v.Error()
	}
	return strings.Join(errStrings, "; ")
}

func (e Error) Error() string {
	return fmt.Sprintf("code: %d, message: %q", e.Code, e.Message)
}
<|endoftext|>


<commit_before>package multispark

import (
	"encoding/json"
	"fmt"
	"log"
	"time"

	H "github.com/dustin/go-humanize"

	ui "github.com/gizak/termui"
	DB "github.com/vrecan/FluxDash/influx"
	SL "github.com/vrecan/FluxDash/sparkline"
	"github.com/vrecan/FluxDash/timecop"
)

const (
	defaultHeight   = 3
	defaultInterval = "5s"
)
const (
	Short   = 1
	Percent = 2
	Bytes   = 3
	Time    = 4
)

type MultiSparkInfo struct {
	From     string
	Time     string
	Title    string
	Where    string
	DataType int
}

type MultiSpark struct {
	SL.SparkLines
	db *DB.Influx
	I  MultiSparkInfo
}

func NewMultiSpark(db *DB.Influx, i MultiSparkInfo) *MultiSpark {
	ms := &MultiSpark{db: db, I: i}
	ms.SetDataAndTitle()
	return ms
}

func (s *MultiSpark) Update() {
	s.SetDataAndTitle()
	// s.SL.BorderLabelFg = ui.ColorGreen | ui.AttrBold
	// s.SL.Border = true
	// s.SL.Height = 10
}

func buildQuery(sel string, from string, where string, time string, groupBy string) string {
	if len(sel) == 0 || len(from) == 0 || len(time) == 0 {
		log.Fatal("invalid query string :", fmt.Sprintf("SELECT %s FROM %s WHERE %s AND time > %s %s", sel, from, where, groupBy))
	}
	if len(where) > 0 {
		return fmt.Sprintf("SELECT %s FROM %s WHERE %s AND time > %s %s", sel, from, where, time, groupBy)
	} else {
		return fmt.Sprintf("SELECT %s FROM %s WHERE time > %s %s", sel, from, time, groupBy)
	}
}

func (s *MultiSpark) SetDataAndTitle() {
	data, labels := getData(s.db, buildQuery("mean(value)", s.I.From, s.I.Where, s.I.Time, fmt.Sprintf("GROUP BY time(%s)", defaultInterval)))
	meanTotal, _ := getData(s.db, buildQuery("mean(value)", s.I.From, s.I.Where, s.I.Time, ""))
	maxTotal, _ := getData(s.db, buildQuery("max(value)", s.I.From, s.I.Where, s.I.Time, ""))
	var uiSparks []ui.Sparkline
	for i, _ := range data {
		line := ui.NewSparkline()
		line.Data = data[i]
		//line.Title = lables[i]
		switch s.I.DataType {
		case Percent:
			line.Title = fmt.Sprintf("%s mean:%v%% max:%v%% cur: %v", labels[i], meanTotal[i][0], maxTotal[i][0], data[i][len(data[i])-1])
		case Bytes:
			line.Title = fmt.Sprintf("%s mean:%v max:%v cur: %v", labels[i], H.Bytes(uint64(meanTotal[i][0])), H.Bytes(uint64(maxTotal[i][0])), H.Bytes(uint64(data[i][len(data[i])-1])))
		case Short:
			line.Title = fmt.Sprintf("%s mean:%v max:%v cur: %v", labels[i], H.Comma(int64(meanTotal[i][0])), H.Comma(int64(maxTotal[i][0])), H.Comma(int64(data[i][len(data[i])-1])))
		case Time:
			line.Title = fmt.Sprintf("%s mean:%v max:%v cur: %v", labels[i], timecop.GetCommaString(float64(meanTotal[i][0]), "nanoseconds"), timecop.GetCommaString(float64(maxTotal[i][0]), "nanoseconds"), timecop.GetCommaString(float64(data[i][len(data[i])-1]), "nanoseconds"))
		default:
			log.Fatal("Data type is invalid: ", s.I.DataType)
		}
		uiSparks = append(uiSparks, line)
	}
	s.SL = ui.NewSparklines(uiSparks...)
	s.SL.Height = 3 + len(data)*2

}
func (s *MultiSpark) GetColumns() []*ui.Row {
	return []*ui.Row{ui.NewCol(12, 0, s.Sparks())}
}
func getData(db *DB.Influx, q string) (data [][]int, labels []string) {
	r, err := db.Query(q)
	if nil != err {
		log.Fatal(err)
	}
	if len(r) == 0 || len(r[0].Series) == 0 {
		log.Fatal(q)
	}
	labels = make([]string, len(r[0].Series))
	data = make([][]int, len(r[0].Series))
	for i, result := range r[0].Series {
		labels[i] = result.Name
		for _, row := range result.Values {
			_, err := time.Parse(time.RFC3339, row[0].(string))
			if err != nil {
				log.Fatal(err)
			}
			if len(row) > 1 {
				if nil != row[1] {
					val, err := row[1].(json.Number).Float64()
					if nil != err {
						fmt.Println("ERR: ", err)
					}
					data[i] = append(data[i], int(val))
				}
			}
		}
	}
	return data, labels
}
<commit_msg>dead code<commit_after>package multispark

import (
	"encoding/json"
	"fmt"
	"log"
	"time"

	H "github.com/dustin/go-humanize"

	ui "github.com/gizak/termui"
	DB "github.com/vrecan/FluxDash/influx"
	SL "github.com/vrecan/FluxDash/sparkline"
	"github.com/vrecan/FluxDash/timecop"
)

const (
	defaultHeight   = 3
	defaultInterval = "5s"
)
const (
	Short   = 1
	Percent = 2
	Bytes   = 3
	Time    = 4
)

type MultiSparkInfo struct {
	From     string
	Time     string
	Title    string
	Where    string
	DataType int
}

type MultiSpark struct {
	SL.SparkLines
	db *DB.Influx
	I  MultiSparkInfo
}

func NewMultiSpark(db *DB.Influx, i MultiSparkInfo) *MultiSpark {
	ms := &MultiSpark{db: db, I: i}
	ms.SetDataAndTitle()
	return ms
}

func (s *MultiSpark) Update() {
	s.SetDataAndTitle()
}

func buildQuery(sel string, from string, where string, time string, groupBy string) string {
	if len(sel) == 0 || len(from) == 0 || len(time) == 0 {
		log.Fatal("invalid query string :", fmt.Sprintf("SELECT %s FROM %s WHERE %s AND time > %s %s", sel, from, where, groupBy))
	}
	if len(where) > 0 {
		return fmt.Sprintf("SELECT %s FROM %s WHERE %s AND time > %s %s", sel, from, where, time, groupBy)
	} else {
		return fmt.Sprintf("SELECT %s FROM %s WHERE time > %s %s", sel, from, time, groupBy)
	}
}

func (s *MultiSpark) SetDataAndTitle() {
	data, labels := getData(s.db, buildQuery("mean(value)", s.I.From, s.I.Where, s.I.Time, fmt.Sprintf("GROUP BY time(%s)", defaultInterval)))
	meanTotal, _ := getData(s.db, buildQuery("mean(value)", s.I.From, s.I.Where, s.I.Time, ""))
	maxTotal, _ := getData(s.db, buildQuery("max(value)", s.I.From, s.I.Where, s.I.Time, ""))
	var uiSparks []ui.Sparkline
	for i, _ := range data {
		line := ui.NewSparkline()
		line.Data = data[i]
		switch s.I.DataType {
		case Percent:
			line.Title = fmt.Sprintf("%s mean:%v%% max:%v%% cur: %v", labels[i], meanTotal[i][0], maxTotal[i][0], data[i][len(data[i])-1])
		case Bytes:
			line.Title = fmt.Sprintf("%s mean:%v max:%v cur: %v", labels[i], H.Bytes(uint64(meanTotal[i][0])), H.Bytes(uint64(maxTotal[i][0])), H.Bytes(uint64(data[i][len(data[i])-1])))
		case Short:
			line.Title = fmt.Sprintf("%s mean:%v max:%v cur: %v", labels[i], H.Comma(int64(meanTotal[i][0])), H.Comma(int64(maxTotal[i][0])), H.Comma(int64(data[i][len(data[i])-1])))
		case Time:
			line.Title = fmt.Sprintf("%s mean:%v max:%v cur: %v", labels[i], timecop.GetCommaString(float64(meanTotal[i][0]), "nanoseconds"), timecop.GetCommaString(float64(maxTotal[i][0]), "nanoseconds"), timecop.GetCommaString(float64(data[i][len(data[i])-1]), "nanoseconds"))
		default:
			log.Fatal("Data type is invalid: ", s.I.DataType)
		}
		uiSparks = append(uiSparks, line)
	}
	s.SL = ui.NewSparklines(uiSparks...)
	s.SL.Height = 3 + len(data)*2

}
func (s *MultiSpark) GetColumns() []*ui.Row {
	return []*ui.Row{ui.NewCol(12, 0, s.Sparks())}
}
func getData(db *DB.Influx, q string) (data [][]int, labels []string) {
	r, err := db.Query(q)
	if nil != err {
		log.Fatal(err)
	}
	if len(r) == 0 || len(r[0].Series) == 0 {
		log.Fatal(q)
	}
	labels = make([]string, len(r[0].Series))
	data = make([][]int, len(r[0].Series))
	for i, result := range r[0].Series {
		labels[i] = result.Name
		for _, row := range result.Values {
			_, err := time.Parse(time.RFC3339, row[0].(string))
			if err != nil {
				log.Fatal(err)
			}
			if len(row) > 1 {
				if nil != row[1] {
					val, err := row[1].(json.Number).Float64()
					if nil != err {
						fmt.Println("ERR: ", err)
					}
					data[i] = append(data[i], int(val))
				}
			}
		}
	}
	return data, labels
}
<|endoftext|>


<commit_before>package sim

import (
	"encoding/json"
	"time"

	"github.com/Queens-Hacks/Propagate/sandbox"
	"github.com/Sirupsen/logrus"
)

type MarshalledState struct {
	State []byte
	Diff  []byte
}

// After calling this function it is no longer safe to do anything with s from
// outside of the simulation
func (s *State) StartSimulate() <-chan MarshalledState {
	ch := make(chan MarshalledState)

	go func() {
		// We want to run the thing every 500 milliseconds
		tick := time.NewTicker(time.Millisecond * 500)

		for {
			s.simulateTick()

			if !s.diff.isEmpty() {
				md, err := json.Marshal(s.diff)
				if err != nil {
					logrus.Fatal(err)
				}
				ms, err := json.Marshal(s.state)
				if err != nil {
					logrus.Fatal(err)
				}

				spores := []spore{}
				if s.diff.Spores != nil {
					spores = s.diff.Spores
				}

				s.diff = diff{[]tileDiff{}, map[string]*Plant{}, []string{}, spores}
				ch <- MarshalledState{ms, md}
			}

			<-tick.C
		}
	}()

	return ch
}

func (s *State) mkWorldState(_ *growthRoot) sandbox.WorldState {
	var ws sandbox.WorldState
	ws.Lighting = map[sandbox.Direction]float64{}

	ws.Lighting[sandbox.Left] = 0
	ws.Lighting[sandbox.Right] = 0
	ws.Lighting[sandbox.Up] = 0
	ws.Lighting[sandbox.Down] = 0

	return ws
}

func MoveGrowthNodeToLocation(loc Location, in sandbox.NewState, mh int, mw int) Location {
	new := loc

	if in.Dir == sandbox.Left {
		new.X -= 1
	} else if in.Dir == sandbox.Right {
		new.X += 1
	} else if in.Dir == sandbox.Up {
		new.Y -= 1
	} else if in.Dir == sandbox.Down {
		new.Y += 1
	} else {
		return loc
	}

	// Can't move there, it's out of bounds!
	if new.Y < 0 || new.Y > mh {
		logrus.Info("newY out of bounds", new.Y)
		return loc
	}
	if new.X < 0 || new.X > mw {
		logrus.Info("newY out of bounds")
		return loc
	}

	return new // new looks good!
}

func (s *State) applyChanges(root *growthRoot, in sandbox.NewState) {
	new := root.Loc

	if in.Operation == sandbox.Move {
		new = MoveGrowthNodeToLocation(root.Loc, in, s.Height(), s.Width())
		s.SetTile(new, Tile{PlantTile, &plantInfo{
			PlantId: root.PlantId,
			Parent:  root.Loc,
		}})
	} else if in.Operation == sandbox.Split {
		tmp := MoveGrowthNodeToLocation(root.Loc, in, s.Height(), s.Width())
		s.SetTile(tmp, Tile{PlantTile, &plantInfo{
			PlantId: root.PlantId,
			Parent:  root.Loc,
		}})
		s.SetTile(root.Loc, Tile{PlantTile, &plantInfo{
			PlantId: root.PlantId,
			Parent:  root.Loc,
		}})
	} else if in.Operation == sandbox.Wait {
		// that was easy
		return
	} else {
		return
	}

	// XXX Should this go through a method rather than direct mutation?
	// Move the growth root to the new location
	root.Loc = new
}

type newStateInfo struct {
	ch   <-chan sandbox.NewState
	root *growthRoot
}

// This is called by a timer every n time units
func (s *State) simulateTick() {
	responses := make([]newStateInfo, len(s.state.roots))

	// Tell each root to run until the next move operation
	for i := range s.state.roots {
		root := s.state.roots[i]
		ch := root.node.Update(s.mkWorldState(root))
		responses[i] = newStateInfo{ch, root}
	}

	for _, response := range responses {
		newState := <-response.ch
		s.applyChanges(response.root, newState)
	}

	spores := []spore{}
	for _, p := range s.diff.Spores {
		// UpdateSpore returns true if it has planted the spore
		spawned := s.UpdateSpore(&p)
		// Unplanted spores are kept for next tick
		if !spawned {
			spores = append(spores, p)
		}
	}
	s.diff.Spores = spores
}
<commit_msg>added bounds checking to tick<commit_after>package sim

import (
	"encoding/json"
	"time"

	"github.com/Queens-Hacks/Propagate/sandbox"
	"github.com/Sirupsen/logrus"
)

type MarshalledState struct {
	State []byte
	Diff  []byte
}

// After calling this function it is no longer safe to do anything with s from
// outside of the simulation
func (s *State) StartSimulate() <-chan MarshalledState {
	ch := make(chan MarshalledState)

	go func() {
		// We want to run the thing every 500 milliseconds
		tick := time.NewTicker(time.Millisecond * 500)

		for {
			s.simulateTick()

			if !s.diff.isEmpty() {
				md, err := json.Marshal(s.diff)
				if err != nil {
					logrus.Fatal(err)
				}
				ms, err := json.Marshal(s.state)
				if err != nil {
					logrus.Fatal(err)
				}

				spores := []spore{}
				if s.diff.Spores != nil {
					spores = s.diff.Spores
				}

				s.diff = diff{[]tileDiff{}, map[string]*Plant{}, []string{}, spores}
				ch <- MarshalledState{ms, md}
			}

			<-tick.C
		}
	}()

	return ch
}

func (s *State) mkWorldState(_ *growthRoot) sandbox.WorldState {
	var ws sandbox.WorldState
	ws.Lighting = map[sandbox.Direction]float64{}

	ws.Lighting[sandbox.Left] = 0
	ws.Lighting[sandbox.Right] = 0
	ws.Lighting[sandbox.Up] = 0
	ws.Lighting[sandbox.Down] = 0

	return ws
}

func boundsCheck(loc Location, mh int, mw int) Location {
	// Can't move there, it's out of bounds!
	if loc.Y < 0 || loc.Y > mh {
		logrus.Info("newY out of bounds", loc.Y)
		return loc
	}

	// loop around loop
	if loc.X < 0 {
		loc.X = mw + loc.X
	} else if loc.X >= mw {
		loc.X = loc.X - mw
	}

	return loc
}

func MoveGrowthNodeToLocation(loc Location, in sandbox.NewState, mh int, mw int) Location {
	new := loc

	if in.Dir == sandbox.Left {
		new.X -= 1
	} else if in.Dir == sandbox.Right {
		new.X += 1
	} else if in.Dir == sandbox.Up {
		new.Y -= 1
	} else if in.Dir == sandbox.Down {
		new.Y += 1
	} else {
		return loc
	}

	new = boundsCheck(new, mh, mw)

	return new // new looks good!
}

func (s *State) applyChanges(root *growthRoot, in sandbox.NewState) {
	new := root.Loc

	if in.Operation == sandbox.Move {
		new = MoveGrowthNodeToLocation(root.Loc, in, s.Height(), s.Width())
		s.SetTile(new, Tile{PlantTile, &plantInfo{
			PlantId: root.PlantId,
			Parent:  root.Loc,
		}})
	} else if in.Operation == sandbox.Split {
		tmp := MoveGrowthNodeToLocation(root.Loc, in, s.Height(), s.Width())
		s.SetTile(tmp, Tile{PlantTile, &plantInfo{
			PlantId: root.PlantId,
			Parent:  root.Loc,
		}})
		s.SetTile(root.Loc, Tile{PlantTile, &plantInfo{
			PlantId: root.PlantId,
			Parent:  root.Loc,
		}})
	} else if in.Operation == sandbox.Wait {
		// that was easy
		return
	} else {
		return
	}

	// XXX Should this go through a method rather than direct mutation?
	// Move the growth root to the new location
	root.Loc = new
}

type newStateInfo struct {
	ch   <-chan sandbox.NewState
	root *growthRoot
}

// This is called by a timer every n time units
func (s *State) simulateTick() {
	responses := make([]newStateInfo, len(s.state.roots))

	// Tell each root to run until the next move operation
	for i := range s.state.roots {
		root := s.state.roots[i]
		ch := root.node.Update(s.mkWorldState(root))
		responses[i] = newStateInfo{ch, root}
	}

	for _, response := range responses {
		newState := <-response.ch
		s.applyChanges(response.root, newState)
	}

	spores := []spore{}
	for _, p := range s.diff.Spores {
		// UpdateSpore returns true if it has planted the spore
		spawned := s.UpdateSpore(&p)
		// Unplanted spores are kept for next tick
		if !spawned {
			spores = append(spores, p)
		}
	}
	s.diff.Spores = spores
}
<|endoftext|>


<commit_before><commit_msg>mat: add benchmark for Cholesky.ToSym<commit_after><|endoftext|>


<commit_before>package main

import "sync"

type Logic struct {
	stateMap map[string]string
	rules    map[string]string
	sync.RWMutex
}

func NewLogic() *Logic {
	l := &Logic{stateMap: make(map[string]string), rules: make(map[string]string)}
	return l
}

func (l *Logic) ListenForChanges() chan interface{} {
	//TODO maybe this should be a buffered channel so we dont block on send in netStart/newClient
	c := make(chan interface{})
	go l.listen(c)
	return c
}

// listen will run in a own goroutine and listen to incoming state changes and Parse them
func (l *Logic) listen(c chan interface{}) {
	for state := range c {
		l.ParseState(state)
	}
}

func (l *Logic) ParseState(state interface{}) {
	//TODO parse all nodes.State here and generate something like this:
	// statemap["Devices[1].State"] = "OFF"
	// this might be usefull: http://play.golang.org/p/JQnry4s6KE
	// http://blog.golang.org/json-and-go
}

/*
Example of state:
State: {
	Devices: {
		1: {
			Id: "1",
			Name: "Dev1",
			State: "OFF",
			Type: ""
		},
		2: {
			Id: "2",
			Name: "Dev2",
			State: "ON",
			Type: ""
		}
	}
}
*/
<commit_msg>added comment about posible solution<commit_after>package main

import "sync"

type Logic struct {
	stateMap map[string]string
	rules    map[string]string
	sync.RWMutex
}

func NewLogic() *Logic {
	l := &Logic{stateMap: make(map[string]string), rules: make(map[string]string)}
	return l
}

func (l *Logic) ListenForChanges() chan interface{} {
	//TODO maybe this should be a buffered channel so we dont block on send in netStart/newClient
	c := make(chan interface{})
	go l.listen(c)
	return c
}

// listen will run in a own goroutine and listen to incoming state changes and Parse them
func (l *Logic) listen(c chan interface{}) {
	for state := range c {
		l.ParseState(state)
	}
}

func (l *Logic) ParseState(state interface{}) {
	//TODO parse all nodes.State here and generate something like this:
	// OR we dont use stateMap and only use rules Devices[2].On == true and parse it using jsonpath example below.
	// statemap["Devices[1].State"] = "OFF"
	// this might be usefull: http://play.golang.org/p/JQnry4s6KE
	// http://blog.golang.org/json-and-go
}

/*
Example of state:
State: {
	Devices: {
		1: {
			Id: "1",
			Name: "Dev1",
			State: "OFF",
			Type: ""
		},
		2: {
			Id: "2",
			Name: "Dev2",
			State: "ON",
			Type: ""
		}
	}
}
*/
<|endoftext|>