<commit_before>// MIT License
//
// Copyright (c) 2017 Stefan Wichmann
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import log "github.com/Sirupsen/logrus"
import "os/signal"
import "syscall"
import "os"
import "sync"
import "flag"

var applicationVersion = "development"
var debug = flag.Bool("debug", false, "Enable debug logging")
var logfile = flag.String("log", "", "Redirect log output to specified file")
var forceUpdate = flag.Bool("forceUpdate", false, "Update to new major version")

func main() {
	flag.Parse()
	configureLogging()
	log.Printf("Kelvin %v starting up... 🚀", applicationVersion)
	go CheckForUpdate(applicationVersion, *forceUpdate)
	go validateSystemTime()
	go handleSIGHUP()

	// load configuration or create a new one
	configuration, err := InitializeConfiguration()
	if err != nil {
		log.Fatal(err)
	}

	// find bridge
	bridge, err := InitializeBridge(configuration)
	if err != nil {
		log.Fatal(err)
	}
	err = bridge.printDevices()
	if err != nil {
		log.Fatal(err)
	}

	// find location
	_, err = InitializeLocation(configuration)
	if err != nil {
		log.Fatal(err)
	}

	// Save configuration if we changed it.
	if configuration.Modified {
		log.Debugf("Configuration changed. Saving...")
		err = configuration.Write()
		if err != nil {
			log.Fatal(err)
		}
	}

	// start routine for every light
	hueLights, err := bridge.Lights()
	if err != nil {
		log.Fatal(err)
	}
	var wg sync.WaitGroup
	for _, hueLight := range hueLights {
		hueLight := hueLight
		wg.Add(1)
		go func() {
			hueLight.updateCyclic(configuration)
			wg.Done()
		}()
	}
	wg.Wait()
	log.Debugf("All routines ended...")
}

func handleSIGHUP() {
	sighup := make(chan os.Signal, 1)
	signal.Notify(sighup, syscall.SIGHUP)
	<-sighup // wait for signal
	log.Printf("Received signal SIGHUP. Restarting...")
	Restart()
}

func configureLogging() {
	formatter := new(log.TextFormatter)
	formatter.FullTimestamp = true
	formatter.TimestampFormat = "2006/02/01 15:04:05"
	log.SetFormatter(formatter)
	if *debug {
		log.SetLevel(log.DebugLevel)
	}
	if logfile != nil {
		file, err := os.OpenFile(*logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err == nil {
			log.SetOutput(file)
		} else {
			log.Info("Failed to log to file, using default stderr")
		}
	}
}

func validateSystemTime() {
	// validate local clock as it forms the basis for all time calculations.
	valid, err := IsLocalTimeValid()
	if err != nil {
		log.Fatal(err)
	}
	if !valid {
		log.Warningf("WARNING: Your local system time seems to be more than one minute off. Timings may be inaccurate.")
	} else {
		log.Debugf("Local system time validated.")
	}
}
<commit_msg>Fix logging to file<commit_after>// MIT License
//
// Copyright (c) 2017 Stefan Wichmann
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import log "github.com/Sirupsen/logrus"
import "os/signal"
import "syscall"
import "os"
import "sync"
import "flag"

var applicationVersion = "development"
var debug = flag.Bool("debug", false, "Enable debug logging")
var logfile = flag.String("log", "", "Redirect log output to specified file")
var forceUpdate = flag.Bool("forceUpdate", false, "Update to new major version")

func main() {
	flag.Parse()
	configureLogging()
	log.Printf("Kelvin %v starting up... 🚀", applicationVersion)
	go CheckForUpdate(applicationVersion, *forceUpdate)
	go validateSystemTime()
	go handleSIGHUP()

	// load configuration or create a new one
	configuration, err := InitializeConfiguration()
	if err != nil {
		log.Fatal(err)
	}

	// find bridge
	bridge, err := InitializeBridge(configuration)
	if err != nil {
		log.Fatal(err)
	}
	err = bridge.printDevices()
	if err != nil {
		log.Fatal(err)
	}

	// find location
	_, err = InitializeLocation(configuration)
	if err != nil {
		log.Fatal(err)
	}

	// Save configuration if we changed it.
	if configuration.Modified {
		log.Debugf("Configuration changed. Saving...")
		err = configuration.Write()
		if err != nil {
			log.Fatal(err)
		}
	}

	// start routine for every light
	hueLights, err := bridge.Lights()
	if err != nil {
		log.Fatal(err)
	}
	var wg sync.WaitGroup
	for _, hueLight := range hueLights {
		hueLight := hueLight
		wg.Add(1)
		go func() {
			hueLight.updateCyclic(configuration)
			wg.Done()
		}()
	}
	wg.Wait()
	log.Debugf("All routines ended...")
}

func handleSIGHUP() {
	sighup := make(chan os.Signal, 1)
	signal.Notify(sighup, syscall.SIGHUP)
	<-sighup // wait for signal
	log.Printf("Received signal SIGHUP. Restarting...")
	Restart()
}

func configureLogging() {
	formatter := new(log.TextFormatter)
	formatter.FullTimestamp = true
	formatter.TimestampFormat = "2006/02/01 15:04:05"
	log.SetFormatter(formatter)
	if *debug {
		log.SetLevel(log.DebugLevel)
	}
	if logfile != nil && *logfile != "" {
		file, err := os.OpenFile(*logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err == nil {
			log.SetOutput(file)
		} else {
			log.Info("Failed to log to file, using default stderr")
		}
	}
}

func validateSystemTime() {
	// validate local clock as it forms the basis for all time calculations.
	valid, err := IsLocalTimeValid()
	if err != nil {
		log.Fatal(err)
	}
	if !valid {
		log.Warningf("WARNING: Your local system time seems to be more than one minute off. Timings may be inaccurate.")
	} else {
		log.Debugf("Local system time validated.")
	}
}
<|endoftext|>


<commit_before>package keymux

import (
	"errors"
	"hash/crc32"
	"sync"
)

var (
	ErrStopped    = errors.New("mux is stopped")
	ErrFullBuffer = errors.New("worker buffer is full")
)

type panicError struct{ reason interface{} }

func (err panicError) Error() string { return "panic occurs" }

type Handler interface {
	Handle(key []byte, val interface{}) (interface{}, error)
}

type FixedHandler interface {
	Handle(val interface{}) (interface{}, error)
}

type job struct {
	key []byte
	val interface{}
	res chan res
}

type res struct {
	val interface{}
	err error
}

func (j job) do(handler Handler) {
	defer close(j.res)
	defer func() {
		if reason := recover(); reason != nil {
			j.res <- res{nil, panicError{reason}}
		}
	}()
	val, err := handler.Handle(j.key, j.val)
	j.res <- res{val, err}
}

type Mux struct {
	mu sync.RWMutex
	wg sync.WaitGroup
	ws []chan job
}

func Start(handler Handler, workers, bufferSize int) *Mux {
	m := new(Mux)
	m.Start(handler, workers, bufferSize)
	return m
}

func (m *Mux) Start(handler Handler, workers, bufferSize int) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.ws != nil {
		panic("mux is already started")
	}
	if workers <= 0 {
		panic("number of workers must be positive")
	}
	if bufferSize < 0 {
		panic("worker buffer size must not be negative")
	}
	m.wg.Add(workers)
	m.ws = make([]chan job, workers)
	for i := range m.ws {
		m.ws[i] = make(chan job, bufferSize)
		go func(jobs <-chan job) {
			defer m.wg.Done()
			for j := range jobs {
				j.do(handler)
			}
		}(m.ws[i])
	}
}

func (m *Mux) Stop() {
	m.mu.Lock()
	defer m.mu.Unlock()
	for _, jobs := range m.ws {
		close(jobs)
	}
	m.ws = nil
	m.wg.Wait()
}

func (m *Mux) handle(crc uint32, j job) error {
	m.mu.RLock()
	defer m.mu.RUnlock()
	if m.ws == nil {
		return ErrStopped
	}
	w := crc % uint32(len(m.ws))
	select {
	case m.ws[w] <- j:
		return nil
	default:
		return ErrFullBuffer
	}
}

func (m *Mux) Handle(key []byte, val interface{}) (interface{}, error) {
	j := job{key, val, make(chan res)}
	err := m.handle(crc32.ChecksumIEEE(key), j)
	if err != nil {
		return nil, err
	}
	r := <-j.res
	if p, ok := r.err.(panicError); ok {
		panic(p.reason)
	}
	return r.val, r.err
}

type fixed struct {
	handler Handler
	key     []byte
}

func Fixed(handler Handler, key []byte) FixedHandler {
	return fixed{handler: handler, key: key}
}

func (f fixed) Handle(val interface{}) (interface{}, error) {
	return f.handler.Handle(f.key, val)
}

type urgent struct {
	handler Handler
}

func Urgent(handler Handler) Handler {
	return urgent{handler: handler}
}

func (u urgent) Handle(key []byte, val interface{}) (interface{}, error) {
	for {
		if val, err := u.handler.Handle(key, val); err != ErrFullBuffer {
			return val, err
		}
	}
}
<commit_msg>It is not necessary to close channel<commit_after>package keymux

import (
	"errors"
	"hash/crc32"
	"sync"
)

var (
	ErrStopped    = errors.New("mux is stopped")
	ErrFullBuffer = errors.New("worker buffer is full")
)

type panicError struct{ reason interface{} }

func (err panicError) Error() string { return "panic occurs" }

type Handler interface {
	Handle(key []byte, val interface{}) (interface{}, error)
}

type FixedHandler interface {
	Handle(val interface{}) (interface{}, error)
}

type job struct {
	key []byte
	val interface{}
	res chan res
}

type res struct {
	val interface{}
	err error
}

func (j job) do(handler Handler) {
	defer func() {
		if reason := recover(); reason != nil {
			j.res <- res{nil, panicError{reason}}
		}
	}()
	val, err := handler.Handle(j.key, j.val)
	j.res <- res{val, err}
}

type Mux struct {
	mu sync.RWMutex
	wg sync.WaitGroup
	ws []chan job
}

func Start(handler Handler, workers, bufferSize int) *Mux {
	m := new(Mux)
	m.Start(handler, workers, bufferSize)
	return m
}

func (m *Mux) Start(handler Handler, workers, bufferSize int) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.ws != nil {
		panic("mux is already started")
	}
	if workers <= 0 {
		panic("number of workers must be positive")
	}
	if bufferSize < 0 {
		panic("worker buffer size must not be negative")
	}
	m.wg.Add(workers)
	m.ws = make([]chan job, workers)
	for i := range m.ws {
		m.ws[i] = make(chan job, bufferSize)
		go func(jobs <-chan job) {
			defer m.wg.Done()
			for j := range jobs {
				j.do(handler)
			}
		}(m.ws[i])
	}
}

func (m *Mux) Stop() {
	m.mu.Lock()
	defer m.mu.Unlock()
	for _, jobs := range m.ws {
		close(jobs)
	}
	m.ws = nil
	m.wg.Wait()
}

func (m *Mux) handle(crc uint32, j job) error {
	m.mu.RLock()
	defer m.mu.RUnlock()
	if m.ws == nil {
		return ErrStopped
	}
	w := crc % uint32(len(m.ws))
	select {
	case m.ws[w] <- j:
		return nil
	default:
		return ErrFullBuffer
	}
}

func (m *Mux) Handle(key []byte, val interface{}) (interface{}, error) {
	j := job{key, val, make(chan res)}
	err := m.handle(crc32.ChecksumIEEE(key), j)
	if err != nil {
		return nil, err
	}
	r := <-j.res
	if p, ok := r.err.(panicError); ok {
		panic(p.reason)
	}
	return r.val, r.err
}

type fixed struct {
	handler Handler
	key     []byte
}

func Fixed(handler Handler, key []byte) FixedHandler {
	return fixed{handler: handler, key: key}
}

func (f fixed) Handle(val interface{}) (interface{}, error) {
	return f.handler.Handle(f.key, val)
}

type urgent struct {
	handler Handler
}

func Urgent(handler Handler) Handler {
	return urgent{handler: handler}
}

func (u urgent) Handle(key []byte, val interface{}) (interface{}, error) {
	for {
		if val, err := u.handler.Handle(key, val); err != ErrFullBuffer {
			return val, err
		}
	}
}
<|endoftext|>


<commit_before>//
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"context"
	"fmt"
	"strings"

	"cloud.google.com/go/spanner"

	sppb "google.golang.org/genproto/googleapis/spanner/v1"
)

// Table represents a Spanner table.
type Table struct {
	Name        string
	Columns     []string
	ChildTables []*Table
}

func (t *Table) String() string {
	return fmt.Sprintf("{Name: %q, Columns: %v, ChildTables: %v}", t.Name, t.Columns, t.ChildTables)
}

func (t *Table) quotedColumnList() string {
	var quoted []string
	for _, c := range t.Columns {
		quoted = append(quoted, fmt.Sprintf("`%s`", c))
	}
	return strings.Join(quoted, ", ")
}

// TableIterator is an iterator to get tables in the database one by one.
type TableIterator struct {
	tables []*Table
}

// Do executes a given func with each table in the database.
func (i *TableIterator) Do(f func(*Table) error) error {
	for _, t := range i.tables {
		if err := f(t); err != nil {
			return err
		}
		if len(t.ChildTables) > 0 {
			childIter := &TableIterator{t.ChildTables}
			if err := childIter.Do(f); err != nil {
				return err
			}
		}
	}
	return nil
}

type tableRow struct {
	name       string
	parentName string
	columns    []string
}

// FetchTables fetches all table information in the database from Spanner.
func FetchTables(ctx context.Context, txn *spanner.ReadOnlyTransaction) (*TableIterator, error) {
	// SQL for fetching table name, parent, and columns
	stmt := spanner.NewStatement(`
SELECT t.TABLE_NAME as table, t.PARENT_TABLE_NAME as parent, c.columns
FROM INFORMATION_SCHEMA.TABLES as t
JOIN (
    SELECT c.TABLE_NAME as table, ARRAY_AGG(c.COLUMN_NAME) as columns
    FROM INFORMATION_SCHEMA.COLUMNS AS c
    WHERE c.TABLE_CATALOG = '' AND c.TABLE_SCHEMA = ''
    GROUP BY c.TABLE_NAME
) as c
ON t.TABLE_NAME = c.table
WHERE t.TABLE_CATALOG = '' AND t.TABLE_SCHEMA = ''
ORDER BY t.TABLE_NAME ASC
`)
	var rows []tableRow
	opts := spanner.QueryOptions{Priority: sppb.RequestOptions_PRIORITY_LOW}
	if err := txn.QueryWithOptions(ctx, stmt, opts).Do(func(r *spanner.Row) error {
		var tableName, parentTableName string
		var columns []string
		var parentTableNamePtr *string // nullable

		if err := r.ColumnByName("table", &tableName); err != nil {
			return err
		}

		if err := r.ColumnByName("parent", &parentTableNamePtr); err != nil {
			return err
		}
		if parentTableNamePtr != nil {
			parentTableName = *parentTableNamePtr
		}

		if err := r.ColumnByName("columns", &columns); err != nil {
			return err
		}

		rows = append(rows, tableRow{
			name:       tableName,
			columns:    columns,
			parentName: parentTableName,
		})
		return nil
	}); err != nil {
		return nil, err
	}

	tables := findChildTables(rows, "") // root
	return &TableIterator{tables}, nil
}

func findChildTables(rows []tableRow, parent string) []*Table {
	var tables []*Table
	for _, row := range rows {
		if row.parentName == parent {
			tables = append(tables, &Table{
				Name:        row.name,
				Columns:     row.columns,
				ChildTables: findChildTables(rows, row.name),
			})
		}
	}
	return tables
}
<commit_msg>Exclude INSERT statements for Views (#15)<commit_after>//
// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"context"
	"fmt"
	"strings"

	"cloud.google.com/go/spanner"

	sppb "google.golang.org/genproto/googleapis/spanner/v1"
)

// Table represents a Spanner table.
type Table struct {
	Name        string
	Columns     []string
	ChildTables []*Table
}

func (t *Table) String() string {
	return fmt.Sprintf("{Name: %q, Columns: %v, ChildTables: %v}", t.Name, t.Columns, t.ChildTables)
}

func (t *Table) quotedColumnList() string {
	var quoted []string
	for _, c := range t.Columns {
		quoted = append(quoted, fmt.Sprintf("`%s`", c))
	}
	return strings.Join(quoted, ", ")
}

// TableIterator is an iterator to get tables in the database one by one.
type TableIterator struct {
	tables []*Table
}

// Do executes a given func with each table in the database.
func (i *TableIterator) Do(f func(*Table) error) error {
	for _, t := range i.tables {
		if err := f(t); err != nil {
			return err
		}
		if len(t.ChildTables) > 0 {
			childIter := &TableIterator{t.ChildTables}
			if err := childIter.Do(f); err != nil {
				return err
			}
		}
	}
	return nil
}

type tableRow struct {
	name       string
	parentName string
	columns    []string
}

// FetchTables fetches all table information in the database from Spanner.
func FetchTables(ctx context.Context, txn *spanner.ReadOnlyTransaction) (*TableIterator, error) {
	// SQL for fetching table name, parent, and columns
	stmt := spanner.NewStatement(`
SELECT t.TABLE_NAME as table, t.PARENT_TABLE_NAME as parent, c.columns
FROM INFORMATION_SCHEMA.TABLES as t
JOIN (
    SELECT c.TABLE_NAME as table, ARRAY_AGG(c.COLUMN_NAME) as columns
    FROM INFORMATION_SCHEMA.COLUMNS AS c
    WHERE c.TABLE_CATALOG = '' AND c.TABLE_SCHEMA = ''
    GROUP BY c.TABLE_NAME
) as c
ON t.TABLE_NAME = c.table
WHERE t.TABLE_CATALOG = '' AND t.TABLE_SCHEMA = '' AND t.TABLE_TYPE = 'BASE TABLE'
ORDER BY t.TABLE_NAME ASC
`)
	var rows []tableRow
	opts := spanner.QueryOptions{Priority: sppb.RequestOptions_PRIORITY_LOW}
	if err := txn.QueryWithOptions(ctx, stmt, opts).Do(func(r *spanner.Row) error {
		var tableName, parentTableName string
		var columns []string
		var parentTableNamePtr *string // nullable

		if err := r.ColumnByName("table", &tableName); err != nil {
			return err
		}

		if err := r.ColumnByName("parent", &parentTableNamePtr); err != nil {
			return err
		}
		if parentTableNamePtr != nil {
			parentTableName = *parentTableNamePtr
		}

		if err := r.ColumnByName("columns", &columns); err != nil {
			return err
		}

		rows = append(rows, tableRow{
			name:       tableName,
			columns:    columns,
			parentName: parentTableName,
		})
		return nil
	}); err != nil {
		return nil, err
	}

	tables := findChildTables(rows, "") // root
	return &TableIterator{tables}, nil
}

func findChildTables(rows []tableRow, parent string) []*Table {
	var tables []*Table
	for _, row := range rows {
		if row.parentName == parent {
			tables = append(tables, &Table{
				Name:        row.name,
				Columns:     row.columns,
				ChildTables: findChildTables(rows, row.name),
			})
		}
	}
	return tables
}
<|endoftext|>


<commit_before>package wal

import (
	"configuration"
	"fmt"
	"os"
	"path"
	"protocol"
	"sort"
	"strings"
)

type WAL struct {
	config             *configuration.Configuration
	logFiles           []*log
	serverId           uint32
	nextLogFileSuffix  int
	requestsPerLogFile int
}

const HOST_ID_OFFSET = uint64(10000)

func NewWAL(config *configuration.Configuration) (*WAL, error) {
	if config.WalDir == "" {
		return nil, fmt.Errorf("wal directory cannot be empty")
	}
	_, err := os.Stat(config.WalDir)

	if os.IsNotExist(err) {
		err = os.MkdirAll(config.WalDir, 0755)
	}

	if err != nil {
		return nil, err
	}

	dir, err := os.Open(config.WalDir)
	if err != nil {
		return nil, err
	}
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return nil, err
	}

	nextLogFileSuffix := 0
	logFiles := make([]*log, 0)
	for _, name := range names {
		if !strings.HasPrefix(name, "log.") {
			continue
		}
		f, err := os.OpenFile(path.Join(config.WalDir, name), os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
		logFile, err := newLog(f, config)
		if err != nil {
			return nil, err
		}
		if suffix := logFile.suffix(); suffix > nextLogFileSuffix {
			nextLogFileSuffix = suffix
		}
		logFiles = append(logFiles, logFile)
	}

	// sort the logfiles by the first request number in the log
	sort.Sort(sortableLogSlice(logFiles))

	wal := &WAL{config: config, logFiles: logFiles, requestsPerLogFile: config.WalRequestsPerLogFile, nextLogFileSuffix: nextLogFileSuffix}

	// if we don't have any log files open yet, open a new one
	if len(logFiles) == 0 {
		_, err = wal.createNewLog()
	}
	return wal, err
}

func (self *WAL) SetServerId(id uint32) {
	self.serverId = id
}

// Marks a given request for a given server as committed
func (self *WAL) Commit(requestNumber uint32, serverId uint32) error {
	lastLogFile := self.logFiles[len(self.logFiles)-1]
	lastLogFile.state.commitRequestNumber(serverId, requestNumber)
	lowestCommitedRequestNumber := lastLogFile.state.LowestCommitedRequestNumber()

	index := self.firstLogFile(lowestCommitedRequestNumber)
	if index == 0 {
		return nil
	}

	var unusedLogFiles []*log
	unusedLogFiles, self.logFiles = self.logFiles[:index], self.logFiles[index:]
	for _, logFile := range unusedLogFiles {
		logFile.close()
		logFile.delete()
	}
	return nil
}

// In the case where this server is running and another one in the cluster stops responding, at some point this server will have to just write
// requests to disk. When the downed server comes back up, it's this server's responsibility to send out any writes that were queued up. If
// the yield function returns nil then the request is committed.
func (self *WAL) RecoverServerFromRequestNumber(requestNumber uint32, shardIds []uint32, yield func(request *protocol.Request, shardId uint32) error) error {
	var firstLogFile int

outer:
	for _, logFile := range self.logFiles[firstLogFile:] {
		ch, stopChan := logFile.replayFromRequestNumber(shardIds, requestNumber)
		for {
			x := <-ch
			if x == nil {
				continue outer
			}

			if x.err != nil {
				return x.err
			}

			if err := yield(x.request, x.shardId); err != nil {
				stopChan <- struct{}{}
				return err
			}
		}
		close(stopChan)
	}
	return nil
}

func (self *WAL) Close() error {
	for _, l := range self.logFiles {
		if err := l.close(); err != nil {
			return err
		}
	}
	return nil
}

// PRIVATE functions

// creates a new log file using the next suffix and initializes its
// state with the state of the last log file
func (self *WAL) createNewLog() (*log, error) {
	self.nextLogFileSuffix++
	logFileName := path.Join(self.config.WalDir, fmt.Sprintf("log.%d", self.nextLogFileSuffix))
	logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return nil, err
	}
	log, err := newLog(logFile, self.config)
	if err != nil {
		return nil, err
	}

	// TODO: this is ugly, we have to copy some of the state to the new
	// log. Find a better way to do this, possibly separating the state
	// that we need to keep between log files in their own file
	if len(self.logFiles) > 0 {
		lastLogFile := self.logFiles[len(self.logFiles)-1]
		// update the new state to continue from where the last log file
		// left off
		log.state.continueFromState(lastLogFile.state)
	}
	self.logFiles = append(self.logFiles, log)
	return log, nil
}

// Will assign sequence numbers if null. Returns a unique id that
// should be marked as committed for each server as it gets confirmed.
func (self *WAL) AssignSequenceNumbersAndLog(request *protocol.Request, shard Shard) (uint32, error) {
	lastLogFile := self.logFiles[len(self.logFiles)-1]
	if lastLogFile.state.TotalNumberOfRequests >= self.requestsPerLogFile {
		err := lastLogFile.forceBookmark(true)
		if err != nil {
			return 0, err
		}
		lastLogFile, err = self.createNewLog()
		if err != nil {
			return 0, err
		}
	}

	requestNumber, err := lastLogFile.appendRequest(request, shard.Id())
	if err != nil {
		return 0, err
	}
	return requestNumber, nil
}

func (self *WAL) doesLogFileContainRequest(requestNumber uint32) func(int) bool {
	return func(i int) bool {
		if self.logFiles[i].firstRequestNumber() > requestNumber {
			return true
		}
		return false
	}
}

// returns the first log file that contains the given request number
func (self *WAL) firstLogFile(requestNumber uint32) int {
	lengthLogFiles := len(self.logFiles)
	if requestNumber >= self.logFiles[lengthLogFiles-1].firstRequestNumber() {
		return lengthLogFiles - 1
	} else if requestNumber <= self.logFiles[0].firstRequestNumber() {
		return 0
	}
	return sort.Search(lengthLogFiles, self.doesLogFileContainRequest(requestNumber)) - 1
}
<commit_msg>make the log file rotation thread safe<commit_after>package wal

import (
	"configuration"
	"fmt"
	"os"
	"path"
	"protocol"
	"sort"
	"strings"
	"sync"
)

type WAL struct {
	config             *configuration.Configuration
	logFiles           []*log
	serverId           uint32
	nextLogFileSuffix  int
	requestsPerLogFile int
	rotateLock         sync.Mutex
}

const HOST_ID_OFFSET = uint64(10000)

func NewWAL(config *configuration.Configuration) (*WAL, error) {
	if config.WalDir == "" {
		return nil, fmt.Errorf("wal directory cannot be empty")
	}
	_, err := os.Stat(config.WalDir)

	if os.IsNotExist(err) {
		err = os.MkdirAll(config.WalDir, 0755)
	}

	if err != nil {
		return nil, err
	}

	dir, err := os.Open(config.WalDir)
	if err != nil {
		return nil, err
	}
	names, err := dir.Readdirnames(-1)
	if err != nil {
		return nil, err
	}

	nextLogFileSuffix := 0
	logFiles := make([]*log, 0)
	for _, name := range names {
		if !strings.HasPrefix(name, "log.") {
			continue
		}
		f, err := os.OpenFile(path.Join(config.WalDir, name), os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
		logFile, err := newLog(f, config)
		if err != nil {
			return nil, err
		}
		if suffix := logFile.suffix(); suffix > nextLogFileSuffix {
			nextLogFileSuffix = suffix
		}
		logFiles = append(logFiles, logFile)
	}

	// sort the logfiles by the first request number in the log
	sort.Sort(sortableLogSlice(logFiles))

	wal := &WAL{config: config, logFiles: logFiles, requestsPerLogFile: config.WalRequestsPerLogFile, nextLogFileSuffix: nextLogFileSuffix}

	// if we don't have any log files open yet, open a new one
	if len(logFiles) == 0 {
		_, err = wal.createNewLog()
	}
	return wal, err
}

func (self *WAL) SetServerId(id uint32) {
	self.serverId = id
}

// Marks a given request for a given server as committed
func (self *WAL) Commit(requestNumber uint32, serverId uint32) error {
	lastLogFile := self.logFiles[len(self.logFiles)-1]
	lastLogFile.state.commitRequestNumber(serverId, requestNumber)
	lowestCommitedRequestNumber := lastLogFile.state.LowestCommitedRequestNumber()

	index := self.firstLogFile(lowestCommitedRequestNumber)
	if index == 0 {
		return nil
	}

	var unusedLogFiles []*log
	unusedLogFiles, self.logFiles = self.logFiles[:index], self.logFiles[index:]
	for _, logFile := range unusedLogFiles {
		logFile.close()
		logFile.delete()
	}
	return nil
}

// In the case where this server is running and another one in the cluster stops responding, at some point this server will have to just write
// requests to disk. When the downed server comes back up, it's this server's responsibility to send out any writes that were queued up. If
// the yield function returns nil then the request is committed.
func (self *WAL) RecoverServerFromRequestNumber(requestNumber uint32, shardIds []uint32, yield func(request *protocol.Request, shardId uint32) error) error {
	var firstLogFile int

outer:
	for _, logFile := range self.logFiles[firstLogFile:] {
		ch, stopChan := logFile.replayFromRequestNumber(shardIds, requestNumber)
		for {
			x := <-ch
			if x == nil {
				continue outer
			}

			if x.err != nil {
				return x.err
			}

			if err := yield(x.request, x.shardId); err != nil {
				stopChan <- struct{}{}
				return err
			}
		}
		close(stopChan)
	}
	return nil
}

func (self *WAL) Close() error {
	for _, l := range self.logFiles {
		if err := l.close(); err != nil {
			return err
		}
	}
	return nil
}

// PRIVATE functions

// creates a new log file using the next suffix and initializes its
// state with the state of the last log file
func (self *WAL) createNewLog() (*log, error) {
	self.nextLogFileSuffix++
	logFileName := path.Join(self.config.WalDir, fmt.Sprintf("log.%d", self.nextLogFileSuffix))
	logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return nil, err
	}
	log, err := newLog(logFile, self.config)
	if err != nil {
		return nil, err
	}

	// TODO: this is ugly, we have to copy some of the state to the new
	// log. Find a better way to do this, possibly separating the state
	// that we need to keep between log files in their own file
	if len(self.logFiles) > 0 {
		lastLogFile := self.logFiles[len(self.logFiles)-1]
		// update the new state to continue from where the last log file
		// left off
		log.state.continueFromState(lastLogFile.state)
	}
	self.logFiles = append(self.logFiles, log)
	return log, nil
}

// Will assign sequence numbers if null. Returns a unique id that
// should be marked as committed for each server as it gets confirmed.
func (self *WAL) AssignSequenceNumbersAndLog(request *protocol.Request, shard Shard) (uint32, error) {
	lastLogFile := self.logFiles[len(self.logFiles)-1]
	if self.shouldRotateTheLogFile() {
		if err := self.rotateTheLogFile(); err != nil {
			return 0, err
		}
	}

	requestNumber, err := lastLogFile.appendRequest(request, shard.Id())
	if err != nil {
		return 0, err
	}
	return requestNumber, nil
}

func (self *WAL) doesLogFileContainRequest(requestNumber uint32) func(int) bool {
	return func(i int) bool {
		if self.logFiles[i].firstRequestNumber() > requestNumber {
			return true
		}
		return false
	}
}

// returns the first log file that contains the given request number
func (self *WAL) firstLogFile(requestNumber uint32) int {
	lengthLogFiles := len(self.logFiles)
	if requestNumber >= self.logFiles[lengthLogFiles-1].firstRequestNumber() {
		return lengthLogFiles - 1
	} else if requestNumber <= self.logFiles[0].firstRequestNumber() {
		return 0
	}
	return sort.Search(lengthLogFiles, self.doesLogFileContainRequest(requestNumber)) - 1
}

func (self *WAL) shouldRotateTheLogFile() bool {
	lastLogFile := self.logFiles[len(self.logFiles)-1]
	return lastLogFile.state.TotalNumberOfRequests >= self.requestsPerLogFile
}

func (self *WAL) rotateTheLogFile() error {
	self.rotateLock.Lock()
	defer self.rotateLock.Unlock()

	if !self.shouldRotateTheLogFile() {
		return nil
	}

	lastLogFile := self.logFiles[len(self.logFiles)-1]
	err := lastLogFile.forceBookmark(true)
	if err != nil {
		return err
	}
	lastLogFile, err = self.createNewLog()
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package stardb

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"path"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Farm struct {

	// Fields with  `json:"-"` are ommited for web display

	ID        bson.ObjectId `bson:"_id,omitempty" json:"-"`
	UniqueID  int           `json:"-"`
	Name      string
	Farmer    string
	Likes     int
	SaveTime  time.Time `json:"-"`
	Thumbnail string
}

func (f *Farm) ScreenshotPath() string {
	return fmt.Sprintf("/screenshot/%v/%d.xml", f.ID.Hex(), f.SaveTime.Unix())
}

func FarmsJSON() ([]byte, error) {
	var result []*Farm

	if err := FarmCollection.Find(nil).Limit(20).All(&result); err != nil {
		return nil, err
	}
	for _, farm := range result {
		farm.Thumbnail = farm.ScreenshotPath()
	}
	return json.Marshal(result)
}

func UpdateFarmTime(c *mgo.Collection, id bson.ObjectId) error {
	return c.Update(bson.M{"_id": id}, bson.M{"savetime": time.Now()})
}

func FindFarm(c *mgo.Collection, uniqueIDForThisGame int, playerName, farmName string) (ret *Farm, existing bool, err error) {
	ret = &Farm{}
	q := c.Find(bson.M{
		"name":     farmName,
		"farmer":   playerName,
		"uniqueid": uniqueIDForThisGame,
	})
	log.Printf("q %v", q)
	if err := q.One(&ret); err != nil {
		log.Println("not found", err)

		farm := &Farm{
			Name:     farmName,
			Farmer:   playerName,
			UniqueID: uniqueIDForThisGame,
			ID:       bson.NewObjectId(),
			SaveTime: time.Now(),
		}
		if err := c.Insert(farm); err != nil {
			log.Println("could not insert", err)
			return nil, false, err
		}
		log.Println("insert ok", farm.ID.String())
		return farm, false, nil
	}
	log.Printf("found ok %v, %v, %v", ret.Name, ret.Farmer, ret.SaveTime)

	return ret, true, nil
}

func WriteSaveFile(farm *Farm, body []byte) error {
	if farm.SaveTime.IsZero() {
		return fmt.Errorf("error writing save file: unexpected zero save time")
	}

	saveFile := path.Join("saveGames", farm.ScreenshotPath())
	g, err := GFS.Create(saveFile)
	if err != nil {
		return fmt.Errorf("Error opening grid saveGames %v: %v", saveFile, err)

	}
	defer g.Close()
	if _, err := g.Write(body); err != nil {
		return fmt.Errorf("Failed to write grid save file at %v: %v", saveFile, err)
	}
	log.Printf("Wrote grid saveGame file %v", saveFile)
	return nil
}

// NewScreenshotWriter saves a screenshot in GFS at screenshots/<hexid>.png
func NewScreenshotWriter(farm *Farm) (io.WriteCloser, error) {
	if farm.SaveTime.IsZero() {
		return nil, fmt.Errorf("error writing screenshot: unexpected zero save time")
	}
	return GFS.Create(farm.ScreenshotPath())
}
<commit_msg>no spam<commit_after>package stardb

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"path"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Farm struct {

	// Fields with  `json:"-"` are ommited for web display

	ID        bson.ObjectId `bson:"_id,omitempty" json:"-"`
	UniqueID  int           `json:"-"`
	Name      string
	Farmer    string
	Likes     int
	SaveTime  time.Time `json:"-"`
	Thumbnail string
}

func (f *Farm) ScreenshotPath() string {
	return fmt.Sprintf("/screenshot/%v/%d.xml", f.ID.Hex(), f.SaveTime.Unix())
}

func FarmsJSON() ([]byte, error) {
	var result []*Farm

	if err := FarmCollection.Find(nil).Limit(20).All(&result); err != nil {
		return nil, err
	}
	for _, farm := range result {
		farm.Thumbnail = farm.ScreenshotPath()
	}
	return json.Marshal(result)
}

func UpdateFarmTime(c *mgo.Collection, id bson.ObjectId) error {
	return c.Update(bson.M{"_id": id}, bson.M{"savetime": time.Now()})
}

func FindFarm(c *mgo.Collection, uniqueIDForThisGame int, playerName, farmName string) (ret *Farm, existing bool, err error) {
	ret = &Farm{}
	q := c.Find(bson.M{
		"name":     farmName,
		"farmer":   playerName,
		"uniqueid": uniqueIDForThisGame,
	})
	if err := q.One(&ret); err != nil {
		log.Println("not found", err)

		farm := &Farm{
			Name:     farmName,
			Farmer:   playerName,
			UniqueID: uniqueIDForThisGame,
			ID:       bson.NewObjectId(),
			SaveTime: time.Now(),
		}
		if err := c.Insert(farm); err != nil {
			log.Println("could not insert", err)
			return nil, false, err
		}
		log.Println("insert ok", farm.ID.String())
		return farm, false, nil
	}
	log.Printf("found ok %v, %v, %v", ret.Name, ret.Farmer, ret.SaveTime)

	return ret, true, nil
}

func WriteSaveFile(farm *Farm, body []byte) error {
	if farm.SaveTime.IsZero() {
		return fmt.Errorf("error writing save file: unexpected zero save time")
	}

	saveFile := path.Join("saveGames", farm.ScreenshotPath())
	g, err := GFS.Create(saveFile)
	if err != nil {
		return fmt.Errorf("Error opening grid saveGames %v: %v", saveFile, err)

	}
	defer g.Close()
	if _, err := g.Write(body); err != nil {
		return fmt.Errorf("Failed to write grid save file at %v: %v", saveFile, err)
	}
	log.Printf("Wrote grid saveGame file %v", saveFile)
	return nil
}

// NewScreenshotWriter saves a screenshot in GFS at screenshots/<hexid>.png
func NewScreenshotWriter(farm *Farm) (io.WriteCloser, error) {
	if farm.SaveTime.IsZero() {
		return nil, fmt.Errorf("error writing screenshot: unexpected zero save time")
	}
	return GFS.Create(farm.ScreenshotPath())
}
<|endoftext|>


<commit_before>package stars

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
)

const (
	pageCount = 1
	apiPath   = "https://api.github.com/users/%v/starred?per_page=" + pageCount
)

// Stars …
type Stars struct {
	Pages    int
	Username string
	stared   []StaredRepos
}

// StaredRepos …
type StaredRepos struct {
	Name        string `json:"name"`
	URL         string `json:"html_url"`
	Description string `json:"description"`
}

// Repos …
func (s *Stars) Repos() ([]StaredRepos, error) {

	if err := s.setPagesCount(); err != nil {
		return nil, err
	}

	r, err := s.starsFromPage(1)
	if err != nil {
		return nil, err
	}
	s.stared = append(s.stared, r...)

	return s.stared, nil
}

func (s *Stars) setPagesCount() error {
	apiURL := fmt.Sprintf(apiPath, s.Username)
	log.Printf("%v\n", apiURL)

	resp, err := http.Get(apiURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	s.Pages, err = pagesCount(resp.Header.Get("Link"))
	log.Printf("%v\n", s.Pages)

	return nil
}

func (s *Stars) starsFromPage(p int) ([]StaredRepos, error) {
	var r []StaredRepos

	apiURL := fmt.Sprintf(apiPath+"&page=%v", s.Username, strconv.Itoa(p))
	log.Printf("%v\n", apiURL)

	resp, err := http.Get(apiURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		log.Printf("%v\n", err)
		return nil, err
	}

	return r, nil
}

func pagesCount(s string) (int, error) {
	if s == "" {
		return 1, nil
	}
	re, err := regexp.Compile("page=\\d+")
	if err != nil {
		return 0, err
	}
	foundStrings := re.FindAllString(s, -1)
	found := foundStrings[len(foundStrings)-1]

	re2, err := regexp.Compile("\\d+")
	if err != nil {
		return 0, err
	}

	i, err := strconv.Atoi(re2.FindString(found))
	if err != nil {
		return 0, err
	}

	return i, nil
}
<commit_msg>Add pseudo random page functionality<commit_after>package stars

import (
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

const (
	pageCount = "1"
	apiPath   = "https://api.github.com/users/%v/starred?per_page=" + pageCount
)

// Stars …
type Stars struct {
	Pages    int
	Username string
	stared   []StaredRepos
}

// StaredRepos …
type StaredRepos struct {
	Name        string `json:"name"`
	URL         string `json:"html_url"`
	Description string `json:"description"`
}

// Repos …
func (s *Stars) Repos() ([]StaredRepos, error) {

	if err := s.setPagesCount(); err != nil {
		return nil, err
	}

	r, err := s.starsFromPage(randomPageNumber(s.Pages))
	if err != nil {
		return nil, err
	}
	s.stared = append(s.stared, r...)

	return s.stared, nil
}

func (s *Stars) setPagesCount() error {
	apiURL := fmt.Sprintf(apiPath, s.Username)
	log.Printf("%v\n", apiURL)

	resp, err := http.Get(apiURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	s.Pages, err = pagesCount(resp.Header.Get("Link"))
	log.Printf("%v\n", s.Pages)

	return nil
}

func (s *Stars) starsFromPage(p int) ([]StaredRepos, error) {
	var r []StaredRepos

	apiURL := fmt.Sprintf(apiPath+"&page=%v", s.Username, strconv.Itoa(p))
	log.Printf("%v\n", apiURL)

	resp, err := http.Get(apiURL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		log.Printf("%v\n", err)
		return nil, err
	}

	return r, nil
}

func pagesCount(s string) (int, error) {
	if s == "" {
		return 1, nil
	}
	re, err := regexp.Compile("page=\\d+")
	if err != nil {
		return 0, err
	}
	foundStrings := re.FindAllString(s, -1)
	found := foundStrings[len(foundStrings)-1]

	re2, err := regexp.Compile("\\d+")
	if err != nil {
		return 0, err
	}

	i, err := strconv.Atoi(re2.FindString(found))
	if err != nil {
		return 0, err
	}

	return i, nil
}

func randomPageNumber(i int) int {
	const shortForm = "2006-January-02"

	year, month, day := time.Now().Date()

	date := fmt.Sprintf("%v-%v-%v", year, month, day)
	t, _ := time.Parse(shortForm, date)
	rand.Seed(t.Unix())

	randomNumber := rand.Intn(i)
	log.Printf("Random: %v\n", randomNumber)

	return randomNumber
}
<|endoftext|>


<commit_before>package stash

//go:generate go-bindata -pkg $GOPACKAGE -o assets.go assets/

import (
	"encoding/json"
	"fmt"
	"strings"
)

// Base list acquired from
// https://poeapp.com/api/item-types.json
//
// The raw response was transformed with
// x as the parsed baseInfo.json
// transformed = x.map(v=> {
// 	v = Object.assign({}, v);
// 	v.root = v.key;
// 	delete v.key;
// 	v.flavors = v.value;
// 	delete v.value;
// 	return v})
// .map(v => {
// 	v.flavors = v.flavors.map(f => {
// 		f = Object.assign({}, f);
// 		f.flavor = f.key;
// 		delete f.key;
// 		f.bases = f.value;
// 		delete f.value;
// 		return f
// 	});
// 	return v;
// })
// JSON.stringify(transformed)

type flavor struct {
	Flavor string   `json:"flavor"`
	Bases  []string `json:"bases"`
}

type root struct {
	Root    string   `json:"root"`
	Flavors []flavor `json:"flavors"`
}

type baseLookup []root

// toInverter converts a baseLookup to a baseInverter
func (bl baseLookup) toInverter() baseInverter {

	basesToFlavors := make(map[string]string)
	flavorsToRoots := make(map[string]string)

	for _, root := range bl {
		for _, flavor := range root.Flavors {
			flavorsToRoots[flavor.Flavor] = root.Root

			for _, base := range flavor.Bases {
				basesToFlavors[base] = flavor.Flavor
			}
		}
	}

	return baseInverter{
		flavorsToRoots: flavorsToRoots,
		basesToFlavors: basesToFlavors,
	}

}

func getBaseLookup() (*baseLookup, error) {
	raw, err := Asset("assets/baseInfo.json")
	if err != nil {
		return nil, fmt.Errorf("failed to fetch baseInfo.json, err=%s", err)
	}

	var bl baseLookup
	err = json.Unmarshal(raw, &bl)
	return &bl, err
}

type baseInverter struct {
	basesToFlavors map[string]string
	flavorsToRoots map[string]string
}

// typeLineToRootAndFlavor takes a given TypeLine on an item
// and converts it to a root and flavor.
//
// This even handles the super mangled typeLines that GGG sometimes includes...
func (inv baseInverter) typeLineToRootAndFlavor(typeLine string) (flavor,
	root string, ok bool) {

	// Try to resolve the typeLine to include a base which
	// indicates a given flavor
	for propBase, propFlavor := range inv.basesToFlavors {
		if strings.Contains(typeLine, propBase) {
			flavor = propFlavor
			break
		}
	}
	if len(flavor) == 0 {
		ok = false
		return
	}

	// Resolve a correct flavor to base easily
	root = inv.flavorsToRoots[flavor]
	ok = true
	return

}

// Singleton inverter the package uses
var inverter baseInverter

// MatchTypeline returns a flavor and root for a discovered base
// or ok is false.
func MatchTypeline(typeline string) (flavor, root string, ok bool) {
	return inverter.typeLineToRootAndFlavor(typeline)
}

// Prep our singleton inverter
func init() {
	bl, err := getBaseLookup()
	if err != nil {
		panic(fmt.Sprintf("failed to deserialize baseLookup on stash init, err=%s", err))
	}

	inverter = bl.toInverter()
}
<commit_msg>stash faster base matching<commit_after>package stash

//go:generate go-bindata -pkg $GOPACKAGE -o assets.go assets/

import (
	"encoding/json"
	"fmt"
	"strings"
)

// Base list acquired from
// https://poeapp.com/api/item-types.json
//
// The raw response was transformed with
// x as the parsed baseInfo.json
// transformed = x.map(v=> {
// 	v = Object.assign({}, v);
// 	v.root = v.key;
// 	delete v.key;
// 	v.flavors = v.value;
// 	delete v.value;
// 	return v})
// .map(v => {
// 	v.flavors = v.flavors.map(f => {
// 		f = Object.assign({}, f);
// 		f.flavor = f.key;
// 		delete f.key;
// 		f.bases = f.value;
// 		delete f.value;
// 		return f
// 	});
// 	return v;
// })
// JSON.stringify(transformed)

type flavor struct {
	Flavor string   `json:"flavor"`
	Bases  []string `json:"bases"`
}

type root struct {
	Root    string   `json:"root"`
	Flavors []flavor `json:"flavors"`
}

type baseLookup []root

// toInverter converts a baseLookup to a baseInverter
func (bl baseLookup) toInverter() baseInverter {

	basesToFlavors := make(map[string]string)
	flavorsToRoots := make(map[string]string)

	for _, root := range bl {
		for _, flavor := range root.Flavors {
			flavorsToRoots[flavor.Flavor] = root.Root

			for _, base := range flavor.Bases {
				basesToFlavors[base] = flavor.Flavor
			}
		}
	}

	return baseInverter{
		flavorsToRoots: flavorsToRoots,
		basesToFlavors: basesToFlavors,
	}

}

func getBaseLookup() (*baseLookup, error) {
	raw, err := Asset("assets/baseInfo.json")
	if err != nil {
		return nil, fmt.Errorf("failed to fetch baseInfo.json, err=%s", err)
	}

	var bl baseLookup
	err = json.Unmarshal(raw, &bl)
	return &bl, err
}

type baseInverter struct {
	basesToFlavors map[string]string
	flavorsToRoots map[string]string
}

// typeLineToRootAndFlavor takes a given TypeLine on an item
// and converts it to a root and flavor.
//
// This even handles the super mangled typeLines that GGG sometimes includes...
func (inv baseInverter) typeLineToRootAndFlavor(typeLine string) (flavor,
	root string, ok bool) {

	// First we try a direct match
	flavor, ok = inv.basesToFlavors[typeLine]
	if ok {
		// Resolve the root and return
		root = inv.flavorsToRoots[flavor]
		return
	}

	// If direct match failed, we need an exhaustive substring search
	// This hurts performance but usually is fairly rare :|
	for propBase, propFlavor := range inv.basesToFlavors {
		if strings.Contains(typeLine, propBase) {
			flavor = propFlavor
			break
		}
	}
	if len(flavor) == 0 {
		ok = false
		return
	}

	// Resolve a correct flavor to base easily
	root = inv.flavorsToRoots[flavor]
	ok = true
	return

}

// Singleton inverter the package uses
var inverter baseInverter

// MatchTypeline returns a flavor and root for a discovered base
// or ok is false.
func MatchTypeline(typeline string) (flavor, root string, ok bool) {
	return inverter.typeLineToRootAndFlavor(typeline)
}

// Prep our singleton inverter
func init() {
	bl, err := getBaseLookup()
	if err != nil {
		panic(fmt.Sprintf("failed to deserialize baseLookup on stash init, err=%s", err))
	}

	inverter = bl.toInverter()
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0
// Details: https://raw.githubusercontent.com/square/quotaservice/master/LICENSE

package stats

import (
	"fmt"
	"time"

	"gopkg.in/redis.v5"

	"github.com/square/quotaservice/events"
	"github.com/square/quotaservice/logging"
)

type redisListener struct {
	client *redis.Client
}

// NewRedisStatsListener creates a redis-backed stats
// listener with the passed in redis.Options.
func NewRedisStatsListener(redisOpts *redis.Options) Listener {
	client := redis.NewClient(redisOpts)
	_, err := client.Ping().Result()

	if err != nil {
		logging.Fatalf("RedisStatsListener: cannot connect to Redis, %v", err)
	}

	return &redisListener{client}
}

func (l *redisListener) redisTopList(key string) []*BucketScore {
	results, err := l.client.ZRevRangeWithScores(key, 0, 10).Result()

	if err != nil && err.Error() != "redis: nil" {
		logging.Printf("RedisStatsListener.TopList error (%s) %v", key, err)
		return emptyArr
	}

	arr := make([]*BucketScore, len(results))

	for i, item := range results {
		arr[i] = &BucketScore{item.Member.(string), int64(item.Score)}
	}

	return arr
}

func statsNamespace(key, namespace string) string {
	return fmt.Sprintf("stats:%s:%s", namespace, key)
}

// TopHits is implemented for stats.Listener
// TopHits returns a sorted list of the 10 buckets with the highest # of hits
// in the specified namespace within the current bucketed hour
func (l *redisListener) TopHits(namespace string) []*BucketScore {
	return l.redisTopList(statsNamespace("hits", namespace))
}

// TopMisses is implemented for stats.Listener
// TopMisses returns a sorted list of the 10 buckets with the highest # of misses
// in the specified namespace within the current bucketed hour
func (l *redisListener) TopMisses(namespace string) []*BucketScore {
	return l.redisTopList(statsNamespace("misses", namespace))
}

// Get is implemented for stats.Listener
// Get returns the hits and misses for a bucket in the specified namespace
// within the current bucketed hour
func (l *redisListener) Get(namespace, bucket string) *BucketScores {
	scores := &BucketScores{0, 0}

	value, err := l.client.ZScore(statsNamespace("misses", namespace), bucket).Result()

	if err != nil && err.Error() != "redis: nil" {
		logging.Printf("RedisStatsListener.Get error (%s, %s) %v", namespace, bucket, err)
	} else {
		scores.Misses = int64(value)
	}

	value, err = l.client.ZScore(statsNamespace("hits", namespace), bucket).Result()

	if err != nil && err.Error() != "redis: nil" {
		logging.Printf("RedisStatsListener.Get error (%s, %s) %v", namespace, bucket, err)
	} else {
		scores.Hits = int64(value)
	}

	return scores
}

func nearestHour() time.Time {
	return time.Now().Add(time.Hour).Truncate(time.Hour)
}

// HandleEvent is implemented for stats.Listener
// HandleEvent consumes dynamic bucket events (see events.Event)
func (l *redisListener) HandleEvent(event events.Event) {
	if !event.Dynamic() {
		return
	}

	var key string
	var numTokens int64 = 1

	switch event.EventType() {
	case events.EVENT_BUCKET_MISS:
		key = "misses"
	case events.EVENT_TOKENS_SERVED:
		numTokens = event.NumTokens()
		key = "hits"
	default:
		return
	}

	namespace := statsNamespace(key, event.Namespace())
	bucket := event.BucketName()

	var incr *redis.FloatCmd
	_, err := l.client.Pipelined(func(pipe *redis.Pipeline) error {
		incr = pipe.ZIncrBy(namespace, float64(numTokens), bucket)
		pipe.ExpireAt(namespace, nearestHour())
		return nil
	})

	if err != nil || incr.Err() != nil {
		logging.Printf("RedisStatsListener.HandleEvent error (%s, %s, %d) %v, %v",
			namespace, bucket, numTokens, err, incr.Err())
	}
	go func() {
		var incr *redis.FloatCmd
		_, err := l.client.Pipelined(func(pipe *redis.Pipeline) error {
			incr = pipe.ZIncrBy(namespace, float64(numTokens), bucket)
			pipe.ExpireAt(namespace, nearestHour())
			return nil
		})

		if err != nil || incr.Err() != nil {
			logging.Printf("RedisStatsListener.HandleEvent error (%s, %s, %d) %v, %v",
				namespace, bucket, numTokens, err, incr.Err())
		}
	}()
}
<commit_msg>Replace individual goroutines with a batcher<commit_after>// Licensed under the Apache License, Version 2.0
// Details: https://raw.githubusercontent.com/square/quotaservice/master/LICENSE

package stats

import (
	"fmt"
	"sync"
	"time"

	"gopkg.in/redis.v5"

	"github.com/square/quotaservice/events"
	"github.com/square/quotaservice/logging"
)

type statsUpdate struct {
	namespace string
	numTokens float64
	bucket    string
}

type redisListener struct {
	client           *redis.Client
	statsUpdates     []*statsUpdate
	statsUpdatesLock *sync.Mutex
	notifyBatcher    chan struct{}
}

const (
	batchSize           = 128
	batchSubmitInterval = 100 * time.Millisecond
)

// NewRedisStatsListener creates a redis-backed stats
// listener with the passed in redis.Options.
func NewRedisStatsListener(redisOpts *redis.Options) Listener {
	client := redis.NewClient(redisOpts)
	_, err := client.Ping().Result()

	if err != nil {
		logging.Fatalf("RedisStatsListener: cannot connect to Redis, %v", err)
	}

	l := &redisListener{
		client:           client,
		statsUpdates:     make([]*statsUpdate, 0, batchSize),
		notifyBatcher:    make(chan struct{}, 1),
		statsUpdatesLock: &sync.Mutex{},
	}

	go l.batcher()

	return l
}

func (l *redisListener) redisTopList(key string) []*BucketScore {
	results, err := l.client.ZRevRangeWithScores(key, 0, 10).Result()

	if err != nil && err.Error() != "redis: nil" {
		logging.Printf("RedisStatsListener.TopList error (%s) %v", key, err)
		return emptyArr
	}

	arr := make([]*BucketScore, len(results))

	for i, item := range results {
		arr[i] = &BucketScore{item.Member.(string), int64(item.Score)}
	}

	return arr
}

func statsNamespace(key, namespace string) string {
	return fmt.Sprintf("stats:%s:%s", namespace, key)
}

// TopHits is implemented for stats.Listener
// TopHits returns a sorted list of the 10 buckets with the highest # of hits
// in the specified namespace within the current bucketed hour
func (l *redisListener) TopHits(namespace string) []*BucketScore {
	return l.redisTopList(statsNamespace("hits", namespace))
}

// TopMisses is implemented for stats.Listener
// TopMisses returns a sorted list of the 10 buckets with the highest # of misses
// in the specified namespace within the current bucketed hour
func (l *redisListener) TopMisses(namespace string) []*BucketScore {
	return l.redisTopList(statsNamespace("misses", namespace))
}

// Get is implemented for stats.Listener
// Get returns the hits and misses for a bucket in the specified namespace
// within the current bucketed hour
func (l *redisListener) Get(namespace, bucket string) *BucketScores {
	scores := &BucketScores{0, 0}

	value, err := l.client.ZScore(statsNamespace("misses", namespace), bucket).Result()

	if err != nil && err.Error() != "redis: nil" {
		logging.Printf("RedisStatsListener.Get error (%s, %s) %v", namespace, bucket, err)
	} else {
		scores.Misses = int64(value)
	}

	value, err = l.client.ZScore(statsNamespace("hits", namespace), bucket).Result()

	if err != nil && err.Error() != "redis: nil" {
		logging.Printf("RedisStatsListener.Get error (%s, %s) %v", namespace, bucket, err)
	} else {
		scores.Hits = int64(value)
	}

	return scores
}

func nearestHour() time.Time {
	return time.Now().Add(time.Hour).Truncate(time.Hour)
}

// HandleEvent is implemented for stats.Listener
// HandleEvent consumes dynamic bucket events (see events.Event)
func (l *redisListener) HandleEvent(event events.Event) {
	if !event.Dynamic() {
		return
	}

	var key string
	var numTokens int64 = 1

	switch event.EventType() {
	case events.EVENT_BUCKET_MISS:
		key = "misses"
	case events.EVENT_TOKENS_SERVED:
		numTokens = event.NumTokens()
		key = "hits"
	default:
		return
	}

	namespace := statsNamespace(key, event.Namespace())
	bucket := event.BucketName()

	l.queueStatsUpdate(namespace, numTokens, bucket)
}

// queueStatsUpdate queues a statsUpdate to be sent to redis via the batcher
func (l *redisListener) queueStatsUpdate(namespace string, numTokens int64, bucket string) {
	l.statsUpdatesLock.Lock()

	l.statsUpdates = append(l.statsUpdates, &statsUpdate{
		namespace: namespace,
		numTokens: float64(numTokens),
		bucket:    bucket,
	})

	l.statsUpdatesLock.Unlock()

	select {
	case l.notifyBatcher <- struct{}{}:
		// Done
	default:
		// There's already a pending notification
	}
}

// batcher aggregates stats updates. Any stats that are aggregated are guaranteed
// to be sent within batchSubmitInterval, or once batchSize stats updates are queued,
// whichever happens first.
func (l *redisListener) batcher() {
	timeout := time.After(batchSubmitInterval)

	for {
		var buffer []*statsUpdate

		select {
		case <-timeout:
			l.statsUpdatesLock.Lock()

			if len(l.statsUpdates) == 0 {
				l.statsUpdatesLock.Unlock()
				timeout = time.After(batchSubmitInterval)
				continue
			}

		case <-l.notifyBatcher:
			l.statsUpdatesLock.Lock()

			if len(l.statsUpdates) < batchSize {
				l.statsUpdatesLock.Unlock()
				continue
			}
		}

		buffer = make([]*statsUpdate, len(l.statsUpdates))
		copy(buffer, l.statsUpdates)

		l.statsUpdatesLock.Unlock()

		go l.submitBatch(buffer)
		timeout = time.After(batchSubmitInterval)
	}
}

// submitBatch sends the provided stats updates to redis in one pipelined query.
func (l *redisListener) submitBatch(batch []*statsUpdate) {
	cmds, err := l.client.Pipelined(func(pipe *redis.Pipeline) error {
		for _, update := range batch {
			pipe.ZIncrBy(update.namespace, update.numTokens, update.bucket)
			pipe.ExpireAt(update.namespace, nearestHour())
		}
		return nil
	})

	if err != nil {
		logging.Printf("RedisStatsListener.HandleEvent pipeline error %v", err)
	}

	for i, cmd := range cmds {
		if cmd.Err() == nil {
			continue
		}

		update := batch[i]
		logging.Printf("RedisStatsListener.HandleEvent error (%s, %s, %d) %v",
			update.namespace, update.bucket, update.numTokens, cmd.Err())
	}
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package stats collects serviced metrics and posts them to the TSDB.

package stats

import (
	"bytes"
	"encoding/json"
	"github.com/rcrowley/go-metrics"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/stats/cgroup"
	"github.com/zenoss/serviced/utils"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/zzk"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// StatsReporter collects and posts serviced stats to the TSDB.
type StatsReporter struct {
	destination  string
	closeChannel chan bool
	zkDAO        *zzk.ZkDao
}

type containerStat struct {
	Metric    string            `json:"metric"`
	Value     string            `json:"value"`
	Timestamp int64             `json:"timestamp"`
	Tags      map[string]string `json:"tags"`
}

var registries map[string]metrics.Registry
var hostID string
var hostRegistry metrics.Registry

// getOrCreateRegistry returns a registry for a given service id or creates it
// if it doesn't exist.
func getOrCreateRegistry(serviceid string) metrics.Registry {
	if registry, ok := registries[serviceid]; ok {
		return registry
	}
	registries[serviceid] = metrics.NewRegistry()
	return registries[serviceid]
}

// NewStatsReporter creates a new StatsReporter and kicks off the reporting goroutine.
func NewStatsReporter(destination string, interval time.Duration, zkDAO *zzk.ZkDao) (*StatsReporter, error) {
	registries = make(map[string]metrics.Registry)
	var err error
	hostID, err = utils.HostID()
	if err != nil {
		glog.Errorf("Could not determine host ID.")
		return nil, err
	}
	hostRegistry = getOrCreateRegistry(hostID)
	sr := StatsReporter{destination, make(chan bool), zkDAO}
	go sr.report(interval)
	return &sr, nil
}

// Close shuts down the reporting goroutine. Blocks waiting for the goroutine to signal that it
// is indeed shutting down.
func (sr StatsReporter) Close() {
	glog.Warningf("######### Closing stats reporter")
	sr.closeChannel <- true
	_ = <-sr.closeChannel
}

// Updates the default registry, fills out the metric consumer format, and posts
// the data to the TSDB. Stops when close signal is received on closeChannel.
func (sr StatsReporter) report(d time.Duration) {
	tc := time.Tick(d)
	for {
		select {
		case _ = <-sr.closeChannel:
			glog.V(3).Info("Ceasing stat reporting.")
			sr.closeChannel <- true
			return
		case t := <-tc:
			glog.V(3).Info("Reporting container stats at:", t)
			sr.updateStats()
			stats := sr.gatherStats(t)
			err := sr.post(stats)
			if err != nil {
				glog.Errorf("Error reporting container stats.")
			}
		}
	}
}

// Updates the default registry.
func (sr StatsReporter) updateStats() {
	// Stats for serviced.
	if cpuacctStat, err := cgroup.ReadCpuacctStat(""); err != nil {
		glog.V(3).Info("Couldn't read CpuacctStat:", err)
	} else {
		metrics.GetOrRegisterGauge("CpuacctStat.system", hostRegistry).Update(cpuacctStat.System)
		metrics.GetOrRegisterGauge("CpuacctStat.user", hostRegistry).Update(cpuacctStat.User)
	}

	if memoryStat, err := cgroup.ReadMemoryStat(""); err != nil {
		glog.V(3).Info("Couldn't read MemoryStat:", err)
	} else {
		metrics.GetOrRegisterGauge("MemoryStat.pgfault", hostRegistry).Update(memoryStat.Pgfault)
		metrics.GetOrRegisterGauge("MemoryStat.rss", hostRegistry).Update(memoryStat.Rss)
	}

	if openFileDescriptorCount, err := GetOpenFileDescriptorCount(); err != nil {
		glog.V(3).Info("Couldn't get open file descriptor count", err)
	} else {
		metrics.GetOrRegisterGauge("Serviced.OpenFileDescriptors", hostRegistry).Update(openFileDescriptorCount)
	}
	// Stats for the containers.
    var running []*dao.RunningService
	sr.zkDAO.GetRunningServicesForHost(hostID, &running)
	for _, rs := range(running) {
		containerRegistry := getOrCreateRegistry(rs.ServiceID)
		if cpuacctStat, err := cgroup.ReadCpuacctStat("/sys/fs/cgroup/cpuacct/docker/" + rs.DockerID + "/cpuacct.stat"); err != nil {
			glog.V(3).Info("Couldn't read CpuacctStat:", err)
		} else {
			metrics.GetOrRegisterGauge("CpuacctStat.system", containerRegistry).Update(cpuacctStat.System)
			metrics.GetOrRegisterGauge("CpuacctStat.user", containerRegistry).Update(cpuacctStat.User)
		}
		if memoryStat, err := cgroup.ReadMemoryStat("/sys/fs/cgroup/memory/docker/" + rs.DockerID + "/memory.stat"); err != nil {
			glog.V(3).Info("Couldn't read MemoryStat:", err)
		} else {
			metrics.GetOrRegisterGauge("MemoryStat.pgfault", containerRegistry).Update(memoryStat.Pgfault)
			metrics.GetOrRegisterGauge("MemoryStat.rss", containerRegistry).Update(memoryStat.Rss)
		}
	}
}

// Fills out the metric consumer format.
func (sr StatsReporter) gatherStats(t time.Time) []containerStat {
	stats := []containerStat{}
	for serviceID, registry := range registries {
		reg, _ := registry.(*metrics.StandardRegistry)
		reg.Each(func(name string, i interface{}) {
			metric := i.(metrics.Gauge)
			tagmap := make(map[string]string)
			if (serviceID != hostID) {
				tagmap["controlplane_service_id"] = serviceID
			}
			tagmap["controlplane_host_id"] = hostID
			stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
		})
	}
	return stats
}

// Send the list of stats to the TSDB.
func (sr StatsReporter) post(stats []containerStat) error {
	payload := map[string][]containerStat{"metrics": stats}
	data, err := json.Marshal(payload)
	if err != nil {
		glog.V(0).Info("Couldn't marshal stats: ", err)
		return err
	}
	statsreq, err := http.NewRequest("POST", sr.destination, bytes.NewBuffer(data))
	if err != nil {
		glog.V(0).Info("Couldn't create stats request: ", err)
		return err
	}
	statsreq.Header["User-Agent"] = []string{"Zenoss Metric Publisher"}
	statsreq.Header["Content-Type"] = []string{"application/json"}
	resp, reqerr := http.DefaultClient.Do(statsreq)
	if reqerr != nil {
		glog.V(0).Info("Couldn't post stats: ", reqerr)
		return reqerr
	}
	if strings.Contains(resp.Status, "200 OK") == false {
		glog.V(0).Info("Non-success: ", resp.Status)
		glog.Errorf("couldn't post stats: ", resp.Status)
		return nil
	}
	resp.Body.Close()
	return nil
}
<commit_msg>remove debug logging<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

// Package stats collects serviced metrics and posts them to the TSDB.

package stats

import (
	"bytes"
	"encoding/json"
	"github.com/rcrowley/go-metrics"
	"github.com/zenoss/glog"
	"github.com/zenoss/serviced/stats/cgroup"
	"github.com/zenoss/serviced/utils"
	"github.com/zenoss/serviced/dao"
	"github.com/zenoss/serviced/zzk"
	"net/http"
	"strconv"
	"strings"
	"time"
)

// StatsReporter collects and posts serviced stats to the TSDB.
type StatsReporter struct {
	destination  string
	closeChannel chan bool
	zkDAO        *zzk.ZkDao
}

type containerStat struct {
	Metric    string            `json:"metric"`
	Value     string            `json:"value"`
	Timestamp int64             `json:"timestamp"`
	Tags      map[string]string `json:"tags"`
}

var registries map[string]metrics.Registry
var hostID string
var hostRegistry metrics.Registry

// getOrCreateRegistry returns a registry for a given service id or creates it
// if it doesn't exist.
func getOrCreateRegistry(serviceid string) metrics.Registry {
	if registry, ok := registries[serviceid]; ok {
		return registry
	}
	registries[serviceid] = metrics.NewRegistry()
	return registries[serviceid]
}

// NewStatsReporter creates a new StatsReporter and kicks off the reporting goroutine.
func NewStatsReporter(destination string, interval time.Duration, zkDAO *zzk.ZkDao) (*StatsReporter, error) {
	registries = make(map[string]metrics.Registry)
	var err error
	hostID, err = utils.HostID()
	if err != nil {
		glog.Errorf("Could not determine host ID.")
		return nil, err
	}
	hostRegistry = getOrCreateRegistry(hostID)
	sr := StatsReporter{destination, make(chan bool), zkDAO}
	go sr.report(interval)
	return &sr, nil
}

// Close shuts down the reporting goroutine. Blocks waiting for the goroutine to signal that it
// is indeed shutting down.
func (sr StatsReporter) Close() {
	sr.closeChannel <- true
	_ = <-sr.closeChannel
}

// Updates the default registry, fills out the metric consumer format, and posts
// the data to the TSDB. Stops when close signal is received on closeChannel.
func (sr StatsReporter) report(d time.Duration) {
	tc := time.Tick(d)
	for {
		select {
		case _ = <-sr.closeChannel:
			glog.V(3).Info("Ceasing stat reporting.")
			sr.closeChannel <- true
			return
		case t := <-tc:
			glog.V(3).Info("Reporting container stats at:", t)
			sr.updateStats()
			stats := sr.gatherStats(t)
			err := sr.post(stats)
			if err != nil {
				glog.Errorf("Error reporting container stats.")
			}
		}
	}
}

// Updates the default registry.
func (sr StatsReporter) updateStats() {
	// Stats for serviced.
	if cpuacctStat, err := cgroup.ReadCpuacctStat(""); err != nil {
		glog.V(3).Info("Couldn't read CpuacctStat:", err)
	} else {
		metrics.GetOrRegisterGauge("CpuacctStat.system", hostRegistry).Update(cpuacctStat.System)
		metrics.GetOrRegisterGauge("CpuacctStat.user", hostRegistry).Update(cpuacctStat.User)
	}

	if memoryStat, err := cgroup.ReadMemoryStat(""); err != nil {
		glog.V(3).Info("Couldn't read MemoryStat:", err)
	} else {
		metrics.GetOrRegisterGauge("MemoryStat.pgfault", hostRegistry).Update(memoryStat.Pgfault)
		metrics.GetOrRegisterGauge("MemoryStat.rss", hostRegistry).Update(memoryStat.Rss)
	}

	if openFileDescriptorCount, err := GetOpenFileDescriptorCount(); err != nil {
		glog.V(3).Info("Couldn't get open file descriptor count", err)
	} else {
		metrics.GetOrRegisterGauge("Serviced.OpenFileDescriptors", hostRegistry).Update(openFileDescriptorCount)
	}
	// Stats for the containers.
    var running []*dao.RunningService
	sr.zkDAO.GetRunningServicesForHost(hostID, &running)
	for _, rs := range(running) {
		containerRegistry := getOrCreateRegistry(rs.ServiceID)
		if cpuacctStat, err := cgroup.ReadCpuacctStat("/sys/fs/cgroup/cpuacct/docker/" + rs.DockerID + "/cpuacct.stat"); err != nil {
			glog.V(3).Info("Couldn't read CpuacctStat:", err)
		} else {
			metrics.GetOrRegisterGauge("CpuacctStat.system", containerRegistry).Update(cpuacctStat.System)
			metrics.GetOrRegisterGauge("CpuacctStat.user", containerRegistry).Update(cpuacctStat.User)
		}
		if memoryStat, err := cgroup.ReadMemoryStat("/sys/fs/cgroup/memory/docker/" + rs.DockerID + "/memory.stat"); err != nil {
			glog.V(3).Info("Couldn't read MemoryStat:", err)
		} else {
			metrics.GetOrRegisterGauge("MemoryStat.pgfault", containerRegistry).Update(memoryStat.Pgfault)
			metrics.GetOrRegisterGauge("MemoryStat.rss", containerRegistry).Update(memoryStat.Rss)
		}
	}
}

// Fills out the metric consumer format.
func (sr StatsReporter) gatherStats(t time.Time) []containerStat {
	stats := []containerStat{}
	for serviceID, registry := range registries {
		reg, _ := registry.(*metrics.StandardRegistry)
		reg.Each(func(name string, i interface{}) {
			metric := i.(metrics.Gauge)
			tagmap := make(map[string]string)
			if (serviceID != hostID) {
				tagmap["controlplane_service_id"] = serviceID
			}
			tagmap["controlplane_host_id"] = hostID
			stats = append(stats, containerStat{name, strconv.FormatInt(metric.Value(), 10), t.Unix(), tagmap})
		})
	}
	return stats
}

// Send the list of stats to the TSDB.
func (sr StatsReporter) post(stats []containerStat) error {
	payload := map[string][]containerStat{"metrics": stats}
	data, err := json.Marshal(payload)
	if err != nil {
		glog.V(0).Info("Couldn't marshal stats: ", err)
		return err
	}
	statsreq, err := http.NewRequest("POST", sr.destination, bytes.NewBuffer(data))
	if err != nil {
		glog.V(0).Info("Couldn't create stats request: ", err)
		return err
	}
	statsreq.Header["User-Agent"] = []string{"Zenoss Metric Publisher"}
	statsreq.Header["Content-Type"] = []string{"application/json"}
	resp, reqerr := http.DefaultClient.Do(statsreq)
	if reqerr != nil {
		glog.V(0).Info("Couldn't post stats: ", reqerr)
		return reqerr
	}
	if strings.Contains(resp.Status, "200 OK") == false {
		glog.V(0).Info("Non-success: ", resp.Status)
		glog.Errorf("couldn't post stats: ", resp.Status)
		return nil
	}
	resp.Body.Close()
	return nil
}
<|endoftext|>


<commit_before>package mpd_test

import (
	. "github.com/todd/gompd"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Status", func() {
  var client Client
  var status Status

  BeforeEach(func() {
    client, _ = Init(host, port, timeoutms)
    status, _ = client.GetStatus()
  })

  AfterEach(func() {
    status.Free()
    client.Close()
  })

  Describe("GetSongID", func() {
    It("Returns the ID of the current song", func() {
      id, err := status.GetSongID()

      Expect(err).ToNot(HaveOccurred())
      Expect(id).ToNot(Equal(-1)) // MPD returns -1 if no current song
    })
  })
})
<commit_msg>Update format<commit_after>package mpd_test

import (
	. "github.com/todd/gompd"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Status", func() {
	var client Client
	var status Status

	BeforeEach(func() {
		client, _ = Init(host, port, timeoutms)
		status, _ = client.GetStatus()
	})

	AfterEach(func() {
		status.Free()
		client.Close()
	})

	Describe("GetSongID", func() {
		It("Returns the ID of the current song", func() {
			id, err := status.GetSongID()

			Expect(err).ToNot(HaveOccurred())
			Expect(id).ToNot(Equal(-1)) // MPD returns -1 if no current song
		})
	})
})
<|endoftext|>


<commit_before>/*
Package framed adds basic support for message framing over streams.

Messages contain an header and a body, both of which are length prefixed.

Here are the bytes (stored in little-endian byte order):

0-2: unsigned 16 bit int header length
2-4: unsinged 16 bit int body length
4+:  message content (header and body)

The use of a uint16 means that the maximum possible header and body lengths
are 65535 each.

Example:

	package main

	import (
		"github.com/oxtoacart/framed"
		"net"
		"log"
	)

	func main() {
		// Replace host:port with an actual TCP server, for example the echo service
		if conn, err := net.Dial("tcp", "host:port"); err == nil {
			framedConn = Framed{conn}
			if err := framedConn.Write([]byte("Hello World")); err == nil {
				if resp, err := framedConn.Read(); err == nil {
					log.Println("We're done!")
				}
			}
		}
	}
*/
package framed

import (
	"encoding/binary"
	"io"
	"io/ioutil"
)

var endianness = binary.LittleEndian

/*
A Framed enhances an io.ReadWriteCloser to provide methods that allow writing
and reading frames.

Although the underlying ReadWriteCloser may be safe to use from multiple
goroutines, a Framed is not.
*/
type Framed struct {
	io.ReadWriteCloser // the raw underlying connection
	hasReadInitial     bool
}

type AlreadyReadError string

func (err AlreadyReadError) Error() string {
	return string(err)
}

type BufferTooSmall string

func (err BufferTooSmall) Error() string {
	return string(err)
}

type Frame struct {
	Header       *FrameSection
	Body         *FrameSection
	framed       *Framed
	headerLength int16
	bodyLength   int16
}

type FrameSection struct {
	frame          *Frame
	init           func() error
	bytesRemaining int
	startedReading bool
}

func NewFramed(readWriteCloser io.ReadWriteCloser) *Framed {
	return &Framed{readWriteCloser, false}
}

/*
ReadInitial reads the initial frame from the framed
*/
func (framed *Framed) ReadInitial() (frame *Frame, err error) {
	if framed.hasReadInitial {
		return nil, AlreadyReadError("Initial Frame already read")
	}
	frame, err = framed.nextFrame()
	framed.hasReadInitial = true
	return
}

func (frame *Frame) NextFrame() (nextFrame *Frame, err error) {
	if err = frame.Header.drain(); err != nil {
		return
	}
	if err = frame.Body.drain(); err != nil {
		return
	}
	nextFrame, err = frame.framed.nextFrame()
	return
}

func (framed *Framed) WriteFrame(header []byte, body []byte) (err error) {
	if err = framed.WriteHeader(int16(len(header)), int16(len(body))); err != nil {
		return err
	}
	if _, err = framed.Write(header); err != nil {
		return err
	}
	_, err = framed.Write(body)
	return
}

func (framed *Framed) WriteHeader(headerLength int16, bodyLength int16) (err error) {
	return writeHeaderTo(framed, headerLength, bodyLength)
}

func (frame *Frame) CopyTo(out io.Writer) (err error) {
	if frame.Header.startedReading || frame.Body.startedReading {
		return AlreadyReadError("Already read from frame, cannot copy")
	}
	if err = writeHeaderTo(out, frame.headerLength, frame.bodyLength); err != nil {
		return
	}
	_, err = io.CopyN(out, frame.framed, int64(frame.headerLength+frame.bodyLength))
	return
}

func (section *FrameSection) Read(p []byte) (n int, err error) {
	if section.bytesRemaining == 0 {
		return 0, err
	}
	if section.init != nil {
		if err = section.init(); err != nil {
			return 0, err
		}
	}
	section.startedReading = true
	if len(p) > section.bytesRemaining {
		p = p[0:section.bytesRemaining]
	}
	n, err = section.frame.framed.Read(p)
	if n > 0 {
		section.bytesRemaining -= n
	}
	if section.bytesRemaining == 0 {
		err = io.EOF
	}
	return
}

func (framed *Framed) nextFrame() (frame *Frame, err error) {
	frame = &Frame{framed: framed}
	frame.Header = &FrameSection{frame: frame}
	frame.Body = &FrameSection{frame: frame, init: frame.Header.drain}
	if err = frame.readLengths(); err != nil {
		return
	}
	return
}

func (frame *Frame) readLengths() (err error) {
	if err = binary.Read(frame.framed, endianness, &frame.headerLength); err != nil {
		return
	}
	if err = binary.Read(frame.framed, endianness, &frame.bodyLength); err != nil {
		return
	}
	frame.Header.bytesRemaining = int(frame.headerLength)
	frame.Body.bytesRemaining = int(frame.bodyLength)
	return
}

func (section *FrameSection) drain() (err error) {
	if section.bytesRemaining > 0 {
		_, err = io.Copy(ioutil.Discard, section)
	}
	return
}

func writeHeaderTo(out io.Writer, headerLength int16, bodyLength int16) (err error) {
	if err = binary.Write(out, endianness, headerLength); err != nil {
		return
	}
	err = binary.Write(out, endianness, bodyLength)
	return
}
<commit_msg>Comment updates<commit_after>/*
Package framed adds basic support for message framing over streams.

Messages contain an header and a body, both of which are length prefixed.

Here are the bytes (stored in little-endian byte order):

0-2: unsigned 16 bit int header length
2-4: unsinged 16 bit int body length
4+:  message content (header and body)

The use of a uint16 means that the maximum possible header and body lengths
are 65535 each.

Example:

	package main

	import (
		"github.com/oxtoacart/framed"
		"io/ioutil"
		"log"
		"net"
	)

	func main() {
		// Replace host:port with an actual TCP server, for example the echo service
		if conn, err := net.Dial("tcp", "host:port"); err == nil {
			framedConn = Framed{conn}
			if err := framedConn.WriteFrame([]byte("Header"), []byte("Hello World")); err == nil {
				if err, frame := framedConn.ReadInitial(); err == nil {
					// Note - Read is just like io.Reader.Read(), so we use ioutil.ReadAll
					if header, err := ioutil.ReadAll(framedConn.Header); err == nil {
						if body, err := ioutil.ReadAll(framedConn.Body); err == nil {
							nextFrame, err := frame.NextFrame()
							// And so on
						}
					}
				}
			}
		}
	}
*/
package framed

import (
	"encoding/binary"
	"io"
	"io/ioutil"
)

var endianness = binary.LittleEndian

/*
A Framed enhances an io.ReadWriteCloser to provide methods that allow writing
and reading frames.

Although the underlying ReadWriteCloser may be safe to use from multiple
goroutines, a Framed is not.
*/
type Framed struct {
	io.ReadWriteCloser // the raw underlying connection
	hasReadInitial     bool
}

// AlreadyReadError is returned when someone has already read from a Frame
type AlreadyReadError string

func (err AlreadyReadError) Error() string {
	return string(err)
}

// Frame encapsulates a frame from a Framed
type Frame struct {
	Header       *FrameSection
	Body         *FrameSection
	framed       *Framed
	headerLength int16
	bodyLength   int16
}

// FrameSection encapsulates a section of a frame (header or body)
type FrameSection struct {
	frame          *Frame
	init           func() error
	bytesRemaining int
	startedReading bool
}

// NewFramed creates a new Framed on top of the given readWriteCloser
func NewFramed(readWriteCloser io.ReadWriteCloser) *Framed {
	return &Framed{readWriteCloser, false}
}

/*
ReadInitial reads the initial frame from the framed.  It returns an
AlreadyReadError if the initial frame has already been read previously.
*/
func (framed *Framed) ReadInitial() (frame *Frame, err error) {
	if framed.hasReadInitial {
		return nil, AlreadyReadError("Initial Frame already read")
	}
	frame, err = framed.nextFrame()
	framed.hasReadInitial = true
	return
}

/*
NextFrame returns the next frame from the Framed underlying the frame on which
it is called.
*/
func (frame *Frame) NextFrame() (nextFrame *Frame, err error) {
	if err = frame.Header.drain(); err != nil {
		return
	}
	if err = frame.Body.drain(); err != nil {
		return
	}
	nextFrame, err = frame.framed.nextFrame()
	return
}

/*
WriteFrame writes the given header and body to the Framed.
Either or both can be nil.
*/
func (framed *Framed) WriteFrame(header []byte, body []byte) (err error) {
	headerLength := 0
	bodyLength := 0
	if header != nil {
		headerLength = len(header)
	}
	if body != nil {
		bodyLength = len(body)
	}

	if err = framed.WriteHeader(int16(headerLength), int16(bodyLength)); err != nil {
		return err
	}

	if header != nil {
		if _, err = framed.Write(header); err != nil {
			return err
		}
	}

	if body != nil {
		_, err = framed.Write(body)
	}

	return
}

// WriteHeader writes a frame header with the given lengths to the Framed.
func (framed *Framed) WriteHeader(headerLength int16, bodyLength int16) (err error) {
	return writeHeaderTo(framed, headerLength, bodyLength)
}

/*
CopyTo copies the given Frame to the given Writer.  If reading of the Frame
has already started before CopyTo is called, CopyTo returns an
AlreadyReadError.
*/
func (frame *Frame) CopyTo(out io.Writer) (err error) {
	if frame.Header.startedReading || frame.Body.startedReading {
		return AlreadyReadError("Already read from frame, cannot copy")
	}
	if err = writeHeaderTo(out, frame.headerLength, frame.bodyLength); err != nil {
		return
	}
	_, err = io.CopyN(out, frame.framed, int64(frame.headerLength+frame.bodyLength))
	return
}

/*
Read implements io.Reader.Read for a FrameSection.  Just like the usual Read(),
this one may read incompletely, so make sure to call it until it returns EOF.
*/
func (section *FrameSection) Read(p []byte) (n int, err error) {
	if section.bytesRemaining == 0 {
		return 0, err
	}
	if section.init != nil {
		if err = section.init(); err != nil {
			return 0, err
		}
	}
	section.startedReading = true
	if len(p) > section.bytesRemaining {
		p = p[0:section.bytesRemaining]
	}
	n, err = section.frame.framed.Read(p)
	if n > 0 {
		section.bytesRemaining -= n
	}
	if section.bytesRemaining == 0 {
		err = io.EOF
	}
	return
}

func (framed *Framed) nextFrame() (frame *Frame, err error) {
	frame = &Frame{framed: framed}
	frame.Header = &FrameSection{frame: frame}
	frame.Body = &FrameSection{frame: frame, init: frame.Header.drain}
	if err = frame.readLengths(); err != nil {
		return
	}
	return
}

func (frame *Frame) readLengths() (err error) {
	if err = binary.Read(frame.framed, endianness, &frame.headerLength); err != nil {
		return
	}
	if err = binary.Read(frame.framed, endianness, &frame.bodyLength); err != nil {
		return
	}
	frame.Header.bytesRemaining = int(frame.headerLength)
	frame.Body.bytesRemaining = int(frame.bodyLength)
	return
}

func (section *FrameSection) drain() (err error) {
	if section.bytesRemaining > 0 {
		_, err = io.Copy(ioutil.Discard, section)
	}
	return
}

func writeHeaderTo(out io.Writer, headerLength int16, bodyLength int16) (err error) {
	if err = binary.Write(out, endianness, headerLength); err != nil {
		return
	}
	err = binary.Write(out, endianness, bodyLength)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
)

//
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	transitive := func() bool {
		transitive := flag.Bool("transitive", false, "Traverse transitive imports, i.e. vendor/")
		flag.Parse()
		return *transitive
	}()

	self := "."
	{
		if path, err := filepath.Abs("."); err != nil {
			fmt.Fprintf(os.Stderr, "abs . failed: %s\n", err)
			os.Exit(1)
		} else {
			self = filepath.Base(path)
		}
	}

	if _, err := os.Stat(".git"); err != nil {
		fmt.Fprintf(os.Stderr, ".git does not exist\n")
		os.Exit(1)
	}

	_git, _go := "", ""
	if git_, err := exec.LookPath("git"); err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	} else {
		_git = git_
	}

	if go_, err := exec.LookPath("go"); err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	} else {
		_go = go_
	}

	imports := exec.Command(_go, `list`, `-f`, `{{$p := .ImportPath}}{{range $imp := .Imports}}{{printf "%s\t%s\n" $p $imp}}{{end}}`, `./...`)

	r, w := io.Pipe()
	imports.Stdout = w

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func(r io.Reader) {
		var (
			err  error = nil
			line string
		)
		reader := bufio.NewReader(r)
		repos := map[string]int{}
		for err == nil {
			if line, err = reader.ReadString('\n'); err == nil {
				pi := strings.Split(strings.TrimSpace(string(line)), "\t")
				p, i := pi[0], pi[1]
				parts := strings.Split(i, "/")
				if len(parts) >= 3 && parts[2] != self {
					if !transitive && strings.Contains(p, "/vendor") {
						continue
					}
					repo := strings.Join(parts[:3], "/")
					repos[repo] += 1
				}
			}
		}
		for repo, _ := range repos {
			b := path.Join("vendor", repo)
			if _, err := os.Stat(b); err == nil {
				fmt.Printf("%s exists\n", repo)
			} else {
				submodule := exec.Command(_git, `submodule`, `add`, `-f`, fmt.Sprintf(`https://%s`, repo), fmt.Sprintf(`vendor/%s`, repo))
				if output, err := submodule.CombinedOutput(); err != nil {
					fmt.Fprintf(os.Stderr, "ERROR: %v\noutput:\n%s", submodule.Args, output)
				} else {
					fmt.Printf("%s", output)
				}
			}
		}
		wg.Done()
	}(r)

	imports.Start()
	imports.Wait()
	w.Close()
	wg.Wait()

}
<commit_msg>+-subtree -branch -dry-run -list -force.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
)

//
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	self := "."
	{
		if path, err := filepath.Abs("."); err != nil {
			fmt.Fprintf(os.Stderr, "abs . failed: %s\n", err)
			os.Exit(1)
		} else {
			self = filepath.Base(path)
		}
	}

	if _, err := os.Stat(".git"); err != nil {
		fmt.Fprintf(os.Stderr, ".git does not exist\n")
		os.Exit(1)
	}

	_git, _go := "", ""
	if git_, err := exec.LookPath("git"); err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	} else {
		_git = git_
	}

	if go_, err := exec.LookPath("go"); err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err)
		os.Exit(1)
	} else {
		_go = go_
	}

	branch := ""
	{
		cmd := exec.Command(_git, "rev-parse", "--abbrev-ref", "HEAD")
		if output, err := cmd.CombinedOutput(); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\noutput:\n%s", cmd.Args, output)
		} else {
			branch = strings.TrimSpace(string(output))
		}
	}

	force, verbose, transitive, subtree, list, dryrun := false, false, false, false, false, false
	flag.BoolVar(&transitive, "transitive", transitive, "Traverse transitive imports, i.e. vendor/")
	flag.BoolVar(&subtree, "subtree", subtree, "Use a subtree instead of a submodule.")
	flag.BoolVar(&dryrun, "dry-run", dryrun, "Just print the command but do not run it.")
	flag.BoolVar(&list, "list", list, "Only list the imports that can be frozen.")
	flag.BoolVar(&verbose, "verbose", verbose, "More output.")
	flag.BoolVar(&force, "force", force, "Force.")
	flag.StringVar(&branch, "branch", branch, "Git branch/commit to submodule/subtree.")
	flag.Usage = func() {
		fmt.Println("Usage:")
		flag.PrintDefaults()
	}

	flag.Parse()
	patterns := make([]*regexp.Regexp, len(flag.Args()))
	for i, a := range flag.Args() {
		patterns[i] = regexp.MustCompile(a)
	}

	imports := exec.Command(_go, "list", "-f", `{{$p := .ImportPath}}{{range $imp := .Imports}}{{printf "%s\t%s\n" $p $imp}}{{end}}`, "./...")

	r, w := io.Pipe()
	imports.Stdout = w

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func(r io.Reader) {
		var (
			err  error = nil
			line string
		)
		reader := bufio.NewReader(r)
		reponly := map[string]int{}
		for err == nil {
			if line, err = reader.ReadString('\n'); err == nil {
				pi := strings.Split(strings.TrimSpace(string(line)), "\t")
				p, i := pi[0], pi[1]
				parts := strings.Split(i, "/")
				if len(parts) >= 3 && parts[2] != self {
					if !transitive && strings.Contains(p, "/vendor") {
						continue
					}
					matched := 0
					for _, p := range patterns {
						if p.MatchString(i) {
							matched += 1
						}
					}
					if matched != len(patterns) {
						continue
					}
					repo := strings.Join(parts[:3], "/")
					reponly[repo] += 1
				}
			}
		}
		repos := make([]string, len(reponly))
		ri := 0
		for repo, _ := range reponly {
			repos[ri] = repo
			ri += 1
		}
		sort.Strings(repos)
		for _, repo := range repos {
			if list {
				fmt.Printf("%s\n", repo)
				continue
			}
			var cmd *exec.Cmd
			b := path.Join("vendor", repo)
			if _, err := os.Stat(b); err == nil {
				if verbose {
					fmt.Printf("%s exists\n", repo)
				}
				continue
			} else if subtree {
				cmd = exec.Command(_git, "subtree", "add", "--prefix", fmt.Sprintf("vendor/%s", repo), fmt.Sprintf("https://%s", repo), branch, "--squash")
			} else {
				fullSubmodule := []string{"-f", "-b", branch, fmt.Sprintf("https://%s", repo), fmt.Sprintf("vendor/%s", repo)}
				var submoduler []string
				if force {
					submoduler = fullSubmodule
				} else {
					submoduler = fullSubmodule[1:]
				}
				cmd = exec.Command(_git, append([]string{"submodule", "add"}, submoduler...)...)
			}

			if dryrun {
				fmt.Println(strings.Join(cmd.Args, " "))
			} else {
				if output, err := cmd.CombinedOutput(); err != nil {
					fmt.Fprintf(os.Stderr, "ERROR: %v\noutput:\n%s", cmd.Args, output)
				} else if verbose {
					fmt.Printf("%s", output)
				}
			}
		}
		wg.Done()
	}(r)

	imports.Start()
	imports.Wait()
	w.Close()
	wg.Wait()

}
<|endoftext|>