<commit_before>package db

import (
	"time"

	"github.com/garyburd/redigo/redis"
)

/******
type Pool struct {
    // 用来创建redis连接的方法
    Dial func() (Conn, error)
    // 如果设置了给func,那么每次p.Get()的时候都会调用该方法来验证连接的可用性
    TestOnBorrow func(c Conn, t time.Time) error
    // 定义连接池中最大连接数（超过这个数会关闭老的链接，总会保持这个数）
    MaxIdle int
    // 当前连接池中可用的链接数
    MaxActive int
    // 定义链接的超时时间，每次p.Get()的时候会检测这个连接是否超时（超时会关闭，并释放可用连接数）
    IdleTimeout time.Duration
    // 当可用连接数为0是，那么当wait=true,那么当调用p.Get()时，会阻塞等待，否则，返回nil.
    Wait bool
}
******/

const (
	IMAGEKEY = "GSERVERKEY"
)

type Cache struct {
	pool *redis.Pool // 连接池
	key  string      // 用于记录redis中所有的key
}

type Redis struct {
	MaxIdle            int
	MaxActive          int
	IdleTimeout        int
	RedisServer        string
	DialConnectTimeout int
	DialReadTimeout    int
	DialWriteTimeout   int
	Auth               string
	DbNum              int
}

// 新建redis-pool
func NewRedis(redisCfg Redis) *Cache {
	cache := &Cache{
		key: IMAGEKEY,
	}
	cache.initRedis(redisCfg)
	conn := cache.pool.Get()
	defer conn.Close()
	return cache
}

func (cache *Cache) initRedis(redisCfg Redis) {
	dialFunc := func() (c redis.Conn, err error) {
		c, err = redis.Dial("tcp", redisCfg.RedisServer)
		if err != nil {
			return nil, err
		}
		if redisCfg.Auth != "" {
			if _, err := c.Do("AUTH", redisCfg.Auth); err != nil {
				c.Close()
				return nil, err
			}
		}
		_, selecterr := c.Do("SELECT", redisCfg.DbNum)
		if selecterr != nil {
			c.Close()
			return nil, selecterr
		}
		return
	}
	var maxIdle, maxActive int
	var idleTimeout time.Duration
	if redisCfg.MaxIdle <= 0 {
		maxIdle = 3
	} else {
		maxIdle = redisCfg.MaxIdle
	}
	if redisCfg.MaxActive <= 0 {
		maxActive = 32
	} else {
		maxActive = redisCfg.MaxActive
	}
	if redisCfg.IdleTimeout <= 0 {
		idleTimeout = time.Duration(180) * time.Second
	} else {
		idleTimeout = time.Duration(redisCfg.IdleTimeout) * time.Second
	}
	cache.pool = &redis.Pool{
		MaxIdle:     maxIdle,
		MaxActive:   maxActive,
		IdleTimeout: idleTimeout,
		Dial:        dialFunc,
	}
}

/*******************************封装调用接口*******************************/

// 执行redis命令
func (cache *Cache) do(commandName string, args ...interface{}) (reply interface{}, err error) {
	conn := cache.pool.Get()
	defer conn.Close()
	return conn.Do(commandName, args...)
}

// 获取指定key
func (cache *Cache) Get(key string) interface{} {
	if v, err := cache.do("GET", key); err == nil {
		return v
	}
	return nil
}

// 获取多个key
func (cache *Cache) GetMulti(keys []string) []interface{} {
	size := len(keys)
	var rv []interface{}
	conn := cache.pool.Get()
	defer conn.Close()
	var err error
	for _, key := range keys {
		err = conn.Send("GET", key)
		if err != nil {
			goto ERROR
		}
	}
	if err = conn.Flush(); err != nil {
		goto ERROR
	}
	for i := 0; i < size; i++ {
		if v, err := conn.Receive(); err == nil {
			rv = append(rv, v.([]byte))
		} else {
			rv = append(rv, err)
		}
	}
	return rv
ERROR:
	rv = rv[0:0]
	for i := 0; i < size; i++ {
		rv = append(rv, nil)
	}
	return rv
}

// 存储一对k-v
func (cache *Cache) Put(key string, val interface{}, timeout time.Duration) error {
	var err error
	if _, err = cache.do("SETEX", key, int64(timeout/time.Second), val); err != nil {
		return err
	}
	if _, err = cache.do("HSET", cache.key, key, true); err != nil {
		return err
	}
	return err
}

// 删除指定key
func (cache *Cache) Delete(key string) error {
	var err error
	if _, err = cache.do("DEL", key); err != nil {
		return err
	}
	_, err = cache.do("HDEL", cache.key, key)
	return err
}

// 检查指定key是否存在
func (cache *Cache) IsExist(key string) bool {
	v, err := redis.Bool(cache.do("EXISTS", key))
	if err != nil {
		return false
	}
	if !v {
		if _, err = cache.do("HDEL", cache.key, key); err != nil {
			return false
		}
	}
	return v
}

// 自增指定key
func (cache *Cache) Incr(key string) error {
	_, err := redis.Bool(cache.do("INCRBY", key, 1))
	return err
}

// 自减指定key
func (cache *Cache) Decr(key string) error {
	_, err := redis.Bool(cache.do("INCRBY", key, -1))
	return err
}

// 清理所有缓存
func (cache *Cache) ClearAll() error {
	cachedKeys, err := redis.Strings(cache.do("HKEYS", cache.key))
	if err != nil {
		return err
	}
	for _, str := range cachedKeys {
		if _, err = cache.do("DEL", str); err != nil {
			return err
		}
	}
	_, err = cache.do("DEL", cache.key)
	return err
}
<commit_msg>update const INFINITE<commit_after>package db

import (
	"time"

	"github.com/garyburd/redigo/redis"
)

/******
type Pool struct {
    // 用来创建redis连接的方法
    Dial func() (Conn, error)
    // 如果设置了给func,那么每次p.Get()的时候都会调用该方法来验证连接的可用性
    TestOnBorrow func(c Conn, t time.Time) error
    // 定义连接池中最大连接数（超过这个数会关闭老的链接，总会保持这个数）
    MaxIdle int
    // 当前连接池中可用的链接数
    MaxActive int
    // 定义链接的超时时间，每次p.Get()的时候会检测这个连接是否超时（超时会关闭，并释放可用连接数）
    IdleTimeout time.Duration
    // 当可用连接数为0是，那么当wait=true,那么当调用p.Get()时，会阻塞等待，否则，返回nil.
    Wait bool
}
******/

const (
	IMAGEKEY = "GSERVERKEY"
)

const (
	INFINITE = 1 << 32
)

type Cache struct {
	pool *redis.Pool // 连接池
	key  string      // 用于记录redis中所有的key
}

type Redis struct {
	MaxIdle            int
	MaxActive          int
	IdleTimeout        int
	RedisServer        string
	DialConnectTimeout int
	DialReadTimeout    int
	DialWriteTimeout   int
	Auth               string
	DbNum              int
}

// 新建redis-pool
func NewRedis(redisCfg Redis) *Cache {
	cache := &Cache{
		key: IMAGEKEY,
	}
	cache.initRedis(redisCfg)
	conn := cache.pool.Get()
	defer conn.Close()
	return cache
}

func (cache *Cache) initRedis(redisCfg Redis) {
	dialFunc := func() (c redis.Conn, err error) {
		c, err = redis.Dial("tcp", redisCfg.RedisServer)
		if err != nil {
			return nil, err
		}
		if redisCfg.Auth != "" {
			if _, err := c.Do("AUTH", redisCfg.Auth); err != nil {
				c.Close()
				return nil, err
			}
		}
		_, selecterr := c.Do("SELECT", redisCfg.DbNum)
		if selecterr != nil {
			c.Close()
			return nil, selecterr
		}
		return
	}
	var maxIdle, maxActive int
	var idleTimeout time.Duration
	if redisCfg.MaxIdle <= 0 {
		maxIdle = 3
	} else {
		maxIdle = redisCfg.MaxIdle
	}
	if redisCfg.MaxActive <= 0 {
		maxActive = 32
	} else {
		maxActive = redisCfg.MaxActive
	}
	if redisCfg.IdleTimeout <= 0 {
		idleTimeout = time.Duration(180) * time.Second
	} else {
		idleTimeout = time.Duration(redisCfg.IdleTimeout) * time.Second
	}
	cache.pool = &redis.Pool{
		MaxIdle:     maxIdle,
		MaxActive:   maxActive,
		IdleTimeout: idleTimeout,
		Dial:        dialFunc,
	}
}

/*******************************封装调用接口*******************************/

// 执行redis命令
func (cache *Cache) do(commandName string, args ...interface{}) (reply interface{}, err error) {
	conn := cache.pool.Get()
	defer conn.Close()
	return conn.Do(commandName, args...)
}

// 获取指定key
func (cache *Cache) Get(key string) interface{} {
	if v, err := cache.do("GET", key); err == nil {
		return v
	}
	return nil
}

// 获取多个key
func (cache *Cache) GetMulti(keys []string) []interface{} {
	size := len(keys)
	var rv []interface{}
	conn := cache.pool.Get()
	defer conn.Close()
	var err error
	for _, key := range keys {
		err = conn.Send("GET", key)
		if err != nil {
			goto ERROR
		}
	}
	if err = conn.Flush(); err != nil {
		goto ERROR
	}
	for i := 0; i < size; i++ {
		if v, err := conn.Receive(); err == nil {
			rv = append(rv, v.([]byte))
		} else {
			rv = append(rv, err)
		}
	}
	return rv
ERROR:
	rv = rv[0:0]
	for i := 0; i < size; i++ {
		rv = append(rv, nil)
	}
	return rv
}

// 存储一对k-v
func (cache *Cache) Put(key string, val interface{}, timeout time.Duration) error {
	var err error
	if _, err = cache.do("SETEX", key, int64(timeout/time.Second), val); err != nil {
		return err
	}
	if _, err = cache.do("HSET", cache.key, key, true); err != nil {
		return err
	}
	return err
}

// 删除指定key
func (cache *Cache) Delete(key string) error {
	var err error
	if _, err = cache.do("DEL", key); err != nil {
		return err
	}
	_, err = cache.do("HDEL", cache.key, key)
	return err
}

// 检查指定key是否存在
func (cache *Cache) IsExist(key string) bool {
	v, err := redis.Bool(cache.do("EXISTS", key))
	if err != nil {
		return false
	}
	if !v {
		if _, err = cache.do("HDEL", cache.key, key); err != nil {
			return false
		}
	}
	return v
}

// 自增指定key
func (cache *Cache) Incr(key string) error {
	_, err := redis.Bool(cache.do("INCRBY", key, 1))
	return err
}

// 自减指定key
func (cache *Cache) Decr(key string) error {
	_, err := redis.Bool(cache.do("INCRBY", key, -1))
	return err
}

// 清理所有缓存
func (cache *Cache) ClearAll() error {
	cachedKeys, err := redis.Strings(cache.do("HKEYS", cache.key))
	if err != nil {
		return err
	}
	for _, str := range cachedKeys {
		if _, err = cache.do("DEL", str); err != nil {
			return err
		}
	}
	_, err = cache.do("DEL", cache.key)
	return err
}
<|endoftext|>


<commit_before>/*
   Copyright (C) 2013  Salsita s.r.o.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program. If not, see {http://www.gnu.org/licenses/}.
*/

package receiver

import (
	"net"
	"net/http"
	"os"
	"os/signal"

	"github.com/cider/go-cider/cider/services/logging"
	"github.com/cider/go-cider/cider/services/pubsub"
	zlogging "github.com/cider/go-cider/cider/transports/zmq3/logging"
	zpubsub "github.com/cider/go-cider/cider/transports/zmq3/pubsub"

	zmq "github.com/pebbe/zmq3"
)

// API functions ---------------------------------------------------------------

var (
	Logger *logging.Service
	PubSub *pubsub.Service
)

// Serve POST requests using the handler passed into ListenAndServe.
// This function blocks until a signal is received. So signals are being
// handled by this function, no need to do it manually.
func ListenAndServe(handler http.Handler) {
	// Load all the required environment variables, panic if any is not set.
	// This is placed here and not outside to make testing easier (possible).
	// The applications do not have to really connect to Cider to run tests.
	var (
		alias = mustBeSet(os.Getenv("CIDER_ALIAS"))
		addr  = mustBeSet(os.Getenv("LISTEN_ADDRESS"))
		token = mustBeSet(os.Getenv("ACCESS_TOKEN"))
	)

	// Start catching interrupts.
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt)

	// Initialise Logging service from environmental variables.
	var err error
	Logger, err = logging.NewService(func() (logging.Transport, error) {
		factory := zlogging.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_LOGGING_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(err)
	}
	Logger.Info("Logging service initialised\n")

	// Make sure ZeroMQ is terminated properly.
	defer func() {
		Logger.Info("Waiting for ZeroMQ context to terminate...\n")
		Logger.Flush()
		zmq.Term()
	}()

	// Initialise PubSub service from environmental variables.
	PubSub, err = pubsub.NewService(func() (pubsub.Transport, error) {
		factory := zpubsub.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_PUBSUB_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(Logger.Critical(err))
	}
	defer PubSub.Close()
	Logger.Info("PubSub service initialised\n")

	// Listen.
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		panic(Logger.Critical(err))
	}

	// Start processing interrupts.
	var interrupted bool
	go func() {
		<-signalCh
		interrupted = true
		listener.Close()
	}()

	// Keep serving until interrupted.
	err = http.Serve(listener, authenticatedServer(token, handler))
	if err != nil && !interrupted {
		panic(Logger.Critical(err))
	}
}

// Helpers ---------------------------------------------------------------------

func mustBeSet(v string) string {
	if v == "" {
		panic("Required variable is not set")
	}
	return v
}

func authenticatedServer(token string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Allow the POST method only.
		if r.Method != "POST" {
			http.Error(w, "POST Method Expected", http.StatusMethodNotAllowed)
			return
		}

		// Make sure that the token query parameter is set correctly.
		if r.FormValue("token") != token {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}

		// If everything is ok, serve the user-defined handler.
		handler.ServeHTTP(w, r)
	})
}
<commit_msg>Remove extra newlines in log records<commit_after>/*
   Copyright (C) 2013  Salsita s.r.o.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program. If not, see {http://www.gnu.org/licenses/}.
*/

package receiver

import (
	"net"
	"net/http"
	"os"
	"os/signal"

	"github.com/cider/go-cider/cider/services/logging"
	"github.com/cider/go-cider/cider/services/pubsub"
	zlogging "github.com/cider/go-cider/cider/transports/zmq3/logging"
	zpubsub "github.com/cider/go-cider/cider/transports/zmq3/pubsub"

	zmq "github.com/pebbe/zmq3"
)

// API functions ---------------------------------------------------------------

var (
	Logger *logging.Service
	PubSub *pubsub.Service
)

// Serve POST requests using the handler passed into ListenAndServe.
// This function blocks until a signal is received. So signals are being
// handled by this function, no need to do it manually.
func ListenAndServe(handler http.Handler) {
	// Load all the required environment variables, panic if any is not set.
	// This is placed here and not outside to make testing easier (possible).
	// The applications do not have to really connect to Cider to run tests.
	var (
		alias = mustBeSet(os.Getenv("CIDER_ALIAS"))
		addr  = mustBeSet(os.Getenv("LISTEN_ADDRESS"))
		token = mustBeSet(os.Getenv("ACCESS_TOKEN"))
	)

	// Start catching interrupts.
	signalCh := make(chan os.Signal, 1)
	signal.Notify(signalCh, os.Interrupt)

	// Initialise Logging service from environmental variables.
	var err error
	Logger, err = logging.NewService(func() (logging.Transport, error) {
		factory := zlogging.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_LOGGING_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(err)
	}
	Logger.Info("Logging service initialised")

	// Make sure ZeroMQ is terminated properly.
	defer func() {
		Logger.Info("Waiting for ZeroMQ context to terminate...")
		Logger.Flush()
		zmq.Term()
	}()

	// Initialise PubSub service from environmental variables.
	PubSub, err = pubsub.NewService(func() (pubsub.Transport, error) {
		factory := zpubsub.NewTransportFactory()
		factory.MustReadConfigFromEnv("CIDER_ZMQ3_PUBSUB_").MustBeFullyConfigured()
		return factory.NewTransport(alias)
	})
	if err != nil {
		panic(Logger.Critical(err))
	}
	defer PubSub.Close()
	Logger.Info("PubSub service initialised")

	// Listen.
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		panic(Logger.Critical(err))
	}

	// Start processing interrupts.
	var interrupted bool
	go func() {
		<-signalCh
		interrupted = true
		listener.Close()
	}()

	// Keep serving until interrupted.
	err = http.Serve(listener, authenticatedServer(token, handler))
	if err != nil && !interrupted {
		panic(Logger.Critical(err))
	}
}

// Helpers ---------------------------------------------------------------------

func mustBeSet(v string) string {
	if v == "" {
		panic("Required variable is not set")
	}
	return v
}

func authenticatedServer(token string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Allow the POST method only.
		if r.Method != "POST" {
			http.Error(w, "POST Method Expected", http.StatusMethodNotAllowed)
			return
		}

		// Make sure that the token query parameter is set correctly.
		if r.FormValue("token") != token {
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}

		// If everything is ok, serve the user-defined handler.
		handler.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>package data_types

import (
	"time"
)

type SoftLayer_Virtual_Guest_Parameters struct {
	Parameters []SoftLayer_Virtual_Guest `json:"parameters"`
}

type SoftLayer_Virtual_Guest struct {
	AccountId                    int        `json:"accountId,omitempty"`
	CreateDate                   *time.Time `json:"createDate,omitempty"`
	DedicatedAccountHostOnlyFlag bool       `json:"dedicatedAccountHostOnlyFlag,omitempty"`
	Domain                       string     `json:"domain,omitempty"`
	FullyQualifiedDomainName     string     `json:"fullyQualifiedDomainName,omitempty"`
	Hostname                     string     `json:"hostname,omitempty"`
	Id                           int        `json:"id,omitempty"`
	LastPowerStateId             int        `json:"lastPowerStateId,omitempty"`
	LastVerifiedDate             *time.Time `json:"lastVerifiedDate,omitempty"`
	MaxCpu                       int        `json:"maxCpu,omitempty"`
	MaxCpuUnits                  string     `json:"maxCpuUnits,omitempty"`
	MaxMemory                    int        `json:"maxMemory,omitempty"`
	MetricPollDate               *time.Time `json:"metricPollDate,omitempty"`
	ModifyDate                   *time.Time `json:"modifyDate,omitempty"`
	Notes                        string     `json:"notes,omitempty"`
	PostInstallScriptUri         string     `json:"postInstallScriptUri,omitempty"`
	PrivateNetworkOnlyFlag       bool       `json:"privateNetworkOnlyFlag,omitempty"`
	StartCpus                    int        `json:"startCpus,omitempty"`
	StatusId                     int        `json:"statusId,omitempty"`
	Uuid                         string     `json:"uuid,omitempty"`
}

type SoftLayer_Virtual_Guest_Template_Parameters struct {
	Parameters []SoftLayer_Virtual_Guest_Template `json:"parameters"`
}

type SoftLayer_Virtual_Guest_Template struct {
	//Required
	Hostname          string     `json:"hostname"`
	Domain            string     `json:"domain"`
	StartCpus         int        `json:"startCpus"`
	MaxMemory         int        `json:"maxMemory"`
	Datacenter        Datacenter `json:"datacenter"`
	HourlyBillingFlag bool       `json:"hourlyBillingFlag"`
	LocalDiskFlag     bool       `json:"localDiskFlag"`

	//Conditionally required
	OperatingSystemReferenceCode string                    `json:"operatingSystemReferenceCode"`
	BlockDeviceTemplateGroup     *BlockDeviceTemplateGroup `json:"blockDeviceTemplateGroup,omitempty"`

	//Optional
	DedicatedAccountHostOnlyFlag   bool                            `json:"dedicatedAccountHostOnlyFlag,omitempty"`
	NetworkComponents              []NetworkComponents             `json:"networkComponents,omitempty"`
	PrivateNetworkOnlyFlag         bool                            `json:"privateNetworkOnlyFlag,omitempty"`
	PrimaryNetworkComponent        *PrimaryNetworkComponent        `json:"primaryNetworkComponent,omitempty"`
	PrimaryBackendNetworkComponent *PrimaryBackendNetworkComponent `json:"primaryBackendNetworkComponent,omitempty"`

	BlockDevices []BlockDevice `json:"blockDevices,omitempty"`
	UserData     []UserData    `json:"userData,omitempty"`
	SshKeys      []SshKey      `json:"sshKeys,omitempty"`

	PostInstallScriptUri string `json:"postInstallScriptUri,omitempty"`
}

type Datacenter struct {
	//Required
	Name string `json:"name"`
}

type BlockDeviceTemplateGroup struct {
	//Required
	GlobalIdentifier string `json:"globalIdentifier,omitempty"`
}

type NetworkComponents struct {
	//Required, defaults to 10
	MaxSpeed int `json:"maxSpeed,omitempty"`
}

type NetworkVlan struct {
	//Required
	Id int `json:"id,omitempty"`
}

type PrimaryNetworkComponent struct {
	//Required
	NetworkVlan NetworkVlan `json:"networkVlan,omitempty"`
}

type PrimaryBackendNetworkComponent struct {
	//Required
	NetworkVlan NetworkVlan `json:"networkVlan,omitempty"`
}

type DiskImage struct {
	//Required
	capacity int `json:"capacity,omitempty"`
}

type BlockDevice struct {
	//Required
	Device    string    `json:"device,omitempty"`
	DiskImage DiskImage `json:"diskImage,omitempty"`
}

type UserData struct {
	//Required
	Value string `json:"value,omitempty"`
}

type SshKey struct {
	//Required
	Id int `json:"id,omitempty"`
}
<commit_msg>added missing fields<commit_after>package data_types

import (
	"time"
)

type SoftLayer_Virtual_Guest_Parameters struct {
	Parameters []SoftLayer_Virtual_Guest `json:"parameters"`
}

type SoftLayer_Virtual_Guest struct {
	AccountId                    int        `json:"accountId,omitempty"`
	CreateDate                   *time.Time `json:"createDate,omitempty"`
	DedicatedAccountHostOnlyFlag bool       `json:"dedicatedAccountHostOnlyFlag,omitempty"`
	Domain                       string     `json:"domain,omitempty"`
	FullyQualifiedDomainName     string     `json:"fullyQualifiedDomainName,omitempty"`
	Hostname                     string     `json:"hostname,omitempty"`
	Id                           int        `json:"id,omitempty"`
	LastPowerStateId             int        `json:"lastPowerStateId,omitempty"`
	LastVerifiedDate             *time.Time `json:"lastVerifiedDate,omitempty"`
	MaxCpu                       int        `json:"maxCpu,omitempty"`
	MaxCpuUnits                  string     `json:"maxCpuUnits,omitempty"`
	MaxMemory                    int        `json:"maxMemory,omitempty"`
	MetricPollDate               *time.Time `json:"metricPollDate,omitempty"`
	ModifyDate                   *time.Time `json:"modifyDate,omitempty"`
	Notes                        string     `json:"notes,omitempty"`
	PostInstallScriptUri         string     `json:"postInstallScriptUri,omitempty"`
	PrivateNetworkOnlyFlag       bool       `json:"privateNetworkOnlyFlag,omitempty"`
	StartCpus                    int        `json:"startCpus,omitempty"`
	StatusId                     int        `json:"statusId,omitempty"`
	Uuid                         string     `json:"uuid,omitempty"`

	GlobalIdentifier        string `json:"globalIdentifier,omitempty"`
	ManagedResourceFlag     bool   `json:"managedResourceFlag,omitempty"`
	PrimaryBackendIpAddress string `json:"primaryBackendIpAddress,omitempty"`
	PrimaryIpAddress        string `json:"primaryIpAddress,omitempty"`
}

type SoftLayer_Virtual_Guest_Template_Parameters struct {
	Parameters []SoftLayer_Virtual_Guest_Template `json:"parameters"`
}

type SoftLayer_Virtual_Guest_Template struct {
	//Required
	Hostname          string     `json:"hostname"`
	Domain            string     `json:"domain"`
	StartCpus         int        `json:"startCpus"`
	MaxMemory         int        `json:"maxMemory"`
	Datacenter        Datacenter `json:"datacenter"`
	HourlyBillingFlag bool       `json:"hourlyBillingFlag"`
	LocalDiskFlag     bool       `json:"localDiskFlag"`

	//Conditionally required
	OperatingSystemReferenceCode string                    `json:"operatingSystemReferenceCode"`
	BlockDeviceTemplateGroup     *BlockDeviceTemplateGroup `json:"blockDeviceTemplateGroup,omitempty"`

	//Optional
	DedicatedAccountHostOnlyFlag   bool                            `json:"dedicatedAccountHostOnlyFlag,omitempty"`
	NetworkComponents              []NetworkComponents             `json:"networkComponents,omitempty"`
	PrivateNetworkOnlyFlag         bool                            `json:"privateNetworkOnlyFlag,omitempty"`
	PrimaryNetworkComponent        *PrimaryNetworkComponent        `json:"primaryNetworkComponent,omitempty"`
	PrimaryBackendNetworkComponent *PrimaryBackendNetworkComponent `json:"primaryBackendNetworkComponent,omitempty"`

	BlockDevices []BlockDevice `json:"blockDevices,omitempty"`
	UserData     []UserData    `json:"userData,omitempty"`
	SshKeys      []SshKey      `json:"sshKeys,omitempty"`

	PostInstallScriptUri string `json:"postInstallScriptUri,omitempty"`
}

type Datacenter struct {
	//Required
	Name string `json:"name"`
}

type BlockDeviceTemplateGroup struct {
	//Required
	GlobalIdentifier string `json:"globalIdentifier,omitempty"`
}

type NetworkComponents struct {
	//Required, defaults to 10
	MaxSpeed int `json:"maxSpeed,omitempty"`
}

type NetworkVlan struct {
	//Required
	Id int `json:"id,omitempty"`
}

type PrimaryNetworkComponent struct {
	//Required
	NetworkVlan NetworkVlan `json:"networkVlan,omitempty"`
}

type PrimaryBackendNetworkComponent struct {
	//Required
	NetworkVlan NetworkVlan `json:"networkVlan,omitempty"`
}

type DiskImage struct {
	//Required
	capacity int `json:"capacity,omitempty"`
}

type BlockDevice struct {
	//Required
	Device    string    `json:"device,omitempty"`
	DiskImage DiskImage `json:"diskImage,omitempty"`
}

type UserData struct {
	//Required
	Value string `json:"value,omitempty"`
}

type SshKey struct {
	//Required
	Id int `json:"id,omitempty"`
}
<|endoftext|>


<commit_before>package updater

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/inconshreveable/go-update"
	"github.com/kr/binarydist"

	"github.com/kardianos/osext"
)

// Update protocol:
//
//   GET hk.heroku.com/hk/linux-amd64.json
//
//   200 ok
//   {
//       "Version": "2",
//       "Sha256": "..." // base64
//   }
//
// then
//
//   GET hkpatch.s3.amazonaws.com/hk/1/2/linux-amd64
//
//   200 ok
//   [bsdiff data]
//
// or
//
//   GET hkdist.s3.amazonaws.com/hk/2/linux-amd64.gz
//
//   200 ok
//   [gzipped executable data]
//
//

const (
	plat = runtime.GOOS + "-" + runtime.GOARCH
)

const devValidTime = 7 * 24 * time.Hour

var errHashMismatch = errors.New("new file hash mismatch after patch")
var up = update.New()

// Updater is the configuration and runtime data for doing an update.
//
// Note that ApiURL, BinURL and DiffURL should have the same value if all files are available at the same location.
//
// Example:
//
//  updater := &selfupdate.Updater{
//  	CurrentVersion: version,
//  	ApiURL:         "http://updates.yourdomain.com/",
//  	BinURL:         "http://updates.yourdownmain.com/",
//  	DiffURL:        "http://updates.yourdomain.com/",
//  	Dir:            "update/",
//  	CmdName:        "myapp", // app name
//  }
//  if updater != nil {
//  	go updater.BackgroundRun()
//  }
type Updater struct {
	CurrentVersion string // Currently running version.
	APIURL         string // Base URL for API requests (json files).
	CmdName        string // Command name is appended to the ApiURL like http://apiurl/CmdName/. This represents one binary.
	BinURL         string // Base URL for full binary downloads.
	DiffURL        string // Base URL for diff downloads.
	Dir            string // Directory to store selfupdate state.
	Info           struct {
		Version string
		Sha256  []byte
	}
}

// BackgroundRun starts the update check and apply cycle.
func (u *Updater) BackgroundRun() error {
	os.MkdirAll(u.getExecRelativeDir(u.Dir), 0777)
	if err := up.CanUpdate(); err != nil {
		log.Println(err)
		return err
	}
	//self, err := osext.Executable()
	//if err != nil {
	// fail update, couldn't figure out path to self
	//return
	//}
	// TODO(bgentry): logger isn't on Windows. Replace w/ proper error reports.
	if err := u.update(); err != nil {
		return err
	}
	return nil
}

func fetch(url string) (io.ReadCloser, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		log.Errorf("bad http status from %s: %v", url, resp.Status)
		return nil, fmt.Errorf("bad http status from %s: %v", url, resp.Status)
	}
	return resp.Body, nil
}

func verifySha(bin []byte, sha []byte) bool {
	h := sha256.New()
	h.Write(bin)
	return bytes.Equal(h.Sum(nil), sha)
}

func (u *Updater) fetchAndApplyPatch(old io.Reader) ([]byte, error) {
	r, err := fetch(u.DiffURL + u.CmdName + "/" + u.CurrentVersion + "/" + u.Info.Version + "/" + plat)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	var buf bytes.Buffer
	err = binarydist.Patch(old, &buf, r)
	return buf.Bytes(), err
}

func (u *Updater) fetchAndVerifyPatch(old io.Reader) ([]byte, error) {
	bin, err := u.fetchAndApplyPatch(old)
	if err != nil {
		return nil, err
	}
	if !verifySha(bin, u.Info.Sha256) {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchAndVerifyFullBin() ([]byte, error) {
	bin, err := u.fetchBin()
	if err != nil {
		return nil, err
	}
	verified := verifySha(bin, u.Info.Sha256)
	if !verified {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchBin() ([]byte, error) {
	r, err := fetch(u.BinURL + u.CmdName + "/" + u.Info.Version + "/" + plat + ".gz")
	if err != nil {
		return nil, err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	if _, err = io.Copy(buf, gz); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func (u *Updater) fetchInfo() error {
	r, err := fetch(u.APIURL + u.CmdName + "/" + plat + ".json")
	if err != nil {
		return err
	}
	defer r.Close()
	err = json.NewDecoder(r).Decode(&u.Info)
	if err != nil {
		return err
	}
	if len(u.Info.Sha256) != sha256.Size {
		return errors.New("bad cmd hash in info")
	}
	return nil
}

func (u *Updater) getExecRelativeDir(dir string) string {
	filename, _ := osext.Executable()
	path := filepath.Join(filepath.Dir(filename), dir)
	return path
}

func (u *Updater) update() error {
	path, err := osext.Executable()
	if err != nil {
		return err
	}
	old, err := os.Open(path)
	if err != nil {
		return err
	}
	defer old.Close()

	err = u.fetchInfo()
	if err != nil {
		log.Println(err)
		return err
	}
	if u.Info.Version == u.CurrentVersion {
		return nil
	}
	bin, err := u.fetchAndVerifyPatch(old)
	if err != nil {
		if err == errHashMismatch {
			log.Println("update: hash mismatch from patched binary")
		} else {
			if u.DiffURL != "" {
				log.Println("update: patching binary,", err)
			}
		}

		bin, err = u.fetchAndVerifyFullBin()
		if err != nil {
			if err == errHashMismatch {
				log.Println("update: hash mismatch from full binary")
			} else {
				log.Println("update: fetching full binary,", err)
			}
			return err
		}
	}

	// close the old binary before installing because on windows
	// it can't be renamed if a handle to the file is still open
	old.Close()

	err, errRecover := up.FromStream(bytes.NewBuffer(bin))
	if errRecover != nil {
		log.Errorf("update and recovery errors: %q %q", err, errRecover)
		return fmt.Errorf("update and recovery errors: %q %q", err, errRecover)
	}
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>fix dependency version for go-update<commit_after>package updater

import (
	"bytes"
	"compress/gzip"
	"crypto/sha256"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"time"

	log "github.com/Sirupsen/logrus"
	"gopkg.in/inconshreveable/go-update.v0"
	"github.com/kr/binarydist"

	"github.com/kardianos/osext"
)

// Update protocol:
//
//   GET hk.heroku.com/hk/linux-amd64.json
//
//   200 ok
//   {
//       "Version": "2",
//       "Sha256": "..." // base64
//   }
//
// then
//
//   GET hkpatch.s3.amazonaws.com/hk/1/2/linux-amd64
//
//   200 ok
//   [bsdiff data]
//
// or
//
//   GET hkdist.s3.amazonaws.com/hk/2/linux-amd64.gz
//
//   200 ok
//   [gzipped executable data]
//
//

const (
	plat = runtime.GOOS + "-" + runtime.GOARCH
)

const devValidTime = 7 * 24 * time.Hour

var errHashMismatch = errors.New("new file hash mismatch after patch")
var up = update.New()

// Updater is the configuration and runtime data for doing an update.
//
// Note that ApiURL, BinURL and DiffURL should have the same value if all files are available at the same location.
//
// Example:
//
//  updater := &selfupdate.Updater{
//  	CurrentVersion: version,
//  	ApiURL:         "http://updates.yourdomain.com/",
//  	BinURL:         "http://updates.yourdownmain.com/",
//  	DiffURL:        "http://updates.yourdomain.com/",
//  	Dir:            "update/",
//  	CmdName:        "myapp", // app name
//  }
//  if updater != nil {
//  	go updater.BackgroundRun()
//  }
type Updater struct {
	CurrentVersion string // Currently running version.
	APIURL         string // Base URL for API requests (json files).
	CmdName        string // Command name is appended to the ApiURL like http://apiurl/CmdName/. This represents one binary.
	BinURL         string // Base URL for full binary downloads.
	DiffURL        string // Base URL for diff downloads.
	Dir            string // Directory to store selfupdate state.
	Info           struct {
		Version string
		Sha256  []byte
	}
}

// BackgroundRun starts the update check and apply cycle.
func (u *Updater) BackgroundRun() error {
	os.MkdirAll(u.getExecRelativeDir(u.Dir), 0777)
	if err := up.CanUpdate(); err != nil {
		log.Println(err)
		return err
	}
	//self, err := osext.Executable()
	//if err != nil {
	// fail update, couldn't figure out path to self
	//return
	//}
	// TODO(bgentry): logger isn't on Windows. Replace w/ proper error reports.
	if err := u.update(); err != nil {
		return err
	}
	return nil
}

func fetch(url string) (io.ReadCloser, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		log.Errorf("bad http status from %s: %v", url, resp.Status)
		return nil, fmt.Errorf("bad http status from %s: %v", url, resp.Status)
	}
	return resp.Body, nil
}

func verifySha(bin []byte, sha []byte) bool {
	h := sha256.New()
	h.Write(bin)
	return bytes.Equal(h.Sum(nil), sha)
}

func (u *Updater) fetchAndApplyPatch(old io.Reader) ([]byte, error) {
	r, err := fetch(u.DiffURL + u.CmdName + "/" + u.CurrentVersion + "/" + u.Info.Version + "/" + plat)
	if err != nil {
		return nil, err
	}
	defer r.Close()
	var buf bytes.Buffer
	err = binarydist.Patch(old, &buf, r)
	return buf.Bytes(), err
}

func (u *Updater) fetchAndVerifyPatch(old io.Reader) ([]byte, error) {
	bin, err := u.fetchAndApplyPatch(old)
	if err != nil {
		return nil, err
	}
	if !verifySha(bin, u.Info.Sha256) {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchAndVerifyFullBin() ([]byte, error) {
	bin, err := u.fetchBin()
	if err != nil {
		return nil, err
	}
	verified := verifySha(bin, u.Info.Sha256)
	if !verified {
		return nil, errHashMismatch
	}
	return bin, nil
}

func (u *Updater) fetchBin() ([]byte, error) {
	r, err := fetch(u.BinURL + u.CmdName + "/" + u.Info.Version + "/" + plat + ".gz")
	if err != nil {
		return nil, err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	gz, err := gzip.NewReader(r)
	if err != nil {
		return nil, err
	}
	if _, err = io.Copy(buf, gz); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func (u *Updater) fetchInfo() error {
	r, err := fetch(u.APIURL + u.CmdName + "/" + plat + ".json")
	if err != nil {
		return err
	}
	defer r.Close()
	err = json.NewDecoder(r).Decode(&u.Info)
	if err != nil {
		return err
	}
	if len(u.Info.Sha256) != sha256.Size {
		return errors.New("bad cmd hash in info")
	}
	return nil
}

func (u *Updater) getExecRelativeDir(dir string) string {
	filename, _ := osext.Executable()
	path := filepath.Join(filepath.Dir(filename), dir)
	return path
}

func (u *Updater) update() error {
	path, err := osext.Executable()
	if err != nil {
		return err
	}
	old, err := os.Open(path)
	if err != nil {
		return err
	}
	defer old.Close()

	err = u.fetchInfo()
	if err != nil {
		log.Println(err)
		return err
	}
	if u.Info.Version == u.CurrentVersion {
		return nil
	}
	bin, err := u.fetchAndVerifyPatch(old)
	if err != nil {
		if err == errHashMismatch {
			log.Println("update: hash mismatch from patched binary")
		} else {
			if u.DiffURL != "" {
				log.Println("update: patching binary,", err)
			}
		}

		bin, err = u.fetchAndVerifyFullBin()
		if err != nil {
			if err == errHashMismatch {
				log.Println("update: hash mismatch from full binary")
			} else {
				log.Println("update: fetching full binary,", err)
			}
			return err
		}
	}

	// close the old binary before installing because on windows
	// it can't be renamed if a handle to the file is still open
	old.Close()

	err, errRecover := up.FromStream(bytes.NewBuffer(bin))
	if errRecover != nil {
		log.Errorf("update and recovery errors: %q %q", err, errRecover)
		return fmt.Errorf("update and recovery errors: %q %q", err, errRecover)
	}
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package application

import (
	"context"
	"os"
	"time"

	"github.com/vardius/golog"
	"github.com/vardius/shutdown"
)

// Adapter interface
type Adapter interface {
	Start(ctx context.Context) error
	Stop(ctx context.Context) error
}

// App represents application service
type App struct {
	logger   golog.Logger
	adapters []Adapter
}

// New provides new service application
func New(logger golog.Logger) *App {
	return &App{
		logger: logger,
	}
}

// AddAdapters adds adapters to application service
func (app *App) AddAdapters(adapters ...Adapter) *App {
	return &App{
		adapters: append(app.adapters, adapters...),
	}
}

// Run runs the service application
func (app *App) Run(ctx context.Context) {
	stop := func() {
		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
		defer cancel()

		app.logger.Info(ctx, "shutting down...\n")

		for _, adapter := range app.adapters {
			go func(adapter Adapter) {
				if err := adapter.Stop(ctx); err != nil {
					app.logger.Critical(ctx, "shutdown error: %v\n", err)
					os.Exit(1)
				}
			}(adapter)
		}

		app.logger.Info(ctx, "gracefully stopped\n")
	}

	for _, adapter := range app.adapters {
		go func(adapter Adapter) {
			err := adapter.Start(ctx)

			stop()

			if err != nil {
				app.logger.Critical(ctx, "%v\n", adapter.Start(ctx))
				os.Exit(1)
			}
		}(adapter)
	}

	shutdown.GracefulStop(stop)
}
<commit_msg>Add shutdown timeout configuration<commit_after>package application

import (
	"context"
	"os"
	"time"

	"github.com/vardius/golog"
	"github.com/vardius/shutdown"
)

// Adapter interface
type Adapter interface {
	Start(ctx context.Context) error
	Stop(ctx context.Context) error
}

// App represents application service
type App struct {
	adapters        []Adapter
	shutdownTimeout time.Duration

	logger golog.Logger
}

// New provides new service application
func New(logger golog.Logger) *App {
	return &App{
		shutdownTimeout: 5 * time.Second, // Default shutdown timeout
		logger:          logger,
	}
}

// AddAdapters adds adapters to application service
func (app *App) AddAdapters(adapters ...Adapter) *App {
	return &App{
		adapters: append(app.adapters, adapters...),
	}
}

// WithShutdownTimeout overrides default shutdown timout
func (app *App) WithShutdownTimeout(timeout time.Duration) {
	app.shutdownTimeout = timeout
}

// Run runs the service application
func (app *App) Run(ctx context.Context) {
	stop := func() {
		ctx, cancel := context.WithTimeout(ctx, app.shutdownTimeout)
		defer cancel()

		app.logger.Info(ctx, "shutting down...\n")

		for _, adapter := range app.adapters {
			go func(adapter Adapter) {
				if err := adapter.Stop(ctx); err != nil {
					app.logger.Critical(ctx, "shutdown error: %v\n", err)
					os.Exit(1)
				}
			}(adapter)
		}

		app.logger.Info(ctx, "gracefully stopped\n")
	}

	for _, adapter := range app.adapters {
		go func(adapter Adapter) {
			if err := adapter.Start(ctx); err != nil {
				app.logger.Critical(ctx, "%v\n", err)
				stop()
			}
		}(adapter)
	}

	shutdown.GracefulStop(stop)
}
<|endoftext|>


<commit_before>// +build linux
// Code in this package is heavily adapted from https://github.com/opencontainers/runc/blob/7362fa2d282feffb9b19911150e01e390a23899d/libcontainer/cgroups/systemd
// Credit goes to the runc authors.

package dbusmgr

import (
	"context"
	"errors"
	"strings"
	"sync"

	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
	dbus "github.com/godbus/dbus/v5"
)

var (
	dbusC        *systemdDbus.Conn
	dbusMu       sync.RWMutex
	dbusInited   bool
	dbusRootless bool
)

type DbusConnManager struct{}

// NewDbusConnManager initializes systemd dbus connection manager.
func NewDbusConnManager(rootless bool) *DbusConnManager {
	dbusMu.Lock()
	defer dbusMu.Unlock()
	if dbusInited && rootless != dbusRootless {
		panic("can't have both root and rootless dbus")
	}
	dbusRootless = rootless
	dbusInited = true
	return &DbusConnManager{}
}

// getConnection lazily initializes and returns systemd dbus connection.
func (d *DbusConnManager) GetConnection() (*systemdDbus.Conn, error) {
	// In the case where dbusC != nil
	// Use the read lock the first time to ensure
	// that Conn can be acquired at the same time.
	dbusMu.RLock()
	if conn := dbusC; conn != nil {
		dbusMu.RUnlock()
		return conn, nil
	}
	dbusMu.RUnlock()

	// In the case where dbusC == nil
	// Use write lock to ensure that only one
	// will be created
	dbusMu.Lock()
	defer dbusMu.Unlock()
	if conn := dbusC; conn != nil {
		return conn, nil
	}

	conn, err := d.newConnection()
	if err != nil {
		return nil, err
	}
	dbusC = conn
	return conn, nil
}

func (d *DbusConnManager) newConnection() (*systemdDbus.Conn, error) {
	if dbusRootless {
		return newUserSystemdDbus()
	}
	return systemdDbus.NewWithContext(context.TODO())
}

var errDbusConnClosed = dbus.ErrClosed.Error()

// RetryOnDisconnect calls op, and if the error it returns is about closed dbus
// connection, the connection is re-established and the op is retried. This helps
// with the situation when dbus is restarted and we have a stale connection.
func (d *DbusConnManager) RetryOnDisconnect(op func(*systemdDbus.Conn) error) error {
	for {
		conn, err := d.GetConnection()
		if err != nil {
			return err
		}
		err = op(conn)
		if !isDbusError(err, errDbusConnClosed) {
			return err
		}
		d.resetConnection(conn)
	}
}

// resetConnection resets the connection to its initial state
// (so it can be reconnected if necessary).
func (d *DbusConnManager) resetConnection(conn *systemdDbus.Conn) {
	dbusMu.Lock()
	defer dbusMu.Unlock()
	if dbusC != nil && dbusC == conn {
		dbusC.Close()
		dbusC = nil
	}
}

// isDbusError returns true if the error is a specific dbus error.
func isDbusError(err error, name string) bool {
	if err != nil {
		var derr dbus.Error
		if errors.As(err, &derr) {
			return strings.Contains(derr.Name, name)
		}
	}
	return false
}
<commit_msg>simplify checking for dbus error<commit_after>// +build linux
// Code in this package is heavily adapted from https://github.com/opencontainers/runc/blob/7362fa2d282feffb9b19911150e01e390a23899d/libcontainer/cgroups/systemd
// Credit goes to the runc authors.

package dbusmgr

import (
	"context"
	"errors"
	"sync"

	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
	dbus "github.com/godbus/dbus/v5"
)

var (
	dbusC        *systemdDbus.Conn
	dbusMu       sync.RWMutex
	dbusInited   bool
	dbusRootless bool
)

type DbusConnManager struct{}

// NewDbusConnManager initializes systemd dbus connection manager.
func NewDbusConnManager(rootless bool) *DbusConnManager {
	dbusMu.Lock()
	defer dbusMu.Unlock()
	if dbusInited && rootless != dbusRootless {
		panic("can't have both root and rootless dbus")
	}
	dbusRootless = rootless
	dbusInited = true
	return &DbusConnManager{}
}

// getConnection lazily initializes and returns systemd dbus connection.
func (d *DbusConnManager) GetConnection() (*systemdDbus.Conn, error) {
	// In the case where dbusC != nil
	// Use the read lock the first time to ensure
	// that Conn can be acquired at the same time.
	dbusMu.RLock()
	if conn := dbusC; conn != nil {
		dbusMu.RUnlock()
		return conn, nil
	}
	dbusMu.RUnlock()

	// In the case where dbusC == nil
	// Use write lock to ensure that only one
	// will be created
	dbusMu.Lock()
	defer dbusMu.Unlock()
	if conn := dbusC; conn != nil {
		return conn, nil
	}

	conn, err := d.newConnection()
	if err != nil {
		return nil, err
	}
	dbusC = conn
	return conn, nil
}

func (d *DbusConnManager) newConnection() (*systemdDbus.Conn, error) {
	if dbusRootless {
		return newUserSystemdDbus()
	}
	return systemdDbus.NewWithContext(context.TODO())
}

// RetryOnDisconnect calls op, and if the error it returns is about closed dbus
// connection, the connection is re-established and the op is retried. This helps
// with the situation when dbus is restarted and we have a stale connection.
func (d *DbusConnManager) RetryOnDisconnect(op func(*systemdDbus.Conn) error) error {
	for {
		conn, err := d.GetConnection()
		if err != nil {
			return err
		}
		err = op(conn)
		if !errors.Is(err, dbus.ErrClosed) {
			return err
		}
		d.resetConnection(conn)
	}
}

// resetConnection resets the connection to its initial state
// (so it can be reconnected if necessary).
func (d *DbusConnManager) resetConnection(conn *systemdDbus.Conn) {
	dbusMu.Lock()
	defer dbusMu.Unlock()
	if dbusC != nil && dbusC == conn {
		dbusC.Close()
		dbusC = nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"bytes"
	"context"
	"go/ast"
	"go/scanner"
	"go/types"
	"sync"

	"golang.org/x/tools/go/analysis"
	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

type importer struct {
	view   *view
	ctx    context.Context
	config *packages.Config

	// seen maintains the set of previously imported packages.
	// If we have seen a package that is already in this map, we have a circular import.
	seen map[packageID]struct{}

	// topLevelPackageID is the ID of the package from which type-checking began.
	topLevelPackageID packageID

	// parentPkg is the package that imports the current package.
	parentPkg *pkg

	// parentCheckPackageHandle is the check package handle that imports the current package.
	parentCheckPackageHandle *checkPackageHandle
}

// checkPackageKey uniquely identifies a package and its config.
type checkPackageKey struct {
	files  string
	config string

	// TODO: For now, we don't include dependencies in the key.
	// This will be necessary when we change the cache invalidation logic.
}

// checkPackageHandle implements source.CheckPackageHandle.
type checkPackageHandle struct {
	handle *memoize.Handle

	files   []source.ParseGoHandle
	imports map[packagePath]*checkPackageHandle

	m      *metadata
	config *packages.Config
}

// checkPackageData contains the data produced by type-checking a package.
type checkPackageData struct {
	memoize.NoCopy

	pkg *pkg
	err error
}

func (pkg *pkg) GetImport(ctx context.Context, pkgPath string) (source.Package, error) {
	if imp := pkg.imports[packagePath(pkgPath)]; imp != nil {
		return imp, nil
	}
	// Don't return a nil pointer because that still satisfies the interface.
	return nil, errors.Errorf("no imported package for %s", pkgPath)
}

// checkPackageHandle returns a source.CheckPackageHandle for a given package and config.
func (imp *importer) checkPackageHandle(m *metadata) (*checkPackageHandle, error) {
	phs, err := imp.parseGoHandles(m)
	if err != nil {
		return nil, err
	}
	key := checkPackageKey{
		files:  hashParseKeys(phs),
		config: hashConfig(imp.config),
	}
	cph := &checkPackageHandle{
		m:       m,
		files:   phs,
		config:  imp.config,
		imports: make(map[packagePath]*checkPackageHandle),
	}
	h := imp.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} {
		data := &checkPackageData{}
		data.pkg, data.err = func() (*pkg, error) {
			return imp.typeCheck(cph, m)
		}()
		return data
	})
	cph.handle = h
	return cph, nil
}

// hashConfig returns the hash for the *packages.Config.
func hashConfig(config *packages.Config) string {
	b := bytes.NewBuffer(nil)

	// Dir, Mode, Env, BuildFlags are the parts of the config that can change.
	b.WriteString(config.Dir)
	b.WriteString(string(config.Mode))

	for _, e := range config.Env {
		b.WriteString(e)
	}
	for _, f := range config.BuildFlags {
		b.WriteString(f)
	}
	return hashContents(b.Bytes())
}

func (cph *checkPackageHandle) Check(ctx context.Context) (source.Package, error) {
	return cph.check(ctx)
}

func (cph *checkPackageHandle) check(ctx context.Context) (*pkg, error) {
	v := cph.handle.Get(ctx)
	if v == nil {
		return nil, ctx.Err()
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (cph *checkPackageHandle) Config() *packages.Config {
	return cph.config
}

func (cph *checkPackageHandle) Files() []source.ParseGoHandle {
	return cph.files
}

func (cph *checkPackageHandle) Cached(ctx context.Context) (source.Package, error) {
	v := cph.handle.Cached()
	if v == nil {
		return nil, errors.Errorf("no cached value for %s", cph.m.pkgPath)
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (imp *importer) parseGoHandles(m *metadata) ([]source.ParseGoHandle, error) {
	phs := make([]source.ParseGoHandle, 0, len(m.files))
	for _, uri := range m.files {
		// Call the unlocked version of getFile since we are holding the view's mutex.
		f, err := imp.view.GetFile(imp.ctx, uri)
		if err != nil {
			return nil, err
		}
		gof, ok := f.(*goFile)
		if !ok {
			return nil, errors.Errorf("%s is not a Go file", f.URI())
		}
		fh := gof.Handle(imp.ctx)
		mode := source.ParseExported
		if imp.topLevelPackageID == m.id {
			mode = source.ParseFull
		} else if imp.view.session.cache.store.Cached(parseKey{
			file: fh.Identity(),
			mode: source.ParseFull,
		}) != nil {
			// If we have the full AST cached, don't bother getting the trimmed version.
			mode = source.ParseFull
		}
		phs = append(phs, imp.view.session.cache.ParseGoHandle(fh, mode))
	}
	return phs, nil
}

func (imp *importer) Import(pkgPath string) (*types.Package, error) {
	// We need to set the parent package's imports, so there should always be one.
	if imp.parentPkg == nil {
		return nil, errors.Errorf("no parent package for import %s", pkgPath)
	}
	// Get the package metadata from the importing package.
	cph, ok := imp.parentCheckPackageHandle.imports[packagePath(pkgPath)]
	if !ok {
		return nil, errors.Errorf("no package data for import path %s", pkgPath)
	}
	// Create a check package handle to get the type information for this package.
	pkg, err := cph.check(imp.ctx)
	if err != nil {
		return nil, err
	}
	imp.parentPkg.imports[packagePath(pkgPath)] = pkg
	// Add every file in this package to our cache.
	if err := imp.cachePackage(cph, pkg, cph.m); err != nil {
		return nil, err
	}
	return pkg.GetTypes(), nil
}

func (imp *importer) typeCheck(cph *checkPackageHandle, m *metadata) (*pkg, error) {
	ctx, done := trace.StartSpan(imp.ctx, "cache.importer.typeCheck")
	defer done()

	pkg := &pkg{
		view:       imp.view,
		id:         m.id,
		pkgPath:    m.pkgPath,
		files:      cph.Files(),
		imports:    make(map[packagePath]*pkg),
		typesSizes: m.typesSizes,
		typesInfo: &types.Info{
			Types:      make(map[ast.Expr]types.TypeAndValue),
			Defs:       make(map[*ast.Ident]types.Object),
			Uses:       make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
			Scopes:     make(map[ast.Node]*types.Scope),
		},
		analyses: make(map[*analysis.Analyzer]*analysisEntry),
	}
	// If the package comes back with errors from `go list`,
	// don't bother type-checking it.
	for _, err := range m.errors {
		pkg.errors = append(m.errors, err)
	}
	// Set imports of package to correspond to cached packages.
	cimp := imp.child(pkg, cph)
	for _, child := range m.children {
		childHandle, err := cimp.checkPackageHandle(child)
		if err != nil {
			log.Error(imp.ctx, "no check package handle", err, telemetry.Package.Of(child.id))
			continue
		}
		cph.imports[child.pkgPath] = childHandle
	}
	var (
		files       = make([]*ast.File, len(pkg.files))
		parseErrors = make([]error, len(pkg.files))
		wg          sync.WaitGroup
	)
	for i, ph := range pkg.files {
		wg.Add(1)
		go func(i int, ph source.ParseGoHandle) {
			defer wg.Done()

			files[i], parseErrors[i] = ph.Parse(ctx)
		}(i, ph)
	}
	wg.Wait()

	for _, err := range parseErrors {
		if err == context.Canceled {
			return nil, errors.Errorf("parsing files for %s: %v", m.pkgPath, err)
		}
		if err != nil {
			imp.view.session.cache.appendPkgError(pkg, err)
		}
	}

	var i int
	for _, f := range files {
		if f != nil {
			files[i] = f
			i++
		}
	}
	files = files[:i]

	// Use the default type information for the unsafe package.
	if m.pkgPath == "unsafe" {
		pkg.types = types.Unsafe
	} else if len(files) == 0 { // not the unsafe package, no parsed files
		return nil, errors.Errorf("no parsed files for package %s", pkg.pkgPath)
	} else {
		pkg.types = types.NewPackage(string(m.pkgPath), m.name)
	}

	cfg := &types.Config{
		Error: func(err error) {
			imp.view.session.cache.appendPkgError(pkg, err)
		},
		Importer: cimp,
	}
	check := types.NewChecker(cfg, imp.view.session.cache.FileSet(), pkg.types, pkg.typesInfo)

	// Ignore type-checking errors.
	check.Files(files)

	return pkg, nil
}

func (imp *importer) child(pkg *pkg, cph *checkPackageHandle) *importer {
	// Handle circular imports by copying previously seen imports.
	seen := make(map[packageID]struct{})
	for k, v := range imp.seen {
		seen[k] = v
	}
	seen[pkg.id] = struct{}{}
	return &importer{
		view:                     imp.view,
		ctx:                      imp.ctx,
		config:                   imp.config,
		seen:                     seen,
		topLevelPackageID:        imp.topLevelPackageID,
		parentPkg:                pkg,
		parentCheckPackageHandle: cph,
	}
}

func (imp *importer) cachePackage(cph *checkPackageHandle, pkg *pkg, m *metadata) error {
	for _, ph := range pkg.files {
		uri := ph.File().Identity().URI
		f, err := imp.view.GetFile(imp.ctx, uri)
		if err != nil {
			return errors.Errorf("no such file %s: %v", uri, err)
		}
		gof, ok := f.(*goFile)
		if !ok {
			return errors.Errorf("%s is not a Go file", uri)
		}
		if err := imp.cachePerFile(gof, ph, cph, m); err != nil {
			return errors.Errorf("failed to cache file %s: %v", gof.URI(), err)
		}
	}
	return nil
}

func (imp *importer) cachePerFile(gof *goFile, ph source.ParseGoHandle, cph source.CheckPackageHandle, m *metadata) error {
	gof.mu.Lock()
	defer gof.mu.Unlock()

	// Set the package even if we failed to parse the file.
	if gof.pkgs == nil {
		gof.pkgs = make(map[packageID]source.CheckPackageHandle)
	}
	gof.pkgs[m.id] = cph

	file, err := ph.Parse(imp.ctx)
	if file == nil {
		return errors.Errorf("no AST for %s: %v", ph.File().Identity().URI, err)
	}
	gof.imports = file.Imports
	return nil
}

func (c *cache) appendPkgError(pkg *pkg, err error) {
	if err == nil {
		return
	}
	var errs []packages.Error
	switch err := err.(type) {
	case *scanner.Error:
		errs = append(errs, packages.Error{
			Pos:  err.Pos.String(),
			Msg:  err.Msg,
			Kind: packages.ParseError,
		})
	case scanner.ErrorList:
		// The first parser error is likely the root cause of the problem.
		if err.Len() > 0 {
			errs = append(errs, packages.Error{
				Pos:  err[0].Pos.String(),
				Msg:  err[0].Msg,
				Kind: packages.ParseError,
			})
		}
	case types.Error:
		errs = append(errs, packages.Error{
			Pos:  c.FileSet().Position(err.Pos).String(),
			Msg:  err.Msg,
			Kind: packages.TypeError,
		})
	}
	pkg.errors = append(pkg.errors, errs...)
}
<commit_msg>internal/lsp: use memoize store's context when type checking<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"bytes"
	"context"
	"go/ast"
	"go/scanner"
	"go/types"
	"sync"

	"golang.org/x/tools/go/analysis"
	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/memoize"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

type importer struct {
	view   *view
	ctx    context.Context
	config *packages.Config

	// seen maintains the set of previously imported packages.
	// If we have seen a package that is already in this map, we have a circular import.
	seen map[packageID]struct{}

	// topLevelPackageID is the ID of the package from which type-checking began.
	topLevelPackageID packageID

	// parentPkg is the package that imports the current package.
	parentPkg *pkg

	// parentCheckPackageHandle is the check package handle that imports the current package.
	parentCheckPackageHandle *checkPackageHandle
}

// checkPackageKey uniquely identifies a package and its config.
type checkPackageKey struct {
	files  string
	config string

	// TODO: For now, we don't include dependencies in the key.
	// This will be necessary when we change the cache invalidation logic.
}

// checkPackageHandle implements source.CheckPackageHandle.
type checkPackageHandle struct {
	handle *memoize.Handle

	files   []source.ParseGoHandle
	imports map[packagePath]*checkPackageHandle

	m      *metadata
	config *packages.Config
}

// checkPackageData contains the data produced by type-checking a package.
type checkPackageData struct {
	memoize.NoCopy

	pkg *pkg
	err error
}

func (pkg *pkg) GetImport(ctx context.Context, pkgPath string) (source.Package, error) {
	if imp := pkg.imports[packagePath(pkgPath)]; imp != nil {
		return imp, nil
	}
	// Don't return a nil pointer because that still satisfies the interface.
	return nil, errors.Errorf("no imported package for %s", pkgPath)
}

// checkPackageHandle returns a source.CheckPackageHandle for a given package and config.
func (imp *importer) checkPackageHandle(m *metadata) (*checkPackageHandle, error) {
	phs, err := imp.parseGoHandles(m)
	if err != nil {
		return nil, err
	}
	key := checkPackageKey{
		files:  hashParseKeys(phs),
		config: hashConfig(imp.config),
	}
	cph := &checkPackageHandle{
		m:       m,
		files:   phs,
		config:  imp.config,
		imports: make(map[packagePath]*checkPackageHandle),
	}
	h := imp.view.session.cache.store.Bind(key, func(ctx context.Context) interface{} {
		origCtx := imp.ctx
		defer func() { imp.ctx = origCtx }()

		// We must use the store's detached context to avoid poisoning the
		// cache with context.Canceled if the request is cancelled.
		imp.ctx = ctx

		data := &checkPackageData{}
		data.pkg, data.err = imp.typeCheck(cph, m)
		return data
	})
	cph.handle = h
	return cph, nil
}

// hashConfig returns the hash for the *packages.Config.
func hashConfig(config *packages.Config) string {
	b := bytes.NewBuffer(nil)

	// Dir, Mode, Env, BuildFlags are the parts of the config that can change.
	b.WriteString(config.Dir)
	b.WriteString(string(config.Mode))

	for _, e := range config.Env {
		b.WriteString(e)
	}
	for _, f := range config.BuildFlags {
		b.WriteString(f)
	}
	return hashContents(b.Bytes())
}

func (cph *checkPackageHandle) Check(ctx context.Context) (source.Package, error) {
	return cph.check(ctx)
}

func (cph *checkPackageHandle) check(ctx context.Context) (*pkg, error) {
	v := cph.handle.Get(ctx)
	if v == nil {
		return nil, ctx.Err()
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (cph *checkPackageHandle) Config() *packages.Config {
	return cph.config
}

func (cph *checkPackageHandle) Files() []source.ParseGoHandle {
	return cph.files
}

func (cph *checkPackageHandle) Cached(ctx context.Context) (source.Package, error) {
	v := cph.handle.Cached()
	if v == nil {
		return nil, errors.Errorf("no cached value for %s", cph.m.pkgPath)
	}
	data := v.(*checkPackageData)
	return data.pkg, data.err
}

func (imp *importer) parseGoHandles(m *metadata) ([]source.ParseGoHandle, error) {
	phs := make([]source.ParseGoHandle, 0, len(m.files))
	for _, uri := range m.files {
		// Call the unlocked version of getFile since we are holding the view's mutex.
		f, err := imp.view.GetFile(imp.ctx, uri)
		if err != nil {
			return nil, err
		}
		gof, ok := f.(*goFile)
		if !ok {
			return nil, errors.Errorf("%s is not a Go file", f.URI())
		}
		fh := gof.Handle(imp.ctx)
		mode := source.ParseExported
		if imp.topLevelPackageID == m.id {
			mode = source.ParseFull
		} else if imp.view.session.cache.store.Cached(parseKey{
			file: fh.Identity(),
			mode: source.ParseFull,
		}) != nil {
			// If we have the full AST cached, don't bother getting the trimmed version.
			mode = source.ParseFull
		}
		phs = append(phs, imp.view.session.cache.ParseGoHandle(fh, mode))
	}
	return phs, nil
}

func (imp *importer) Import(pkgPath string) (*types.Package, error) {
	// We need to set the parent package's imports, so there should always be one.
	if imp.parentPkg == nil {
		return nil, errors.Errorf("no parent package for import %s", pkgPath)
	}
	// Get the package metadata from the importing package.
	cph, ok := imp.parentCheckPackageHandle.imports[packagePath(pkgPath)]
	if !ok {
		return nil, errors.Errorf("no package data for import path %s", pkgPath)
	}
	// Create a check package handle to get the type information for this package.
	pkg, err := cph.check(imp.ctx)
	if err != nil {
		return nil, err
	}
	imp.parentPkg.imports[packagePath(pkgPath)] = pkg
	// Add every file in this package to our cache.
	if err := imp.cachePackage(cph, pkg, cph.m); err != nil {
		return nil, err
	}
	return pkg.GetTypes(), nil
}

func (imp *importer) typeCheck(cph *checkPackageHandle, m *metadata) (*pkg, error) {
	ctx, done := trace.StartSpan(imp.ctx, "cache.importer.typeCheck")
	defer done()

	pkg := &pkg{
		view:       imp.view,
		id:         m.id,
		pkgPath:    m.pkgPath,
		files:      cph.Files(),
		imports:    make(map[packagePath]*pkg),
		typesSizes: m.typesSizes,
		typesInfo: &types.Info{
			Types:      make(map[ast.Expr]types.TypeAndValue),
			Defs:       make(map[*ast.Ident]types.Object),
			Uses:       make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
			Scopes:     make(map[ast.Node]*types.Scope),
		},
		analyses: make(map[*analysis.Analyzer]*analysisEntry),
	}
	// If the package comes back with errors from `go list`,
	// don't bother type-checking it.
	for _, err := range m.errors {
		pkg.errors = append(m.errors, err)
	}
	// Set imports of package to correspond to cached packages.
	cimp := imp.child(pkg, cph)
	for _, child := range m.children {
		childHandle, err := cimp.checkPackageHandle(child)
		if err != nil {
			log.Error(imp.ctx, "no check package handle", err, telemetry.Package.Of(child.id))
			continue
		}
		cph.imports[child.pkgPath] = childHandle
	}
	var (
		files       = make([]*ast.File, len(pkg.files))
		parseErrors = make([]error, len(pkg.files))
		wg          sync.WaitGroup
	)
	for i, ph := range pkg.files {
		wg.Add(1)
		go func(i int, ph source.ParseGoHandle) {
			defer wg.Done()

			files[i], parseErrors[i] = ph.Parse(ctx)
		}(i, ph)
	}
	wg.Wait()

	for _, err := range parseErrors {
		if err == context.Canceled {
			return nil, errors.Errorf("parsing files for %s: %v", m.pkgPath, err)
		}
		if err != nil {
			imp.view.session.cache.appendPkgError(pkg, err)
		}
	}

	var i int
	for _, f := range files {
		if f != nil {
			files[i] = f
			i++
		}
	}
	files = files[:i]

	// Use the default type information for the unsafe package.
	if m.pkgPath == "unsafe" {
		pkg.types = types.Unsafe
	} else if len(files) == 0 { // not the unsafe package, no parsed files
		return nil, errors.Errorf("no parsed files for package %s", pkg.pkgPath)
	} else {
		pkg.types = types.NewPackage(string(m.pkgPath), m.name)
	}

	cfg := &types.Config{
		Error: func(err error) {
			imp.view.session.cache.appendPkgError(pkg, err)
		},
		Importer: cimp,
	}
	check := types.NewChecker(cfg, imp.view.session.cache.FileSet(), pkg.types, pkg.typesInfo)

	// Ignore type-checking errors.
	check.Files(files)

	return pkg, nil
}

func (imp *importer) child(pkg *pkg, cph *checkPackageHandle) *importer {
	// Handle circular imports by copying previously seen imports.
	seen := make(map[packageID]struct{})
	for k, v := range imp.seen {
		seen[k] = v
	}
	seen[pkg.id] = struct{}{}
	return &importer{
		view:                     imp.view,
		ctx:                      imp.ctx,
		config:                   imp.config,
		seen:                     seen,
		topLevelPackageID:        imp.topLevelPackageID,
		parentPkg:                pkg,
		parentCheckPackageHandle: cph,
	}
}

func (imp *importer) cachePackage(cph *checkPackageHandle, pkg *pkg, m *metadata) error {
	for _, ph := range pkg.files {
		uri := ph.File().Identity().URI
		f, err := imp.view.GetFile(imp.ctx, uri)
		if err != nil {
			return errors.Errorf("no such file %s: %v", uri, err)
		}
		gof, ok := f.(*goFile)
		if !ok {
			return errors.Errorf("%s is not a Go file", uri)
		}
		if err := imp.cachePerFile(gof, ph, cph, m); err != nil {
			return errors.Errorf("failed to cache file %s: %v", gof.URI(), err)
		}
	}
	return nil
}

func (imp *importer) cachePerFile(gof *goFile, ph source.ParseGoHandle, cph source.CheckPackageHandle, m *metadata) error {
	gof.mu.Lock()
	defer gof.mu.Unlock()

	// Set the package even if we failed to parse the file.
	if gof.pkgs == nil {
		gof.pkgs = make(map[packageID]source.CheckPackageHandle)
	}
	gof.pkgs[m.id] = cph

	file, err := ph.Parse(imp.ctx)
	if file == nil {
		return errors.Errorf("no AST for %s: %v", ph.File().Identity().URI, err)
	}
	gof.imports = file.Imports
	return nil
}

func (c *cache) appendPkgError(pkg *pkg, err error) {
	if err == nil {
		return
	}
	var errs []packages.Error
	switch err := err.(type) {
	case *scanner.Error:
		errs = append(errs, packages.Error{
			Pos:  err.Pos.String(),
			Msg:  err.Msg,
			Kind: packages.ParseError,
		})
	case scanner.ErrorList:
		// The first parser error is likely the root cause of the problem.
		if err.Len() > 0 {
			errs = append(errs, packages.Error{
				Pos:  err[0].Pos.String(),
				Msg:  err[0].Msg,
				Kind: packages.ParseError,
			})
		}
	case types.Error:
		errs = append(errs, packages.Error{
			Pos:  c.FileSet().Position(err.Pos).String(),
			Msg:  err.Msg,
			Kind: packages.TypeError,
		})
	}
	pkg.errors = append(pkg.errors, errs...)
}
<|endoftext|>


<commit_before>package git

import (
	"errors"
	"fmt"
)

// ErrInvalidVersionFormat is return when the version isnt in a valid format
type ErrInvalidVersionFormat struct {
	version string
}

func (e ErrInvalidVersionFormat) Error() string {
	return fmt.Sprintf("%v is not in a valid version format", e.version)
}

// ErrDirty happens when the repo has uncommitted/unstashed changes
type ErrDirty struct {
	status string
}

func (e ErrDirty) Error() string {
	return fmt.Sprintf("git is currently in a dirty state:\n%v", e.status)
}

// ErrWrongRef happens when the HEAD reference is different from the tag being built
type ErrWrongRef struct {
	commit, tag string
}

func (e ErrWrongRef) Error() string {
	return fmt.Sprintf("git tag %v was not made against commit %v", e.tag, e.commit)
}

// ErrNoTag happens if the underlying git repository doesn't contain any tags
// but no snapshot-release was requested.
var ErrNoTag = errors.New("git doesn't contain any tags. Either add a tag or use --snapshot")

// ErrNotRepository happens if you try to run goreleaser against a folder
// which is not a git repository.
var ErrNotRepository = errors.New("current folder is not a git repository")

// ErrNoGit happens when git is not present in PATH.
var ErrNoGit = errors.New("git not present in PATH")
<commit_msg>clean: unused err<commit_after>package git

import (
	"errors"
	"fmt"
)

// ErrDirty happens when the repo has uncommitted/unstashed changes
type ErrDirty struct {
	status string
}

func (e ErrDirty) Error() string {
	return fmt.Sprintf("git is currently in a dirty state:\n%v", e.status)
}

// ErrWrongRef happens when the HEAD reference is different from the tag being built
type ErrWrongRef struct {
	commit, tag string
}

func (e ErrWrongRef) Error() string {
	return fmt.Sprintf("git tag %v was not made against commit %v", e.tag, e.commit)
}

// ErrNoTag happens if the underlying git repository doesn't contain any tags
// but no snapshot-release was requested.
var ErrNoTag = errors.New("git doesn't contain any tags. Either add a tag or use --snapshot")

// ErrNotRepository happens if you try to run goreleaser against a folder
// which is not a git repository.
var ErrNotRepository = errors.New("current folder is not a git repository")

// ErrNoGit happens when git is not present in PATH.
var ErrNoGit = errors.New("git not present in PATH")
<|endoftext|>


<commit_before>package restic

import (
	"fmt"
	"os"
	"sync"
	"time"

	"golang.org/x/crypto/ssh/terminal"
)

const minTickerTime = time.Second / 60

var isTerminal = terminal.IsTerminal(int(os.Stdout.Fd()))
var forceUpdateProgress = make(chan bool)

// Progress reports progress on an operation.
type Progress struct {
	OnStart  func()
	OnUpdate ProgressFunc
	OnDone   ProgressFunc
	fnM      sync.Mutex

	cur        Stat
	curM       sync.Mutex
	start      time.Time
	c          *time.Ticker
	cancel     chan struct{}
	o          *sync.Once
	d          time.Duration
	lastUpdate time.Time

	running bool
}

// Stat captures newly done parts of the operation.
type Stat struct {
	Files  uint64
	Dirs   uint64
	Bytes  uint64
	Trees  uint64
	Blobs  uint64
	Errors uint64
}

// ProgressFunc is used to report progress back to the user.
type ProgressFunc func(s Stat, runtime time.Duration, ticker bool)

// NewProgress returns a new progress reporter. When Start() is called, the
// function OnStart is executed once. Afterwards the function OnUpdate is
// called when new data arrives or at least every d interval. The function
// OnDone is called when Done() is called. Both functions are called
// synchronously and can use shared state.
func NewProgress() *Progress {
	var d time.Duration
	if isTerminal {
		d = time.Second
	}
	return &Progress{d: d}
}

// Start resets and runs the progress reporter.
func (p *Progress) Start() {
	if p == nil || p.running {
		return
	}

	p.o = &sync.Once{}
	p.cancel = make(chan struct{})
	p.running = true
	p.Reset()
	p.start = time.Now()
	p.c = nil
	if p.d != 0 {
		p.c = time.NewTicker(p.d)
	}

	if p.OnStart != nil {
		p.OnStart()
	}

	go p.reporter()
}

// Reset resets all statistic counters to zero.
func (p *Progress) Reset() {
	if p == nil {
		return
	}

	if !p.running {
		panic("resetting a non-running Progress")
	}

	p.curM.Lock()
	p.cur = Stat{}
	p.curM.Unlock()
}

// Report adds the statistics from s to the current state and tries to report
// the accumulated statistics via the feedback channel.
func (p *Progress) Report(s Stat) {
	if p == nil {
		return
	}

	if !p.running {
		panic("reporting in a non-running Progress")
	}

	p.curM.Lock()
	p.cur.Add(s)
	cur := p.cur
	needUpdate := false
	if isTerminal && time.Since(p.lastUpdate) > minTickerTime {
		p.lastUpdate = time.Now()
		needUpdate = true
	}
	p.curM.Unlock()

	if needUpdate {
		p.updateProgress(cur, false)
	}

}

func (p *Progress) updateProgress(cur Stat, ticker bool) {
	if p.OnUpdate == nil {
		return
	}

	p.fnM.Lock()
	p.OnUpdate(cur, time.Since(p.start), ticker)
	p.fnM.Unlock()
}

func (p *Progress) reporter() {
	if p == nil {
		return
	}

	updateProgress := func() {
		p.curM.Lock()
		cur := p.cur
		p.curM.Unlock()
		p.updateProgress(cur, true)
	}

	var ticker <-chan time.Time
	if p.c != nil {
		ticker = p.c.C
	}

	for {
		select {
		case <-ticker:
			updateProgress()
		case <-forceUpdateProgress:
			updateProgress()
		case <-p.cancel:
			if p.c != nil {
				p.c.Stop()
			}
			return
		}
	}
}

// Done closes the progress report.
func (p *Progress) Done() {
	if p == nil || !p.running {
		return
	}

	p.running = false
	p.o.Do(func() {
		close(p.cancel)
	})

	cur := p.cur

	if p.OnDone != nil {
		p.fnM.Lock()
		p.OnUpdate(cur, time.Since(p.start), false)
		p.OnDone(cur, time.Since(p.start), false)
		p.fnM.Unlock()
	}
}

// Add accumulates other into s.
func (s *Stat) Add(other Stat) {
	s.Bytes += other.Bytes
	s.Dirs += other.Dirs
	s.Files += other.Files
	s.Trees += other.Trees
	s.Blobs += other.Blobs
	s.Errors += other.Errors
}

func (s Stat) String() string {
	b := float64(s.Bytes)
	var str string

	switch {
	case s.Bytes > 1<<40:
		str = fmt.Sprintf("%.3f TiB", b/(1<<40))
	case s.Bytes > 1<<30:
		str = fmt.Sprintf("%.3f GiB", b/(1<<30))
	case s.Bytes > 1<<20:
		str = fmt.Sprintf("%.3f MiB", b/(1<<20))
	case s.Bytes > 1<<10:
		str = fmt.Sprintf("%.3f KiB", b/(1<<10))
	default:
		str = fmt.Sprintf("%dB", s.Bytes)
	}

	return fmt.Sprintf("Stat(%d files, %d dirs, %v trees, %v blobs, %d errors, %v)",
		s.Files, s.Dirs, s.Trees, s.Blobs, s.Errors, str)
}
<commit_msg>Control progress rate with RESTIC_PROGRESS_FPS env<commit_after>package restic

import (
	"fmt"
	"os"
	"strconv"
	"sync"
	"time"

	"golang.org/x/crypto/ssh/terminal"
)

// minTickerTime limits how often the progress ticker is updated. It can be
// overridden using the RESTIC_PROGRESS_FPS (frames per second) environment
// variable.
var minTickerTime = time.Second / 60

var isTerminal = terminal.IsTerminal(int(os.Stdout.Fd()))
var forceUpdateProgress = make(chan bool)

func init() {
	fps, err := strconv.ParseInt(os.Getenv("RESTIC_PROGRESS_FPS"), 10, 64)
	if err == nil && fps >= 1 {
		if fps > 60 {
			fps = 60
		}
		minTickerTime = time.Second / time.Duration(fps)
	}
}

// Progress reports progress on an operation.
type Progress struct {
	OnStart  func()
	OnUpdate ProgressFunc
	OnDone   ProgressFunc
	fnM      sync.Mutex

	cur        Stat
	curM       sync.Mutex
	start      time.Time
	c          *time.Ticker
	cancel     chan struct{}
	o          *sync.Once
	d          time.Duration
	lastUpdate time.Time

	running bool
}

// Stat captures newly done parts of the operation.
type Stat struct {
	Files  uint64
	Dirs   uint64
	Bytes  uint64
	Trees  uint64
	Blobs  uint64
	Errors uint64
}

// ProgressFunc is used to report progress back to the user.
type ProgressFunc func(s Stat, runtime time.Duration, ticker bool)

// NewProgress returns a new progress reporter. When Start() is called, the
// function OnStart is executed once. Afterwards the function OnUpdate is
// called when new data arrives or at least every d interval. The function
// OnDone is called when Done() is called. Both functions are called
// synchronously and can use shared state.
func NewProgress() *Progress {
	var d time.Duration
	if isTerminal {
		d = time.Second
	}
	return &Progress{d: d}
}

// Start resets and runs the progress reporter.
func (p *Progress) Start() {
	if p == nil || p.running {
		return
	}

	p.o = &sync.Once{}
	p.cancel = make(chan struct{})
	p.running = true
	p.Reset()
	p.start = time.Now()
	p.c = nil
	if p.d != 0 {
		p.c = time.NewTicker(p.d)
	}

	if p.OnStart != nil {
		p.OnStart()
	}

	go p.reporter()
}

// Reset resets all statistic counters to zero.
func (p *Progress) Reset() {
	if p == nil {
		return
	}

	if !p.running {
		panic("resetting a non-running Progress")
	}

	p.curM.Lock()
	p.cur = Stat{}
	p.curM.Unlock()
}

// Report adds the statistics from s to the current state and tries to report
// the accumulated statistics via the feedback channel.
func (p *Progress) Report(s Stat) {
	if p == nil {
		return
	}

	if !p.running {
		panic("reporting in a non-running Progress")
	}

	p.curM.Lock()
	p.cur.Add(s)
	cur := p.cur
	needUpdate := false
	if isTerminal && time.Since(p.lastUpdate) > minTickerTime {
		p.lastUpdate = time.Now()
		needUpdate = true
	}
	p.curM.Unlock()

	if needUpdate {
		p.updateProgress(cur, false)
	}

}

func (p *Progress) updateProgress(cur Stat, ticker bool) {
	if p.OnUpdate == nil {
		return
	}

	p.fnM.Lock()
	p.OnUpdate(cur, time.Since(p.start), ticker)
	p.fnM.Unlock()
}

func (p *Progress) reporter() {
	if p == nil {
		return
	}

	updateProgress := func() {
		p.curM.Lock()
		cur := p.cur
		p.curM.Unlock()
		p.updateProgress(cur, true)
	}

	var ticker <-chan time.Time
	if p.c != nil {
		ticker = p.c.C
	}

	for {
		select {
		case <-ticker:
			updateProgress()
		case <-forceUpdateProgress:
			updateProgress()
		case <-p.cancel:
			if p.c != nil {
				p.c.Stop()
			}
			return
		}
	}
}

// Done closes the progress report.
func (p *Progress) Done() {
	if p == nil || !p.running {
		return
	}

	p.running = false
	p.o.Do(func() {
		close(p.cancel)
	})

	cur := p.cur

	if p.OnDone != nil {
		p.fnM.Lock()
		p.OnUpdate(cur, time.Since(p.start), false)
		p.OnDone(cur, time.Since(p.start), false)
		p.fnM.Unlock()
	}
}

// Add accumulates other into s.
func (s *Stat) Add(other Stat) {
	s.Bytes += other.Bytes
	s.Dirs += other.Dirs
	s.Files += other.Files
	s.Trees += other.Trees
	s.Blobs += other.Blobs
	s.Errors += other.Errors
}

func (s Stat) String() string {
	b := float64(s.Bytes)
	var str string

	switch {
	case s.Bytes > 1<<40:
		str = fmt.Sprintf("%.3f TiB", b/(1<<40))
	case s.Bytes > 1<<30:
		str = fmt.Sprintf("%.3f GiB", b/(1<<30))
	case s.Bytes > 1<<20:
		str = fmt.Sprintf("%.3f MiB", b/(1<<20))
	case s.Bytes > 1<<10:
		str = fmt.Sprintf("%.3f KiB", b/(1<<10))
	default:
		str = fmt.Sprintf("%dB", s.Bytes)
	}

	return fmt.Sprintf("Stat(%d files, %d dirs, %v trees, %v blobs, %d errors, %v)",
		s.Files, s.Dirs, s.Trees, s.Blobs, s.Errors, str)
}
<|endoftext|>


<commit_before>package graph

import (
	"encoding/json"
	"fmt"
	"strings"

	"nimona.io/pkg/hash"
	"nimona.io/pkg/object"
	"nimona.io/pkg/stream"
)

type graphObject struct {
	ID       string
	NodeType string
	Context  string
	Display  string
	Parents  []string
	Data     string
}

func toGraphObject(v object.Object) (*graphObject, error) {
	b, err := json.Marshal(v.ToMap())
	if err != nil {
		return nil, err
	}
	nType := "object:root"
	parents := stream.Parents(v)
	if len(parents) > 0 {
		nType = "object"
	}
	o := &graphObject{
		ID:       hash.New(v).String(),
		NodeType: nType,
		Context:  v.GetType(),
		Parents:  []string{},
		Data:     string(b),
	}
	if d, ok := v.Get("@display").(string); ok {
		o.Display = d
	}
	for _, p := range stream.Parents(v) {
		o.Parents = append(o.Parents, p.String())
	}
	return o, nil
}

// Dot returns a graphviz representation of a graph
func Dot(objects []object.Object) (string, error) {
	graphObjects := make([]graphObject, len(objects))
	for i, o := range objects {
		igo, err := toGraphObject(o)
		if err != nil {
			return "", err
		}
		graphObjects[i] = *igo
	}
	return dot(graphObjects), nil
}

func dot(objects []graphObject) string {
	idSize := 5
	s := ""
	objectIDs := []string{}
	mutationIDs := []string{}
	for _, o := range objects {
		parents := make([]string, len(o.Parents))
		for i, p := range o.Parents {
			parents[i] = fmt.Sprintf(
				`<%s>`,
				p[1:idSize+1],
			)
		}
		id := fmt.Sprintf(
			`<%s>`,
			o.ID[1:idSize+1],
		)
		if len(parents) == 0 {
			s += fmt.Sprintf(
				"\t%s -> {} [shape=doublecircle];\n",
				id,
			)
			objectIDs = append(objectIDs, id)
		} else {
			s += fmt.Sprintf(
				"\t%s -> {%s} [shape=circle,label=\" mutates\"];\n",
				id,
				strings.Join(parents, " "),
			)
			mutationIDs = append(mutationIDs, id)
		}
	}
	m := "\trankdir=TB;\n"
	m += "\tsize=\"5,4\"\n"
	m += "\tgraph [bgcolor=white, fontname=Helvetica, fontsize=11];\n"
	m += "\tedge [fontname=Helvetica, fontcolor=grey, fontsize=9];\n"
	m += fmt.Sprintf(
		"\tnode [shape=doublecircle, fontname=Monospace, fontsize=11]; %s\n",
		strings.Join(objectIDs, " "),
	)
	m += fmt.Sprintf(
		"\tnode [shape=circle, fontname=Monospace, fontsize=11]; %s\n",
		strings.Join(mutationIDs, " "),
	)
	return fmt.Sprintf("digraph G {\n%s%s}", m, s)
}
<commit_msg>fix(store/graph): update dot for new hashes<commit_after>package graph

import (
	"encoding/json"
	"fmt"
	"strings"

	"nimona.io/pkg/hash"
	"nimona.io/pkg/object"
	"nimona.io/pkg/stream"
)

type graphObject struct {
	ID       string
	NodeType string
	Context  string
	Display  string
	Parents  []string
	Data     string
}

func toGraphObject(v object.Object) (*graphObject, error) {
	b, err := json.Marshal(v.ToMap())
	if err != nil {
		return nil, err
	}
	nType := "object:root"
	parents := stream.Parents(v)
	if len(parents) > 0 {
		nType = "object"
	}
	o := &graphObject{
		ID:       hash.New(v).String(),
		NodeType: nType,
		Context:  v.GetType(),
		Parents:  []string{},
		Data:     string(b),
	}
	if d, ok := v.Get("@display").(string); ok {
		o.Display = d
	}
	for _, p := range stream.Parents(v) {
		o.Parents = append(o.Parents, p.String())
	}
	return o, nil
}

// Dot returns a graphviz representation of a graph
func Dot(objects []object.Object) (string, error) {
	graphObjects := make([]graphObject, len(objects))
	for i, o := range objects {
		igo, err := toGraphObject(o)
		if err != nil {
			return "", err
		}
		graphObjects[i] = *igo
	}
	return dot(graphObjects), nil
}

func dot(objects []graphObject) string {
	clean := func(s string) string {
		return strings.Replace(s, "hash:oh1.", "", 1)[:5]
	}
	s := ""
	objectIDs := []string{}
	mutationIDs := []string{}
	for _, o := range objects {
		parents := make([]string, len(o.Parents))
		for i, p := range o.Parents {
			parents[i] = fmt.Sprintf(
				`<%s>`,
				clean(p),
			)
		}
		id := fmt.Sprintf(
			`<%s>`,
			clean(o.ID),
		)
		if len(parents) == 0 {
			s += fmt.Sprintf(
				"\t%s -> {} [shape=doublecircle];\n",
				id,
			)
			objectIDs = append(objectIDs, id)
		} else {
			s += fmt.Sprintf(
				"\t%s -> {%s} [shape=circle,label=\"  dep\"];\n",
				id,
				strings.Join(parents, " "),
			)
			mutationIDs = append(mutationIDs, id)
		}
	}
	m := "\trankdir=TB;\n"
	m += "\tsize=\"5,4\"\n"
	m += "\tgraph [bgcolor=white, fontname=Helvetica, fontsize=11];\n"
	m += "\tedge [fontname=Helvetica, fontcolor=grey, fontsize=9];\n"
	m += fmt.Sprintf(
		"\tnode [shape=doublecircle, fontname=Monospace, fontsize=11]; %s\n",
		strings.Join(objectIDs, " "),
	)
	m += fmt.Sprintf(
		"\tnode [shape=circle, fontname=Monospace, fontsize=11]; %s\n",
		strings.Join(mutationIDs, " "),
	)
	return fmt.Sprintf("digraph G {\n%s%s}", m, s)
}
<|endoftext|>


<commit_before>package commandline

import "strings"

func Parse(s string) *Command {
	ss := strings.Split(s, " ")
	if ss[0] == "" {
		return nil
	}
	return &Command{
		Name: ss[0],
		Args: ss[1:],
	}
}

type Command struct {
	Name string
	Args []string
}

type scanner struct {
	src []byte
}

func (s scanner) scan() {
}
<commit_msg>Add tokenType<commit_after>package commandline

import "strings"

func Parse(s string) *Command {
	ss := strings.Split(s, " ")
	if ss[0] == "" {
		return nil
	}
	return &Command{
		Name: ss[0],
		Args: ss[1:],
	}
}

type Command struct {
	Name string
	Args []string
}

type scanner struct {
	src []byte
}

func (s scanner) scan() {
}

type tokenType int
<|endoftext|>


<commit_before>package sse

import (
	"bufio"
	"bytes"
	"io"
)

const (
	defaultBufferSize = 8096
)

var (
	// DefaultDecoder is the decoder used by EventSource by default.
	DefaultDecoder = &decoder{defaultBufferSize}

	bytesLF    = []byte("\n")
	bytesCRLF  = []byte("\r\n")
	bytesSPACE = []byte(" ")
	bytesCOLON = []byte(":")
)

type (
	// Decoder interface decodes events from a reader input
	Decoder interface {
		Decode(in io.Reader) (out <-chan Event)
	}
	decoder struct {
		bufferSize int
	}
)

// NewDecoder builds an SSE decoder with the specified buffer size.
func NewDecoder(bufferSize int) Decoder {
	d := &decoder{}
	d.initialise(bufferSize)
	return d
}

func (d *decoder) initialise(bufferSize int) {
	d.bufferSize = bufferSize
}

// Returns a channel of SSE events from a reader input.
func (d *decoder) Decode(in io.Reader) <-chan Event {
	buffIn := bufio.NewReaderSize(in, d.bufferSize)
	out := make(chan Event)
	go process(buffIn, out)
	return out
}

// Processes a reader and sends the parsed SSE events
// to the output channel.
// This function is intended to run in a go-routine.
func process(in *bufio.Reader, out chan Event) {
	// Stores event data, which is filled after one or many lines from the reader
	var eventID, eventType, dataBuffer = new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)

	// Stores data about the current line being processed
	var field, value = new(bytes.Buffer), new(bytes.Buffer)

	for {
		line, err := in.ReadSlice('\n')
		if err != nil {
			close(out)
			return
		}

		// Dispatch event
		if bytes.Equal(line, bytesLF) || bytes.Equal(line, bytesCRLF) {
			// Skip event if Data buffer its empty
			if dataBuffer.Len() == 0 {
				dataBuffer.Reset()
				eventType.Reset()
				continue
			}

			data := dataBuffer.Bytes()

			// Trim last byte if line feed
			data = bytes.TrimSuffix(data, bytesLF)

			// Create event
			event := newEvent(eventID.String(), eventType.String(), data)

			// Clear event buffers
			eventType.Reset()
			dataBuffer.Reset()

			// Dispatch event
			out <- event
			continue
		}

		colonIndex := bytes.Index(line, bytesCOLON)

		// Sanitise line feeds
		line = sanitise(line)

		// Extract field/value for current line
		field.Reset()
		value.Reset()

		switch colonIndex {
		case 0:
			continue
		case -1:
			field.Write(line)
		default:
			field.Write(line[:colonIndex])
			line = line[colonIndex+1:]
			line = bytes.TrimPrefix(line, bytesSPACE)
			value.Write(line)
		}

		// Process field
		fieldName := field.String()
		switch fieldName {
		case "event":
			eventType.WriteString(value.String())
		case "data":
			dataBuffer.Write(value.Bytes())
			dataBuffer.WriteByte('\n')
		case "id":
			eventID.Reset()
			eventID.Write(value.Bytes())
		case "retry":
			// TODO(alevinval): unused at the moment, will need refactor
			// or change on the internal API, as decoder has no knowledge on the underlying connection.
		default:
			// Ignore field
		}
	}
}

// Sanitises line feed ending.
func sanitise(line []byte) []byte {
	if bytes.HasSuffix(line, bytesCRLF) {
		line = bytes.TrimSuffix(line, bytesCRLF)
	} else {
		line = bytes.TrimSuffix(line, bytesLF)
	}
	return line
}
<commit_msg>Avoid calling String() many times, just write the bytes, as String() will be called anyway when dispatching the event.<commit_after>package sse

import (
	"bufio"
	"bytes"
	"io"
)

const (
	defaultBufferSize = 8096
)

var (
	// DefaultDecoder is the decoder used by EventSource by default.
	DefaultDecoder = &decoder{defaultBufferSize}

	bytesLF    = []byte("\n")
	bytesCRLF  = []byte("\r\n")
	bytesSPACE = []byte(" ")
	bytesCOLON = []byte(":")
)

type (
	// Decoder interface decodes events from a reader input
	Decoder interface {
		Decode(in io.Reader) (out <-chan Event)
	}
	decoder struct {
		bufferSize int
	}
)

// NewDecoder builds an SSE decoder with the specified buffer size.
func NewDecoder(bufferSize int) Decoder {
	d := &decoder{}
	d.initialise(bufferSize)
	return d
}

func (d *decoder) initialise(bufferSize int) {
	d.bufferSize = bufferSize
}

// Returns a channel of SSE events from a reader input.
func (d *decoder) Decode(in io.Reader) <-chan Event {
	buffIn := bufio.NewReaderSize(in, d.bufferSize)
	out := make(chan Event)
	go process(buffIn, out)
	return out
}

// Processes a reader and sends the parsed SSE events
// to the output channel.
// This function is intended to run in a go-routine.
func process(in *bufio.Reader, out chan Event) {
	// Stores event data, which is filled after one or many lines from the reader
	var eventID, eventType, dataBuffer = new(bytes.Buffer), new(bytes.Buffer), new(bytes.Buffer)

	// Stores data about the current line being processed
	var field, value = new(bytes.Buffer), new(bytes.Buffer)

	for {
		line, err := in.ReadSlice('\n')
		if err != nil {
			close(out)
			return
		}

		// Dispatch event
		if bytes.Equal(line, bytesLF) || bytes.Equal(line, bytesCRLF) {
			// Skip event if Data buffer its empty
			if dataBuffer.Len() == 0 {
				dataBuffer.Reset()
				eventType.Reset()
				continue
			}

			data := dataBuffer.Bytes()

			// Trim last byte if line feed
			data = bytes.TrimSuffix(data, bytesLF)

			// Create event
			event := newEvent(eventID.String(), eventType.String(), data)

			// Clear event buffers
			eventType.Reset()
			dataBuffer.Reset()

			// Dispatch event
			out <- event
			continue
		}

		colonIndex := bytes.Index(line, bytesCOLON)

		// Sanitise line feeds
		line = sanitise(line)

		// Extract field/value for current line
		field.Reset()
		value.Reset()

		switch colonIndex {
		case 0:
			continue
		case -1:
			field.Write(line)
		default:
			field.Write(line[:colonIndex])
			line = line[colonIndex+1:]
			line = bytes.TrimPrefix(line, bytesSPACE)
			value.Write(line)
		}

		// Process field
		fieldName := field.String()
		switch fieldName {
		case "event":
			eventType.Write(value.Bytes())
		case "data":
			dataBuffer.Write(value.Bytes())
			dataBuffer.WriteByte('\n')
		case "id":
			eventID.Reset()
			eventID.Write(value.Bytes())
		case "retry":
			// TODO(alevinval): unused at the moment, will need refactor
			// or change on the internal API, as decoder has no knowledge on the underlying connection.
		default:
			// Ignore field
		}
	}
}

// Sanitises line feed ending.
func sanitise(line []byte) []byte {
	if bytes.HasSuffix(line, bytesCRLF) {
		line = bytes.TrimSuffix(line, bytesCRLF)
	} else {
		line = bytes.TrimSuffix(line, bytesLF)
	}
	return line
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math"
)

type PacketConfig struct {
	DataRate                    int
	BlockSize, BlockSize2       int
	SymbolLength, SymbolLength2 int
	SampleRate                  int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	BufferLength                   int
	Preamble                       string
}

func (cfg PacketConfig) Log() {
	log.Println("BlockSize:", cfg.BlockSize)
	log.Println("SampleRate:", cfg.SampleRate)
	log.Println("DataRate:", cfg.DataRate)
	log.Println("SymbolLength:", cfg.SymbolLength)
	log.Println("PreambleSymbols:", cfg.PreambleSymbols)
	log.Println("PreambleLength:", cfg.PreambleLength)
	log.Println("PacketSymbols:", cfg.PacketSymbols)
	log.Println("PacketLength:", cfg.PacketLength)
	log.Println("Preamble:", cfg.Preamble)
}

type Decoder struct {
	cfg PacketConfig

	iq        []byte
	signal    []float64
	quantized []byte

	lut MagLUT

	preamble []byte
	slices   [][]byte

	pkt []byte
}

func NewDecoder(cfg PacketConfig) (d Decoder) {
	d.cfg = cfg

	d.iq = make([]byte, d.cfg.BufferLength<<1)
	d.signal = make([]float64, d.cfg.BufferLength)
	d.quantized = make([]byte, d.cfg.BufferLength)

	d.lut = NewMagLUT()

	d.preamble = make([]byte, len(d.cfg.Preamble))
	for idx := range d.cfg.Preamble {
		if d.cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	d.slices = make([][]byte, d.cfg.SymbolLength2)
	flat := make([]byte, d.cfg.BlockSize2-(d.cfg.BlockSize2%d.cfg.SymbolLength2))

	for symbolOffset := range d.slices {
		lower := symbolOffset * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		upper := (symbolOffset + 1) * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		d.slices[symbolOffset] = flat[lower:upper]
	}

	d.pkt = make([]byte, d.cfg.PacketSymbols>>3)

	return
}

func (d Decoder) Decode(input []byte) (pkts [][]byte) {
	// Shift new block into buffers.
	copy(d.iq, d.iq[d.cfg.BlockSize<<1:])
	copy(d.signal, d.signal[d.cfg.BlockSize:])
	copy(d.quantized, d.quantized[d.cfg.BlockSize:])
	copy(d.iq[d.cfg.PacketLength<<1:], input[:])

	iqBlock := d.iq[d.cfg.PacketLength<<1:]
	signalBlock := d.signal[d.cfg.PacketLength:]
	d.lut.Execute(iqBlock, signalBlock)

	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	d.Filter(signalBlock)
	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	Quantize(signalBlock, d.quantized[d.cfg.PacketLength-d.cfg.SymbolLength2:])
	d.Pack(d.quantized[:d.cfg.BlockSize2], d.slices)

	indexes := d.Search(d.slices, d.preamble)

	seen := make(map[string]bool)

	for _, qIdx := range indexes {
		if qIdx > d.cfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.cfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.quantized[qIdx+(pIdx*d.cfg.SymbolLength2)]
		}

		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}
	return
}

type MagLUT []float64

func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = 127.4 - float64(idx)
		lut[idx] *= lut[idx]
	}
	return
}

func (lut MagLUT) Execute(input []byte, output []float64) {
	for idx := range output {
		lutIdx := idx << 1
		output[idx] = math.Sqrt(lut[input[lutIdx]] + lut[input[lutIdx+1]])
	}
}

func (d Decoder) Filter(input []float64) {
	csum := make([]float64, len(input)+1)

	var sum float64
	for idx, v := range input {
		sum += v
		csum[idx+1] = sum
	}

	lower := csum[d.cfg.SymbolLength:]
	upper := csum[d.cfg.SymbolLength2:]
	for idx := range input[:len(input)-d.cfg.SymbolLength2] {
		input[idx] = (lower[idx] - csum[idx]) - (upper[idx] - lower[idx])
	}

	return
}

func Quantize(input []float64, output []byte) {
	for idx, val := range input {
		output[idx] = byte(math.Float64bits(val)>>63) ^ 0x01
	}

	return
}

func (d Decoder) Pack(input []byte, slices [][]byte) {
	for symbolOffset, slice := range slices {
		for symbolIdx := range slice {
			slice[symbolIdx] = input[symbolIdx*d.cfg.SymbolLength2+symbolOffset]
		}
	}

	return
}

func (d Decoder) Search(slices [][]byte, preamble []byte) (indexes []int) {
	for symbolOffset, slice := range slices {
		for symbolIdx := range slice[:len(slice)-len(preamble)] {
			found := true
			for bitIdx := range preamble {
				found = found && preamble[bitIdx] == slice[symbolIdx+bitIdx]
				if !found {
					break
				}
			}
			if found {
				indexes = append(indexes, symbolIdx*d.cfg.SymbolLength2+symbolOffset)
			}
		}
	}

	return
}

func NextPowerOf2(v int) int {
	return 1 << uint(math.Ceil(math.Log2(float64(v))))
}
<commit_msg>Optimize Search method ~x1.4 faster.<commit_after>package main

import (
	"fmt"
	"log"
	"math"
)

type PacketConfig struct {
	DataRate                    int
	BlockSize, BlockSize2       int
	SymbolLength, SymbolLength2 int
	SampleRate                  int

	PreambleSymbols, PacketSymbols int
	PreambleLength, PacketLength   int
	BufferLength                   int
	Preamble                       string
}

func (cfg PacketConfig) Log() {
	log.Println("BlockSize:", cfg.BlockSize)
	log.Println("SampleRate:", cfg.SampleRate)
	log.Println("DataRate:", cfg.DataRate)
	log.Println("SymbolLength:", cfg.SymbolLength)
	log.Println("PreambleSymbols:", cfg.PreambleSymbols)
	log.Println("PreambleLength:", cfg.PreambleLength)
	log.Println("PacketSymbols:", cfg.PacketSymbols)
	log.Println("PacketLength:", cfg.PacketLength)
	log.Println("Preamble:", cfg.Preamble)
}

type Decoder struct {
	cfg PacketConfig

	iq        []byte
	signal    []float64
	quantized []byte

	lut MagLUT

	preamble []byte
	slices   [][]byte

	pkt []byte
}

func NewDecoder(cfg PacketConfig) (d Decoder) {
	d.cfg = cfg

	d.iq = make([]byte, d.cfg.BufferLength<<1)
	d.signal = make([]float64, d.cfg.BufferLength)
	d.quantized = make([]byte, d.cfg.BufferLength)

	d.lut = NewMagLUT()

	d.preamble = make([]byte, len(d.cfg.Preamble))
	for idx := range d.cfg.Preamble {
		if d.cfg.Preamble[idx] == '1' {
			d.preamble[idx] = 1
		}
	}

	d.slices = make([][]byte, d.cfg.SymbolLength2)
	flat := make([]byte, d.cfg.BlockSize2-(d.cfg.BlockSize2%d.cfg.SymbolLength2))

	for symbolOffset := range d.slices {
		lower := symbolOffset * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		upper := (symbolOffset + 1) * (d.cfg.BlockSize2 / d.cfg.SymbolLength2)
		d.slices[symbolOffset] = flat[lower:upper]
	}

	d.pkt = make([]byte, d.cfg.PacketSymbols>>3)

	return
}

func (d Decoder) Decode(input []byte) (pkts [][]byte) {
	// Shift new block into buffers.
	copy(d.iq, d.iq[d.cfg.BlockSize<<1:])
	copy(d.signal, d.signal[d.cfg.BlockSize:])
	copy(d.quantized, d.quantized[d.cfg.BlockSize:])
	copy(d.iq[d.cfg.PacketLength<<1:], input[:])

	iqBlock := d.iq[d.cfg.PacketLength<<1:]
	signalBlock := d.signal[d.cfg.PacketLength:]
	d.lut.Execute(iqBlock, signalBlock)

	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	d.Filter(signalBlock)
	signalBlock = d.signal[d.cfg.PacketLength-d.cfg.SymbolLength2:]
	Quantize(signalBlock, d.quantized[d.cfg.PacketLength-d.cfg.SymbolLength2:])
	d.Pack(d.quantized[:d.cfg.BlockSize2], d.slices)

	indexes := d.Search(d.slices, d.preamble)

	seen := make(map[string]bool)

	for _, qIdx := range indexes {
		if qIdx > d.cfg.BlockSize {
			continue
		}

		// Packet is 1 bit per byte, pack to 8-bits per byte.
		for pIdx := 0; pIdx < d.cfg.PacketSymbols; pIdx++ {
			d.pkt[pIdx>>3] <<= 1
			d.pkt[pIdx>>3] |= d.quantized[qIdx+(pIdx*d.cfg.SymbolLength2)]
		}

		pktStr := fmt.Sprintf("%02X", d.pkt)
		if !seen[pktStr] {
			seen[pktStr] = true
			pkts = append(pkts, make([]byte, len(d.pkt)))
			copy(pkts[len(pkts)-1], d.pkt)
		}
	}
	return
}

type MagLUT []float64

func NewMagLUT() (lut MagLUT) {
	lut = make([]float64, 0x100)
	for idx := range lut {
		lut[idx] = 127.4 - float64(idx)
		lut[idx] *= lut[idx]
	}
	return
}

func (lut MagLUT) Execute(input []byte, output []float64) {
	for idx := range output {
		lutIdx := idx << 1
		output[idx] = math.Sqrt(lut[input[lutIdx]] + lut[input[lutIdx+1]])
	}
}

func (d Decoder) Filter(input []float64) {
	csum := make([]float64, len(input)+1)

	var sum float64
	for idx, v := range input {
		sum += v
		csum[idx+1] = sum
	}

	lower := csum[d.cfg.SymbolLength:]
	upper := csum[d.cfg.SymbolLength2:]
	for idx := range input[:len(input)-d.cfg.SymbolLength2] {
		input[idx] = (lower[idx] - csum[idx]) - (upper[idx] - lower[idx])
	}

	return
}

func Quantize(input []float64, output []byte) {
	for idx, val := range input {
		output[idx] = byte(math.Float64bits(val)>>63) ^ 0x01
	}

	return
}

func (d Decoder) Pack(input []byte, slices [][]byte) {
	for symbolOffset, slice := range slices {
		for symbolIdx := range slice {
			slice[symbolIdx] = input[symbolIdx*d.cfg.SymbolLength2+symbolOffset]
		}
	}

	return
}

func (d Decoder) Search(slices [][]byte, preamble []byte) (indexes []int) {
	for symbolOffset, slice := range slices {
		for symbolIdx := range slice[:len(slice)-len(preamble)] {
			var result uint8
			for bitIdx, bit := range preamble {
				result |= bit ^ slice[symbolIdx+bitIdx]
				if result != 0 {
					break
				}
			}
			if result == 0 {
				indexes = append(indexes, symbolIdx*d.cfg.SymbolLength2+symbolOffset)
			}
		}
	}

	return
}

func NextPowerOf2(v int) int {
	return 1 << uint(math.Ceil(math.Log2(float64(v))))
}
<|endoftext|>