<commit_before>package command

import (
	"flag"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/funkygao/fae/config"
	"github.com/funkygao/fae/servant/mysql"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jm "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/manager/dummy"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/ryanuber/columnize"
)

type Job struct {
	Ui  cli.Ui
	Cmd string

	mc     *mysql.MysqlCluster
	zkzone *zk.ZkZone
	due    int
}

func (this *Job) Run(args []string) (exitCode int) {
	var (
		zone  string
		appid string
	)
	cmdFlags := flag.NewFlagSet("job", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.DefaultZone(), "")
	cmdFlags.StringVar(&appid, "app", "", "")
	cmdFlags.IntVar(&this.due, "d", 0, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if appid != "" {
		this.displayAppJobs(appid)
		return
	}

	this.printResourcesAndActors()

	return
}

// TODO diagnose all app's jobs status
func (this *Job) displayAppJobs(appid string) {
	aid := this.connectMysqlCluster(appid)
	lines := make([]string, 0)
	header := "Topic|Type|JobId|Due|Payload"
	lines = append(lines, header)

	// FIXME does not respect appid, show all now
	this.forSortedJobQueues(func(topic string) {
		table := jm.JobTable(topic)
		archiveTable := jm.HistoryTable(topic)

		sqlRealTime := fmt.Sprintf("SELECT job_id,app_id,payload,due_time FROM %s ORDER BY due_time DESC", table)
		if this.due > 0 {
			sqlRealTime = fmt.Sprintf("SELECT job_id,app_id,payload,due_time FROM %s WHERE due_time<=? ORDER BY due_time DESC",
				table, time.Now().Unix()+int64(this.due))
		}
		rows, err := this.mc.Query(jm.AppPool, table, aid, sqlRealTime)
		swallow(err)

		var item job.JobItem
		for rows.Next() {
			err = rows.Scan(&item.JobId, &item.AppId, &item.Payload, &item.DueTime)
			swallow(err)
			lines = append(lines, fmt.Sprintf("%s|RT|%d|%d|%s", topic, item.JobId, item.DueTime, item.PayloadString(50)))
		}
		rows.Close()

		sqlArchive := fmt.Sprintf("SELECT job_id,app_id,payload,due_time FROM %s ORDER BY due_time ASC LIMIT 100", archiveTable)
		rows, err = this.mc.Query(jm.AppPool, table, aid, sqlArchive)
		swallow(err)

		for rows.Next() {
			err = rows.Scan(&item.JobId, &item.AppId, &item.Payload, &item.DueTime)
			swallow(err)
			lines = append(lines, fmt.Sprintf("%s|AR|%d|%d|%s", topic, item.JobId, item.DueTime, item.PayloadString(50)))
		}
		rows.Close()
	})

	if len(lines) > 1 {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Job) forSortedJobQueues(f func(jobQueue string)) {
	jobQueues := this.zkzone.ChildrenWithData(zk.PubsubJobQueues)
	sortedName := make([]string, 0, len(jobQueues))
	for name := range jobQueues {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, name := range sortedName {
		f(name)
	}
}

func (this *Job) printResourcesAndActors() {
	lines := make([]string, 0)
	header := "Topic|Cluster|Ctime|Mtime"
	lines = append(lines, header)
	// jobs
	jobQueues := this.zkzone.ChildrenWithData(zk.PubsubJobQueues)
	sortedName := make([]string, 0, len(jobQueues))
	for name := range jobQueues {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	manager.Default = dummy.New("")
	for _, topic := range sortedName {
		zdata := jobQueues[topic]
		if appid := manager.Default.TopicAppid(topic); appid == "" {
			lines = append(lines, fmt.Sprintf("?%s|%s|%s|%s", topic,
				string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic,
				string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
		}

	}
	if len(lines) > 1 {
		this.Ui.Info("Jobs")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// webhooks
	lines = lines[:0]
	header = "Topic|Endpoints|Ctime|Mtime"
	lines = append(lines, header)
	webhoooks := this.zkzone.ChildrenWithData(zk.PubsubWebhooks)
	sortedName = sortedName[:0]
	for name := range webhoooks {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, topic := range sortedName {
		zdata := webhoooks[topic]
		var hook zk.WebhookMeta
		hook.From(zdata.Data())
		if appid := manager.Default.TopicAppid(topic); appid == "" {
			lines = append(lines, fmt.Sprintf("?%s|%+v|%s|%s", topic, hook.Endpoints,
				zdata.Ctime(), zdata.Mtime()))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%+v|%s|%s", topic, hook.Endpoints,
				zdata.Ctime(), zdata.Mtime()))
		}
	}
	if len(lines) > 1 {
		this.Ui.Info("Webhooks")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// actors
	lines = lines[:0]
	header = "Actor|Ctime|Mtime"
	lines = append(lines, header)
	actors := this.zkzone.ChildrenWithData(zk.PubsubActors)
	sortedName = sortedName[:0]
	for name := range actors {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, name := range sortedName {
		zdata := actors[name]
		lines = append(lines, fmt.Sprintf("%s|%s|%s", name,
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Actors")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// job <-> actor
	lines = lines[:0]
	header = "Topic|Actor|Ctime|Mtime"
	lines = append(lines, header)
	jobQueueOwners := this.zkzone.ChildrenWithData(zk.PubsubJobQueueOwners)
	sortedName = sortedName[:0]
	for name := range jobQueueOwners {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, jobQueue := range sortedName {
		zdata := jobQueueOwners[jobQueue]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", jobQueue, string(zdata.Data()),
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Job<->Actor")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// webhook <-> actor
	lines = lines[:0]
	header = "Topic|Actor|Ctime|Mtime"
	lines = append(lines, header)
	webhookOwners := this.zkzone.ChildrenWithData(zk.PubsubWebhookOwners)
	sortedName = sortedName[:0]
	for name := range webhookOwners {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, topic := range sortedName {
		zdata := webhookOwners[topic]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic, string(zdata.Data()),
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Webhook<->Actor")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Job) connectMysqlCluster(appid string) int {
	b, err := this.zkzone.KatewayJobClusterConfig()
	if err != nil {
		panic(err)
	}
	var mcc = &config.ConfigMysql{}
	if err = mcc.From(b); err != nil {
		panic(err)
	}

	this.mc = mysql.New(mcc)
	return jm.App_id(appid)
}

func (*Job) Synopsis() string {
	return "Display job/actor related znodes for PubSub system."
}

func (this *Job) Help() string {
	help := fmt.Sprintf(`
Usage: %s job [options]

    %s

Options:

    -z zone

    -c cluster

    -app <app id>
      List app's real-time and archive job table contents.

    -d <due time in seconds>
      List jobs due from now within how many seconds.

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>fix compile err: appid was removed from job table<commit_after>package command

import (
	"flag"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/funkygao/fae/config"
	"github.com/funkygao/fae/servant/mysql"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jm "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/manager/dummy"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/ryanuber/columnize"
)

type Job struct {
	Ui  cli.Ui
	Cmd string

	mc     *mysql.MysqlCluster
	zkzone *zk.ZkZone
	due    int
}

func (this *Job) Run(args []string) (exitCode int) {
	var (
		zone  string
		appid string
	)
	cmdFlags := flag.NewFlagSet("job", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.DefaultZone(), "")
	cmdFlags.StringVar(&appid, "app", "", "")
	cmdFlags.IntVar(&this.due, "d", 0, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	this.zkzone = zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if appid != "" {
		this.displayAppJobs(appid)
		return
	}

	this.printResourcesAndActors()

	return
}

// TODO diagnose all app's jobs status
func (this *Job) displayAppJobs(appid string) {
	aid := this.connectMysqlCluster(appid)
	lines := make([]string, 0)
	header := "Topic|Type|JobId|Due|Payload"
	lines = append(lines, header)

	// FIXME does not respect appid, show all now
	this.forSortedJobQueues(func(topic string) {
		table := jm.JobTable(topic)
		archiveTable := jm.HistoryTable(topic)

		sqlRealTime := fmt.Sprintf("SELECT job_id,payload,due_time FROM %s ORDER BY due_time DESC", table)
		if this.due > 0 {
			sqlRealTime = fmt.Sprintf("SELECT job_id,payload,due_time FROM %s WHERE due_time<=? ORDER BY due_time DESC",
				table, time.Now().Unix()+int64(this.due))
		}
		rows, err := this.mc.Query(jm.AppPool, table, aid, sqlRealTime)
		swallow(err)

		var item job.JobItem
		for rows.Next() {
			err = rows.Scan(&item.JobId, &item.Payload, &item.DueTime)
			swallow(err)
			lines = append(lines, fmt.Sprintf("%s|RT|%d|%d|%s", topic, item.JobId, item.DueTime, item.PayloadString(50)))
		}
		rows.Close()

		sqlArchive := fmt.Sprintf("SELECT job_id,payload,due_time FROM %s ORDER BY due_time ASC LIMIT 100", archiveTable)
		rows, err = this.mc.Query(jm.AppPool, table, aid, sqlArchive)
		swallow(err)

		for rows.Next() {
			err = rows.Scan(&item.JobId, &item.Payload, &item.DueTime)
			swallow(err)
			lines = append(lines, fmt.Sprintf("%s|AR|%d|%d|%s", topic, item.JobId, item.DueTime, item.PayloadString(50)))
		}
		rows.Close()
	})

	if len(lines) > 1 {
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Job) forSortedJobQueues(f func(jobQueue string)) {
	jobQueues := this.zkzone.ChildrenWithData(zk.PubsubJobQueues)
	sortedName := make([]string, 0, len(jobQueues))
	for name := range jobQueues {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, name := range sortedName {
		f(name)
	}
}

func (this *Job) printResourcesAndActors() {
	lines := make([]string, 0)
	header := "Topic|Cluster|Ctime|Mtime"
	lines = append(lines, header)
	// jobs
	jobQueues := this.zkzone.ChildrenWithData(zk.PubsubJobQueues)
	sortedName := make([]string, 0, len(jobQueues))
	for name := range jobQueues {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	manager.Default = dummy.New("")
	for _, topic := range sortedName {
		zdata := jobQueues[topic]
		if appid := manager.Default.TopicAppid(topic); appid == "" {
			lines = append(lines, fmt.Sprintf("?%s|%s|%s|%s", topic,
				string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic,
				string(zdata.Data()), zdata.Ctime(), zdata.Mtime()))
		}

	}
	if len(lines) > 1 {
		this.Ui.Info("Jobs")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// webhooks
	lines = lines[:0]
	header = "Topic|Endpoints|Ctime|Mtime"
	lines = append(lines, header)
	webhoooks := this.zkzone.ChildrenWithData(zk.PubsubWebhooks)
	sortedName = sortedName[:0]
	for name := range webhoooks {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, topic := range sortedName {
		zdata := webhoooks[topic]
		var hook zk.WebhookMeta
		hook.From(zdata.Data())
		if appid := manager.Default.TopicAppid(topic); appid == "" {
			lines = append(lines, fmt.Sprintf("?%s|%+v|%s|%s", topic, hook.Endpoints,
				zdata.Ctime(), zdata.Mtime()))
		} else {
			lines = append(lines, fmt.Sprintf("%s|%+v|%s|%s", topic, hook.Endpoints,
				zdata.Ctime(), zdata.Mtime()))
		}
	}
	if len(lines) > 1 {
		this.Ui.Info("Webhooks")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// actors
	lines = lines[:0]
	header = "Actor|Ctime|Mtime"
	lines = append(lines, header)
	actors := this.zkzone.ChildrenWithData(zk.PubsubActors)
	sortedName = sortedName[:0]
	for name := range actors {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, name := range sortedName {
		zdata := actors[name]
		lines = append(lines, fmt.Sprintf("%s|%s|%s", name,
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Actors")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// job <-> actor
	lines = lines[:0]
	header = "Topic|Actor|Ctime|Mtime"
	lines = append(lines, header)
	jobQueueOwners := this.zkzone.ChildrenWithData(zk.PubsubJobQueueOwners)
	sortedName = sortedName[:0]
	for name := range jobQueueOwners {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, jobQueue := range sortedName {
		zdata := jobQueueOwners[jobQueue]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", jobQueue, string(zdata.Data()),
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Job<->Actor")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}

	// webhook <-> actor
	lines = lines[:0]
	header = "Topic|Actor|Ctime|Mtime"
	lines = append(lines, header)
	webhookOwners := this.zkzone.ChildrenWithData(zk.PubsubWebhookOwners)
	sortedName = sortedName[:0]
	for name := range webhookOwners {
		sortedName = append(sortedName, name)
	}
	sort.Strings(sortedName)

	for _, topic := range sortedName {
		zdata := webhookOwners[topic]
		lines = append(lines, fmt.Sprintf("%s|%s|%s|%s", topic, string(zdata.Data()),
			zdata.Ctime(), zdata.Mtime()))
	}
	if len(lines) > 1 {
		this.Ui.Info("Webhook<->Actor")
		this.Ui.Output(columnize.SimpleFormat(lines))
	}
}

func (this *Job) connectMysqlCluster(appid string) int {
	b, err := this.zkzone.KatewayJobClusterConfig()
	if err != nil {
		panic(err)
	}
	var mcc = &config.ConfigMysql{}
	if err = mcc.From(b); err != nil {
		panic(err)
	}

	this.mc = mysql.New(mcc)
	return jm.App_id(appid)
}

func (*Job) Synopsis() string {
	return "Display job/actor related znodes for PubSub system."
}

func (this *Job) Help() string {
	help := fmt.Sprintf(`
Usage: %s job [options]

    %s

Options:

    -z zone

    -c cluster

    -app <app id>
      List app's real-time and archive job table contents.

    -d <due time in seconds>
      List jobs due from now within how many seconds.

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>// Copyright 2013 Beego Authors
// Copyright 2014 The Macaron Authors
// Copyright 2015 The Tango Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package cache is a middleware that provides the cache management of Tango.
package cache

import (
	"fmt"
	"github.com/lunny/tango"
)

// Adapter is the interface that operates the cache data.
type Adapter interface {
	// Put puts value into cache with key and expire time.
	Put(key string, val interface{}, timeout int64) error
	// Get gets cached value by given key.
	Get(key string) interface{}
	// Delete deletes cached value by given key.
	Delete(key string) error
	// Incr increases cached int-type value by given key as a counter.
	Incr(key string) error
	// Decr decreases cached int-type value by given key as a counter.
	Decr(key string) error
	// IsExist returns true if cached value exists.
	IsExist(key string) bool
	// Flush deletes all cached data.
	Flush() error
	// StartAndGC starts GC routine based on config string settings.
	StartAndGC(opt Options) error
}

// Options represents a struct for specifying configuration options for the cache middleware.
type Options struct {
	// Name of adapter. Default is "memory".
	Adapter string
	// Adapter configuration, it's corresponding to adapter.
	AdapterConfig string
	// GC interval time in seconds. Default is 60.
	Interval int
	// Occupy entire database. Default is false.
	OccupyMode bool
	// Configuration section name. Default is "cache".
	Section string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}
	if len(opt.Section) == 0 {
		opt.Section = "cache"
	}
	if len(opt.Adapter) == 0 {
		opt.Adapter = "memory"
	}
	if opt.Interval == 0 {
		opt.Interval = 60
	}
	if len(opt.AdapterConfig) == 0 {
		opt.AdapterConfig = "data/caches"
	}
	return opt
}

// NewAdapter creates and returns a new cacheAdapter by given adapter name and configuration.
// It panics when given adapter isn't registered and starts GC automatically.
func NewAdapter(name string, opt Options) (Adapter, error) {
	adapter, ok := adapters[name]
	if !ok {
		return nil, fmt.Errorf("cache: unknown adapter '%s'(forgot to import?)", name)
	}
	return adapter, adapter.StartAndGC(opt)
}

var adapters = make(map[string]Adapter)

// Register registers a adapter.
func Register(name string, adapter Adapter) {
	if adapter == nil {
		panic("cache: cannot register adapter with nil value")
	}
	if _, dup := adapters[name]; dup {
		panic(fmt.Errorf("cache: cannot register adapter '%s' twice", name))
	}
	adapters[name] = adapter
}

// Cacher provides tango action handler to get proper handler
type Cacher interface {
	SetCache(*Caches)
}

// Cache maintains cache adapter and tango handler interface
type Cache struct {
	adapter Adapter
	options Options
}

// Put puts value into cache with key and expire time.
func (c *Cache) Put(key string, val interface{}, timeout int64) error {
	return c.adapter.Put(key, val, timeout)
}

// Get gets cached value by given key.
func (c *Cache) Get(key string) interface{} {
	return c.adapter.Get(key)
}

// Delete deletes cached value by given key.
func (c *Cache) Delete(key string) error {
	return c.adapter.Delete(key)
}

// Incr increases cached int-type value by given key as a counter.
func (c *Cache) Incr(key string) error {
	return c.adapter.Incr(key)
}

// Decr decreases cached int-type value by given key as a counter.
func (c *Cache) Decr(key string) error {
	return c.adapter.Decr(key)
}

// IsExist returns true if cached value exists.
func (c *Cache) IsExist(key string) bool {
	return c.adapter.IsExist(key)
}

// Flush deletes all cached data.
func (c *Cache) Flush() error {
	return c.adapter.Flush()
}

// Options return cache option.
func (c *Cache) Option() Options {
	return c.options
}

// set Caches
func (c *Cache) SetCache(cs *Caches) {
	c.adapter = cs.adapter
	c.options = cs.Options
}

// Caches
type Caches struct {
	Options
	adapter Adapter
}

// Handle implement tango.Handle
func (c *Caches) Handle(ctx *tango.Context) {
	if action := ctx.Action(); ctx != nil {
		if s, ok := action.(Cacher); ok {
			s.SetCache(c)
		}
	}

	ctx.Next()
}

// New is a middleware that maps a cache.Cache service into the tango handler chain.
// An single variadic cache.Options struct can be optionally provided to configure.
func New(options ...Options) *Caches {
	opt := prepareOptions(options)
	adapter, err := NewAdapter(opt.Adapter, opt)
	if err != nil {
		panic(err)
	}
	return &Caches{
		Options: opt,
		adapter: adapter,
	}
}
<commit_msg>refactor interface<commit_after>// Copyright 2013 Beego Authors
// Copyright 2014 The Macaron Authors
// Copyright 2015 The Tango Authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package cache is a middleware that provides the cache management of Tango.
package cache

import (
	"fmt"

	"github.com/lunny/tango"
)

// Adapter is the interface that operates the cache data.
type Adapter interface {
	// Put puts value into cache with key and expire time.
	Put(key string, val interface{}, timeout int64) error
	// Get gets cached value by given key.
	Get(key string) interface{}
	// Delete deletes cached value by given key.
	Delete(key string) error
	// Incr increases cached int-type value by given key as a counter.
	Incr(key string) error
	// Decr decreases cached int-type value by given key as a counter.
	Decr(key string) error
	// IsExist returns true if cached value exists.
	IsExist(key string) bool
	// Flush deletes all cached data.
	Flush() error
	// StartAndGC starts GC routine based on config string settings.
	StartAndGC(opt Options) error
}

// Options represents a struct for specifying configuration options for the cache middleware.
type Options struct {
	// Name of adapter. Default is "memory".
	Adapter string
	// Adapter configuration, it's corresponding to adapter.
	AdapterConfig string
	// GC interval time in seconds. Default is 60.
	Interval int
	// Occupy entire database. Default is false.
	OccupyMode bool
	// Configuration section name. Default is "cache".
	Section string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}
	if len(opt.Section) == 0 {
		opt.Section = "cache"
	}
	if len(opt.Adapter) == 0 {
		opt.Adapter = "memory"
	}
	if opt.Interval == 0 {
		opt.Interval = 60
	}
	if len(opt.AdapterConfig) == 0 {
		opt.AdapterConfig = "data/caches"
	}
	return opt
}

// NewAdapter creates and returns a new cacheAdapter by given adapter name and configuration.
// It panics when given adapter isn't registered and starts GC automatically.
func NewAdapter(name string, opt Options) (Adapter, error) {
	adapter, ok := adapters[name]
	if !ok {
		return nil, fmt.Errorf("cache: unknown adapter '%s'(forgot to import?)", name)
	}
	return adapter, adapter.StartAndGC(opt)
}

var adapters = make(map[string]Adapter)

// Register registers a adapter.
func Register(name string, adapter Adapter) {
	if adapter == nil {
		panic("cache: cannot register adapter with nil value")
	}
	if _, dup := adapters[name]; dup {
		panic(fmt.Errorf("cache: cannot register adapter '%s' twice", name))
	}
	adapters[name] = adapter
}

// Cacher provides tango action handler to get proper handler
type Cacher interface {
	SetCaches(*Caches)
}

// Caches
type Caches struct {
	options Options
	adapter Adapter
}

// Cache maintains cache adapter and tango handler interface
type Cache struct {
	*Caches
}

var _ Cacher = new(Cache)

// set Caches
func (c *Cache) SetCaches(cs *Caches) {
	c.Caches = cs
}

// Put puts value into cache with key and expire time.
func (c *Caches) Put(key string, val interface{}, timeout int64) error {
	return c.adapter.Put(key, val, timeout)
}

// Get gets cached value by given key.
func (c *Caches) Get(key string) interface{} {
	return c.adapter.Get(key)
}

// Delete deletes cached value by given key.
func (c *Caches) Delete(key string) error {
	return c.adapter.Delete(key)
}

// Incr increases cached int-type value by given key as a counter.
func (c *Caches) Incr(key string) error {
	return c.adapter.Incr(key)
}

// Decr decreases cached int-type value by given key as a counter.
func (c *Caches) Decr(key string) error {
	return c.adapter.Decr(key)
}

// IsExist returns true if cached value exists.
func (c *Caches) IsExist(key string) bool {
	return c.adapter.IsExist(key)
}

// Flush deletes all cached data.
func (c *Caches) Flush() error {
	return c.adapter.Flush()
}

// Options return cache option.
func (c *Caches) Option() Options {
	return c.options
}

// Handle implement tango.Handle
func (c *Caches) Handle(ctx *tango.Context) {
	if action := ctx.Action(); ctx != nil {
		if s, ok := action.(Cacher); ok {
			s.SetCaches(c)
		}
	}

	ctx.Next()
}

// New is a middleware that maps a cache.Cache service into the tango handler chain.
// An single variadic cache.Options struct can be optionally provided to configure.
func New(options ...Options) *Caches {
	opt := prepareOptions(options)
	adapter, err := NewAdapter(opt.Adapter, opt)
	if err != nil {
		panic(err)
	}
	return &Caches{
		options: opt,
		adapter: adapter,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bitbucket.org/anacrolix/go.torrent/util"
	"bitbucket.org/anacrolix/go.torrent/util/dirwatch"
	"flag"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"syscall"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"bitbucket.org/anacrolix/go.torrent"
	"bitbucket.org/anacrolix/go.torrent/fs"
)

var (
	downloadDir     string
	torrentPath     string
	mountDir        string
	disableTrackers = flag.Bool("disableTrackers", false, "disables trackers")
	testPeer        = flag.String("testPeer", "", "the address for a test peer")
	httpAddr        = flag.String("httpAddr", "localhost:0", "HTTP server bind address")
	readaheadBytes  = flag.Int("readaheadBytes", 10*1024*1024, "bytes to readahead in each torrent from the last read piece")
	testPeerAddr    *net.TCPAddr
)

func init() {
	flag.StringVar(&downloadDir, "downloadDir", "", "location to save torrent data")
	flag.StringVar(&torrentPath, "torrentPath", func() string {
		_user, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}
		return filepath.Join(_user.HomeDir, ".config/transmission/torrents")
	}(), "torrent files in this location describe the contents of the mounted filesystem")
	flag.StringVar(&mountDir, "mountDir", "", "location the torrent contents are made available")
}

func resolveTestPeerAddr() {
	if *testPeer == "" {
		return
	}
	var err error
	testPeerAddr, err = net.ResolveTCPAddr("tcp4", *testPeer)
	if err != nil {
		log.Fatal(err)
	}
}

func setSignalHandlers() {
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		for {
			<-c
			err := fuse.Unmount(mountDir)
			if err != nil {
				log.Print(err)
			}
		}
	}()
}

func addTestPeer(client *torrent.Client) {
	for _, t := range client.Torrents() {
		if testPeerAddr != nil {
			if err := client.AddPeers(t.InfoHash, []torrent.Peer{{
				IP:   testPeerAddr.IP,
				Port: testPeerAddr.Port,
			}}); err != nil {
				log.Print(err)
			}
		}
	}
}

func main() {
	flag.Parse()
	if flag.NArg() != 0 {
		os.Stderr.WriteString("one does not simply pass positional args\n")
		os.Exit(2)
	}
	if mountDir == "" {
		os.Stderr.WriteString("y u no specify mountpoint?\n")
		os.Exit(2)
	}
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	if *httpAddr != "" {
		util.LoggedHTTPServe(*httpAddr)
	}
	conn, err := fuse.Mount(mountDir)
	if err != nil {
		log.Fatal(err)
	}
	defer fuse.Unmount(mountDir)
	// TODO: Think about the ramifications of exiting not due to a signal.
	setSignalHandlers()
	defer conn.Close()
	client := &torrent.Client{
		DataDir:          downloadDir,
		DisableTrackers:  *disableTrackers,
		DownloadStrategy: &torrent.ResponsiveDownloadStrategy{*readaheadBytes},
	}
	http.DefaultServeMux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	client.Start()
	dw, err := dirwatch.New(torrentPath)
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		for ev := range dw.Events {
			switch ev.Change {
			case dirwatch.Added:
				if ev.TorrentFilePath != "" {
					err := client.AddTorrentFromFile(ev.TorrentFilePath)
					if err != nil {
						log.Printf("error adding torrent to client: %s", err)
					}
				} else if ev.Magnet != "" {
					err := client.AddMagnet(ev.Magnet)
					if err != nil {
						log.Printf("error adding magnet: %s", err)
					}
				}
			case dirwatch.Removed:
				err := client.DropTorrent(ev.InfoHash)
				if err != nil {
					log.Printf("error dropping torrent: %s", err)
				}
			}
		}
	}()
	resolveTestPeerAddr()
	fs := torrentfs.New(client)
	go func() {
		for {
			addTestPeer(client)
			time.Sleep(10 * time.Second)
		}
	}()

	if err := fusefs.Serve(conn, fs); err != nil {
		log.Fatal(err)
	}
	<-conn.Ready
	if err := conn.MountError; err != nil {
		log.Fatal(err)
	}
}
<commit_msg>cmd/torrentfs: Missing listener on client<commit_after>package main

import (
	"bitbucket.org/anacrolix/go.torrent/util"
	"bitbucket.org/anacrolix/go.torrent/util/dirwatch"
	"flag"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"os/user"
	"path/filepath"
	"syscall"
	"time"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
	"bitbucket.org/anacrolix/go.torrent"
	"bitbucket.org/anacrolix/go.torrent/fs"
)

var (
	downloadDir     string
	torrentPath     string
	mountDir        string
	disableTrackers = flag.Bool("disableTrackers", false, "disables trackers")
	testPeer        = flag.String("testPeer", "", "the address for a test peer")
	httpAddr        = flag.String("httpAddr", "localhost:0", "HTTP server bind address")
	readaheadBytes  = flag.Int("readaheadBytes", 10*1024*1024, "bytes to readahead in each torrent from the last read piece")
	testPeerAddr    *net.TCPAddr
	listenAddr      = flag.String("listenAddr", ":6882", "incoming connection address")
)

func init() {
	flag.StringVar(&downloadDir, "downloadDir", "", "location to save torrent data")
	flag.StringVar(&torrentPath, "torrentPath", func() string {
		_user, err := user.Current()
		if err != nil {
			log.Fatal(err)
		}
		return filepath.Join(_user.HomeDir, ".config/transmission/torrents")
	}(), "torrent files in this location describe the contents of the mounted filesystem")
	flag.StringVar(&mountDir, "mountDir", "", "location the torrent contents are made available")
}

func makeListener() net.Listener {
	l, err := net.Listen("tcp", *listenAddr)
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func resolveTestPeerAddr() {
	if *testPeer == "" {
		return
	}
	var err error
	testPeerAddr, err = net.ResolveTCPAddr("tcp4", *testPeer)
	if err != nil {
		log.Fatal(err)
	}
}

func setSignalHandlers() {
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	go func() {
		for {
			<-c
			err := fuse.Unmount(mountDir)
			if err != nil {
				log.Print(err)
			}
		}
	}()
}

func addTestPeer(client *torrent.Client) {
	for _, t := range client.Torrents() {
		if testPeerAddr != nil {
			if err := client.AddPeers(t.InfoHash, []torrent.Peer{{
				IP:   testPeerAddr.IP,
				Port: testPeerAddr.Port,
			}}); err != nil {
				log.Print(err)
			}
		}
	}
}

func main() {
	flag.Parse()
	if flag.NArg() != 0 {
		os.Stderr.WriteString("one does not simply pass positional args\n")
		os.Exit(2)
	}
	if mountDir == "" {
		os.Stderr.WriteString("y u no specify mountpoint?\n")
		os.Exit(2)
	}
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	if *httpAddr != "" {
		util.LoggedHTTPServe(*httpAddr)
	}
	conn, err := fuse.Mount(mountDir)
	if err != nil {
		log.Fatal(err)
	}
	defer fuse.Unmount(mountDir)
	// TODO: Think about the ramifications of exiting not due to a signal.
	setSignalHandlers()
	defer conn.Close()
	client := &torrent.Client{
		DataDir:          downloadDir,
		DisableTrackers:  *disableTrackers,
		DownloadStrategy: torrent.NewResponsiveDownloadStrategy(*readaheadBytes),
		Listener:         makeListener(),
	}
	http.DefaultServeMux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		client.WriteStatus(w)
	})
	client.Start()
	dw, err := dirwatch.New(torrentPath)
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		for ev := range dw.Events {
			switch ev.Change {
			case dirwatch.Added:
				if ev.TorrentFilePath != "" {
					err := client.AddTorrentFromFile(ev.TorrentFilePath)
					if err != nil {
						log.Printf("error adding torrent to client: %s", err)
					}
				} else if ev.Magnet != "" {
					err := client.AddMagnet(ev.Magnet)
					if err != nil {
						log.Printf("error adding magnet: %s", err)
					}
				}
			case dirwatch.Removed:
				err := client.DropTorrent(ev.InfoHash)
				if err != nil {
					log.Printf("error dropping torrent: %s", err)
				}
			}
		}
	}()
	resolveTestPeerAddr()
	fs := torrentfs.New(client)
	go func() {
		for {
			addTestPeer(client)
			time.Sleep(10 * time.Second)
		}
	}()

	if err := fusefs.Serve(conn, fs); err != nil {
		log.Fatal(err)
	}
	<-conn.Ready
	if err := conn.MountError; err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/horgh/gorse"
)

// DBItem holds the information about an input/entry that is in the database.
// TODO(will@summercat.com): Refactor to combine with gorse.DBItem. I think we
// should have one that is the generic item. This one includes a field related
// to a single user.
type DBItem struct {
	gorse.DBItem

	// Name from the rss_feed table.
	FeedName string

	// Read state from rss_item_state table
	ReadState string
}

// connectToDB opens a new connection to the database.
func connectToDB(settings *Config) (*sql.DB, error) {
	dsn := fmt.Sprintf("user=%s password=%s dbname=%s host=%s connect_timeout=10",
		settings.DBUser, settings.DBPass, settings.DBName, settings.DBHost)

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	log.Print("Opened new connection to the database.")
	return db, nil
}

// getDB connects us to the database if necessary, and returns an active
// database connection.
//
// We use the global DB variable to try to ensure we use a single connection.
func getDB(settings *Config) (*sql.DB, error) {
	// If we have a db connection, ensure that it is still available so that we
	// reconnect if it is not.
	if DB != nil {
		err := DB.Ping()
		if err == nil {
			return DB, nil
		}

		log.Printf("Database ping failed: %s", err)

		// Continue on, but set us so that we attempt to reconnect.

		DBLock.Lock()
		if DB != nil {
			_ = DB.Close()
			DB = nil
		}
		DBLock.Unlock()
	}

	DBLock.Lock()
	defer DBLock.Unlock()

	if DB != nil {
		return DB, nil
	}

	db, err := connectToDB(settings)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	// Set global
	DB = db

	return DB, nil
}

// dbCountItems retrieves a count of items.
//
// This is for pagination.
func dbCountItems(db *sql.DB, userID int, state gorse.ReadState) (int, error) {
	query := `
		SELECT COUNT(*)
		FROM rss_item ri
		LEFT JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE rf.active = true AND
			COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $3
`

	rows, err := db.Query(query, state.String(), userID, userID)
	if err != nil {
		return -1, err
	}

	if !rows.Next() {
		return -1, errors.New("count not found")
	}

	var count int
	if err := rows.Scan(&count); err != nil {
		_ = rows.Close()
		return -1, err
	}

	if err := rows.Close(); err != nil {
		return -1, fmt.Errorf("problem closing rows: %s", err)
	}

	return count, nil
}

// dbRetrieveFeedItems retrieves feed items from the database which are marked
// a given state.
func dbRetrieveFeedItems(db *sql.DB, settings *Config, order sortOrder,
	page, userID int, state gorse.ReadState) ([]DBItem, error) {

	if page < 1 {
		return nil, errors.New("invalid page number")
	}

	query := `
		SELECT
			rf.name,
			ri.id,
			ri.title,
			ri.link,
			ri.description,
			ri.publication_date
		FROM rss_item ri
		JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE rf.active = true AND
			COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $2
`

	if order == sortAscending {
		query += "ORDER BY ri.publication_date ASC, rf.name, ri.title"
	} else {
		query += "ORDER BY ri.publication_date DESC, rf.name, ri.title"
	}

	query += " LIMIT $3 OFFSET $4"

	offset := (page - 1) * pageSize

	rows, err := db.Query(query, state.String(), userID, pageSize, offset)
	if err != nil {
		return nil, err
	}

	var items []DBItem
	for rows.Next() {
		item := DBItem{}

		err := rows.Scan(&item.FeedName, &item.ID, &item.Title, &item.Link,
			&item.Description, &item.PublicationDate)
		if err != nil {
			log.Printf("Failed to scan row information: %s", err)
			_ = rows.Close()
			return nil, err
		}

		items = append(items, item)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("failure fetching rows: %s", err)
	}

	return items, nil
}

// Retrieve an item's information from the database. This includes the item's
// state for the given user.
func dbGetItem(db *sql.DB, itemID int64, userID int) (DBItem, error) {
	query := `
		SELECT
			ri.id,
			ri.title,
			ri.description,
			ri.link,
			ri.publication_date,
			ri.guid,
			ri.rss_feed_id,
			rf.name,
			COALESCE(ris.state, 'unread')
		FROM rss_item ri
		JOIN rss_feed rf ON ri.rss_feed_id = rf.id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE ri.id = $1 AND
			COALESCE(ris.user_id, $2) = $3
`
	rows, err := db.Query(query, itemID, userID, userID)
	if err != nil {
		return DBItem{}, err
	}

	for rows.Next() {
		item := DBItem{}

		if err := rows.Scan(&item.ID, &item.Title, &item.Description, &item.Link,
			&item.PublicationDate, &item.GUID, &item.RSSFeedID, &item.FeedName,
			&item.ReadState); err != nil {
			_ = rows.Close()
			return DBItem{}, fmt.Errorf("failed to scan row: %s", err)
		}

		if err := rows.Close(); err != nil {
			return DBItem{}, fmt.Errorf("error closing rows: %s", err)
		}

		return item, nil
	}

	if err := rows.Err(); err != nil {
		return DBItem{}, fmt.Errorf("failure fetching rows: %s", err)
	}

	return DBItem{}, fmt.Errorf("item not found")
}

// Record the item was read after having been saved to read later.
//
// It is useful to be able to refer back to such items as it is likely they were
// looked at more closely than others.
func dbRecordReadAfterReadLater(db *sql.DB, userID int, item DBItem) error {
	query := `
		INSERT INTO rss_item_read_after_archive
		(user_id, rss_feed_id, rss_item_id)
		VALUES ($1, $2, $3)
`
	if _, err := db.Exec(query, userID, item.RSSFeedID, item.ID); err != nil {
		return fmt.Errorf("unable to insert: %s", err)
	}

	return nil
}
<commit_msg>Remove unnecessary condition<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"log"

	"github.com/horgh/gorse"
)

// DBItem holds the information about an input/entry that is in the database.
// TODO(will@summercat.com): Refactor to combine with gorse.DBItem. I think we
// should have one that is the generic item. This one includes a field related
// to a single user.
type DBItem struct {
	gorse.DBItem

	// Name from the rss_feed table.
	FeedName string

	// Read state from rss_item_state table
	ReadState string
}

// connectToDB opens a new connection to the database.
func connectToDB(settings *Config) (*sql.DB, error) {
	dsn := fmt.Sprintf("user=%s password=%s dbname=%s host=%s connect_timeout=10",
		settings.DBUser, settings.DBPass, settings.DBName, settings.DBHost)

	db, err := sql.Open("postgres", dsn)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	log.Print("Opened new connection to the database.")
	return db, nil
}

// getDB connects us to the database if necessary, and returns an active
// database connection.
//
// We use the global DB variable to try to ensure we use a single connection.
func getDB(settings *Config) (*sql.DB, error) {
	// If we have a db connection, ensure that it is still available so that we
	// reconnect if it is not.
	if DB != nil {
		err := DB.Ping()
		if err == nil {
			return DB, nil
		}

		log.Printf("Database ping failed: %s", err)

		// Continue on, but set us so that we attempt to reconnect.

		DBLock.Lock()
		if DB != nil {
			_ = DB.Close()
			DB = nil
		}
		DBLock.Unlock()
	}

	DBLock.Lock()
	defer DBLock.Unlock()

	if DB != nil {
		return DB, nil
	}

	db, err := connectToDB(settings)
	if err != nil {
		log.Printf("Failed to connect to the database: %s", err)
		return nil, err
	}

	// Set global
	DB = db

	return DB, nil
}

// dbCountItems retrieves a count of items.
//
// This is for pagination.
func dbCountItems(db *sql.DB, userID int, state gorse.ReadState) (int, error) {
	query := `
		SELECT COUNT(*)
		FROM rss_item ri
		LEFT JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $3
`

	rows, err := db.Query(query, state.String(), userID, userID)
	if err != nil {
		return -1, err
	}

	if !rows.Next() {
		return -1, errors.New("count not found")
	}

	var count int
	if err := rows.Scan(&count); err != nil {
		_ = rows.Close()
		return -1, err
	}

	if err := rows.Close(); err != nil {
		return -1, fmt.Errorf("problem closing rows: %s", err)
	}

	return count, nil
}

// dbRetrieveFeedItems retrieves feed items from the database which are marked
// a given state.
func dbRetrieveFeedItems(db *sql.DB, settings *Config, order sortOrder,
	page, userID int, state gorse.ReadState) ([]DBItem, error) {

	if page < 1 {
		return nil, errors.New("invalid page number")
	}

	query := `
		SELECT
			rf.name,
			ri.id,
			ri.title,
			ri.link,
			ri.description,
			ri.publication_date
		FROM rss_item ri
		JOIN rss_feed rf ON rf.id = ri.rss_feed_id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE COALESCE(ris.state, 'unread') = $1 AND
			COALESCE(ris.user_id, $2) = $2
`

	if order == sortAscending {
		query += "ORDER BY ri.publication_date ASC, rf.name, ri.title"
	} else {
		query += "ORDER BY ri.publication_date DESC, rf.name, ri.title"
	}

	query += " LIMIT $3 OFFSET $4"

	offset := (page - 1) * pageSize

	rows, err := db.Query(query, state.String(), userID, pageSize, offset)
	if err != nil {
		return nil, err
	}

	var items []DBItem
	for rows.Next() {
		item := DBItem{}

		err := rows.Scan(&item.FeedName, &item.ID, &item.Title, &item.Link,
			&item.Description, &item.PublicationDate)
		if err != nil {
			log.Printf("Failed to scan row information: %s", err)
			_ = rows.Close()
			return nil, err
		}

		items = append(items, item)
	}

	if err := rows.Err(); err != nil {
		return nil, fmt.Errorf("failure fetching rows: %s", err)
	}

	return items, nil
}

// Retrieve an item's information from the database. This includes the item's
// state for the given user.
func dbGetItem(db *sql.DB, itemID int64, userID int) (DBItem, error) {
	query := `
		SELECT
			ri.id,
			ri.title,
			ri.description,
			ri.link,
			ri.publication_date,
			ri.guid,
			ri.rss_feed_id,
			rf.name,
			COALESCE(ris.state, 'unread')
		FROM rss_item ri
		JOIN rss_feed rf ON ri.rss_feed_id = rf.id
		LEFT JOIN rss_item_state ris ON ris.item_id = ri.id
		WHERE ri.id = $1 AND
			COALESCE(ris.user_id, $2) = $3
`
	rows, err := db.Query(query, itemID, userID, userID)
	if err != nil {
		return DBItem{}, err
	}

	for rows.Next() {
		item := DBItem{}

		if err := rows.Scan(&item.ID, &item.Title, &item.Description, &item.Link,
			&item.PublicationDate, &item.GUID, &item.RSSFeedID, &item.FeedName,
			&item.ReadState); err != nil {
			_ = rows.Close()
			return DBItem{}, fmt.Errorf("failed to scan row: %s", err)
		}

		if err := rows.Close(); err != nil {
			return DBItem{}, fmt.Errorf("error closing rows: %s", err)
		}

		return item, nil
	}

	if err := rows.Err(); err != nil {
		return DBItem{}, fmt.Errorf("failure fetching rows: %s", err)
	}

	return DBItem{}, fmt.Errorf("item not found")
}

// Record the item was read after having been saved to read later.
//
// It is useful to be able to refer back to such items as it is likely they were
// looked at more closely than others.
func dbRecordReadAfterReadLater(db *sql.DB, userID int, item DBItem) error {
	query := `
		INSERT INTO rss_item_read_after_archive
		(user_id, rss_feed_id, rss_item_id)
		VALUES ($1, $2, $3)
`
	if _, err := db.Exec(query, userID, item.RSSFeedID, item.ID); err != nil {
		return fmt.Errorf("unable to insert: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	mdbs "github.com/msackman/gomdb/server"
	"goshawkdb.io/common"
	"goshawkdb.io/common/certs"
	goshawk "goshawkdb.io/server"
	"goshawkdb.io/server/configuration"
	"goshawkdb.io/server/db"
	"goshawkdb.io/server/network"
	"goshawkdb.io/server/paxos"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"runtime/trace"
	"sync/atomic"
	"syscall"
	"time"
)

func main() {
	log.SetPrefix(common.ProductName + " ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	log.Printf("Version %s; %v", goshawk.ServerVersion, os.Args)

	if s, err := newServer(); err != nil {
		fmt.Printf("\n%v\n\n", err)
		flag.Usage()
		fmt.Println("\nSee https://goshawkdb.io/starting.html for the Getting Started guide.")
		os.Exit(1)
	} else if s != nil {
		s.start()
	}
}

func newServer() (*server, error) {
	var configFile, dataDir, certFile string
	var port int
	var version, genClusterCert, genClientCert bool

	flag.StringVar(&configFile, "config", "", "`Path` to configuration file (required to start server).")
	flag.StringVar(&dataDir, "dir", "", "`Path` to data directory (required to run server).")
	flag.StringVar(&certFile, "cert", "", "`Path` to cluster certificate and key file (required to run server).")
	flag.IntVar(&port, "port", common.DefaultPort, "Port to listen on (required if non-default).")
	flag.BoolVar(&version, "version", false, "Display version and exit.")
	flag.BoolVar(&genClusterCert, "gen-cluster-cert", false, "Generate new cluster certificate key pair.")
	flag.BoolVar(&genClientCert, "gen-client-cert", false, "Generate client certificate key pair.")
	flag.Parse()

	if version {
		log.Printf("%v version %v", common.ProductName, goshawk.ServerVersion)
		return nil, nil
	}

	if genClusterCert {
		certificatePrivateKeyPair, err := certs.NewClusterCertificate()
		if err != nil {
			return nil, err
		}
		fmt.Printf("%v%v", certificatePrivateKeyPair.CertificatePEM, certificatePrivateKeyPair.PrivateKeyPEM)
		return nil, nil
	}

	if len(certFile) == 0 {
		return nil, fmt.Errorf("No certificate supplied (missing -cert parameter). Use -gen-cluster-cert to create cluster certificate.")
	}
	certificate, err := ioutil.ReadFile(certFile)
	if err != nil {
		return nil, err
	}

	if genClientCert {
		certificatePrivateKeyPair, err := certs.NewClientCertificate(certificate)
		if err != nil {
			return nil, err
		}
		fmt.Printf("%v%v", certificatePrivateKeyPair.CertificatePEM, certificatePrivateKeyPair.PrivateKeyPEM)
		fingerprint := sha256.Sum256(certificatePrivateKeyPair.Certificate)
		log.Printf("Fingerprint: %v\n", hex.EncodeToString(fingerprint[:]))
		return nil, nil
	}

	if dataDir == "" {
		dataDir, err = ioutil.TempDir("", common.ProductName+"_Data_")
		if err != nil {
			return nil, err
		}
		log.Printf("No data dir supplied (missing -dir parameter). Using %v for data.\n", dataDir)
	}
	err = os.MkdirAll(dataDir, 0750)
	if err != nil {
		return nil, err
	}

	if configFile != "" {
		_, err := ioutil.ReadFile(configFile)
		if err != nil {
			return nil, err
		}
	}

	if !(0 < port && port < 65536) {
		return nil, fmt.Errorf("Supplied port is illegal (%v). Port must be > 0 and < 65536", port)
	}

	s := &server{
		configFile:   configFile,
		certificate:  certificate,
		dataDir:      dataDir,
		port:         uint16(port),
		onShutdown:   []func(){},
		shutdownChan: make(chan goshawk.EmptyStruct),
	}

	if err = s.ensureRMId(); err != nil {
		return nil, err
	}
	if err = s.ensureBootCount(); err != nil {
		return nil, err
	}

	return s, nil
}

type server struct {
	configFile        string
	certificate       []byte
	dataDir           string
	port              uint16
	rmId              common.RMId
	bootCount         uint32
	connectionManager *network.ConnectionManager
	transmogrifier    *network.TopologyTransmogrifier
	profileFile       *os.File
	traceFile         *os.File
	onShutdown        []func()
	shutdownChan      chan goshawk.EmptyStruct
	shutdownCounter   int32
}

func (s *server) start() {
	os.Stdin.Close()

	procs := runtime.NumCPU()
	if procs < 2 {
		procs = 2
	}
	runtime.GOMAXPROCS(procs)

	commandLineConfig, err := s.commandLineConfig()
	s.maybeShutdown(err)

	nodeCertPrivKeyPair, err := certs.GenerateNodeCertificatePrivateKeyPair(s.certificate)
	for idx := range s.certificate {
		s.certificate[idx] = 0
	}
	s.certificate = nil
	s.maybeShutdown(err)

	disk, err := mdbs.NewMDBServer(s.dataDir, 0, 0600, goshawk.MDBInitialSize, procs/2, time.Millisecond, db.DB)
	s.maybeShutdown(err)
	db := disk.(*db.Databases)
	s.addOnShutdown(db.Shutdown)

	cm, transmogrifier := network.NewConnectionManager(s.rmId, s.bootCount, procs, db, nodeCertPrivKeyPair, s.port, s, commandLineConfig)
	s.addOnShutdown(func() { cm.Shutdown(paxos.Sync) })
	s.addOnShutdown(transmogrifier.Shutdown)
	s.connectionManager = cm
	s.transmogrifier = transmogrifier

	go s.signalHandler()

	listener, err := network.NewListener(s.port, cm)
	s.maybeShutdown(err)
	s.addOnShutdown(listener.Shutdown)

	defer s.shutdown(nil)
	<-s.shutdownChan
}

func (s *server) addOnShutdown(f func()) {
	if f != nil {
		s.onShutdown = append(s.onShutdown, f)
	}
}

func (s *server) shutdown(err error) {
	for idx := len(s.onShutdown) - 1; idx >= 0; idx-- {
		s.onShutdown[idx]()
	}
	if err == nil {
		log.Println("Shutdown.")
	} else {
		log.Fatal("Shutdown due to fatal error: ", err)
	}
}

func (s *server) maybeShutdown(err error) {
	if err != nil {
		s.shutdown(err)
	}
}

func (s *server) ensureRMId() error {
	path := s.dataDir + "/rmid"
	if b, err := ioutil.ReadFile(path); err == nil {
		s.rmId = common.RMId(binary.BigEndian.Uint32(b))
		return nil

	} else {
		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
		for s.rmId == common.RMIdEmpty {
			s.rmId = common.RMId(rng.Uint32())
		}
		b := make([]byte, 4)
		binary.BigEndian.PutUint32(b, uint32(s.rmId))
		return ioutil.WriteFile(path, b, 0400)
	}
}

func (s *server) ensureBootCount() error {
	path := s.dataDir + "/bootcount"
	if b, err := ioutil.ReadFile(path); err == nil {
		s.bootCount = binary.BigEndian.Uint32(b) + 1
	} else {
		s.bootCount = 1
	}
	b := make([]byte, 4)
	binary.BigEndian.PutUint32(b, s.bootCount)
	return ioutil.WriteFile(path, b, 0600)
}

func (s *server) commandLineConfig() (*configuration.Configuration, error) {
	if s.configFile != "" {
		return configuration.LoadConfigurationFromPath(s.configFile)
	}
	return nil, nil
}

func (s *server) SignalShutdown() {
	// this may fail if stdout has died
	log.Println("Shutting down.")
	if atomic.AddInt32(&s.shutdownCounter, 1) == 1 {
		s.shutdownChan <- goshawk.EmptyStructVal
	}
}

func (s *server) signalStatus() {
	sc := goshawk.NewStatusConsumer()
	go sc.Consume(func(str string) {
		log.Printf("System Status for %v\n%v\nStatus End\n", s.rmId, str)
	})
	sc.Emit(fmt.Sprintf("Configuration File: %v", s.configFile))
	sc.Emit(fmt.Sprintf("Data Directory: %v", s.dataDir))
	sc.Emit(fmt.Sprintf("Port: %v", s.port))
	s.connectionManager.Status(sc)
}

func (s *server) signalReloadConfig() {
	if s.configFile == "" {
		log.Println("Attempt to reload config failed as no path to configuration provided on command line.")
		return
	}
	config, err := configuration.LoadConfigurationFromPath(s.configFile)
	if err != nil {
		log.Println("Cannot reload config due to error:", err)
		return
	}
	s.transmogrifier.RequestConfigurationChange(config)
}

func (s *server) signalDumpStacks() {
	size := 16384
	for {
		buf := make([]byte, size)
		if l := runtime.Stack(buf, true); l < size {
			log.Printf("Stacks dump\n%s\nStacks dump end", buf[:l])
			return
		} else {
			size += size
		}
	}
}

func (s *server) signalToggleCpuProfile() {
	if s.profileFile == nil {
		memFile, err := ioutil.TempFile("", common.ProductName+"_Mem_Profile_")
		if goshawk.CheckWarn(err) {
			return
		}
		if goshawk.CheckWarn(pprof.Lookup("heap").WriteTo(memFile, 0)) {
			return
		}
		if !goshawk.CheckWarn(memFile.Close()) {
			log.Println("Memory profile written to", memFile.Name())
		}

		profFile, err := ioutil.TempFile("", common.ProductName+"_CPU_Profile_")
		if goshawk.CheckWarn(err) {
			return
		}
		if goshawk.CheckWarn(pprof.StartCPUProfile(profFile)) {
			return
		}
		s.profileFile = profFile
		log.Println("Profiling started in", profFile.Name())

	} else {
		pprof.StopCPUProfile()
		if !goshawk.CheckWarn(s.profileFile.Close()) {
			log.Println("Profiling stopped in", s.profileFile.Name())
		}
		s.profileFile = nil
	}
}

func (s *server) signalToggleTrace() {
	if s.traceFile == nil {
		traceFile, err := ioutil.TempFile("", common.ProductName+"_Trace_")
		if goshawk.CheckWarn(err) {
			return
		}
		if goshawk.CheckWarn(trace.Start(traceFile)) {
			return
		}
		s.traceFile = traceFile
		log.Println("Tracing started in", traceFile.Name())

	} else {
		trace.Stop()
		if !goshawk.CheckWarn(s.traceFile.Close()) {
			log.Println("Tracing stopped in", s.traceFile.Name())
		}
		s.traceFile = nil
	}
}

func (s *server) signalHandler() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGTERM, syscall.SIGPIPE, syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGUSR2, os.Interrupt)
	for {
		sig := <-sigs
		switch sig {
		case syscall.SIGPIPE:
			if _, err := os.Stdout.WriteString("Socket has closed\n"); err != nil {
				// stdout has errored; probably whatever we were being
				// piped to has died.
				s.SignalShutdown()
			}
		case syscall.SIGTERM, syscall.SIGINT:
			s.SignalShutdown()
		case syscall.SIGHUP:
			s.signalReloadConfig()
		case syscall.SIGQUIT:
			s.signalDumpStacks()
		case syscall.SIGUSR1:
			s.signalStatus()
		case syscall.SIGUSR2:
			s.signalToggleCpuProfile()
			//s.signalToggleTrace()
		}
	}
}
<commit_msg>Report LMDB version on startup.<commit_after>package main

import (
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	mdb "github.com/msackman/gomdb"
	mdbs "github.com/msackman/gomdb/server"
	"goshawkdb.io/common"
	"goshawkdb.io/common/certs"
	goshawk "goshawkdb.io/server"
	"goshawkdb.io/server/configuration"
	"goshawkdb.io/server/db"
	"goshawkdb.io/server/network"
	"goshawkdb.io/server/paxos"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"runtime"
	"runtime/pprof"
	"runtime/trace"
	"sync/atomic"
	"syscall"
	"time"
)

func main() {
	log.SetPrefix(common.ProductName + " ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
	log.Printf("GoshawkDB Version %s with %s; %v", goshawk.ServerVersion, mdb.Version(), os.Args)

	if s, err := newServer(); err != nil {
		fmt.Printf("\n%v\n\n", err)
		flag.Usage()
		fmt.Println("\nSee https://goshawkdb.io/starting.html for the Getting Started guide.")
		os.Exit(1)
	} else if s != nil {
		s.start()
	}
}

func newServer() (*server, error) {
	var configFile, dataDir, certFile string
	var port int
	var version, genClusterCert, genClientCert bool

	flag.StringVar(&configFile, "config", "", "`Path` to configuration file (required to start server).")
	flag.StringVar(&dataDir, "dir", "", "`Path` to data directory (required to run server).")
	flag.StringVar(&certFile, "cert", "", "`Path` to cluster certificate and key file (required to run server).")
	flag.IntVar(&port, "port", common.DefaultPort, "Port to listen on (required if non-default).")
	flag.BoolVar(&version, "version", false, "Display version and exit.")
	flag.BoolVar(&genClusterCert, "gen-cluster-cert", false, "Generate new cluster certificate key pair.")
	flag.BoolVar(&genClientCert, "gen-client-cert", false, "Generate client certificate key pair.")
	flag.Parse()

	if version {
		log.Printf("%v version %v", common.ProductName, goshawk.ServerVersion)
		return nil, nil
	}

	if genClusterCert {
		certificatePrivateKeyPair, err := certs.NewClusterCertificate()
		if err != nil {
			return nil, err
		}
		fmt.Printf("%v%v", certificatePrivateKeyPair.CertificatePEM, certificatePrivateKeyPair.PrivateKeyPEM)
		return nil, nil
	}

	if len(certFile) == 0 {
		return nil, fmt.Errorf("No certificate supplied (missing -cert parameter). Use -gen-cluster-cert to create cluster certificate.")
	}
	certificate, err := ioutil.ReadFile(certFile)
	if err != nil {
		return nil, err
	}

	if genClientCert {
		certificatePrivateKeyPair, err := certs.NewClientCertificate(certificate)
		if err != nil {
			return nil, err
		}
		fmt.Printf("%v%v", certificatePrivateKeyPair.CertificatePEM, certificatePrivateKeyPair.PrivateKeyPEM)
		fingerprint := sha256.Sum256(certificatePrivateKeyPair.Certificate)
		log.Printf("Fingerprint: %v\n", hex.EncodeToString(fingerprint[:]))
		return nil, nil
	}

	if dataDir == "" {
		dataDir, err = ioutil.TempDir("", common.ProductName+"_Data_")
		if err != nil {
			return nil, err
		}
		log.Printf("No data dir supplied (missing -dir parameter). Using %v for data.\n", dataDir)
	}
	err = os.MkdirAll(dataDir, 0750)
	if err != nil {
		return nil, err
	}

	if configFile != "" {
		_, err := ioutil.ReadFile(configFile)
		if err != nil {
			return nil, err
		}
	}

	if !(0 < port && port < 65536) {
		return nil, fmt.Errorf("Supplied port is illegal (%v). Port must be > 0 and < 65536", port)
	}

	s := &server{
		configFile:   configFile,
		certificate:  certificate,
		dataDir:      dataDir,
		port:         uint16(port),
		onShutdown:   []func(){},
		shutdownChan: make(chan goshawk.EmptyStruct),
	}

	if err = s.ensureRMId(); err != nil {
		return nil, err
	}
	if err = s.ensureBootCount(); err != nil {
		return nil, err
	}

	return s, nil
}

type server struct {
	configFile        string
	certificate       []byte
	dataDir           string
	port              uint16
	rmId              common.RMId
	bootCount         uint32
	connectionManager *network.ConnectionManager
	transmogrifier    *network.TopologyTransmogrifier
	profileFile       *os.File
	traceFile         *os.File
	onShutdown        []func()
	shutdownChan      chan goshawk.EmptyStruct
	shutdownCounter   int32
}

func (s *server) start() {
	os.Stdin.Close()

	procs := runtime.NumCPU()
	if procs < 2 {
		procs = 2
	}
	runtime.GOMAXPROCS(procs)

	commandLineConfig, err := s.commandLineConfig()
	s.maybeShutdown(err)

	nodeCertPrivKeyPair, err := certs.GenerateNodeCertificatePrivateKeyPair(s.certificate)
	for idx := range s.certificate {
		s.certificate[idx] = 0
	}
	s.certificate = nil
	s.maybeShutdown(err)

	disk, err := mdbs.NewMDBServer(s.dataDir, 0, 0600, goshawk.MDBInitialSize, procs/2, time.Millisecond, db.DB)
	s.maybeShutdown(err)
	db := disk.(*db.Databases)
	s.addOnShutdown(db.Shutdown)

	cm, transmogrifier := network.NewConnectionManager(s.rmId, s.bootCount, procs, db, nodeCertPrivKeyPair, s.port, s, commandLineConfig)
	s.addOnShutdown(func() { cm.Shutdown(paxos.Sync) })
	s.addOnShutdown(transmogrifier.Shutdown)
	s.connectionManager = cm
	s.transmogrifier = transmogrifier

	go s.signalHandler()

	listener, err := network.NewListener(s.port, cm)
	s.maybeShutdown(err)
	s.addOnShutdown(listener.Shutdown)

	defer s.shutdown(nil)
	<-s.shutdownChan
}

func (s *server) addOnShutdown(f func()) {
	if f != nil {
		s.onShutdown = append(s.onShutdown, f)
	}
}

func (s *server) shutdown(err error) {
	for idx := len(s.onShutdown) - 1; idx >= 0; idx-- {
		s.onShutdown[idx]()
	}
	if err == nil {
		log.Println("Shutdown.")
	} else {
		log.Fatal("Shutdown due to fatal error: ", err)
	}
}

func (s *server) maybeShutdown(err error) {
	if err != nil {
		s.shutdown(err)
	}
}

func (s *server) ensureRMId() error {
	path := s.dataDir + "/rmid"
	if b, err := ioutil.ReadFile(path); err == nil {
		s.rmId = common.RMId(binary.BigEndian.Uint32(b))
		return nil

	} else {
		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
		for s.rmId == common.RMIdEmpty {
			s.rmId = common.RMId(rng.Uint32())
		}
		b := make([]byte, 4)
		binary.BigEndian.PutUint32(b, uint32(s.rmId))
		return ioutil.WriteFile(path, b, 0400)
	}
}

func (s *server) ensureBootCount() error {
	path := s.dataDir + "/bootcount"
	if b, err := ioutil.ReadFile(path); err == nil {
		s.bootCount = binary.BigEndian.Uint32(b) + 1
	} else {
		s.bootCount = 1
	}
	b := make([]byte, 4)
	binary.BigEndian.PutUint32(b, s.bootCount)
	return ioutil.WriteFile(path, b, 0600)
}

func (s *server) commandLineConfig() (*configuration.Configuration, error) {
	if s.configFile != "" {
		return configuration.LoadConfigurationFromPath(s.configFile)
	}
	return nil, nil
}

func (s *server) SignalShutdown() {
	// this may fail if stdout has died
	log.Println("Shutting down.")
	if atomic.AddInt32(&s.shutdownCounter, 1) == 1 {
		s.shutdownChan <- goshawk.EmptyStructVal
	}
}

func (s *server) signalStatus() {
	sc := goshawk.NewStatusConsumer()
	go sc.Consume(func(str string) {
		log.Printf("System Status for %v\n%v\nStatus End\n", s.rmId, str)
	})
	sc.Emit(fmt.Sprintf("Configuration File: %v", s.configFile))
	sc.Emit(fmt.Sprintf("Data Directory: %v", s.dataDir))
	sc.Emit(fmt.Sprintf("Port: %v", s.port))
	s.connectionManager.Status(sc)
}

func (s *server) signalReloadConfig() {
	if s.configFile == "" {
		log.Println("Attempt to reload config failed as no path to configuration provided on command line.")
		return
	}
	config, err := configuration.LoadConfigurationFromPath(s.configFile)
	if err != nil {
		log.Println("Cannot reload config due to error:", err)
		return
	}
	s.transmogrifier.RequestConfigurationChange(config)
}

func (s *server) signalDumpStacks() {
	size := 16384
	for {
		buf := make([]byte, size)
		if l := runtime.Stack(buf, true); l < size {
			log.Printf("Stacks dump\n%s\nStacks dump end", buf[:l])
			return
		} else {
			size += size
		}
	}
}

func (s *server) signalToggleCpuProfile() {
	if s.profileFile == nil {
		memFile, err := ioutil.TempFile("", common.ProductName+"_Mem_Profile_")
		if goshawk.CheckWarn(err) {
			return
		}
		if goshawk.CheckWarn(pprof.Lookup("heap").WriteTo(memFile, 0)) {
			return
		}
		if !goshawk.CheckWarn(memFile.Close()) {
			log.Println("Memory profile written to", memFile.Name())
		}

		profFile, err := ioutil.TempFile("", common.ProductName+"_CPU_Profile_")
		if goshawk.CheckWarn(err) {
			return
		}
		if goshawk.CheckWarn(pprof.StartCPUProfile(profFile)) {
			return
		}
		s.profileFile = profFile
		log.Println("Profiling started in", profFile.Name())

	} else {
		pprof.StopCPUProfile()
		if !goshawk.CheckWarn(s.profileFile.Close()) {
			log.Println("Profiling stopped in", s.profileFile.Name())
		}
		s.profileFile = nil
	}
}

func (s *server) signalToggleTrace() {
	if s.traceFile == nil {
		traceFile, err := ioutil.TempFile("", common.ProductName+"_Trace_")
		if goshawk.CheckWarn(err) {
			return
		}
		if goshawk.CheckWarn(trace.Start(traceFile)) {
			return
		}
		s.traceFile = traceFile
		log.Println("Tracing started in", traceFile.Name())

	} else {
		trace.Stop()
		if !goshawk.CheckWarn(s.traceFile.Close()) {
			log.Println("Tracing stopped in", s.traceFile.Name())
		}
		s.traceFile = nil
	}
}

func (s *server) signalHandler() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.SIGTERM, syscall.SIGPIPE, syscall.SIGQUIT, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGUSR2, os.Interrupt)
	for {
		sig := <-sigs
		switch sig {
		case syscall.SIGPIPE:
			if _, err := os.Stdout.WriteString("Socket has closed\n"); err != nil {
				// stdout has errored; probably whatever we were being
				// piped to has died.
				s.SignalShutdown()
			}
		case syscall.SIGTERM, syscall.SIGINT:
			s.SignalShutdown()
		case syscall.SIGHUP:
			s.signalReloadConfig()
		case syscall.SIGQUIT:
			s.signalDumpStacks()
		case syscall.SIGUSR1:
			s.signalStatus()
		case syscall.SIGUSR2:
			s.signalToggleCpuProfile()
			//s.signalToggleTrace()
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"testing"

	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

func TestTryToChangeLogLevelWithoutParameter(t *testing.T) {
	teamName := "TestTeam"
	channelName := "TestChannel"
	userName := os.Getenv("CLAIMR_SUPERUSER")

	currentLogLevel := log.GetLevel()
	message := fmt.Sprintf("No log level provided, keeping in `%s`", currentLogLevel)
	mockResponse, patchReply := createMockReply(t, message)
	patchGetEvent := createMockEvent(t, teamName, channelName, userName)

	mockRequest, patchParam := createMockRequest(t, map[string]string{"level": ""})

	changeLogLevel(mockRequest, mockResponse)

	assert.Equal(t, currentLogLevel, log.GetLevel())

	patchReply.Unpatch()
	patchGetEvent.Unpatch()
	patchParam.Unpatch()
}
<commit_msg>Unknown log level test<commit_after>package cmd

import (
	"fmt"
	"os"
	"testing"

	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

func TestTryToChangeLogLevelWithoutParameter(t *testing.T) {
	teamName := "TestTeam"
	channelName := "TestChannel"
	userName := os.Getenv("CLAIMR_SUPERUSER")

	currentLogLevel := log.GetLevel().String()
	message := fmt.Sprintf("No log level provided, keeping in `%s`", currentLogLevel)
	mockResponse, patchReply := createMockReply(t, message)
	patchGetEvent := createMockEvent(t, teamName, channelName, userName)

	mockRequest, patchParam := createMockRequest(t, map[string]string{"level": ""})

	changeLogLevel(mockRequest, mockResponse)

	assert.Equal(t, currentLogLevel, log.GetLevel().String())

	patchReply.Unpatch()
	patchGetEvent.Unpatch()
	patchParam.Unpatch()
}

func TestTryToChangeLogLevelToUnknownLevel(t *testing.T) {
	teamName := "TestTeam"
	channelName := "TestChannel"
	userName := os.Getenv("CLAIMR_SUPERUSER")

	currentLogLevel := log.GetLevel().String()
	unknownLogLevel := "unknown"
	message := fmt.Sprintf("not a valid logrus Level: \"%s\"", unknownLogLevel)
	mockResponse, patchReply := createMockReply(t, message)
	patchGetEvent := createMockEvent(t, teamName, channelName, userName)

	mockRequest, patchParam := createMockRequest(t, map[string]string{"level": unknownLogLevel})

	changeLogLevel(mockRequest, mockResponse)

	assert.Equal(t, currentLogLevel, log.GetLevel().String())

	patchReply.Unpatch()
	patchGetEvent.Unpatch()
	patchParam.Unpatch()
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"log"
	"strings"

	"github.com/spf13/cobra"
	"github.com/xanzy/go-gitlab"
	"github.com/zaquestion/lab/internal/git"
	lab "github.com/zaquestion/lab/internal/gitlab"
)

// projectCreateCmd represents the create command
var projectCreateCmd = &cobra.Command{
	Use:   "create [path]",
	Short: "Create a new project on GitLab",
	Long: `If no path or name is provided the name of the git repo working directory

path refers the path on gitlab including the full group/namespace/project. If no path or name is provided and the directory is a git repo the name of the current working directory will be used.`,
	Args: cobra.MaximumNArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		var (
			name, _ = cmd.Flags().GetString("name")
			desc, _ = cmd.Flags().GetString("description")
		)
		path := determinePath(args, name)
		if path == "" && name == "" {
			log.Fatal("path or name must be set")
		}

		visibility := gitlab.InternalVisibility
		switch {
		case private:
			visibility = gitlab.PrivateVisibility
		case public:
			visibility = gitlab.PublicVisibility
		}

		opts := gitlab.CreateProjectOptions{
			Path:        gitlab.String(path),
			Name:        gitlab.String(name),
			Description: gitlab.String(desc),
			Visibility:  &visibility,
		}
		p, err := lab.ProjectCreate(&opts)
		if err != nil {
			log.Fatal(err)
		}
		if git.InsideGitRepo() {
			err = git.RemoteAdd("origin", p.SSHURLToRepo, ".")
			if err != nil {
				log.Fatal(err)
			}
		}
		fmt.Println(strings.TrimSuffix(p.HTTPURLToRepo, ".git"))
	},
}

func determinePath(args []string, name string) string {
	var path string
	if len(args) > 0 {
		path = args[0]
	}
	if path == "" && name == "" && git.InsideGitRepo() {
		wd, err := git.WorkingDir()
		if err != nil {
			log.Fatal(err)
		}
		p := strings.Split(wd, "/")
		path = p[len(p)-1]
	}
	return path
}

func init() {
	projectCreateCmd.Flags().StringP("name", "n", "", "name to use for the new project")
	projectCreateCmd.Flags().StringP("description", "d", "", "description to use for the new project")
	projectCreateCmd.Flags().BoolVarP(&private, "private", "p", false, "Make project private; visible only to project members (default: internal)")
	projectCreateCmd.Flags().BoolVar(&public, "public", false, "Make project public; can be accessed without any authentication (default: internal)")
	projectCmd.AddCommand(projectCreateCmd)
}
<commit_msg>Pass required ApprovelsBeforeMerge parameter<commit_after>package cmd

import (
	"fmt"
	"log"
	"strings"

	"github.com/spf13/cobra"
	"github.com/xanzy/go-gitlab"
	"github.com/zaquestion/lab/internal/git"
	lab "github.com/zaquestion/lab/internal/gitlab"
)

// projectCreateCmd represents the create command
var projectCreateCmd = &cobra.Command{
	Use:   "create [path]",
	Short: "Create a new project on GitLab",
	Long: `If no path or name is provided the name of the git repo working directory

path refers the path on gitlab including the full group/namespace/project. If no path or name is provided and the directory is a git repo the name of the current working directory will be used.`,
	Args: cobra.MaximumNArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		var (
			name, _ = cmd.Flags().GetString("name")
			desc, _ = cmd.Flags().GetString("description")
		)
		path := determinePath(args, name)
		if path == "" && name == "" {
			log.Fatal("path or name must be set")
		}

		visibility := gitlab.InternalVisibility
		switch {
		case private:
			visibility = gitlab.PrivateVisibility
		case public:
			visibility = gitlab.PublicVisibility
		}

		opts := gitlab.CreateProjectOptions{
			Path:                 gitlab.String(path),
			Name:                 gitlab.String(name),
			Description:          gitlab.String(desc),
			Visibility:           &visibility,
			ApprovalsBeforeMerge: gitlab.Int(0),
		}
		p, err := lab.ProjectCreate(&opts)
		if err != nil {
			log.Fatal(err)
		}
		if git.InsideGitRepo() {
			err = git.RemoteAdd("origin", p.SSHURLToRepo, ".")
			if err != nil {
				log.Fatal(err)
			}
		}
		fmt.Println(strings.TrimSuffix(p.HTTPURLToRepo, ".git"))
	},
}

func determinePath(args []string, name string) string {
	var path string
	if len(args) > 0 {
		path = args[0]
	}
	if path == "" && name == "" && git.InsideGitRepo() {
		wd, err := git.WorkingDir()
		if err != nil {
			log.Fatal(err)
		}
		p := strings.Split(wd, "/")
		path = p[len(p)-1]
	}
	return path
}

func init() {
	projectCreateCmd.Flags().StringP("name", "n", "", "name to use for the new project")
	projectCreateCmd.Flags().StringP("description", "d", "", "description to use for the new project")
	projectCreateCmd.Flags().BoolVarP(&private, "private", "p", false, "Make project private; visible only to project members (default: internal)")
	projectCreateCmd.Flags().BoolVar(&public, "public", false, "Make project public; can be accessed without any authentication (default: internal)")
	projectCmd.AddCommand(projectCreateCmd)
}
<|endoftext|>


<commit_before>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

// We need Go 1.9+ for proper uint64 handling: https://github.com/kshvakov/clickhouse/issues/69
// +build go1.9

package main

import (
	"bytes"
	"context"
	_ "expvar"
	"html/template"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/Percona-Lab/PromHouse/handlers"
	"github.com/Percona-Lab/PromHouse/storages/clickhouse"
)

const (
	shutdownTimeout = 3 * time.Second
)

// runPromServer runs Prometheus API server until context is canceled, then gracefully stops it.
func runPromServer(ctx context.Context, addr string, params *clickhouse.Params) {
	l := logrus.WithField("component", "api")

	storage, err := clickhouse.New(params)
	if err != nil {
		l.Panic(err)
	}

	prometheus.MustRegister(storage)

	promAPI := handlers.NewPromAPI(storage, l)
	prometheus.MustRegister(promAPI)

	mux := http.NewServeMux()
	mux.HandleFunc("/read", promAPI.Read())
	mux.HandleFunc("/write", promAPI.Write())

	l.Printf("Starting server on http://%s/", addr)
	server := &http.Server{
		Addr:     addr,
		ErrorLog: log.New(os.Stderr, "runPromServer: ", 0),
		Handler:  mux,
	}
	go func() {
		if err := server.ListenAndServe(); err != http.ErrServerClosed {
			l.Panic(err)
		}
		l.Print("Server stopped.")
	}()

	<-ctx.Done()
	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
	if err := server.Shutdown(ctx); err != nil {
		l.Errorf("Failed to shutdown gracefully: %s", err)
	}
	cancel()
}

// runDebugServer runs debug server until context is canceled, then gracefully stops it.
func runDebugServer(ctx context.Context, addr string) {
	l := logrus.WithField("component", "debug")

	http.Handle("/debug/metrics", promhttp.Handler())

	handlers := []string{
		"/debug/metrics", "/debug/vars", "/debug/requests", "/debug/events", "/debug/pprof",
	}
	for i, h := range handlers {
		handlers[i] = "http://" + addr + h
	}

	var buf bytes.Buffer
	err := template.Must(template.New("debug").Parse(`

	{{ range . }}

{{ . }}

	{{ end }}

	`)).Execute(&buf, handlers)
	if err != nil {
		l.Panic(err)
	}
	http.HandleFunc("/debug", func(rw http.ResponseWriter, req *http.Request) {
		rw.Write(buf.Bytes())
	})
	l.Printf("Starting server on http://%s/debug\nRegistered handlers:\n\t%s", addr, strings.Join(handlers, "\n\t"))

	server := &http.Server{
		Addr:     addr,
		ErrorLog: log.New(os.Stderr, "runDebugServer: ", 0),
	}
	go func() {
		if err := server.ListenAndServe(); err != http.ErrServerClosed {
			l.Panic(err)
		}
		l.Print("Server stopped.")
	}()

	<-ctx.Done()
	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
	if err := server.Shutdown(ctx); err != nil {
		l.Errorf("Failed to shutdown gracefully: %s", err)
	}
	cancel()
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("stdlog: ")

	var (
		promAddrF     = kingpin.Flag("listen-prom-addr", "Prometheus remote API server listen address").Default("127.0.0.1:7781").String()
		debugAddrF    = kingpin.Flag("listen-debug-addr", "Debug server listen address").Default("127.0.0.1:7782").String()
		dropF         = kingpin.Flag("db.drop-schema", "Drop existing database schema").Bool()
		maxOpenConnsF = kingpin.Flag("db.max-open-conns", "Maximum number of open connections to the database").Default("75").Int()
		logLevelF     = kingpin.Flag("log.level", "Log level").Default("warn").String()
	)
	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Parse()

	level, err := logrus.ParseLevel(*logLevelF)
	if err != nil {
		logrus.Fatal(err)
	}
	logrus.SetLevel(level)

	l := logrus.WithField("component", "main")
	ctx, cancel := context.WithCancel(context.Background())
	defer l.Print("Done.")

	// handle termination signals: first one gracefully, force exit on the second one
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
	go func() {
		s := <-signals
		l.Warnf("Got %s, shutting down...", unix.SignalName(s.(syscall.Signal)))
		cancel()

		s = <-signals
		l.Panicf("Got %s, exiting!", s, unix.SignalName(s.(syscall.Signal)))
	}()

	// start servers, wait for them to exit
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		params := &clickhouse.Params{
			DSN:                  "tcp://127.0.0.1:9000/?database=prometheus",
			DropDatabase:         *dropF,
			MaxOpenConns:         *maxOpenConnsF,
			MaxTimeSeriesInQuery: 50,
		}
		runPromServer(ctx, *promAddrF, params)
	}()
	go func() {
		defer wg.Done()
		runDebugServer(ctx, *debugAddrF)
	}()
	wg.Wait()
}
<commit_msg>Tweak logging.<commit_after>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

// We need Go 1.9+ for proper uint64 handling: https://github.com/kshvakov/clickhouse/issues/69
// +build go1.9

package main

import (
	"bytes"
	"context"
	_ "expvar"
	"html/template"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
	"gopkg.in/alecthomas/kingpin.v2"

	"github.com/Percona-Lab/PromHouse/handlers"
	"github.com/Percona-Lab/PromHouse/storages/clickhouse"
)

const (
	shutdownTimeout = 3 * time.Second
)

// runPromServer runs Prometheus API server until context is canceled, then gracefully stops it.
func runPromServer(ctx context.Context, addr string, params *clickhouse.Params) {
	l := logrus.WithField("component", "api")

	storage, err := clickhouse.New(params)
	if err != nil {
		l.Panic(err)
	}

	prometheus.MustRegister(storage)

	promAPI := handlers.NewPromAPI(storage, l)
	prometheus.MustRegister(promAPI)

	mux := http.NewServeMux()
	mux.HandleFunc("/read", promAPI.Read())
	mux.HandleFunc("/write", promAPI.Write())

	l.Infof("Starting server on http://%s/", addr)
	server := &http.Server{
		Addr:     addr,
		ErrorLog: log.New(os.Stderr, "runPromServer: ", 0),
		Handler:  mux,
	}
	go func() {
		if err := server.ListenAndServe(); err != http.ErrServerClosed {
			l.Panic(err)
		}
		l.Info("Server stopped.")
	}()

	<-ctx.Done()
	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
	if err := server.Shutdown(ctx); err != nil {
		l.Errorf("Failed to shutdown gracefully: %s", err)
	}
	cancel()
}

// runDebugServer runs debug server until context is canceled, then gracefully stops it.
func runDebugServer(ctx context.Context, addr string) {
	l := logrus.WithField("component", "debug")

	http.Handle("/debug/metrics", promhttp.Handler())

	handlers := []string{
		"/debug/metrics", "/debug/vars", "/debug/requests", "/debug/events", "/debug/pprof",
	}
	for i, h := range handlers {
		handlers[i] = "http://" + addr + h
	}

	var buf bytes.Buffer
	err := template.Must(template.New("debug").Parse(`

	{{ range . }}

{{ . }}

	{{ end }}

	`)).Execute(&buf, handlers)
	if err != nil {
		l.Panic(err)
	}
	http.HandleFunc("/debug", func(rw http.ResponseWriter, req *http.Request) {
		rw.Write(buf.Bytes())
	})
	l.Infof("Starting server on http://%s/debug\nRegistered handlers:\n\t%s", addr, strings.Join(handlers, "\n\t"))

	server := &http.Server{
		Addr:     addr,
		ErrorLog: log.New(os.Stderr, "runDebugServer: ", 0),
	}
	go func() {
		if err := server.ListenAndServe(); err != http.ErrServerClosed {
			l.Panic(err)
		}
		l.Info("Server stopped.")
	}()

	<-ctx.Done()
	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
	if err := server.Shutdown(ctx); err != nil {
		l.Errorf("Failed to shutdown gracefully: %s", err)
	}
	cancel()
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("stdlog: ")

	var (
		promAddrF     = kingpin.Flag("listen-prom-addr", "Prometheus remote API server listen address").Default("127.0.0.1:7781").String()
		debugAddrF    = kingpin.Flag("listen-debug-addr", "Debug server listen address").Default("127.0.0.1:7782").String()
		dropF         = kingpin.Flag("db.drop-schema", "Drop existing database schema").Bool()
		maxOpenConnsF = kingpin.Flag("db.max-open-conns", "Maximum number of open connections to the database").Default("75").Int()
		logLevelF     = kingpin.Flag("log.level", "Log level").Default("warn").String()
	)
	kingpin.CommandLine.HelpFlag.Short('h')
	kingpin.Parse()

	level, err := logrus.ParseLevel(*logLevelF)
	if err != nil {
		logrus.Fatal(err)
	}
	logrus.SetLevel(level)

	l := logrus.WithField("component", "main")
	ctx, cancel := context.WithCancel(context.Background())
	defer l.Info("Done.")

	// handle termination signals: first one gracefully, force exit on the second one
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
	go func() {
		s := <-signals
		l.Warnf("Got %s, shutting down...", unix.SignalName(s.(syscall.Signal)))
		cancel()

		s = <-signals
		l.Panicf("Got %s, exiting!", s, unix.SignalName(s.(syscall.Signal)))
	}()

	// start servers, wait for them to exit
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		params := &clickhouse.Params{
			DSN:                  "tcp://127.0.0.1:9000/?database=prometheus",
			DropDatabase:         *dropF,
			MaxOpenConns:         *maxOpenConnsF,
			MaxTimeSeriesInQuery: 50,
		}
		runPromServer(ctx, *promAddrF, params)
	}()
	go func() {
		defer wg.Done()
		runDebugServer(ctx, *debugAddrF)
	}()
	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strings"

	lru "github.com/hashicorp/golang-lru"
	"github.com/hashicorp/hcl"
	opensmtpd "gopkg.in/opensmtpd.v0"
)

var (
	cache   *lru.Cache
	ignored []*net.IPNet
	config  struct {
		Cache  int
		Ignore []string
		Accept []string
		Reject []string
	}
)

func debugf(format string, args ...interface{}) {
	log.Printf("debug: "+format, args...)
}

func update() (int, error) {
	log.Println("table-rbl: update")
	return 1, nil
}

func reverse(ip net.IP) net.IP {
	log.Printf("ip: %#+v", ip)
	return net.IP{ip[3], ip[2], ip[1], ip[0]}
}

func lookup(rbl string, host net.IP) (result string, listed bool, err error) {
	var (
		query   = fmt.Sprintf("%s.%s", host, rbl)
		results []string
	)
	log.Printf("table-rbl: lookup %q", query)
	if results, err = net.LookupHost(query); err != nil {
		if strings.HasSuffix(err.Error(), ": no such host") {
			err = nil
		}
		return
	}

	if listed = len(results) > 0; listed {
		txts, _ := net.LookupTXT(query)
		if len(txts) > 0 {
			result = txts[0]
		}
	}
	return
}

func check(service int, params opensmtpd.Dict, key string) (int, error) {
	log.Printf("table-rbl: check key=%q", key)
	if key == "local" {
		return 1, nil
	}

	ips, err := net.LookupIP(key)
	if err != nil {
		log.Printf("table-rbl: error looking up %q: %v", key, err)
		return -1, err
	}

	for _, ip := range ips {
		if ip = ip.To4(); ip == nil {
			continue
		}
		log.Printf("table-rbl: %q resolved to %s (%s)", key, ip, reverse(ip))
		for _, network := range ignored {
			if network.Contains(ip) {
				log.Printf("table-rbl: %s is ignored", ip)
				return 1, nil
			}
		}

		if result, block := cache.Get(key); block && result.(string) != "" {
			log.Printf("table-rbl: reject %s (reason %q)", ip, result)
			return 0, nil
		}

		var (
			result string
			listed bool
			host   = reverse(ip)
			err    error
		)
		for _, rbl := range config.Accept {
			if result, listed, err = lookup(rbl, host); err != nil {
				log.Printf("table-rbl: error looking up %q in %q: %v", host, rbl, err)
				return -1, nil
			} else if listed {
				log.Printf("table-rbl: accept %q (reason %q)", ip, result)
				return 1, nil
			}
		}
		for _, rbl := range config.Reject {
			if result, listed, err = lookup(rbl, host); err != nil {
				log.Printf("table-rbl: error looking up %q in %q: %v", host, rbl, err)
				return -1, nil
			} else if listed {
				log.Printf("table-rbl: reject %q (reason %q)", ip, result)
				return 0, nil
			}
		}
	}

	log.Printf("table-rbl: accept %q (not rejected)", key)
	return 1, nil
}

func main() {
	flag.Parse()
	if flag.NArg() != 1 {
		panic(fmt.Sprintf("%s <config>\n", os.Args[0]))
	}
	log.Printf("table-rbl: args=%v", flag.Args())

	b, err := ioutil.ReadFile(flag.Arg(0))
	if err != nil {
		log.Fatalln("table-rbl", err)
	}
	if err = hcl.Unmarshal(b, &config); err != nil {
		log.Fatalln("table-rbl", err)
	}
	if len(config.Reject) == 0 {
		log.Fatalln("table-rbl: no reject rules configured")
	}

	// Setup cache
	if config.Cache == 0 {
		cache, err = lru.New(1024)
	} else {
		cache, err = lru.New(config.Cache)
	}
	if err != nil {
		log.Fatalln("table-rbl", err)
	}

	// Parse ignore rules
	for _, prefix := range config.Ignore {
		var ipnet *net.IPNet
		if _, ipnet, err = net.ParseCIDR(prefix); err != nil {
			panic(err)
		}
		ignored = append(ignored, ipnet)
		debugf("ignore %s", ipnet)
	}

	opensmtpd.Debug = true

	table := &opensmtpd.Table{
		Update: update,
		Check:  check,
		Close: func() error {
			log.Println("table-rbl: close")
			return nil
		},
	}
	log.Fatalln(table.Serve())
}
<commit_msg>Bump API version<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"strings"

	lru "github.com/hashicorp/golang-lru"
	"github.com/hashicorp/hcl"
	opensmtpd "gopkg.in/opensmtpd.v52"
)

var (
	cache   *lru.Cache
	ignored []*net.IPNet
	config  struct {
		Cache  int
		Ignore []string
		Accept []string
		Reject []string
	}
)

func debugf(format string, args ...interface{}) {
	log.Printf("debug: "+format, args...)
}

func update() (int, error) {
	log.Println("table-rbl: update")
	return 1, nil
}

func reverse(ip net.IP) net.IP {
	log.Printf("ip: %#+v", ip)
	return net.IP{ip[3], ip[2], ip[1], ip[0]}
}

func lookup(rbl string, host net.IP) (result string, listed bool, err error) {
	var (
		query   = fmt.Sprintf("%s.%s", host, rbl)
		results []string
	)
	log.Printf("table-rbl: lookup %q", query)
	if results, err = net.LookupHost(query); err != nil {
		if strings.HasSuffix(err.Error(), ": no such host") {
			err = nil
		}
		return
	}

	if listed = len(results) > 0; listed {
		txts, _ := net.LookupTXT(query)
		if len(txts) > 0 {
			result = txts[0]
		}
	}
	return
}

func check(service int, params opensmtpd.Dict, key string) (int, error) {
	log.Printf("table-rbl: check key=%q", key)
	if key == "local" {
		return 1, nil
	}

	ips, err := net.LookupIP(key)
	if err != nil {
		log.Printf("table-rbl: error looking up %q: %v", key, err)
		return -1, err
	}

	for _, ip := range ips {
		if ip = ip.To4(); ip == nil {
			continue
		}
		log.Printf("table-rbl: %q resolved to %s (%s)", key, ip, reverse(ip))
		for _, network := range ignored {
			if network.Contains(ip) {
				log.Printf("table-rbl: %s is ignored", ip)
				return 1, nil
			}
		}

		if result, block := cache.Get(key); block && result.(string) != "" {
			log.Printf("table-rbl: reject %s (reason %q)", ip, result)
			return 0, nil
		}

		var (
			result string
			listed bool
			host   = reverse(ip)
			err    error
		)
		for _, rbl := range config.Accept {
			if result, listed, err = lookup(rbl, host); err != nil {
				log.Printf("table-rbl: error looking up %q in %q: %v", host, rbl, err)
				return -1, nil
			} else if listed {
				log.Printf("table-rbl: accept %q (reason %q)", ip, result)
				return 1, nil
			}
		}
		for _, rbl := range config.Reject {
			if result, listed, err = lookup(rbl, host); err != nil {
				log.Printf("table-rbl: error looking up %q in %q: %v", host, rbl, err)
				return -1, nil
			} else if listed {
				log.Printf("table-rbl: reject %q (reason %q)", ip, result)
				return 0, nil
			}
		}
	}

	log.Printf("table-rbl: accept %q (not rejected)", key)
	return 1, nil
}

func main() {
	flag.Parse()
	if flag.NArg() != 1 {
		panic(fmt.Sprintf("%s <config>\n", os.Args[0]))
	}
	log.Printf("table-rbl: args=%v", flag.Args())

	b, err := ioutil.ReadFile(flag.Arg(0))
	if err != nil {
		log.Fatalln("table-rbl", err)
	}
	if err = hcl.Unmarshal(b, &config); err != nil {
		log.Fatalln("table-rbl", err)
	}
	if len(config.Reject) == 0 {
		log.Fatalln("table-rbl: no reject rules configured")
	}

	// Setup cache
	if config.Cache == 0 {
		cache, err = lru.New(1024)
	} else {
		cache, err = lru.New(config.Cache)
	}
	if err != nil {
		log.Fatalln("table-rbl", err)
	}

	// Parse ignore rules
	for _, prefix := range config.Ignore {
		var ipnet *net.IPNet
		if _, ipnet, err = net.ParseCIDR(prefix); err != nil {
			panic(err)
		}
		ignored = append(ignored, ipnet)
		debugf("ignore %s", ipnet)
	}

	opensmtpd.Debug = true

	table := &opensmtpd.Table{
		Update: update,
		Check:  check,
		Close: func() error {
			log.Println("table-rbl: close")
			return nil
		},
	}
	log.Fatalln(table.Serve())
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!plan9

package syslog

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"testing"
	"time"
)

var serverAddr string

func runSyslog(c net.PacketConn, done chan<- string) {
	var buf [4096]byte
	var rcvd string = ""
	for {
		n, _, err := c.ReadFrom(buf[0:])
		if err != nil || n == 0 {
			break
		}
		rcvd += string(buf[0:n])
	}
	done <- rcvd
}

func startServer(done chan<- string) {
	c, e := net.ListenPacket("udp", "127.0.0.1:0")
	if e != nil {
		log.Fatalf("net.ListenPacket failed udp :0 %v", e)
	}
	serverAddr = c.LocalAddr().String()
	c.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
	go runSyslog(c, done)
}

func skipNetTest(t *testing.T) bool {
	if testing.Short() {
		// Depends on syslog daemon running, and sometimes it's not.
		t.Logf("skipping syslog test during -short")
		return true
	}
	return false
}

func TestNew(t *testing.T) {
	if LOG_LOCAL7 != 23<<3 {
		t.Fatalf("LOG_LOCAL7 has wrong value")
	}
	if skipNetTest(t) {
		return
	}

	s, err := New(LOG_INFO|LOG_USER, "")
	if err != nil {
		t.Fatalf("New() failed: %s", err)
	}
	// Don't send any messages.
	s.Close()
}

func TestNewLogger(t *testing.T) {
	if skipNetTest(t) {
		return
	}
	f, err := NewLogger(LOG_USER|LOG_INFO, 0)
	if f == nil {
		t.Error(err)
	}
}

func TestDial(t *testing.T) {
	if skipNetTest(t) {
		return
	}
	f, err := Dial("", "", (LOG_LOCAL7|LOG_DEBUG)+1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	f, err = Dial("", "", -1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	l, err := Dial("", "", LOG_USER|LOG_ERR, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %s", err)
	}
	l.Close()
}

func TestUDPDial(t *testing.T) {
	done := make(chan string)
	startServer(done)
	l, err := Dial("udp", serverAddr, LOG_USER|LOG_INFO, "syslog_test")
	if err != nil {
		t.Fatalf("syslog.Dial() failed: %s", err)
	}
	msg := "udp test"
	l.Info(msg)
	expected := fmt.Sprintf("<%d>1 ", LOG_USER+LOG_INFO) + "%s %s syslog_test[%d]: udp test\n"
	rcvd := <-done
	var parsedHostname, timestamp string
	var pid int
	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		if n, err := fmt.Sscanf(rcvd, expected, &timestamp, &parsedHostname, &pid); n != 3 ||
			err != nil || hostname != parsedHostname {
			t.Fatalf("s.Info() = '%q', didn't match '%q'", rcvd, expected)
		}
	}
}

func TestWrite(t *testing.T) {
	tests := []struct {
		pri Priority
		pre string
		msg string
		exp string
	}{
		{LOG_USER | LOG_ERR, "syslog_test", "", "%s %s syslog_test[%d]: \n"},
		{LOG_USER | LOG_ERR, "syslog_test", "write test", "%s %s syslog_test[%d]: write test\n"},
		// Write should not add \n if there already is one
		{LOG_USER | LOG_ERR, "syslog_test", "write test 2\n", "%s %s syslog_test[%d]: write test 2\n"},
	}

	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		for _, test := range tests {
			done := make(chan string)
			startServer(done)
			l, err := Dial("udp", serverAddr, test.pri, test.pre)
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %s", err)
			}
			_, err = io.WriteString(l, test.msg)
			if err != nil {
				t.Fatalf("WriteString() failed: %s", err)
			}
			rcvd := <-done
			test.exp = fmt.Sprintf("<%d>1 ", test.pri) + test.exp
			var parsedHostname, timestamp string
			var pid int
			if n, err := fmt.Sscanf(rcvd, test.exp, &timestamp, &parsedHostname, &pid); n != 3 ||
				err != nil || hostname != parsedHostname {
				t.Fatalf("s.Info() = '%q', didn't match '%q'", rcvd, test.exp)
			}
		}
	}
}
<commit_msg>log/syslog: fix flakey test on slow hosts<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!plan9

package syslog

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"testing"
	"time"
)

var serverAddr string

func runSyslog(c net.PacketConn, done chan<- string) {
	var buf [4096]byte
	var rcvd string
	for {
		c.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
		n, _, err := c.ReadFrom(buf[:])
		rcvd += string(buf[:n])
		if err != nil {
			break
		}
	}
	done <- rcvd
}

func startServer(done chan<- string) {
	c, e := net.ListenPacket("udp", "127.0.0.1:0")
	if e != nil {
		log.Fatalf("net.ListenPacket failed udp :0 %v", e)
	}
	serverAddr = c.LocalAddr().String()
	go runSyslog(c, done)
}

func skipNetTest(t *testing.T) bool {
	if testing.Short() {
		// Depends on syslog daemon running, and sometimes it's not.
		t.Logf("skipping syslog test during -short")
		return true
	}
	return false
}

func TestNew(t *testing.T) {
	if LOG_LOCAL7 != 23<<3 {
		t.Fatalf("LOG_LOCAL7 has wrong value")
	}
	if skipNetTest(t) {
		return
	}

	s, err := New(LOG_INFO|LOG_USER, "")
	if err != nil {
		t.Fatalf("New() failed: %s", err)
	}
	// Don't send any messages.
	s.Close()
}

func TestNewLogger(t *testing.T) {
	if skipNetTest(t) {
		return
	}
	f, err := NewLogger(LOG_USER|LOG_INFO, 0)
	if f == nil {
		t.Error(err)
	}
}

func TestDial(t *testing.T) {
	if skipNetTest(t) {
		return
	}
	f, err := Dial("", "", (LOG_LOCAL7|LOG_DEBUG)+1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	f, err = Dial("", "", -1, "syslog_test")
	if f != nil {
		t.Fatalf("Should have trapped bad priority")
	}
	l, err := Dial("", "", LOG_USER|LOG_ERR, "syslog_test")
	if err != nil {
		t.Fatalf("Dial() failed: %s", err)
	}
	l.Close()
}

func TestUDPDial(t *testing.T) {
	done := make(chan string)
	startServer(done)
	l, err := Dial("udp", serverAddr, LOG_USER|LOG_INFO, "syslog_test")
	if err != nil {
		t.Fatalf("syslog.Dial() failed: %s", err)
	}
	msg := "udp test"
	l.Info(msg)
	expected := fmt.Sprintf("<%d>1 ", LOG_USER+LOG_INFO) + "%s %s syslog_test[%d]: udp test\n"
	rcvd := <-done
	var parsedHostname, timestamp string
	var pid int
	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		if n, err := fmt.Sscanf(rcvd, expected, &timestamp, &parsedHostname, &pid); n != 3 ||
			err != nil || hostname != parsedHostname {
			t.Fatalf("s.Info() = '%q', didn't match '%q'", rcvd, expected)
		}
	}
}

func TestWrite(t *testing.T) {
	tests := []struct {
		pri Priority
		pre string
		msg string
		exp string
	}{
		{LOG_USER | LOG_ERR, "syslog_test", "", "%s %s syslog_test[%d]: \n"},
		{LOG_USER | LOG_ERR, "syslog_test", "write test", "%s %s syslog_test[%d]: write test\n"},
		// Write should not add \n if there already is one
		{LOG_USER | LOG_ERR, "syslog_test", "write test 2\n", "%s %s syslog_test[%d]: write test 2\n"},
	}

	if hostname, err := os.Hostname(); err != nil {
		t.Fatalf("Error retrieving hostname")
	} else {
		for _, test := range tests {
			done := make(chan string)
			startServer(done)
			l, err := Dial("udp", serverAddr, test.pri, test.pre)
			if err != nil {
				t.Fatalf("syslog.Dial() failed: %s", err)
			}
			_, err = io.WriteString(l, test.msg)
			if err != nil {
				t.Fatalf("WriteString() failed: %s", err)
			}
			rcvd := <-done
			test.exp = fmt.Sprintf("<%d>1 ", test.pri) + test.exp
			var parsedHostname, timestamp string
			var pid int
			if n, err := fmt.Sscanf(rcvd, test.exp, &timestamp, &parsedHostname, &pid); n != 3 ||
				err != nil || hostname != parsedHostname {
				t.Fatalf("s.Info() = '%q', didn't match '%q'", rcvd, test.exp)
			}
		}
	}
}
<|endoftext|>