<commit_before>package susigo

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"time"
)

type Callback func(*Event)

type Susi struct {
	cert             tls.Certificate
	addr             string
	connected        bool
	conn             net.Conn
	encoder          *json.Encoder
	decoder          *json.Decoder
	callbacks        map[string]Callback
	consumers        map[string]map[int64]Callback
	processors       map[string]map[int64]Callback
	publishProcesses map[string][]Callback
}

type Event struct {
	Topic     string              `json:"topic"`
	Payload   interface{}         `json:"payload"`
	Headers   []map[string]string `json:"headers"`
	ID        string              `json:"id"`
	SessionID string              `json:"sessionid"`
}

type Message struct {
	Type string `json:"type"`
	Data *Event `json:"data"`
}

func NewSusi(addr, certFile, keyFile string) (*Susi, error) {
	susi := new(Susi)
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return nil, err
	}
	susi.cert = cert
	susi.callbacks = make(map[string]Callback)
	susi.addr = addr
	susi.consumers = make(map[string]map[int64]Callback)
	susi.processors = make(map[string]map[int64]Callback)
	susi.publishProcesses = make(map[string][]Callback)
	susi.connected = false
	go susi.backend()
	return susi, nil
}

func (susi *Susi) Publish(event Event, callback Callback) error {
	var id = event.ID
	if event.ID == "" {
		id = strconv.FormatInt(time.Now().UnixNano(), 10)
		event.ID = id
	}
	event.ID = id
	packet := map[string]interface{}{
		"type": "publish",
		"data": event,
	}
	susi.callbacks[id] = callback
	return susi.encoder.Encode(packet)
}

func (susi *Susi) RegisterConsumer(topic string, callback Callback) (int64, error) {
	if susi.consumers[topic] == nil {
		susi.consumers[topic] = make(map[int64]Callback)
	}
	consumers := susi.consumers[topic]
	id := time.Now().UnixNano()
	consumers[id] = callback
	if len(consumers) == 1 {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		return id, susi.encoder.Encode(packet)
	}
	return -1, nil
}

func (susi *Susi) RegisterProcessor(topic string, callback Callback) (int64, error) {
	if susi.processors[topic] == nil {
		susi.processors[topic] = make(map[int64]Callback)
	}
	processors := susi.processors[topic]
	id := time.Now().UnixNano()
	processors[id] = callback
	if len(processors) == 1 {
		packet := map[string]interface{}{
			"type": "registerProcessor",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		return id, susi.encoder.Encode(packet)
	}
	return -1, nil
}

func (susi *Susi) UnregisterConsumer(id int64) error {
	for _, consumers := range susi.consumers {
		if topic, ok := consumers[id]; ok {
			delete(consumers, id)
			if len(consumers) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterConsumer",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				return susi.encoder.Encode(packet)
			}
			return nil
		}
	}
	return errors.New("no such consumer")
}

func (susi *Susi) UnregisterProcessor(id int64) error {
	for _, processors := range susi.processors {
		if topic, ok := processors[id]; ok {
			delete(processors, id)
			if len(processors) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterProcessor",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				return susi.encoder.Encode(packet)
			}
			return nil
		}
	}
	return errors.New("no such processor")
}

func (susi *Susi) connect() error {
	conn, err := tls.Dial("tcp", susi.addr, &tls.Config{
		Certificates:       []tls.Certificate{susi.cert},
		InsecureSkipVerify: true,
	})
	if err != nil {
		log.Printf("failed connecting susi-core (%v), retry...", err)
		return err
	}
	susi.conn = conn
	susi.connected = true
	susi.encoder = json.NewEncoder(susi.conn)
	susi.decoder = json.NewDecoder(susi.conn)
	for consumerTopic := range susi.consumers {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": consumerTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	for processorTopic := range susi.processors {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": processorTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	return nil
}

func (susi *Susi) backend() {
	packet := Message{}
	for {
		if !susi.connected {
			err := susi.connect()
			if err != nil {
				time.Sleep(1 * time.Second)
				continue
			}
		}
		err := susi.decoder.Decode(&packet)
		if err != nil {
			log.Println("Error reading from susi json decoder: ", err)
			susi.connected = false
			continue
		}
		switch packet.Type {
		case "ack":
			{
				event := packet.Data
				id := event.ID
				callback := susi.callbacks[id]
				callback(event)
				delete(susi.callbacks, id)
			}
		case "consumerEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingConsumers []Callback
				for pattern, consumers := range susi.consumers {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, consumer := range consumers {
							matchingConsumers = append(matchingConsumers, consumer)
						}
					}
				}
				for _, consumer := range matchingConsumers {
					consumer(event)
				}
			}
		case "processorEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingProcessors []Callback
				for pattern, processors := range susi.processors {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, processor := range processors {
							matchingProcessors = append(matchingProcessors, processor)
						}
					}
				}
				susi.publishProcesses[event.ID] = matchingProcessors
				susi.Ack(event)
			}
		}
	}
}

func (susi *Susi) Ack(event *Event) error {
	if process, ok := susi.publishProcesses[event.ID]; ok {
		if len(process) == 0 {
			packet := map[string]interface{}{
				"type": "ack",
				"data": event,
			}
			delete(susi.publishProcesses, event.ID)
			return susi.encoder.Encode(packet)
		}
		cb := process[0]
		process = process[1:]
		cb(event)
		return nil
	}
	return errors.New("no publish process found")
}

func (susi *Susi) Dismiss(event *Event) error {
	if _, ok := susi.publishProcesses[event.ID]; ok {
		packet := map[string]interface{}{
			"type": "dismiss",
			"data": event,
		}
		delete(susi.publishProcesses, event.ID)
		return susi.encoder.Encode(packet)
	}
	return errors.New("no publish process found")
}
<commit_msg>dont send if susi is not connected;<commit_after>package susigo

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"time"
)

type Callback func(*Event)

type Susi struct {
	cert             tls.Certificate
	addr             string
	connected        bool
	conn             net.Conn
	encoder          *json.Encoder
	decoder          *json.Decoder
	callbacks        map[string]Callback
	consumers        map[string]map[int64]Callback
	processors       map[string]map[int64]Callback
	publishProcesses map[string][]Callback
}

type Event struct {
	Topic     string              `json:"topic"`
	Payload   interface{}         `json:"payload"`
	Headers   []map[string]string `json:"headers"`
	ID        string              `json:"id"`
	SessionID string              `json:"sessionid"`
}

type Message struct {
	Type string `json:"type"`
	Data *Event `json:"data"`
}

func NewSusi(addr, certFile, keyFile string) (*Susi, error) {
	susi := new(Susi)
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return nil, err
	}
	susi.cert = cert
	susi.callbacks = make(map[string]Callback)
	susi.addr = addr
	susi.consumers = make(map[string]map[int64]Callback)
	susi.processors = make(map[string]map[int64]Callback)
	susi.publishProcesses = make(map[string][]Callback)
	susi.connected = false
	go susi.backend()
	return susi, nil
}

func (susi *Susi) Publish(event Event, callback Callback) error {
	if !susi.connected {
		return errors.New("susi not connected")
	}
	var id = event.ID
	if event.ID == "" {
		id = strconv.FormatInt(time.Now().UnixNano(), 10)
		event.ID = id
	}
	event.ID = id
	packet := map[string]interface{}{
		"type": "publish",
		"data": event,
	}
	susi.callbacks[id] = callback
	return susi.encoder.Encode(packet)
}

func (susi *Susi) RegisterConsumer(topic string, callback Callback) (int64, error) {
	if susi.consumers[topic] == nil {
		susi.consumers[topic] = make(map[int64]Callback)
	}
	consumers := susi.consumers[topic]
	id := time.Now().UnixNano()
	consumers[id] = callback
	if len(consumers) == 1 {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		if susi.connected {
			return id, susi.encoder.Encode(packet)
		}
		return id, nil
	}
	return -1, nil
}

func (susi *Susi) RegisterProcessor(topic string, callback Callback) (int64, error) {
	if susi.processors[topic] == nil {
		susi.processors[topic] = make(map[int64]Callback)
	}
	processors := susi.processors[topic]
	id := time.Now().UnixNano()
	processors[id] = callback
	if len(processors) == 1 {
		packet := map[string]interface{}{
			"type": "registerProcessor",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		if susi.connected {
			return id, susi.encoder.Encode(packet)
		}
		return id, nil
	}
	return -1, nil
}

func (susi *Susi) UnregisterConsumer(id int64) error {
	for _, consumers := range susi.consumers {
		if topic, ok := consumers[id]; ok {
			delete(consumers, id)
			if len(consumers) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterConsumer",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				if susi.connected {
					return susi.encoder.Encode(packet)
				}
				return nil
			}
			return nil
		}
	}
	return errors.New("no such consumer")
}

func (susi *Susi) UnregisterProcessor(id int64) error {
	for _, processors := range susi.processors {
		if topic, ok := processors[id]; ok {
			delete(processors, id)
			if len(processors) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterProcessor",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				if susi.connected {
					return susi.encoder.Encode(packet)
				}
				return nil
			}
			return nil
		}
	}
	return errors.New("no such processor")
}

func (susi *Susi) connect() error {
	conn, err := tls.Dial("tcp", susi.addr, &tls.Config{
		Certificates:       []tls.Certificate{susi.cert},
		InsecureSkipVerify: true,
	})
	if err != nil {
		log.Printf("failed connecting susi-core (%v), retry...", err)
		return err
	}
	susi.conn = conn
	susi.connected = true
	susi.encoder = json.NewEncoder(susi.conn)
	susi.decoder = json.NewDecoder(susi.conn)
	for consumerTopic := range susi.consumers {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": consumerTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	for processorTopic := range susi.processors {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": processorTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	return nil
}

func (susi *Susi) backend() {
	packet := Message{}
	for {
		if !susi.connected {
			err := susi.connect()
			if err != nil {
				time.Sleep(1 * time.Second)
				continue
			}
		}
		err := susi.decoder.Decode(&packet)
		if err != nil {
			log.Println("Error reading from susi json decoder: ", err)
			susi.connected = false
			continue
		}
		switch packet.Type {
		case "ack":
			{
				event := packet.Data
				id := event.ID
				callback := susi.callbacks[id]
				callback(event)
				delete(susi.callbacks, id)
			}
		case "consumerEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingConsumers []Callback
				for pattern, consumers := range susi.consumers {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, consumer := range consumers {
							matchingConsumers = append(matchingConsumers, consumer)
						}
					}
				}
				for _, consumer := range matchingConsumers {
					consumer(event)
				}
			}
		case "processorEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingProcessors []Callback
				for pattern, processors := range susi.processors {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, processor := range processors {
							matchingProcessors = append(matchingProcessors, processor)
						}
					}
				}
				susi.publishProcesses[event.ID] = matchingProcessors
				susi.Ack(event)
			}
		}
	}
}

func (susi *Susi) Ack(event *Event) error {
	if process, ok := susi.publishProcesses[event.ID]; ok {
		if len(process) == 0 {
			packet := map[string]interface{}{
				"type": "ack",
				"data": event,
			}
			delete(susi.publishProcesses, event.ID)
			if susi.connected {
				return susi.encoder.Encode(packet)
			}
			return nil
		}
		cb := process[0]
		process = process[1:]
		cb(event)
		return nil
	}
	return errors.New("no publish process found")
}

func (susi *Susi) Dismiss(event *Event) error {
	if _, ok := susi.publishProcesses[event.ID]; ok {
		packet := map[string]interface{}{
			"type": "dismiss",
			"data": event,
		}
		delete(susi.publishProcesses, event.ID)
		if susi.connected {
			return susi.encoder.Encode(packet)
		}
		return nil
	}
	return errors.New("no publish process found")
}
<|endoftext|>


<commit_before>package main

import "github.com/jangler/edit"

func mustCompile(pattern string, id int) edit.Rule {
	rule, err := edit.NewRule(pattern, id)
	if err != nil {
		panic(err)
	}
	return rule
}

const (
	commentId = iota
	keywordId
	literalId
)

var (
	goRules = []edit.Rule{
		mustCompile(`//.+?$`, commentId),
		mustCompile(`/*.*?\*/`, commentId),
		mustCompile(`\b(break|case|chan|const|continue|default|defer|else|`+
			`fallthrough|for|func|go|goto|if|import|interface|map|package|`+
			`range|return|select|struct|switch|type|var)\b`, keywordId),
		mustCompile(`\b(close|len|cap|new|make|append|copy|delete|complex|`+
			`real|imag|panic|recover|print|println)\b`, keywordId),
		mustCompile(`'.*?'`, literalId),
		mustCompile(`".*?"`, literalId),
		mustCompile("`.*?`", literalId),
		mustCompile(`\b\d+\b`, literalId),
		mustCompile(`\bnil\b`, literalId),
	}
	jsonRules = []edit.Rule{
		mustCompile(`"([^"]|\\")*?":`, keywordId),
		mustCompile(`"([^"]|\\")*?"`, literalId),
		mustCompile(`\b(\d*\.)?\d+\b`, literalId),
	}
)
<commit_msg>Improve golang syntax rules<commit_after>package main

import "github.com/jangler/edit"

func mustCompile(pattern string, id int) edit.Rule {
	rule, err := edit.NewRule(pattern, id)
	if err != nil {
		panic(err)
	}
	return rule
}

const (
	commentId = iota
	keywordId
	literalId
)

var (
	goRules = []edit.Rule{
		mustCompile(`//.+?$`, commentId),
		mustCompile(`/\*.*?\*/`, commentId),
		mustCompile(`\b(break|case|chan|const|continue|default|defer|else|`+
			`fallthrough|for|func|go|goto|if|import|interface|map|package|`+
			`range|return|select|struct|switch|type|var)\b`, keywordId),
		mustCompile(`\b(append|cap|close|complex|copy|delete|imag|len|make|`+
			`new|panic|print|println|real|recover)\b`, keywordId),
		mustCompile(`\b(bool|byte|complex(64|128)|error|float(32|64)|`+
			`u?int(8|16|32|64)?|rune|string|uintptr)\b`, keywordId),
		mustCompile(`'([^']|\\')*?'`, literalId),
		mustCompile(`"([^"]|\\")*?"`, literalId),
		mustCompile("`.*?`", literalId),
		mustCompile(`\b(0[bx])?(\d*\.)?\d+\b`, literalId),
		mustCompile(`\btrue|false|iota|nil\b`, literalId),
	}
	jsonRules = []edit.Rule{
		mustCompile(`"([^"]|\\")*?":`, keywordId),
		mustCompile(`"([^"]|\\")*?"`, literalId),
		mustCompile(`\b(\d*\.)?\d+\b`, literalId),
	}
)
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

// Package fileupdater 编辑内容时更新相关引用文件的关联关系
// 用法：fileupdater.New(ctx,fileModel.NewEmbedded(ctx)).Set(`表名称`,`字段名称`,主键ID).Add(`/test/image.jpg`,false)
package fileupdater

func New(reler Reler) *FileUpdater {
	return &FileUpdater{
		rel: reler,
	}
}

type FileUpdater struct {
	rel     Reler
	project string
	table   string
	field   string
	tableID uint64
}

func (f *FileUpdater) Add(content string, embedded bool) (err error) {
	if len(content) == 0 {
		return
	}
	err = f.Edit(content, embedded)
	return
}

func (f *FileUpdater) Edit(content string, embedded bool) (err error) {
	if !embedded {
		err = f.rel.RelationFiles(f.project, f.table, f.field, f.tableID, content)
		return
	}
	err = f.rel.RelationEmbeddedFiles(f.project, f.table, f.field, f.tableID, content)
	return
}

func (f *FileUpdater) Delete() (err error) {
	err = f.rel.DeleteByTableID(f.project, f.table, f.tableID)
	return
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

// Package fileupdater 编辑内容时更新相关引用文件的关联关系
// 用法：fileupdater.New(fileModel.NewEmbedded(ctx)).Set(`表名称`,`字段名称`,主键ID).Add(`/test/image.jpg`,false)
package fileupdater

func New(reler Reler) *FileUpdater {
	return &FileUpdater{
		rel: reler,
	}
}

type FileUpdater struct {
	rel     Reler
	project string
	table   string
	field   string
	tableID uint64
}

func (f *FileUpdater) Add(content string, embedded bool) (err error) {
	if len(content) == 0 {
		return
	}
	err = f.Edit(content, embedded)
	return
}

func (f *FileUpdater) Edit(content string, embedded bool) (err error) {
	if !embedded {
		err = f.rel.RelationFiles(f.project, f.table, f.field, f.tableID, content)
		return
	}
	err = f.rel.RelationEmbeddedFiles(f.project, f.table, f.field, f.tableID, content)
	return
}

func (f *FileUpdater) Delete() (err error) {
	err = f.rel.DeleteByTableID(f.project, f.table, f.tableID)
	return
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!nacl,!plan9

package syslog

import (
	"errors"
	"log"
	"net"
	"os"
	"strconv"
	"sync"
	"time"
)

// The Priority is a combination of the syslog facility and
// severity. For example, LOG_ALERT | LOG_FTP sends an alert severity
// message from the FTP facility. The default severity is LOG_EMERG;
// the default facility is LOG_KERN.
type Priority int

const severityMask = 0x07
const facilityMask = 0xf8

const (
	// Severity.

	// From /usr/include/sys/syslog.h.
	// These are the same on Linux, BSD, and OS X.
	LOG_EMERG Priority = iota
	LOG_ALERT
	LOG_CRIT
	LOG_ERR
	LOG_WARNING
	LOG_NOTICE
	LOG_INFO
	LOG_DEBUG
)

const (
	// Facility.

	// From /usr/include/sys/syslog.h.
	// These are the same up to LOG_FTP on Linux, BSD, and OS X.
	LOG_KERN Priority = iota << 3
	LOG_USER
	LOG_MAIL
	LOG_DAEMON
	LOG_AUTH
	LOG_SYSLOG
	LOG_LPR
	LOG_NEWS
	LOG_UUCP
	LOG_CRON
	LOG_AUTHPRIV
	LOG_FTP
	_ // unused
	_ // unused
	_ // unused
	_ // unused
	LOG_LOCAL0
	LOG_LOCAL1
	LOG_LOCAL2
	LOG_LOCAL3
	LOG_LOCAL4
	LOG_LOCAL5
	LOG_LOCAL6
	LOG_LOCAL7
)

const maxBufSize = 1024 * 1024 * 20 // 20MB

// A Writer is a connection to a syslog server.
type Writer struct {
	priority Priority
	tag      string
	hostname string
	network  string
	raddr    string

	mu   sync.Mutex // guards conn
	conn *netConn

	b    []byte // buffer for formatted messages
	pid  int    // cached process pid
	bpid []byte
}

// This interface and the separate syslog_unix.go file exist for
// Solaris support as implemented by gccgo. On Solaris you cannot
// simply open a TCP connection to the syslog daemon. The gccgo
// sources have a syslog_solaris.go file that implements unixSyslog to
// return a type that satisfies this interface and simply calls the C
// library syslog function.
type serverConn interface {
	write(p []byte) (int, error)
	close() error
}

type netConn struct {
	conn  net.Conn
	b     []byte // buffer for formatted messages
	pid   int    // cached process pid
	local bool
}

// New establishes a new connection to the system log daemon. Each
// write to the returned writer sends a log message with the given
// priority and prefix.
func New(priority Priority, tag string) (*Writer, error) {
	return Dial("", "", priority, tag)
}

// Dial establishes a connection to a log daemon by connecting to
// address raddr on the specified network. Each write to the returned
// writer sends a log message with the given facility, severity and
// tag.
// If network is empty, Dial will connect to the local syslog server.
func Dial(network, raddr string, priority Priority, tag string) (*Writer, error) {
	if priority < 0 || priority > LOG_LOCAL7|LOG_DEBUG {
		return nil, errors.New("log/syslog: invalid priority")
	}

	if tag == "" {
		tag = os.Args[0]
	}
	hostname, _ := os.Hostname()

	w := &Writer{
		priority: priority,
		tag:      tag,
		hostname: hostname,
		network:  network,
		raddr:    raddr,
	}

	w.mu.Lock()
	defer w.mu.Unlock()

	err := w.connect()
	if err != nil {
		return nil, err
	}
	return w, err
}

// connect makes a connection to the syslog server.
// It must be called with w.mu held.
func (w *Writer) connect() (err error) {
	if w.conn != nil {
		// ignore err from close, it makes sense to continue anyway
		w.conn.close()
		w.conn = nil
	}

	if w.network == "" {
		w.conn, err = unixSyslog()
		if w.hostname == "" {
			w.hostname = "localhost"
		}
	} else {
		var c net.Conn
		c, err = net.Dial(w.network, w.raddr)
		if err == nil {
			w.conn = &netConn{conn: c}
			if w.hostname == "" {
				w.hostname = c.LocalAddr().String()
			}
		}
	}
	return
}

// Write sends a log message to the syslog daemon.
func (w *Writer) Write(b []byte) (int, error) {
	return w.writeAndRetry(w.priority, string(b))
}

// Close closes a connection to the syslog daemon.
func (w *Writer) Close() error {
	w.mu.Lock()
	defer w.mu.Unlock()

	if w.conn != nil {
		err := w.conn.close()
		w.conn = nil
		return err
	}
	return nil
}

// Emerg logs a message with severity LOG_EMERG, ignoring the severity
// passed to New.
func (w *Writer) Emerg(m string) error {
	_, err := w.writeAndRetry(LOG_EMERG, m)
	return err
}

// Alert logs a message with severity LOG_ALERT, ignoring the severity
// passed to New.
func (w *Writer) Alert(m string) error {
	_, err := w.writeAndRetry(LOG_ALERT, m)
	return err
}

// Crit logs a message with severity LOG_CRIT, ignoring the severity
// passed to New.
func (w *Writer) Crit(m string) error {
	_, err := w.writeAndRetry(LOG_CRIT, m)
	return err
}

// Err logs a message with severity LOG_ERR, ignoring the severity
// passed to New.
func (w *Writer) Err(m string) error {
	_, err := w.writeAndRetry(LOG_ERR, m)
	return err
}

// Warning logs a message with severity LOG_WARNING, ignoring the
// severity passed to New.
func (w *Writer) Warning(m string) error {
	_, err := w.writeAndRetry(LOG_WARNING, m)
	return err
}

// Notice logs a message with severity LOG_NOTICE, ignoring the
// severity passed to New.
func (w *Writer) Notice(m string) error {
	_, err := w.writeAndRetry(LOG_NOTICE, m)
	return err
}

// Info logs a message with severity LOG_INFO, ignoring the severity
// passed to New.
func (w *Writer) Info(m string) error {
	_, err := w.writeAndRetry(LOG_INFO, m)
	return err
}

// Debug logs a message with severity LOG_DEBUG, ignoring the severity
// passed to New.
func (w *Writer) Debug(m string) error {
	_, err := w.writeAndRetry(LOG_DEBUG, m)
	return err
}

func (w *Writer) writeAndRetry(p Priority, s string) (n int, err error) {
	pr := (w.priority & facilityMask) | (p & severityMask)
	var msg []byte

	w.mu.Lock()
	if w.conn != nil {
		msg = w.format(pr, w.hostname, w.tag, s)
		if n, err = w.conn.write(msg); err == nil {
			w.mu.Unlock()
			return
		}
	}
	if err = w.connect(); err != nil {
		w.mu.Unlock()
		return
	}
	if msg == nil {
		msg = w.format(pr, w.hostname, w.tag, s)
	}
	n, err = w.conn.write(msg)
	w.mu.Unlock()
	return
}

// func (w *Writer) getpid() int {
func (w *Writer) getpid() []byte {
	if w.pid == 0 {
		w.pid = os.Getpid()
		w.bpid = strconv.AppendInt(make([]byte, 0, 8), int64(w.pid), 10)
	}
	return w.bpid
}

func formatBits(dst []byte, u uint64) []byte {
	const digits = "0123456789abcdefghijklmnopqrstuvwxyz"

	var a [64]byte // +1 for sign of 64bit value in base 2
	i := len(a)

	if ^uintptr(0)>>32 == 0 {
		for u > uint64(^uintptr(0)) {
			q := u / 1e9
			us := uintptr(u - q*1e9) // us % 1e9 fits into a uintptr
			for j := 9; j > 0; j-- {
				i--
				qs := us / 10
				a[i] = byte(us - qs*10 + '0')
				us = qs
			}
			u = q
		}
	}

	// u guaranteed to fit into a uintptr
	us := uintptr(u)
	for us >= 10 {
		i--
		q := us / 10
		a[i] = byte(us - q*10 + '0')
		us = q
	}
	// u < 10
	i--
	a[i] = byte(us + '0')

	return append(dst, a[i:]...)
}

func (w *Writer) format(p Priority, hostname, tag, msg string) []byte {
	b := w.b[0:0]
	b = append(b, '<')
	b = formatBits(b, uint64(p))
	b = append(b, '>')
	b = time.Now().AppendFormat(b, time.RFC3339)
	b = append(b, ' ')
	b = append(b, hostname...)
	b = append(b, ' ')
	b = append(b, tag...)
	b = append(b, '[')

	b = append(b, w.getpid()...)
	// b = strconv.AppendInt(b, int64(w.getpid()), 10)

	b = append(b, "]: "...)
	b = append(b, msg...)
	if b[len(b)-1] != '\n' {
		b = append(b, '\n')
	}
	if len(b) < maxBufSize {
		w.b = b
	}
	return b
}

func (n *netConn) write(p []byte) (int, error) {
	return n.conn.Write(p)
}

func (n *netConn) close() error {
	return n.conn.Close()
}

// NewLogger creates a log.Logger whose output is written to
// the system log service with the specified priority. The logFlag
// argument is the flag set passed through to log.New to create
// the Logger.
func NewLogger(p Priority, logFlag int) (*log.Logger, error) {
	s, err := New(p, "")
	if err != nil {
		return nil, err
	}
	return log.New(s, "", logFlag), nil
}
<commit_msg>Remove defered calls<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !windows,!nacl,!plan9

package syslog

import (
	"errors"
	"log"
	"net"
	"os"
	"strconv"
	"sync"
	"time"
)

// The Priority is a combination of the syslog facility and
// severity. For example, LOG_ALERT | LOG_FTP sends an alert severity
// message from the FTP facility. The default severity is LOG_EMERG;
// the default facility is LOG_KERN.
type Priority int

const severityMask = 0x07
const facilityMask = 0xf8

const (
	// Severity.

	// From /usr/include/sys/syslog.h.
	// These are the same on Linux, BSD, and OS X.
	LOG_EMERG Priority = iota
	LOG_ALERT
	LOG_CRIT
	LOG_ERR
	LOG_WARNING
	LOG_NOTICE
	LOG_INFO
	LOG_DEBUG
)

const (
	// Facility.

	// From /usr/include/sys/syslog.h.
	// These are the same up to LOG_FTP on Linux, BSD, and OS X.
	LOG_KERN Priority = iota << 3
	LOG_USER
	LOG_MAIL
	LOG_DAEMON
	LOG_AUTH
	LOG_SYSLOG
	LOG_LPR
	LOG_NEWS
	LOG_UUCP
	LOG_CRON
	LOG_AUTHPRIV
	LOG_FTP
	_ // unused
	_ // unused
	_ // unused
	_ // unused
	LOG_LOCAL0
	LOG_LOCAL1
	LOG_LOCAL2
	LOG_LOCAL3
	LOG_LOCAL4
	LOG_LOCAL5
	LOG_LOCAL6
	LOG_LOCAL7
)

const maxBufSize = 1024 * 1024 * 20 // 20MB

// A Writer is a connection to a syslog server.
type Writer struct {
	priority Priority
	tag      string
	hostname string
	network  string
	raddr    string

	mu   sync.Mutex // guards conn
	conn *netConn

	b    []byte // buffer for formatted messages
	pid  int    // cached process pid
	bpid []byte
}

// This interface and the separate syslog_unix.go file exist for
// Solaris support as implemented by gccgo. On Solaris you cannot
// simply open a TCP connection to the syslog daemon. The gccgo
// sources have a syslog_solaris.go file that implements unixSyslog to
// return a type that satisfies this interface and simply calls the C
// library syslog function.
type serverConn interface {
	write(p []byte) (int, error)
	close() error
}

type netConn struct {
	conn  net.Conn
	b     []byte // buffer for formatted messages
	pid   int    // cached process pid
	local bool
}

// New establishes a new connection to the system log daemon. Each
// write to the returned writer sends a log message with the given
// priority and prefix.
func New(priority Priority, tag string) (*Writer, error) {
	return Dial("", "", priority, tag)
}

// Dial establishes a connection to a log daemon by connecting to
// address raddr on the specified network. Each write to the returned
// writer sends a log message with the given facility, severity and
// tag.
// If network is empty, Dial will connect to the local syslog server.
func Dial(network, raddr string, priority Priority, tag string) (*Writer, error) {
	if priority < 0 || priority > LOG_LOCAL7|LOG_DEBUG {
		return nil, errors.New("log/syslog: invalid priority")
	}

	if tag == "" {
		tag = os.Args[0]
	}
	hostname, _ := os.Hostname()

	w := &Writer{
		priority: priority,
		tag:      tag,
		hostname: hostname,
		network:  network,
		raddr:    raddr,
	}

	w.mu.Lock()

	err := w.connect()
	if err != nil {
		w.mu.Unlock()
		return nil, err
	}
	w.mu.Unlock()
	return w, err
}

// connect makes a connection to the syslog server.
// It must be called with w.mu held.
func (w *Writer) connect() (err error) {
	if w.conn != nil {
		// ignore err from close, it makes sense to continue anyway
		w.conn.close()
		w.conn = nil
	}

	if w.network == "" {
		w.conn, err = unixSyslog()
		if w.hostname == "" {
			w.hostname = "localhost"
		}
	} else {
		var c net.Conn
		c, err = net.Dial(w.network, w.raddr)
		if err == nil {
			w.conn = &netConn{conn: c}
			if w.hostname == "" {
				w.hostname = c.LocalAddr().String()
			}
		}
	}
	return
}

// Write sends a log message to the syslog daemon.
func (w *Writer) Write(b []byte) (int, error) {
	return w.writeAndRetry(w.priority, string(b))
}

// Close closes a connection to the syslog daemon.
func (w *Writer) Close() (err error) {
	w.mu.Lock()
	defer w.mu.Unlock()

	if w.conn != nil {
		err = w.conn.close()
		w.conn = nil
	}
	return err
}

// Emerg logs a message with severity LOG_EMERG, ignoring the severity
// passed to New.
func (w *Writer) Emerg(m string) error {
	_, err := w.writeAndRetry(LOG_EMERG, m)
	return err
}

// Alert logs a message with severity LOG_ALERT, ignoring the severity
// passed to New.
func (w *Writer) Alert(m string) error {
	_, err := w.writeAndRetry(LOG_ALERT, m)
	return err
}

// Crit logs a message with severity LOG_CRIT, ignoring the severity
// passed to New.
func (w *Writer) Crit(m string) error {
	_, err := w.writeAndRetry(LOG_CRIT, m)
	return err
}

// Err logs a message with severity LOG_ERR, ignoring the severity
// passed to New.
func (w *Writer) Err(m string) error {
	_, err := w.writeAndRetry(LOG_ERR, m)
	return err
}

// Warning logs a message with severity LOG_WARNING, ignoring the
// severity passed to New.
func (w *Writer) Warning(m string) error {
	_, err := w.writeAndRetry(LOG_WARNING, m)
	return err
}

// Notice logs a message with severity LOG_NOTICE, ignoring the
// severity passed to New.
func (w *Writer) Notice(m string) error {
	_, err := w.writeAndRetry(LOG_NOTICE, m)
	return err
}

// Info logs a message with severity LOG_INFO, ignoring the severity
// passed to New.
func (w *Writer) Info(m string) error {
	_, err := w.writeAndRetry(LOG_INFO, m)
	return err
}

// Debug logs a message with severity LOG_DEBUG, ignoring the severity
// passed to New.
func (w *Writer) Debug(m string) error {
	_, err := w.writeAndRetry(LOG_DEBUG, m)
	return err
}

func (w *Writer) writeAndRetry(p Priority, s string) (n int, err error) {
	pr := (w.priority & facilityMask) | (p & severityMask)
	var msg []byte

	w.mu.Lock()
	if w.conn != nil {
		msg = w.format(pr, w.hostname, w.tag, s)
		if n, err = w.conn.write(msg); err == nil {
			w.mu.Unlock()
			return
		}
	}
	if err = w.connect(); err != nil {
		w.mu.Unlock()
		return
	}
	if msg == nil {
		msg = w.format(pr, w.hostname, w.tag, s)
	}
	n, err = w.conn.write(msg)
	w.mu.Unlock()
	return
}

// func (w *Writer) getpid() int {
func (w *Writer) getpid() []byte {
	if w.pid == 0 {
		w.pid = os.Getpid()
		w.bpid = strconv.AppendInt(make([]byte, 0, 8), int64(w.pid), 10)
	}
	return w.bpid
}

func formatBits(dst []byte, u uint64) []byte {
	const digits = "0123456789abcdefghijklmnopqrstuvwxyz"

	var a [64]byte // +1 for sign of 64bit value in base 2
	i := len(a)

	if ^uintptr(0)>>32 == 0 {
		for u > uint64(^uintptr(0)) {
			q := u / 1e9
			us := uintptr(u - q*1e9) // us % 1e9 fits into a uintptr
			for j := 9; j > 0; j-- {
				i--
				qs := us / 10
				a[i] = byte(us - qs*10 + '0')
				us = qs
			}
			u = q
		}
	}

	// u guaranteed to fit into a uintptr
	us := uintptr(u)
	for us >= 10 {
		i--
		q := us / 10
		a[i] = byte(us - q*10 + '0')
		us = q
	}
	// u < 10
	i--
	a[i] = byte(us + '0')

	return append(dst, a[i:]...)
}

func (w *Writer) format(p Priority, hostname, tag, msg string) []byte {
	b := w.b[0:0]
	b = append(b, '<')
	b = formatBits(b, uint64(p))
	b = append(b, '>')
	b = time.Now().AppendFormat(b, time.RFC3339)
	b = append(b, ' ')
	b = append(b, hostname...)
	b = append(b, ' ')
	b = append(b, tag...)
	b = append(b, '[')

	b = append(b, w.getpid()...)
	// b = strconv.AppendInt(b, int64(w.getpid()), 10)

	b = append(b, "]: "...)
	b = append(b, msg...)
	if b[len(b)-1] != '\n' {
		b = append(b, '\n')
	}
	if len(b) < maxBufSize {
		w.b = b
	}
	return b
}

func (n *netConn) write(p []byte) (int, error) {
	return n.conn.Write(p)
}

func (n *netConn) close() error {
	return n.conn.Close()
}

// NewLogger creates a log.Logger whose output is written to
// the system log service with the specified priority. The logFlag
// argument is the flag set passed through to log.New to create
// the Logger.
func NewLogger(p Priority, logFlag int) (*log.Logger, error) {
	s, err := New(p, "")
	if err != nil {
		return nil, err
	}
	return log.New(s, "", logFlag), nil
}
<|endoftext|>


<commit_before>package crdt

import (
	"testing"
	"github.com/stretchr/testify/assert"
)
var _ DData = (*GSet)(nil)

func one(i interface{}, err error) interface{} {
	if err == nil {
		return i
	} else {
		return nil
	}
}

func TestGSet(t *testing.T) {
	assert := assert.New(t)

	set1 := NewGSet()
	set1.Add("1")
	set1.Add("2")

	set2 := NewGSet()
	set2.Add("2")
	set2.Add("3")

	assert.True(set1.Contains("1"))
	assert.True(set1.Contains("2"))
	assert.False(set1.Contains("3"))

	assert.False(set2.Contains("1"))
	assert.True(set2.Contains("2"))
	assert.True(set2.Contains("3"))

	set1.MergeIn(set2)

	assert.True(set1.Contains("3"))
	assert.False(set2.Contains("1"))

	assert.Equal([]string{"1", "2", "3",}, one(set1.Marshal()))

	set3 := NewGSet()
	assert.Nil(set3.Unmarshal([]string{"2", "3",}))
	assert.Equal(set2, set3)
}

func TestGSet(t *testing.T) {
	assert := assert.New(t)

}
<commit_msg>Fix tests<commit_after>package crdt

import (
	"testing"
	"github.com/stretchr/testify/assert"
)
var _ DData = (*GSet)(nil)

func one(i interface{}, err error) interface{} {
	if err == nil {
		return i
	} else {
		return nil
	}
}

func TestGSet(t *testing.T) {
	assert := assert.New(t)

	set1 := NewGSet()
	set1.Add("1")
	set1.Add("2")

	set2 := NewGSet()
	set2.Add("2")
	set2.Add("3")

	assert.True(set1.Contains("1"))
	assert.True(set1.Contains("2"))
	assert.False(set1.Contains("3"))

	assert.False(set2.Contains("1"))
	assert.True(set2.Contains("2"))
	assert.True(set2.Contains("3"))

	set1.MergeIn(set2)

	assert.True(set1.Contains("3"))
	assert.False(set2.Contains("1"))

	assert.Equal([]string{"1", "2", "3",}, one(set1.Marshal()))

	set3 := NewGSet()
	assert.Nil(set3.Unmarshal([]string{"2", "3",}))
	assert.Equal(set2, set3)
}

var _ DData = (*TwoPSet)(nil)

func TestTwoPSet(t *testing.T) {
	//assert := assert.New(t)

}
<|endoftext|>


<commit_before>package logic

import (
	"io/ioutil"
	"os"

	"github.com/jasonlvhit/gocron"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/jclebreton/opensirene/conf"
	"github.com/jclebreton/opensirene/database"
	"github.com/jclebreton/opensirene/opendata/gouvfr/sirene"
)

type Crontab struct {
	PgxClient *database.PgxClient
	Config    conf.Crontab
}

// getDatabaseStatus lists the last successful updates in the database
// and returns the slice of filenames which were successfuly imported.
// Returns an empty slice otherwise.
func (ct *Crontab) getDatabaseStatus() ([]string, error) {
	rows, err := ct.PgxClient.Conn.Query("SELECT filename FROM history WHERE is_success=true")
	if err != nil {
		return nil, err
	}

	var result []string
	for rows.Next() {
		var filename string
		if err := rows.Scan(&filename); err != nil {
			return nil, err
		}
		result = append(result, filename)
	}

	return result, nil
}

// getFilesToImport is a function used to filter a slice of RemoteFile with a slice of
// string. If the file name is present in the RemoteFiles, they are evicted from
// the returned slice
func (ct *Crontab) getFilesToImport(localFiles []string, remoteFiles sirene.RemoteFiles) sirene.RemoteFiles {
	result := sirene.RemoteFiles{}
	for _, remoteFile := range remoteFiles {
		if !ct.isAlreadyImported(localFiles, remoteFile.FileName) {
			result = append(result, remoteFile)
		}
	}
	return result
}

func (ct *Crontab) isAlreadyImported(localFiles []string, remoteFile string) bool {
	for _, localFile := range localFiles {
		if remoteFile == localFile {
			return true
		}
	}
	return false
}

//// removeUselessFiles removes from disk old update files
func (ct *Crontab) removeUselessFiles(keepList []string) error {
	var files []os.FileInfo
	var result []string
	var err error
	var keepIt bool

	if files, err = ioutil.ReadDir(ct.Config.DownloadPath); err != nil {
		return errors.Wrap(err, "couldn't remove useless files")
	}

	for _, f := range files {
		for _, keepFile := range keepList {
			keepIt = false
			if f.Name() == keepFile {
				keepIt = true
				break
			}
		}
		if !keepIt {
			f := ct.Config.DownloadPath + "/" + f.Name()
			result = append(result, f)
			if err = os.Remove(f); err != nil {
				return errors.Wrap(err, "couldn't remove useless files")
			}

		}
	}

	if len(result) > 0 {
		logrus.WithField("files", result).Info("Some useless files has been removed")
	}

	return nil
}

// Daily is the cron task that runs every few hours to get and apply the latest
// updates
func (ct *Crontab) startUpdate() {
	var err error
	var remoteFiles sirene.RemoteFiles
	var dbFiles []string

	if remoteFiles, err = sirene.GrabLatestFull(ct.Config.DownloadPath); err != nil {
		logrus.WithError(err).Error("Could not grab latest index from gov")
		return
	}

	if dbFiles, err = ct.getDatabaseStatus(); err != nil {
		logrus.WithError(err).Error("Could not retrieve current database status")
		return
	}

	if err = ct.removeUselessFiles(dbFiles); err != nil {
		logrus.WithError(err).Error("Could not remove useless files")
		return

	}
	toDownload := ct.getFilesToImport(dbFiles, remoteFiles)

	logrus.
		WithField("remoteFiles", remoteFiles).
		WithField("dbFiles", dbFiles).
		WithField("toDownload", toDownload).Info("Crontab status")

	if err = ImportRemoteFiles(ct.PgxClient, toDownload, ct.Config.DownloadPath); err != nil {
		logrus.WithError(err).Error("Could not update database with latest files")
		return
	}
}

// StartCrontab run Daily function every few hours
func (ct *Crontab) Start() {
	gocron.Every(ct.Config.EveryXHours).Hours().Do(ct.startUpdate)
	gocron.RunAll() // Execute the task at startup
	_, t := gocron.NextRun()
	logrus.WithField("next", t).Info("Started cron background task")
	<-gocron.Start()
}
<commit_msg>Fix useless files removing<commit_after>package logic

import (
	"io/ioutil"
	"os"
	"regexp"

	"github.com/jasonlvhit/gocron"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"github.com/jclebreton/opensirene/conf"
	"github.com/jclebreton/opensirene/database"
	"github.com/jclebreton/opensirene/opendata/gouvfr/sirene"
)

type Crontab struct {
	PgxClient *database.PgxClient
	Config    conf.Crontab
}

// getDatabaseStatus lists the last successful updates in the database
// and returns the slice of filenames which were successfuly imported.
// Returns an empty slice otherwise.
func (ct *Crontab) getDatabaseStatus() ([]string, error) {
	rows, err := ct.PgxClient.Conn.Query("SELECT filename FROM history WHERE is_success=true")
	if err != nil {
		return nil, err
	}

	var result []string
	for rows.Next() {
		var filename string
		if err := rows.Scan(&filename); err != nil {
			return nil, err
		}
		result = append(result, filename)
	}

	return result, nil
}

// getFilesToImport is a function used to filter a slice of RemoteFile with a slice of
// string. If the file name is present in the RemoteFiles, they are evicted from
// the returned slice
func (ct *Crontab) getFilesToImport(localFiles []string, remoteFiles sirene.RemoteFiles) sirene.RemoteFiles {
	result := sirene.RemoteFiles{}
	for _, remoteFile := range remoteFiles {
		if !ct.isAlreadyImported(localFiles, remoteFile.FileName) {
			result = append(result, remoteFile)
		}
	}
	return result
}

func (ct *Crontab) isAlreadyImported(localFiles []string, remoteFile string) bool {
	for _, localFile := range localFiles {
		if remoteFile == localFile {
			return true
		}
	}
	return false
}

//// removeUselessFiles removes from disk old update files
func (ct *Crontab) removeUselessFiles(keepList []string) error {
	var files []os.FileInfo
	var result []string
	var err error
	var keepIt bool

	if files, err = ioutil.ReadDir(ct.Config.DownloadPath); err != nil {
		return errors.Wrap(err, "couldn't remove useless files")
	}

	for _, f := range files {
		for _, keepFile := range keepList {
			keepIt = true
			match, err := regexp.MatchString("/\\.(zip|csv)$/", f.Name())
			if f.Name() != keepFile && err == nil && match {
				keepIt = false
				break
			}
		}
		if !keepIt {
			f := ct.Config.DownloadPath + "/" + f.Name()
			result = append(result, f)
			if err = os.Remove(f); err != nil {
				return errors.Wrap(err, "couldn't remove useless files")
			}

		}
	}

	if len(result) > 0 {
		logrus.WithField("files", result).Info("Some useless files has been removed")
	}

	return nil
}

// Daily is the cron task that runs every few hours to get and apply the latest
// updates
func (ct *Crontab) startUpdate() {
	var err error
	var remoteFiles sirene.RemoteFiles
	var dbFiles []string

	// Retrieve the list of update files stored in database
	if dbFiles, err = ct.getDatabaseStatus(); err != nil {
		logrus.WithError(err).Error("Could not retrieve current database status")
		return
	}

	// Remove other ZIP and CSV files
	if err = ct.removeUselessFiles(dbFiles); err != nil {
		logrus.WithError(err).Error("Could not remove useless files")
		return

	}

	// Search updates
	if remoteFiles, err = sirene.GrabLatestFull(ct.Config.DownloadPath); err != nil {
		logrus.WithError(err).Error("Could not grab latest index from gov")
		return
	}

	// Files to grab
	toDownload := ct.getFilesToImport(dbFiles, remoteFiles)

	logrus.
		WithField("remoteFiles", remoteFiles).
		WithField("dbFiles", dbFiles).
		WithField("toDownload", toDownload).Info("Crontab status")

	// Start upgrade
	if err = ImportRemoteFiles(ct.PgxClient, toDownload, ct.Config.DownloadPath); err != nil {
		logrus.WithError(err).Error("Could not update database with latest files")
		return
	}
}

// StartCrontab run Daily function every few hours
func (ct *Crontab) Start() {
	gocron.Every(ct.Config.EveryXHours).Hours().Do(ct.startUpdate)
	gocron.RunAll() // Execute the task at startup
	_, t := gocron.NextRun()
	logrus.WithField("next", t).Info("Started cron background task")
	<-gocron.Start()
}
<|endoftext|>


<commit_before>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package create provides functions to create new content.
package create

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/gohugoio/hugo/hugofs/glob"

	"github.com/gohugoio/hugo/common/paths"

	"github.com/pkg/errors"

	"github.com/gohugoio/hugo/common/hexec"
	"github.com/gohugoio/hugo/hugofs/files"

	"github.com/gohugoio/hugo/hugofs"

	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"
)

const (
	// DefaultArchetypeTemplateTemplate is the template used in 'hugo new site'
	// and the template we use as a fall back.
	DefaultArchetypeTemplateTemplate = `---
title: "{{ replace .Name "-" " " | title }}"
date: {{ .Date }}
draft: true
---

`
)

// NewContent creates a new content file in h (or a full bundle if the archetype is a directory)
// in targetPath.
func NewContent(h *hugolib.HugoSites, kind, targetPath string) error {
	if h.BaseFs.Content.Dirs == nil {
		return errors.New("no existing content directory configured for this project")
	}

	cf := hugolib.NewContentFactory(h)

	if kind == "" {
		kind = cf.SectionFromFilename(targetPath)
	}

	b := &contentBuilder{
		archeTypeFs: h.PathSpec.BaseFs.Archetypes.Fs,
		sourceFs:    h.PathSpec.Fs.Source,
		ps:          h.PathSpec,
		h:           h,
		cf:          cf,

		kind:       kind,
		targetPath: targetPath,
	}

	ext := paths.Ext(targetPath)

	b.setArcheTypeFilenameToUse(ext)

	withBuildLock := func() (string, error) {
		unlock, err := h.BaseFs.LockBuild()
		if err != nil {
			return "", fmt.Errorf("failed to acquire a build lock: %s", err)
		}
		defer unlock()

		if b.isDir {
			return "", b.buildDir()
		}

		if ext == "" {
			return "", errors.Errorf("failed to resolve %q to a archetype template", targetPath)
		}

		if !files.IsContentFile(b.targetPath) {
			return "", errors.Errorf("target path %q is not a known content format", b.targetPath)
		}

		return b.buildFile()

	}

	filename, err := withBuildLock()
	if err != nil {
		return err
	}

	if filename != "" {
		return b.openInEditorIfConfigured(filename)
	}

	return nil

}

type contentBuilder struct {
	archeTypeFs afero.Fs
	sourceFs    afero.Fs

	ps *helpers.PathSpec
	h  *hugolib.HugoSites
	cf hugolib.ContentFactory

	// Builder state
	archetypeFilename string
	targetPath        string
	kind              string
	isDir             bool
	dirMap            archetypeMap
}

func (b *contentBuilder) buildDir() error {
	// Split the dir into content files and the rest.
	if err := b.mapArcheTypeDir(); err != nil {
		return err
	}

	var contentTargetFilenames []string
	var baseDir string

	for _, fi := range b.dirMap.contentFiles {
		targetFilename := filepath.Join(b.targetPath, strings.TrimPrefix(fi.Meta().Path, b.archetypeFilename))
		abs, err := b.cf.CreateContentPlaceHolder(targetFilename)
		if err != nil {
			return err
		}
		if baseDir == "" {
			baseDir = strings.TrimSuffix(abs, targetFilename)
		}

		contentTargetFilenames = append(contentTargetFilenames, abs)
	}

	var contentInclusionFilter *glob.FilenameFilter
	if !b.dirMap.siteUsed {
		// We don't need to build everything.
		contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
			filename = strings.TrimPrefix(filename, string(os.PathSeparator))
			for _, cn := range contentTargetFilenames {
				if strings.Contains(cn, filename) {
					return true
				}
			}
			return false
		})

	}

	if err := b.h.Build(hugolib.BuildCfg{NoBuildLock: true, SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil {
		return err
	}

	for i, filename := range contentTargetFilenames {
		if err := b.applyArcheType(filename, b.dirMap.contentFiles[i].Meta().Path); err != nil {
			return err
		}
	}

	// Copy the rest as is.
	for _, f := range b.dirMap.otherFiles {
		meta := f.Meta()
		filename := meta.Path

		in, err := meta.Open()
		if err != nil {
			return errors.Wrap(err, "failed to open non-content file")
		}

		targetFilename := filepath.Join(baseDir, b.targetPath, strings.TrimPrefix(filename, b.archetypeFilename))
		targetDir := filepath.Dir(targetFilename)

		if err := b.sourceFs.MkdirAll(targetDir, 0o777); err != nil && !os.IsExist(err) {
			return errors.Wrapf(err, "failed to create target directory for %q", targetDir)
		}

		out, err := b.sourceFs.Create(targetFilename)
		if err != nil {
			return err
		}

		_, err = io.Copy(out, in)
		if err != nil {
			return err
		}

		in.Close()
		out.Close()
	}
	return nil
}

func (b *contentBuilder) buildFile() (string, error) {
	contentPlaceholderAbsFilename, err := b.cf.CreateContentPlaceHolder(b.targetPath)
	if err != nil {
		return "", err
	}

	usesSite, err := b.usesSiteVar(b.archetypeFilename)
	if err != nil {
		return "", err
	}

	var contentInclusionFilter *glob.FilenameFilter
	if !usesSite {
		// We don't need to build everything.
		contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
			filename = strings.TrimPrefix(filename, string(os.PathSeparator))
			return strings.Contains(contentPlaceholderAbsFilename, filename)
		})
	}

	if err := b.h.Build(hugolib.BuildCfg{NoBuildLock: true, SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil {
		return "", err
	}

	if err := b.applyArcheType(contentPlaceholderAbsFilename, b.archetypeFilename); err != nil {
		return "", err
	}

	b.h.Log.Printf("Content %q created", contentPlaceholderAbsFilename)

	return contentPlaceholderAbsFilename, nil
}

func (b *contentBuilder) setArcheTypeFilenameToUse(ext string) {
	var pathsToCheck []string

	if b.kind != "" {
		pathsToCheck = append(pathsToCheck, b.kind+ext)
	}
	pathsToCheck = append(pathsToCheck, "default"+ext, "default")

	for _, p := range pathsToCheck {
		fi, err := b.archeTypeFs.Stat(p)
		if err == nil {
			b.archetypeFilename = p
			b.isDir = fi.IsDir()
			return
		}
	}

}

func (b *contentBuilder) applyArcheType(contentFilename, archetypeFilename string) error {
	p := b.h.GetContentPage(contentFilename)
	if p == nil {
		panic(fmt.Sprintf("[BUG] no Page found for %q", contentFilename))
	}

	f, err := b.sourceFs.Create(contentFilename)
	if err != nil {
		return err
	}
	defer f.Close()

	if archetypeFilename == "" {
		return b.cf.AppplyArchetypeTemplate(f, p, b.kind, DefaultArchetypeTemplateTemplate)
	}

	return b.cf.AppplyArchetypeFilename(f, p, b.kind, archetypeFilename)

}

func (b *contentBuilder) mapArcheTypeDir() error {
	var m archetypeMap

	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
		if err != nil {
			return err
		}

		if fi.IsDir() {
			return nil
		}

		fil := fi.(hugofs.FileMetaInfo)

		if files.IsContentFile(path) {
			m.contentFiles = append(m.contentFiles, fil)
			if !m.siteUsed {
				m.siteUsed, err = b.usesSiteVar(path)
				if err != nil {
					return err
				}
			}
			return nil
		}

		m.otherFiles = append(m.otherFiles, fil)

		return nil
	}

	walkCfg := hugofs.WalkwayConfig{
		WalkFn: walkFn,
		Fs:     b.archeTypeFs,
		Root:   b.archetypeFilename,
	}

	w := hugofs.NewWalkway(walkCfg)

	if err := w.Walk(); err != nil {
		return errors.Wrapf(err, "failed to walk archetype dir %q", b.archetypeFilename)
	}

	b.dirMap = m

	return nil
}

func (b *contentBuilder) openInEditorIfConfigured(filename string) error {
	editor := b.h.Cfg.GetString("newContentEditor")
	if editor == "" {
		return nil
	}

	b.h.Log.Printf("Editing %q with %q ...\n", filename, editor)

	cmd, err := hexec.SafeCommand(editor, filename)
	if err != nil {
		return err
	}

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func (b *contentBuilder) usesSiteVar(filename string) (bool, error) {
	if filename == "" {
		return false, nil
	}
	bb, err := afero.ReadFile(b.archeTypeFs, filename)
	if err != nil {
		return false, errors.Wrap(err, "failed to open archetype file")
	}

	return bytes.Contains(bb, []byte(".Site")) || bytes.Contains(bb, []byte("site.")), nil

}

type archetypeMap struct {
	// These needs to be parsed and executed as Go templates.
	contentFiles []hugofs.FileMetaInfo
	// These are just copied to destination.
	otherFiles []hugofs.FileMetaInfo
	// If the templates needs a fully built site. This can potentially be
	// expensive, so only do when needed.
	siteUsed bool
}
<commit_msg>create: Add a log statement when archetype is a directory<commit_after>// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package create provides functions to create new content.
package create

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/gohugoio/hugo/hugofs/glob"

	"github.com/gohugoio/hugo/common/paths"

	"github.com/pkg/errors"

	"github.com/gohugoio/hugo/common/hexec"
	"github.com/gohugoio/hugo/hugofs/files"

	"github.com/gohugoio/hugo/hugofs"

	"github.com/gohugoio/hugo/helpers"
	"github.com/gohugoio/hugo/hugolib"
	"github.com/spf13/afero"
)

const (
	// DefaultArchetypeTemplateTemplate is the template used in 'hugo new site'
	// and the template we use as a fall back.
	DefaultArchetypeTemplateTemplate = `---
title: "{{ replace .Name "-" " " | title }}"
date: {{ .Date }}
draft: true
---

`
)

// NewContent creates a new content file in h (or a full bundle if the archetype is a directory)
// in targetPath.
func NewContent(h *hugolib.HugoSites, kind, targetPath string) error {
	if h.BaseFs.Content.Dirs == nil {
		return errors.New("no existing content directory configured for this project")
	}

	cf := hugolib.NewContentFactory(h)

	if kind == "" {
		kind = cf.SectionFromFilename(targetPath)
	}

	b := &contentBuilder{
		archeTypeFs: h.PathSpec.BaseFs.Archetypes.Fs,
		sourceFs:    h.PathSpec.Fs.Source,
		ps:          h.PathSpec,
		h:           h,
		cf:          cf,

		kind:       kind,
		targetPath: targetPath,
	}

	ext := paths.Ext(targetPath)

	b.setArcheTypeFilenameToUse(ext)

	withBuildLock := func() (string, error) {
		unlock, err := h.BaseFs.LockBuild()
		if err != nil {
			return "", fmt.Errorf("failed to acquire a build lock: %s", err)
		}
		defer unlock()

		if b.isDir {
			return "", b.buildDir()
		}

		if ext == "" {
			return "", errors.Errorf("failed to resolve %q to a archetype template", targetPath)
		}

		if !files.IsContentFile(b.targetPath) {
			return "", errors.Errorf("target path %q is not a known content format", b.targetPath)
		}

		return b.buildFile()

	}

	filename, err := withBuildLock()
	if err != nil {
		return err
	}

	if filename != "" {
		return b.openInEditorIfConfigured(filename)
	}

	return nil

}

type contentBuilder struct {
	archeTypeFs afero.Fs
	sourceFs    afero.Fs

	ps *helpers.PathSpec
	h  *hugolib.HugoSites
	cf hugolib.ContentFactory

	// Builder state
	archetypeFilename string
	targetPath        string
	kind              string
	isDir             bool
	dirMap            archetypeMap
}

func (b *contentBuilder) buildDir() error {
	// Split the dir into content files and the rest.
	if err := b.mapArcheTypeDir(); err != nil {
		return err
	}

	var contentTargetFilenames []string
	var baseDir string

	for _, fi := range b.dirMap.contentFiles {
		targetFilename := filepath.Join(b.targetPath, strings.TrimPrefix(fi.Meta().Path, b.archetypeFilename))
		abs, err := b.cf.CreateContentPlaceHolder(targetFilename)
		if err != nil {
			return err
		}
		if baseDir == "" {
			baseDir = strings.TrimSuffix(abs, targetFilename)
		}

		contentTargetFilenames = append(contentTargetFilenames, abs)
	}

	var contentInclusionFilter *glob.FilenameFilter
	if !b.dirMap.siteUsed {
		// We don't need to build everything.
		contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
			filename = strings.TrimPrefix(filename, string(os.PathSeparator))
			for _, cn := range contentTargetFilenames {
				if strings.Contains(cn, filename) {
					return true
				}
			}
			return false
		})

	}

	if err := b.h.Build(hugolib.BuildCfg{NoBuildLock: true, SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil {
		return err
	}

	for i, filename := range contentTargetFilenames {
		if err := b.applyArcheType(filename, b.dirMap.contentFiles[i].Meta().Path); err != nil {
			return err
		}
	}

	// Copy the rest as is.
	for _, f := range b.dirMap.otherFiles {
		meta := f.Meta()
		filename := meta.Path

		in, err := meta.Open()
		if err != nil {
			return errors.Wrap(err, "failed to open non-content file")
		}

		targetFilename := filepath.Join(baseDir, b.targetPath, strings.TrimPrefix(filename, b.archetypeFilename))
		targetDir := filepath.Dir(targetFilename)

		if err := b.sourceFs.MkdirAll(targetDir, 0o777); err != nil && !os.IsExist(err) {
			return errors.Wrapf(err, "failed to create target directory for %q", targetDir)
		}

		out, err := b.sourceFs.Create(targetFilename)
		if err != nil {
			return err
		}

		_, err = io.Copy(out, in)
		if err != nil {
			return err
		}

		in.Close()
		out.Close()
	}

	b.h.Log.Printf("Content dir %q created", filepath.Join(baseDir, b.targetPath))

	return nil
}

func (b *contentBuilder) buildFile() (string, error) {
	contentPlaceholderAbsFilename, err := b.cf.CreateContentPlaceHolder(b.targetPath)
	if err != nil {
		return "", err
	}

	usesSite, err := b.usesSiteVar(b.archetypeFilename)
	if err != nil {
		return "", err
	}

	var contentInclusionFilter *glob.FilenameFilter
	if !usesSite {
		// We don't need to build everything.
		contentInclusionFilter = glob.NewFilenameFilterForInclusionFunc(func(filename string) bool {
			filename = strings.TrimPrefix(filename, string(os.PathSeparator))
			return strings.Contains(contentPlaceholderAbsFilename, filename)
		})
	}

	if err := b.h.Build(hugolib.BuildCfg{NoBuildLock: true, SkipRender: true, ContentInclusionFilter: contentInclusionFilter}); err != nil {
		return "", err
	}

	if err := b.applyArcheType(contentPlaceholderAbsFilename, b.archetypeFilename); err != nil {
		return "", err
	}

	b.h.Log.Printf("Content %q created", contentPlaceholderAbsFilename)

	return contentPlaceholderAbsFilename, nil
}

func (b *contentBuilder) setArcheTypeFilenameToUse(ext string) {
	var pathsToCheck []string

	if b.kind != "" {
		pathsToCheck = append(pathsToCheck, b.kind+ext)
	}
	pathsToCheck = append(pathsToCheck, "default"+ext, "default")

	for _, p := range pathsToCheck {
		fi, err := b.archeTypeFs.Stat(p)
		if err == nil {
			b.archetypeFilename = p
			b.isDir = fi.IsDir()
			return
		}
	}

}

func (b *contentBuilder) applyArcheType(contentFilename, archetypeFilename string) error {
	p := b.h.GetContentPage(contentFilename)
	if p == nil {
		panic(fmt.Sprintf("[BUG] no Page found for %q", contentFilename))
	}

	f, err := b.sourceFs.Create(contentFilename)
	if err != nil {
		return err
	}
	defer f.Close()

	if archetypeFilename == "" {
		return b.cf.AppplyArchetypeTemplate(f, p, b.kind, DefaultArchetypeTemplateTemplate)
	}

	return b.cf.AppplyArchetypeFilename(f, p, b.kind, archetypeFilename)

}

func (b *contentBuilder) mapArcheTypeDir() error {
	var m archetypeMap

	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
		if err != nil {
			return err
		}

		if fi.IsDir() {
			return nil
		}

		fil := fi.(hugofs.FileMetaInfo)

		if files.IsContentFile(path) {
			m.contentFiles = append(m.contentFiles, fil)
			if !m.siteUsed {
				m.siteUsed, err = b.usesSiteVar(path)
				if err != nil {
					return err
				}
			}
			return nil
		}

		m.otherFiles = append(m.otherFiles, fil)

		return nil
	}

	walkCfg := hugofs.WalkwayConfig{
		WalkFn: walkFn,
		Fs:     b.archeTypeFs,
		Root:   b.archetypeFilename,
	}

	w := hugofs.NewWalkway(walkCfg)

	if err := w.Walk(); err != nil {
		return errors.Wrapf(err, "failed to walk archetype dir %q", b.archetypeFilename)
	}

	b.dirMap = m

	return nil
}

func (b *contentBuilder) openInEditorIfConfigured(filename string) error {
	editor := b.h.Cfg.GetString("newContentEditor")
	if editor == "" {
		return nil
	}

	b.h.Log.Printf("Editing %q with %q ...\n", filename, editor)

	cmd, err := hexec.SafeCommand(editor, filename)
	if err != nil {
		return err
	}

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}

func (b *contentBuilder) usesSiteVar(filename string) (bool, error) {
	if filename == "" {
		return false, nil
	}
	bb, err := afero.ReadFile(b.archeTypeFs, filename)
	if err != nil {
		return false, errors.Wrap(err, "failed to open archetype file")
	}

	return bytes.Contains(bb, []byte(".Site")) || bytes.Contains(bb, []byte("site.")), nil

}

type archetypeMap struct {
	// These needs to be parsed and executed as Go templates.
	contentFiles []hugofs.FileMetaInfo
	// These are just copied to destination.
	otherFiles []hugofs.FileMetaInfo
	// If the templates needs a fully built site. This can potentially be
	// expensive, so only do when needed.
	siteUsed bool
}
<|endoftext|>


<commit_before>package nodb

import (
	"encoding/binary"
	"errors"
	"time"

	"github.com/lunny/nodb/store"
)

const (
	listHeadSeq int32 = 1
	listTailSeq int32 = 2

	listMinSeq     int32 = 1000
	listMaxSeq     int32 = 1<<31 - 1000
	listInitialSeq int32 = listMinSeq + (listMaxSeq-listMinSeq)/2
)

var errLMetaKey = errors.New("invalid lmeta key")
var errListKey = errors.New("invalid list key")
var errListSeq = errors.New("invalid list sequence, overflow")

func (db *DB) lEncodeMetaKey(key []byte) []byte {
	buf := make([]byte, len(key)+2)
	buf[0] = db.index
	buf[1] = LMetaType

	copy(buf[2:], key)
	return buf
}

func (db *DB) lDecodeMetaKey(ek []byte) ([]byte, error) {
	if len(ek) < 2 || ek[0] != db.index || ek[1] != LMetaType {
		return nil, errLMetaKey
	}

	return ek[2:], nil
}

func (db *DB) lEncodeListKey(key []byte, seq int32) []byte {
	buf := make([]byte, len(key)+8)

	pos := 0
	buf[pos] = db.index
	pos++
	buf[pos] = ListType
	pos++

	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
	pos += 2

	copy(buf[pos:], key)
	pos += len(key)

	binary.BigEndian.PutUint32(buf[pos:], uint32(seq))

	return buf
}

func (db *DB) lDecodeListKey(ek []byte) (key []byte, seq int32, err error) {
	if len(ek) < 8 || ek[0] != db.index || ek[1] != ListType {
		err = errListKey
		return
	}

	keyLen := int(binary.BigEndian.Uint16(ek[2:]))
	if keyLen+8 != len(ek) {
		err = errListKey
		return
	}

	key = ek[4 : 4+keyLen]
	seq = int32(binary.BigEndian.Uint32(ek[4+keyLen:]))
	return
}

func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	var headSeq int32
	var tailSeq int32
	var size int32
	var err error

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	metaKey := db.lEncodeMetaKey(key)
	headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey)
	if err != nil {
		return 0, err
	}

	var pushCnt int = len(args)
	if pushCnt == 0 {
		return int64(size), nil
	}

	var seq int32 = headSeq
	var delta int32 = -1
	if whereSeq == listTailSeq {
		seq = tailSeq
		delta = 1
	}

	//	append elements
	if size > 0 {
		seq += delta
	}

	for i := 0; i < pushCnt; i++ {
		ek := db.lEncodeListKey(key, seq+int32(i)*delta)
		t.Put(ek, args[i])
	}

	seq += int32(pushCnt-1) * delta
	if seq <= listMinSeq || seq >= listMaxSeq {
		return 0, errListSeq
	}

	//	set meta info
	if whereSeq == listHeadSeq {
		headSeq = seq
	} else {
		tailSeq = seq
	}

	db.lSetMeta(metaKey, headSeq, tailSeq)

	err = t.Commit()
	return int64(size) + int64(pushCnt), err
}

func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) {
	if err := checkKeySize(key); err != nil {
		return nil, err
	}

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	var headSeq int32
	var tailSeq int32
	var err error

	metaKey := db.lEncodeMetaKey(key)
	headSeq, tailSeq, _, err = db.lGetMeta(nil, metaKey)
	if err != nil {
		return nil, err
	}

	var value []byte

	var seq int32 = headSeq
	if whereSeq == listTailSeq {
		seq = tailSeq
	}

	itemKey := db.lEncodeListKey(key, seq)
	value, err = db.bucket.Get(itemKey)
	if err != nil {
		return nil, err
	}

	if whereSeq == listHeadSeq {
		headSeq += 1
	} else {
		tailSeq -= 1
	}

	t.Delete(itemKey)
	size := db.lSetMeta(metaKey, headSeq, tailSeq)
	if size == 0 {
		db.rmExpire(t, HashType, key)
	}

	err = t.Commit()
	return value, err
}

//	ps : here just focus on deleting the list data,
//		 any other likes expire is ignore.
func (db *DB) lDelete(t *batch, key []byte) int64 {
	mk := db.lEncodeMetaKey(key)

	var headSeq int32
	var tailSeq int32
	var err error

	it := db.bucket.NewIterator()
	defer it.Close()

	headSeq, tailSeq, _, err = db.lGetMeta(it, mk)
	if err != nil {
		return 0
	}

	var num int64 = 0
	startKey := db.lEncodeListKey(key, headSeq)
	stopKey := db.lEncodeListKey(key, tailSeq)

	rit := store.NewRangeIterator(it, &store.Range{startKey, stopKey, store.RangeClose})
	for ; rit.Valid(); rit.Next() {
		t.Delete(rit.RawKey())
		num++
	}

	t.Delete(mk)

	return num
}

func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq int32, size int32, err error) {
	var v []byte
	if it != nil {
		v = it.Find(ek)
	} else {
		v, err = db.bucket.Get(ek)
	}
	if err != nil {
		return
	} else if v == nil {
		headSeq = listInitialSeq
		tailSeq = listInitialSeq
		size = 0
		return
	} else {
		headSeq = int32(binary.LittleEndian.Uint32(v[0:4]))
		tailSeq = int32(binary.LittleEndian.Uint32(v[4:8]))
		size = tailSeq - headSeq + 1
	}
	return
}

func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 {
	t := db.listBatch

	var size int32 = tailSeq - headSeq + 1
	if size < 0 {
		//	todo : log error + panic
	} else if size == 0 {
		t.Delete(ek)
	} else {
		buf := make([]byte, 8)

		binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq))
		binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq))

		t.Put(ek, buf)
	}

	return size
}

func (db *DB) lExpireAt(key []byte, when int64) (int64, error) {
	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	if llen, err := db.LLen(key); err != nil || llen == 0 {
		return 0, err
	} else {
		db.expireAt(t, ListType, key, when)
		if err := t.Commit(); err != nil {
			return 0, err
		}
	}
	return 1, nil
}

func (db *DB) LIndex(key []byte, index int32) ([]byte, error) {
	if err := checkKeySize(key); err != nil {
		return nil, err
	}

	var seq int32
	var headSeq int32
	var tailSeq int32
	var err error

	metaKey := db.lEncodeMetaKey(key)

	it := db.bucket.NewIterator()
	defer it.Close()

	headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey)
	if err != nil {
		return nil, err
	}

	if index >= 0 {
		seq = headSeq + index
	} else {
		seq = tailSeq + index + 1
	}

	sk := db.lEncodeListKey(key, seq)
	v := it.Find(sk)

	return v, nil
}

func (db *DB) LLen(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	ek := db.lEncodeMetaKey(key)
	_, _, size, err := db.lGetMeta(nil, ek)
	return int64(size), err
}

func (db *DB) LPop(key []byte) ([]byte, error) {
	return db.lpop(key, listHeadSeq)
}

func (db *DB) LPush(key []byte, args ...[]byte) (int64, error) {
	return db.lpush(key, listHeadSeq, args...)
}

func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) {
	if err := checkKeySize(key); err != nil {
		return nil, err
	}

	var headSeq int32
	var llen int32
	var err error

	metaKey := db.lEncodeMetaKey(key)

	it := db.bucket.NewIterator()
	defer it.Close()

	if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil {
		return nil, err
	}

	if start < 0 {
		start = llen + start
	}
	if stop < 0 {
		stop = llen + stop
	}
	if start < 0 {
		start = 0
	}

	if start > stop || start >= llen {
		return [][]byte{}, nil
	}

	if stop >= llen {
		stop = llen - 1
	}

	limit := (stop - start) + 1
	headSeq += start

	v := make([][]byte, 0, limit)

	startKey := db.lEncodeListKey(key, headSeq)
	rit := store.NewRangeLimitIterator(it,
		&store.Range{
			Min:  startKey,
			Max:  nil,
			Type: store.RangeClose},
		&store.Limit{
			Offset: 0,
			Count:  int(limit)})

	for ; rit.Valid(); rit.Next() {
		v = append(v, rit.Value())
	}

	return v, nil
}

func (db *DB) RPop(key []byte) ([]byte, error) {
	return db.lpop(key, listTailSeq)
}

func (db *DB) RPush(key []byte, args ...[]byte) (int64, error) {
	return db.lpush(key, listTailSeq, args...)
}

func (db *DB) LClear(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	num := db.lDelete(t, key)
	db.rmExpire(t, ListType, key)

	err := t.Commit()
	return num, err
}

func (db *DB) LMclear(keys ...[]byte) (int64, error) {
	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	for _, key := range keys {
		if err := checkKeySize(key); err != nil {
			return 0, err
		}

		db.lDelete(t, key)
		db.rmExpire(t, ListType, key)

	}

	err := t.Commit()
	return int64(len(keys)), err
}

func (db *DB) lFlush() (drop int64, err error) {
	t := db.listBatch
	t.Lock()
	defer t.Unlock()
	return db.flushType(t, ListType)
}

func (db *DB) LExpire(key []byte, duration int64) (int64, error) {
	if duration <= 0 {
		return 0, errExpireValue
	}

	return db.lExpireAt(key, time.Now().Unix()+duration)
}

func (db *DB) LExpireAt(key []byte, when int64) (int64, error) {
	if when <= time.Now().Unix() {
		return 0, errExpireValue
	}

	return db.lExpireAt(key, when)
}

func (db *DB) LTTL(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return -1, err
	}

	return db.ttl(ListType, key)
}

func (db *DB) LPersist(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	n, err := db.rmExpire(t, ListType, key)
	if err != nil {
		return 0, err
	}

	err = t.Commit()
	return n, err
}

func (db *DB) LScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
	return db.scan(LMetaType, key, count, inclusive, match)
}

func (db *DB) lEncodeMinKey() []byte {
	return db.lEncodeMetaKey(nil)
}

func (db *DB) lEncodeMaxKey() []byte {
	ek := db.lEncodeMetaKey(nil)
	ek[len(ek)-1] = LMetaType + 1
	return ek
}
<commit_msg>LPush & RPush must has less one value<commit_after>package nodb

import (
	"encoding/binary"
	"errors"
	"time"

	"github.com/lunny/nodb/store"
)

const (
	listHeadSeq int32 = 1
	listTailSeq int32 = 2

	listMinSeq     int32 = 1000
	listMaxSeq     int32 = 1<<31 - 1000
	listInitialSeq int32 = listMinSeq + (listMaxSeq-listMinSeq)/2
)

var errLMetaKey = errors.New("invalid lmeta key")
var errListKey = errors.New("invalid list key")
var errListSeq = errors.New("invalid list sequence, overflow")

func (db *DB) lEncodeMetaKey(key []byte) []byte {
	buf := make([]byte, len(key)+2)
	buf[0] = db.index
	buf[1] = LMetaType

	copy(buf[2:], key)
	return buf
}

func (db *DB) lDecodeMetaKey(ek []byte) ([]byte, error) {
	if len(ek) < 2 || ek[0] != db.index || ek[1] != LMetaType {
		return nil, errLMetaKey
	}

	return ek[2:], nil
}

func (db *DB) lEncodeListKey(key []byte, seq int32) []byte {
	buf := make([]byte, len(key)+8)

	pos := 0
	buf[pos] = db.index
	pos++
	buf[pos] = ListType
	pos++

	binary.BigEndian.PutUint16(buf[pos:], uint16(len(key)))
	pos += 2

	copy(buf[pos:], key)
	pos += len(key)

	binary.BigEndian.PutUint32(buf[pos:], uint32(seq))

	return buf
}

func (db *DB) lDecodeListKey(ek []byte) (key []byte, seq int32, err error) {
	if len(ek) < 8 || ek[0] != db.index || ek[1] != ListType {
		err = errListKey
		return
	}

	keyLen := int(binary.BigEndian.Uint16(ek[2:]))
	if keyLen+8 != len(ek) {
		err = errListKey
		return
	}

	key = ek[4 : 4+keyLen]
	seq = int32(binary.BigEndian.Uint32(ek[4+keyLen:]))
	return
}

func (db *DB) lpush(key []byte, whereSeq int32, args ...[]byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	var headSeq int32
	var tailSeq int32
	var size int32
	var err error

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	metaKey := db.lEncodeMetaKey(key)
	headSeq, tailSeq, size, err = db.lGetMeta(nil, metaKey)
	if err != nil {
		return 0, err
	}

	var pushCnt int = len(args)
	if pushCnt == 0 {
		return int64(size), nil
	}

	var seq int32 = headSeq
	var delta int32 = -1
	if whereSeq == listTailSeq {
		seq = tailSeq
		delta = 1
	}

	//	append elements
	if size > 0 {
		seq += delta
	}

	for i := 0; i < pushCnt; i++ {
		ek := db.lEncodeListKey(key, seq+int32(i)*delta)
		t.Put(ek, args[i])
	}

	seq += int32(pushCnt-1) * delta
	if seq <= listMinSeq || seq >= listMaxSeq {
		return 0, errListSeq
	}

	//	set meta info
	if whereSeq == listHeadSeq {
		headSeq = seq
	} else {
		tailSeq = seq
	}

	db.lSetMeta(metaKey, headSeq, tailSeq)

	err = t.Commit()
	return int64(size) + int64(pushCnt), err
}

func (db *DB) lpop(key []byte, whereSeq int32) ([]byte, error) {
	if err := checkKeySize(key); err != nil {
		return nil, err
	}

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	var headSeq int32
	var tailSeq int32
	var err error

	metaKey := db.lEncodeMetaKey(key)
	headSeq, tailSeq, _, err = db.lGetMeta(nil, metaKey)
	if err != nil {
		return nil, err
	}

	var value []byte

	var seq int32 = headSeq
	if whereSeq == listTailSeq {
		seq = tailSeq
	}

	itemKey := db.lEncodeListKey(key, seq)
	value, err = db.bucket.Get(itemKey)
	if err != nil {
		return nil, err
	}

	if whereSeq == listHeadSeq {
		headSeq += 1
	} else {
		tailSeq -= 1
	}

	t.Delete(itemKey)
	size := db.lSetMeta(metaKey, headSeq, tailSeq)
	if size == 0 {
		db.rmExpire(t, HashType, key)
	}

	err = t.Commit()
	return value, err
}

//	ps : here just focus on deleting the list data,
//		 any other likes expire is ignore.
func (db *DB) lDelete(t *batch, key []byte) int64 {
	mk := db.lEncodeMetaKey(key)

	var headSeq int32
	var tailSeq int32
	var err error

	it := db.bucket.NewIterator()
	defer it.Close()

	headSeq, tailSeq, _, err = db.lGetMeta(it, mk)
	if err != nil {
		return 0
	}

	var num int64 = 0
	startKey := db.lEncodeListKey(key, headSeq)
	stopKey := db.lEncodeListKey(key, tailSeq)

	rit := store.NewRangeIterator(it, &store.Range{startKey, stopKey, store.RangeClose})
	for ; rit.Valid(); rit.Next() {
		t.Delete(rit.RawKey())
		num++
	}

	t.Delete(mk)

	return num
}

func (db *DB) lGetMeta(it *store.Iterator, ek []byte) (headSeq int32, tailSeq int32, size int32, err error) {
	var v []byte
	if it != nil {
		v = it.Find(ek)
	} else {
		v, err = db.bucket.Get(ek)
	}
	if err != nil {
		return
	} else if v == nil {
		headSeq = listInitialSeq
		tailSeq = listInitialSeq
		size = 0
		return
	} else {
		headSeq = int32(binary.LittleEndian.Uint32(v[0:4]))
		tailSeq = int32(binary.LittleEndian.Uint32(v[4:8]))
		size = tailSeq - headSeq + 1
	}
	return
}

func (db *DB) lSetMeta(ek []byte, headSeq int32, tailSeq int32) int32 {
	t := db.listBatch

	var size int32 = tailSeq - headSeq + 1
	if size < 0 {
		//	todo : log error + panic
	} else if size == 0 {
		t.Delete(ek)
	} else {
		buf := make([]byte, 8)

		binary.LittleEndian.PutUint32(buf[0:4], uint32(headSeq))
		binary.LittleEndian.PutUint32(buf[4:8], uint32(tailSeq))

		t.Put(ek, buf)
	}

	return size
}

func (db *DB) lExpireAt(key []byte, when int64) (int64, error) {
	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	if llen, err := db.LLen(key); err != nil || llen == 0 {
		return 0, err
	} else {
		db.expireAt(t, ListType, key, when)
		if err := t.Commit(); err != nil {
			return 0, err
		}
	}
	return 1, nil
}

func (db *DB) LIndex(key []byte, index int32) ([]byte, error) {
	if err := checkKeySize(key); err != nil {
		return nil, err
	}

	var seq int32
	var headSeq int32
	var tailSeq int32
	var err error

	metaKey := db.lEncodeMetaKey(key)

	it := db.bucket.NewIterator()
	defer it.Close()

	headSeq, tailSeq, _, err = db.lGetMeta(it, metaKey)
	if err != nil {
		return nil, err
	}

	if index >= 0 {
		seq = headSeq + index
	} else {
		seq = tailSeq + index + 1
	}

	sk := db.lEncodeListKey(key, seq)
	v := it.Find(sk)

	return v, nil
}

func (db *DB) LLen(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	ek := db.lEncodeMetaKey(key)
	_, _, size, err := db.lGetMeta(nil, ek)
	return int64(size), err
}

func (db *DB) LPop(key []byte) ([]byte, error) {
	return db.lpop(key, listHeadSeq)
}

func (db *DB) LPush(key []byte, arg1 []byte, args ...[]byte) (int64, error) {
	return db.lpush(key, listHeadSeq, arg1, args...)
}

func (db *DB) LRange(key []byte, start int32, stop int32) ([][]byte, error) {
	if err := checkKeySize(key); err != nil {
		return nil, err
	}

	var headSeq int32
	var llen int32
	var err error

	metaKey := db.lEncodeMetaKey(key)

	it := db.bucket.NewIterator()
	defer it.Close()

	if headSeq, _, llen, err = db.lGetMeta(it, metaKey); err != nil {
		return nil, err
	}

	if start < 0 {
		start = llen + start
	}
	if stop < 0 {
		stop = llen + stop
	}
	if start < 0 {
		start = 0
	}

	if start > stop || start >= llen {
		return [][]byte{}, nil
	}

	if stop >= llen {
		stop = llen - 1
	}

	limit := (stop - start) + 1
	headSeq += start

	v := make([][]byte, 0, limit)

	startKey := db.lEncodeListKey(key, headSeq)
	rit := store.NewRangeLimitIterator(it,
		&store.Range{
			Min:  startKey,
			Max:  nil,
			Type: store.RangeClose},
		&store.Limit{
			Offset: 0,
			Count:  int(limit)})

	for ; rit.Valid(); rit.Next() {
		v = append(v, rit.Value())
	}

	return v, nil
}

func (db *DB) RPop(key []byte) ([]byte, error) {
	return db.lpop(key, listTailSeq)
}

func (db *DB) RPush(key []byte, arg1 []byte, args ...[]byte) (int64, error) {
	return db.lpush(key, listTailSeq, arg1, args...)
}

func (db *DB) LClear(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	num := db.lDelete(t, key)
	db.rmExpire(t, ListType, key)

	err := t.Commit()
	return num, err
}

func (db *DB) LMclear(keys ...[]byte) (int64, error) {
	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	for _, key := range keys {
		if err := checkKeySize(key); err != nil {
			return 0, err
		}

		db.lDelete(t, key)
		db.rmExpire(t, ListType, key)

	}

	err := t.Commit()
	return int64(len(keys)), err
}

func (db *DB) lFlush() (drop int64, err error) {
	t := db.listBatch
	t.Lock()
	defer t.Unlock()
	return db.flushType(t, ListType)
}

func (db *DB) LExpire(key []byte, duration int64) (int64, error) {
	if duration <= 0 {
		return 0, errExpireValue
	}

	return db.lExpireAt(key, time.Now().Unix()+duration)
}

func (db *DB) LExpireAt(key []byte, when int64) (int64, error) {
	if when <= time.Now().Unix() {
		return 0, errExpireValue
	}

	return db.lExpireAt(key, when)
}

func (db *DB) LTTL(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return -1, err
	}

	return db.ttl(ListType, key)
}

func (db *DB) LPersist(key []byte) (int64, error) {
	if err := checkKeySize(key); err != nil {
		return 0, err
	}

	t := db.listBatch
	t.Lock()
	defer t.Unlock()

	n, err := db.rmExpire(t, ListType, key)
	if err != nil {
		return 0, err
	}

	err = t.Commit()
	return n, err
}

func (db *DB) LScan(key []byte, count int, inclusive bool, match string) ([][]byte, error) {
	return db.scan(LMetaType, key, count, inclusive, match)
}

func (db *DB) lEncodeMinKey() []byte {
	return db.lEncodeMetaKey(nil)
}

func (db *DB) lEncodeMaxKey() []byte {
	ek := db.lEncodeMetaKey(nil)
	ek[len(ek)-1] = LMetaType + 1
	return ek
}
<|endoftext|>


<commit_before>package controllers

import (
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/nuveo/prest/adapters/postgres"
	"github.com/nuveo/prest/statements"
)

// GetTables list all (or filter) tables
func GetTables(w http.ResponseWriter, r *http.Request) {
	requestWhere, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	order, err := postgres.OrderByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform OrderByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if order == "" {
		order = statements.TablesOrderBy
	}

	sqlTables := fmt.Sprint(
		statements.TablesSelect,
		statements.TablesWhere)

	if requestWhere != "" {
		sqlTables = fmt.Sprintf("%s AND %s", sqlTables, requestWhere)
	}

	sqlTables = fmt.Sprint(sqlTables, order)

	object, err := postgres.Query(sqlTables, values...)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// GetTablesByDatabaseAndSchema list all (or filter) tables based on database and schema
func GetTablesByDatabaseAndSchema(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]

	requestWhere, values, err := postgres.WhereByRequest(r, 3)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sqlSchemaTables := fmt.Sprint(
		statements.SchemaTablesSelect,
		statements.SchemaTablesWhere)

	if requestWhere != "" {
		sqlSchemaTables = fmt.Sprint(sqlSchemaTables, " AND ", requestWhere)
	}

	order, err := postgres.OrderByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform OrderByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if order != "" {
		sqlSchemaTables = fmt.Sprint(sqlSchemaTables, order)
	} else {
		sqlSchemaTables = fmt.Sprint(sqlSchemaTables, statements.SchemaTablesOrderBy)
	}

	page, err := postgres.PaginateIfPossible(r)
	if err != nil {
		err = fmt.Errorf("could not perform PaginateIfPossible: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sqlSchemaTables = fmt.Sprint(sqlSchemaTables, " ", page)

	valuesAux := make([]interface{}, 0)
	valuesAux = append(valuesAux, database)
	valuesAux = append(valuesAux, schema)
	valuesAux = append(valuesAux, values...)

	object, err := postgres.Query(sqlSchemaTables, valuesAux...)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// SelectFromTables perform select in database
func SelectFromTables(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]

	// get selected columns, "*" if empty "_columns"
	cols := postgres.ColumnsByRequest(r)
	cols = postgres.FieldsPermissions(table, cols, "read")

	if len(cols) == 0 {
		err := fmt.Errorf("you don't have permission for this action, please check the permitted fields for this table")
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	selectStr, err := postgres.SelectFields(cols)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	query := fmt.Sprintf("%s %s.%s.%s", selectStr, database, schema, table)

	countQuery, err := postgres.CountByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform CountByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if countQuery != "" {
		query = fmt.Sprintf("%s %s.%s.%s", countQuery, database, schema, table)
	}

	joinValues, err := postgres.JoinByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform JoinByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	for _, j := range joinValues {
		query = fmt.Sprint(query, j)
	}

	requestWhere, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sqlSelect := query
	if requestWhere != "" {
		sqlSelect = fmt.Sprint(
			query,
			" WHERE ",
			requestWhere)
	}

	order, err := postgres.OrderByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform OrderByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if order != "" {
		sqlSelect = fmt.Sprintf("%s %s", sqlSelect, order)
	}

	page, err := postgres.PaginateIfPossible(r)
	if err != nil {
		err = fmt.Errorf("could not perform PaginateIfPossible: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	sqlSelect = fmt.Sprint(sqlSelect, " ", page)

	runQuery := postgres.Query
	if countQuery != "" {
		runQuery = postgres.QueryCount
	}

	object, err := runQuery(sqlSelect, values...)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// InsertInTables perform insert in specific table
func InsertInTables(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]
	object, err := postgres.Insert(database, schema, table, r)
	if err != nil {
		err = fmt.Errorf("could not perform InsertInTables: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// DeleteFromTable perform delete sql
func DeleteFromTable(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]

	where, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	object, err := postgres.Delete(database, schema, table, where, values)
	if err != nil {
		err = fmt.Errorf("could not perform DELETE: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// UpdateTable perform update table
func UpdateTable(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]

	where, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	object, err := postgres.Update(database, schema, table, where, values, r)
	if err != nil {
		err = fmt.Errorf("could not perform UPDATE: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}
<commit_msg>go fmt<commit_after>package controllers

import (
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/nuveo/prest/adapters/postgres"
	"github.com/nuveo/prest/statements"
)

// GetTables list all (or filter) tables
func GetTables(w http.ResponseWriter, r *http.Request) {
	requestWhere, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	order, err := postgres.OrderByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform OrderByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if order == "" {
		order = statements.TablesOrderBy
	}

	sqlTables := fmt.Sprint(
		statements.TablesSelect,
		statements.TablesWhere)

	if requestWhere != "" {
		sqlTables = fmt.Sprintf("%s AND %s", sqlTables, requestWhere)
	}

	sqlTables = fmt.Sprint(sqlTables, order)

	object, err := postgres.Query(sqlTables, values...)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// GetTablesByDatabaseAndSchema list all (or filter) tables based on database and schema
func GetTablesByDatabaseAndSchema(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]

	requestWhere, values, err := postgres.WhereByRequest(r, 3)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sqlSchemaTables := fmt.Sprint(
		statements.SchemaTablesSelect,
		statements.SchemaTablesWhere)

	if requestWhere != "" {
		sqlSchemaTables = fmt.Sprint(sqlSchemaTables, " AND ", requestWhere)
	}

	order, err := postgres.OrderByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform OrderByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if order != "" {
		sqlSchemaTables = fmt.Sprint(sqlSchemaTables, order)
	} else {
		sqlSchemaTables = fmt.Sprint(sqlSchemaTables, statements.SchemaTablesOrderBy)
	}

	page, err := postgres.PaginateIfPossible(r)
	if err != nil {
		err = fmt.Errorf("could not perform PaginateIfPossible: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sqlSchemaTables = fmt.Sprint(sqlSchemaTables, " ", page)

	valuesAux := make([]interface{}, 0)
	valuesAux = append(valuesAux, database)
	valuesAux = append(valuesAux, schema)
	valuesAux = append(valuesAux, values...)

	object, err := postgres.Query(sqlSchemaTables, valuesAux...)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// SelectFromTables perform select in database
func SelectFromTables(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]

	// get selected columns, "*" if empty "_columns"
	cols := postgres.ColumnsByRequest(r)
	cols = postgres.FieldsPermissions(table, cols, "read")

	if len(cols) == 0 {
		err := fmt.Errorf("you don't have permission for this action, please check the permitted fields for this table")
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	selectStr, err := postgres.SelectFields(cols)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	query := fmt.Sprintf("%s %s.%s.%s", selectStr, database, schema, table)

	countQuery, err := postgres.CountByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform CountByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if countQuery != "" {
		query = fmt.Sprintf("%s %s.%s.%s", countQuery, database, schema, table)
	}

	joinValues, err := postgres.JoinByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform JoinByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	for _, j := range joinValues {
		query = fmt.Sprint(query, j)
	}

	requestWhere, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sqlSelect := query
	if requestWhere != "" {
		sqlSelect = fmt.Sprint(
			query,
			" WHERE ",
			requestWhere)
	}

	order, err := postgres.OrderByRequest(r)
	if err != nil {
		err = fmt.Errorf("could not perform OrderByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	if order != "" {
		sqlSelect = fmt.Sprintf("%s %s", sqlSelect, order)
	}

	page, err := postgres.PaginateIfPossible(r)
	if err != nil {
		err = fmt.Errorf("could not perform PaginateIfPossible: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	sqlSelect = fmt.Sprint(sqlSelect, " ", page)

	runQuery := postgres.Query
	if countQuery != "" {
		runQuery = postgres.QueryCount
	}

	object, err := runQuery(sqlSelect, values...)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// InsertInTables perform insert in specific table
func InsertInTables(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]
	object, err := postgres.Insert(database, schema, table, r)
	if err != nil {
		err = fmt.Errorf("could not perform InsertInTables: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// DeleteFromTable perform delete sql
func DeleteFromTable(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]

	where, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	object, err := postgres.Delete(database, schema, table, where, values)
	if err != nil {
		err = fmt.Errorf("could not perform DELETE: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}

// UpdateTable perform update table
func UpdateTable(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	database := vars["database"]
	schema := vars["schema"]
	table := vars["table"]

	where, values, err := postgres.WhereByRequest(r, 1)
	if err != nil {
		err = fmt.Errorf("could not perform WhereByRequest: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	object, err := postgres.Update(database, schema, table, where, values, r)
	if err != nil {
		err = fmt.Errorf("could not perform UPDATE: %v", err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(object)
}
<|endoftext|>


<commit_before>package cron

import (
	"time"

	"github.com/open-falcon/agent/funcs"
	"github.com/open-falcon/agent/g"
	"github.com/open-falcon/common/model"
)

func InitDataHistory() {
	for {
		funcs.UpdateCpuStat()
		funcs.UpdateDiskStats()
		time.Sleep(g.COLLECT_INTERVAL)
	}
}

func Collect() {

	if !g.Config().Transfer.Enabled {
		return
	}

	if len(g.Config().Transfer.Addrs) == 0 {
		return
	}

	for _, v := range funcs.Mappers {
		go collect(int64(v.Interval), v.Fs)
	}
}

func collect(sec int64, fns []func() []*model.MetricValue) {

	for {
		time.Sleep(time.Duration(sec) * time.Second)

		hostname, err := g.Hostname()
		if err != nil {
			continue
		}

		mvs := []*model.MetricValue{}
		ignoreMetrics := g.Config().IgnoreMetrics

		for _, fn := range fns {
			items := fn()
			if items == nil {
				continue
			}

			if len(items) == 0 {
				continue
			}

			for _, mv := range items {
				if b, ok := ignoreMetrics[mv.Metric]; ok && b {
					continue
				} else {
					mvs = append(mvs, mv)
				}
			}
		}

		now := time.Now().Unix()
		for j := 0; j < len(mvs); j++ {
			mvs[j].Step = sec
			mvs[j].Endpoint = hostname
			mvs[j].Timestamp = now
		}

		g.SendToTransfer(mvs)

	}
}
<commit_msg>change sleep -> tikcer<commit_after>package cron

import (
	"time"

	"github.com/open-falcon/agent/funcs"
	"github.com/open-falcon/agent/g"
	"github.com/open-falcon/common/model"
)

func InitDataHistory() {
	for {
		funcs.UpdateCpuStat()
		funcs.UpdateDiskStats()
		time.Sleep(g.COLLECT_INTERVAL)
	}
}

func Collect() {

	if !g.Config().Transfer.Enabled {
		return
	}

	if len(g.Config().Transfer.Addrs) == 0 {
		return
	}

	for _, v := range funcs.Mappers {
		go collect(int64(v.Interval), v.Fs)
	}
}

func collect(sec int64, fns []func() []*model.MetricValue) {
	t := time.NewTicker(time.Second * time.Duration(sec)).C
	for {
		<-t

		hostname, err := g.Hostname()
		if err != nil {
			continue
		}

		mvs := []*model.MetricValue{}
		ignoreMetrics := g.Config().IgnoreMetrics

		for _, fn := range fns {
			items := fn()
			if items == nil {
				continue
			}

			if len(items) == 0 {
				continue
			}

			for _, mv := range items {
				if b, ok := ignoreMetrics[mv.Metric]; ok && b {
					continue
				} else {
					mvs = append(mvs, mv)
				}
			}
		}

		now := time.Now().Unix()
		for j := 0; j < len(mvs); j++ {
			mvs[j].Step = sec
			mvs[j].Endpoint = hostname
			mvs[j].Timestamp = now
		}

		g.SendToTransfer(mvs)

	}
}
<|endoftext|>