<commit_before>// Copyright 2014 Vic Demuzere
//
// Use of this source code is governed by the MIT license.

package irc

import (
	"bytes"
	"strings"
)

// Various constants used for formatting IRC messages.
const (
	prefix     byte = 0x3A // Prefix or last argument
	prefixUser byte = 0x21 // Username
	prefixHost byte = 0x40 // Hostname
	space      byte = 0x20 // Separator

	maxLength = 510 // Maximum length is 512 - 2 for the line endings.
)

func cutsetFunc(r rune) bool {
	// Characters to trim from prefixes/messages.
	return r == '\r' || r == '\n'
}

// Objects implementing the Sender interface are able to send messages to an IRC server.
//
// As there might be a message queue, it is possible that Send returns a nil
// error, but the message is not sent (yet). The error value is only used when
// it is certain that sending the message is impossible.
//
// This interface is not used inside this package, and shouldn't have been
// defined here in the first place. For backwards compatibility only.
type Sender interface {
	Send(*Message) error
}

// Prefix represents the prefix (sender) of an IRC message.
// See RFC1459 section 2.3.1.
//
//    <servername> | <nick> [ '!' <user> ] [ '@' <host> ]
//
type Prefix struct {
	Name string // Nick- or servername
	User string // Username
	Host string // Hostname
}

// ParsePrefix takes a string and attempts to create a Prefix struct.
func ParsePrefix(raw string) (p *Prefix) {

	p = new(Prefix)

	user := indexByte(raw, prefixUser)
	host := indexByte(raw, prefixHost)

	switch {

	case user > 0 && host > user:
		p.Name = raw[:user]
		p.User = raw[user+1 : host]
		p.Host = raw[host+1:]

	case user > 0:
		p.Name = raw[:user]
		p.User = raw[user+1:]

	case host > 0:
		p.Name = raw[:host]
		p.Host = raw[host+1:]

	default:
		p.Name = raw

	}

	return p
}

// Len calculates the length of the string representation of this prefix.
func (p *Prefix) Len() (length int) {
	length = len(p.Name)
	if len(p.User) > 0 {
		length = length + len(p.User) + 1
	}
	if len(p.Host) > 0 {
		length = length + len(p.Host) + 1
	}
	return
}

// Bytes returns a []byte representation of this prefix.
func (p *Prefix) Bytes() []byte {
	buffer := new(bytes.Buffer)
	p.writeTo(buffer)
	return buffer.Bytes()
}

// String returns a string representation of this prefix.
func (p *Prefix) String() (s string) {
	// Benchmarks revealed that in this case simple string concatenation
	// is actually faster than using a ByteBuffer as in (*Message).String()
	s = p.Name
	if len(p.User) > 0 {
		s = s + string(prefixUser) + p.User
	}
	if len(p.Host) > 0 {
		s = s + string(prefixHost) + p.Host
	}
	return
}

// IsHostmask returns true if this prefix looks like a user hostmask.
func (p *Prefix) IsHostmask() bool {
	return len(p.User) > 0 && len(p.Host) > 0
}

// IsServer returns true if this prefix looks like a server name.
func (p *Prefix) IsServer() bool {
	return len(p.User) <= 0 && len(p.Host) <= 0 // && indexByte(p.Name, '.') > 0
}

// writeTo is an utility function to write the prefix to the bytes.Buffer in Message.String().
func (p *Prefix) writeTo(buffer *bytes.Buffer) {
	buffer.WriteString(p.Name)
	if len(p.User) > 0 {
		buffer.WriteByte(prefixUser)
		buffer.WriteString(p.User)
	}
	if len(p.Host) > 0 {
		buffer.WriteByte(prefixHost)
		buffer.WriteString(p.Host)
	}
	return
}

// Message represents an IRC protocol message.
// See RFC1459 section 2.3.1.
//
//    <message>  ::= [':' <prefix> <SPACE> ] <command> <params> <crlf>
//    <prefix>   ::= <servername> | <nick> [ '!' <user> ] [ '@' <host> ]
//    <command>  ::= <letter> { <letter> } | <number> <number> <number>
//    <SPACE>    ::= ' ' { ' ' }
//    <params>   ::= <SPACE> [ ':' <trailing> | <middle> <params> ]
//
//    <middle>   ::= <Any *non-empty* sequence of octets not including SPACE
//                   or NUL or CR or LF, the first of which may not be ':'>
//    <trailing> ::= <Any, possibly *empty*, sequence of octets not including
//                   NUL or CR or LF>
//
//    <crlf>     ::= CR LF
type Message struct {
	*Prefix
	Command  string
	Params   []string
	Trailing string

	// When set to true, the trailing prefix (:) will be added even if the trailing message is empty.
	EmptyTrailing bool
}

// ParseMessage takes a string and attempts to create a Message struct.
// Returns nil if the Message is invalid.
func ParseMessage(raw string) (m *Message) {

	// Ignore empty messages.
	if raw = strings.TrimFunc(raw, cutsetFunc); len(raw) < 2 {
		return nil
	}

	i, j := 0, 0

	m = new(Message)

	if raw[0] == prefix {

		// Prefix ends with a space.
		i = indexByte(raw, space)

		// Prefix string must not be empty if the indicator is present.
		if i < 2 {
			return nil
		}

		m.Prefix = ParsePrefix(raw[1:i])

		// Skip space at the end of the prefix
		i++
	}

	// Find end of command
	j = i + indexByte(raw[i:], space)

	// Extract command
	if j > i {
		m.Command = strings.ToUpper(raw[i:j])
	} else {
		m.Command = strings.ToUpper(raw[i:])

		// We're done here!
		return m
	}

	// Skip space after command
	j++

	// Find prefix for trailer
	i = indexByte(raw[j:], prefix)

	if i < 0 || raw[j+i-1] != space {

		// There is no trailing argument!
		m.Params = strings.Split(raw[j:], string(space))

		// We're done here!
		return m
	}

	// Compensate for index on substring
	i = i + j

	// Check if we need to parse arguments.
	if i > j {
		m.Params = strings.Split(raw[j:i-1], string(space))
	}

	m.Trailing = raw[i+1:]

	// We need to re-encode the trailing argument even if it was empty.
	if len(m.Trailing) <= 0 {
		m.EmptyTrailing = true
	}

	return m

}

// Len calculates the length of the string representation of this message.
func (m *Message) Len() (length int) {

	if m.Prefix != nil {
		length = m.Prefix.Len() + 2 // Include prefix and trailing space
	}

	length = length + len(m.Command)

	if len(m.Params) > 0 {
		length = length + len(m.Params)
		for _, param := range m.Params {
			length = length + len(param)
		}
	}

	if len(m.Trailing) > 0 || m.EmptyTrailing {
		length = length + len(m.Trailing) + 2 // Include prefix and space
	}

	return
}

// Bytes returns a []byte representation of this message.
//
// As noted in rfc2812 section 2.3, messages should not exceed 512 characters
// in length. This method forces that limit by discarding any characters
// exceeding the length limit.
func (m *Message) Bytes() []byte {

	buffer := new(bytes.Buffer)

	// Message prefix
	if m.Prefix != nil {
		buffer.WriteByte(prefix)
		m.Prefix.writeTo(buffer)
		buffer.WriteByte(space)
	}

	// Command is required
	buffer.WriteString(m.Command)

	// Space separated list of arguments
	if len(m.Params) > 0 {
		buffer.WriteByte(space)
		buffer.WriteString(strings.Join(m.Params, string(space)))
	}

	if len(m.Trailing) > 0 || m.EmptyTrailing {
		buffer.WriteByte(space)
		buffer.WriteByte(prefix)
		buffer.WriteString(m.Trailing)
	}

	// We need the limit the buffer length.
	if buffer.Len() > (maxLength) {
		buffer.Truncate(maxLength)
	}

	return buffer.Bytes()
}

// String returns a string representation of this message.
//
// As noted in rfc2812 section 2.3, messages should not exceed 512 characters
// in length. This method forces that limit by discarding any characters
// exceeding the length limit.
func (m *Message) String() string {
	return string(m.Bytes())
}
<commit_msg>Makes go lint happy.<commit_after>// Copyright 2014 Vic Demuzere
//
// Use of this source code is governed by the MIT license.

package irc

import (
	"bytes"
	"strings"
)

// Various constants used for formatting IRC messages.
const (
	prefix     byte = 0x3A // Prefix or last argument
	prefixUser byte = 0x21 // Username
	prefixHost byte = 0x40 // Hostname
	space      byte = 0x20 // Separator

	maxLength = 510 // Maximum length is 512 - 2 for the line endings.
)

func cutsetFunc(r rune) bool {
	// Characters to trim from prefixes/messages.
	return r == '\r' || r == '\n'
}

// Sender represents objects that are able to send messages to an IRC server.
//
// As there might be a message queue, it is possible that Send returns a nil
// error, but the message is not sent (yet). The error value is only used when
// it is certain that sending the message is impossible.
//
// This interface is not used inside this package, and shouldn't have been
// defined here in the first place. For backwards compatibility only.
type Sender interface {
	Send(*Message) error
}

// Prefix represents the prefix (sender) of an IRC message.
// See RFC1459 section 2.3.1.
//
//    <servername> | <nick> [ '!' <user> ] [ '@' <host> ]
//
type Prefix struct {
	Name string // Nick- or servername
	User string // Username
	Host string // Hostname
}

// ParsePrefix takes a string and attempts to create a Prefix struct.
func ParsePrefix(raw string) (p *Prefix) {

	p = new(Prefix)

	user := indexByte(raw, prefixUser)
	host := indexByte(raw, prefixHost)

	switch {

	case user > 0 && host > user:
		p.Name = raw[:user]
		p.User = raw[user+1 : host]
		p.Host = raw[host+1:]

	case user > 0:
		p.Name = raw[:user]
		p.User = raw[user+1:]

	case host > 0:
		p.Name = raw[:host]
		p.Host = raw[host+1:]

	default:
		p.Name = raw

	}

	return p
}

// Len calculates the length of the string representation of this prefix.
func (p *Prefix) Len() (length int) {
	length = len(p.Name)
	if len(p.User) > 0 {
		length = length + len(p.User) + 1
	}
	if len(p.Host) > 0 {
		length = length + len(p.Host) + 1
	}
	return
}

// Bytes returns a []byte representation of this prefix.
func (p *Prefix) Bytes() []byte {
	buffer := new(bytes.Buffer)
	p.writeTo(buffer)
	return buffer.Bytes()
}

// String returns a string representation of this prefix.
func (p *Prefix) String() (s string) {
	// Benchmarks revealed that in this case simple string concatenation
	// is actually faster than using a ByteBuffer as in (*Message).String()
	s = p.Name
	if len(p.User) > 0 {
		s = s + string(prefixUser) + p.User
	}
	if len(p.Host) > 0 {
		s = s + string(prefixHost) + p.Host
	}
	return
}

// IsHostmask returns true if this prefix looks like a user hostmask.
func (p *Prefix) IsHostmask() bool {
	return len(p.User) > 0 && len(p.Host) > 0
}

// IsServer returns true if this prefix looks like a server name.
func (p *Prefix) IsServer() bool {
	return len(p.User) <= 0 && len(p.Host) <= 0 // && indexByte(p.Name, '.') > 0
}

// writeTo is an utility function to write the prefix to the bytes.Buffer in Message.String().
func (p *Prefix) writeTo(buffer *bytes.Buffer) {
	buffer.WriteString(p.Name)
	if len(p.User) > 0 {
		buffer.WriteByte(prefixUser)
		buffer.WriteString(p.User)
	}
	if len(p.Host) > 0 {
		buffer.WriteByte(prefixHost)
		buffer.WriteString(p.Host)
	}
	return
}

// Message represents an IRC protocol message.
// See RFC1459 section 2.3.1.
//
//    <message>  ::= [':' <prefix> <SPACE> ] <command> <params> <crlf>
//    <prefix>   ::= <servername> | <nick> [ '!' <user> ] [ '@' <host> ]
//    <command>  ::= <letter> { <letter> } | <number> <number> <number>
//    <SPACE>    ::= ' ' { ' ' }
//    <params>   ::= <SPACE> [ ':' <trailing> | <middle> <params> ]
//
//    <middle>   ::= <Any *non-empty* sequence of octets not including SPACE
//                   or NUL or CR or LF, the first of which may not be ':'>
//    <trailing> ::= <Any, possibly *empty*, sequence of octets not including
//                   NUL or CR or LF>
//
//    <crlf>     ::= CR LF
type Message struct {
	*Prefix
	Command  string
	Params   []string
	Trailing string

	// When set to true, the trailing prefix (:) will be added even if the trailing message is empty.
	EmptyTrailing bool
}

// ParseMessage takes a string and attempts to create a Message struct.
// Returns nil if the Message is invalid.
func ParseMessage(raw string) (m *Message) {

	// Ignore empty messages.
	if raw = strings.TrimFunc(raw, cutsetFunc); len(raw) < 2 {
		return nil
	}

	i, j := 0, 0

	m = new(Message)

	if raw[0] == prefix {

		// Prefix ends with a space.
		i = indexByte(raw, space)

		// Prefix string must not be empty if the indicator is present.
		if i < 2 {
			return nil
		}

		m.Prefix = ParsePrefix(raw[1:i])

		// Skip space at the end of the prefix
		i++
	}

	// Find end of command
	j = i + indexByte(raw[i:], space)

	// Extract command
	if j > i {
		m.Command = strings.ToUpper(raw[i:j])
	} else {
		m.Command = strings.ToUpper(raw[i:])

		// We're done here!
		return m
	}

	// Skip space after command
	j++

	// Find prefix for trailer
	i = indexByte(raw[j:], prefix)

	if i < 0 || raw[j+i-1] != space {

		// There is no trailing argument!
		m.Params = strings.Split(raw[j:], string(space))

		// We're done here!
		return m
	}

	// Compensate for index on substring
	i = i + j

	// Check if we need to parse arguments.
	if i > j {
		m.Params = strings.Split(raw[j:i-1], string(space))
	}

	m.Trailing = raw[i+1:]

	// We need to re-encode the trailing argument even if it was empty.
	if len(m.Trailing) <= 0 {
		m.EmptyTrailing = true
	}

	return m

}

// Len calculates the length of the string representation of this message.
func (m *Message) Len() (length int) {

	if m.Prefix != nil {
		length = m.Prefix.Len() + 2 // Include prefix and trailing space
	}

	length = length + len(m.Command)

	if len(m.Params) > 0 {
		length = length + len(m.Params)
		for _, param := range m.Params {
			length = length + len(param)
		}
	}

	if len(m.Trailing) > 0 || m.EmptyTrailing {
		length = length + len(m.Trailing) + 2 // Include prefix and space
	}

	return
}

// Bytes returns a []byte representation of this message.
//
// As noted in rfc2812 section 2.3, messages should not exceed 512 characters
// in length. This method forces that limit by discarding any characters
// exceeding the length limit.
func (m *Message) Bytes() []byte {

	buffer := new(bytes.Buffer)

	// Message prefix
	if m.Prefix != nil {
		buffer.WriteByte(prefix)
		m.Prefix.writeTo(buffer)
		buffer.WriteByte(space)
	}

	// Command is required
	buffer.WriteString(m.Command)

	// Space separated list of arguments
	if len(m.Params) > 0 {
		buffer.WriteByte(space)
		buffer.WriteString(strings.Join(m.Params, string(space)))
	}

	if len(m.Trailing) > 0 || m.EmptyTrailing {
		buffer.WriteByte(space)
		buffer.WriteByte(prefix)
		buffer.WriteString(m.Trailing)
	}

	// We need the limit the buffer length.
	if buffer.Len() > (maxLength) {
		buffer.Truncate(maxLength)
	}

	return buffer.Bytes()
}

// String returns a string representation of this message.
//
// As noted in rfc2812 section 2.3, messages should not exceed 512 characters
// in length. This method forces that limit by discarding any characters
// exceeding the length limit.
func (m *Message) String() string {
	return string(m.Bytes())
}
<|endoftext|>


<commit_before>package main

type MetricsMap map[string]int64

type Metrics []*Metric

func (list Metrics) Len() int {
	return len(list)
}

func (list Metrics) Swap(a, b int) {
	list[a], list[b] = list[b], list[a]
}

func (list Metrics) Less(a, b int) bool {
	return list[a].Key < list[b].Key
}
<commit_msg>look at tags when sorting metrics with same key<commit_after>package main

import (
	"fmt"
	"sort"
	"strings"
)

type MetricsMap map[string]int64

type Metrics []*Metric

func (list Metrics) Len() int {
	return len(list)
}

func (list Metrics) Swap(a, b int) {
	list[a], list[b] = list[b], list[a]
}

func (list Metrics) Less(a, b int) bool {
	if list[a].Key == list[b].Key {
		return flattenTags(list[a].Tags) < flattenTags(list[b].Tags)
	} else {
		return list[a].Key < list[b].Key
	}
}

func flattenTags(tags map[string]string) string {
	out := []string{}
	for k, v := range tags {
		out = append(out, fmt.Sprintf("%s=%s", k, v))
	}
	sort.Strings(out)
	return strings.Join(out, " ")
}
<|endoftext|>


<commit_before>package mgr

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/qiniu/logkit/cleaner"
	config "github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/parser"
	"github.com/qiniu/logkit/sender"
	"github.com/qiniu/logkit/utils"

	"github.com/howeyc/fsnotify"
	"github.com/qiniu/log"
)

var DIR_NOT_EXIST_SLEEP_TIME = 300 //300 s

type ManagerConfig struct {
	BindHost string `json:"bind_host"`
	Idc      string `json:"idc"`
	Zone     string `json:"zone"`
}

type cleanQueue struct {
	cleanerCount int
	filecount    map[string]int
}

type Manager struct {
	ManagerConfig
	lock        sync.RWMutex
	cleanlock   sync.Mutex
	cleanChan   chan cleaner.CleanSignal
	cleanQueues map[string]*cleanQueue
	runners     map[string]Runner
	watchers    map[uint64]*fsnotify.Watcher // inode到watcher的映射表
	pregistry   *parser.ParserRegistry
	sregistry   *sender.SenderRegistry
}

func NewManager(conf ManagerConfig) (*Manager, error) {
	ps := parser.NewParserRegistry()
	sr := sender.NewSenderRegistry()
	return NewCustomManager(conf, ps, sr)
}

func NewCustomManager(conf ManagerConfig, pr *parser.ParserRegistry, sr *sender.SenderRegistry) (*Manager, error) {
	m := &Manager{
		ManagerConfig: conf,
		cleanChan:     make(chan cleaner.CleanSignal),
		cleanQueues:   make(map[string]*cleanQueue),
		runners:       make(map[string]Runner),
		watchers:      make(map[uint64]*fsnotify.Watcher),
		pregistry:     pr,
		sregistry:     sr,
	}
	return m, nil
}

func (m *Manager) Stop() error {
	m.lock.Lock()
	defer m.lock.Unlock()
	for _, runner := range m.runners {
		runner.Stop()
	}
	for _, w := range m.watchers {
		if w != nil {
			w.Close()
		}
	}
	close(m.cleanChan)
	return nil
}

func (m *Manager) Remove(confPath string) {
	if !strings.HasSuffix(confPath, ".conf") {
		log.Warn(confPath, "not end with .conf, skipped")
		return
	}
	log.Info("try remove", confPath)
	confPathAbs, err := filepath.Abs(confPath)
	if err != nil {
		log.Warnf("filepath.Abs(%s) failed: %v", confPath)
		return
	}
	confPath = confPathAbs
	m.lock.Lock()
	defer m.lock.Unlock()

	runner, ok := m.runners[confPath]
	if !ok {
		log.Warnf("%s not added, nothing to do", confPath)
		return
	}

	m.removeCleanQueue(runner.Cleaner())
	runner.Stop()
	delete(m.runners, confPath)
	log.Infof("runner %s be removed, total %d", runner.Name(), len(m.runners))
	return
}

func (m *Manager) addCleanQueue(info CleanInfo) {
	if !info.enable {
		return
	}
	m.cleanlock.Lock()
	defer m.cleanlock.Unlock()
	cq, ok := m.cleanQueues[info.logdir]
	if ok {
		cq.cleanerCount++
	} else {
		cq = &cleanQueue{
			cleanerCount: 1,
			filecount:    make(map[string]int),
		}
	}
	log.Info(">>>>>>>>>>>> add clean queue", cq.cleanerCount, info.logdir)
	m.cleanQueues[info.logdir] = cq
	return
}

func (m *Manager) removeCleanQueue(info CleanInfo) {
	if !info.enable {
		return
	}
	m.cleanlock.Lock()
	defer m.cleanlock.Unlock()
	cq, ok := m.cleanQueues[info.logdir]
	if !ok {
		log.Errorf("can't find clean queue %v to remove", info.logdir)
		return
	}
	cq.cleanerCount--
	if cq.cleanerCount <= 0 {
		delete(m.cleanQueues, info.logdir)
	}
	log.Info(">>>>>>>>>>>> remove clean queue", cq.cleanerCount, info.logdir)
	return
}

func (m *Manager) Add(confPath string) {
	if !strings.HasSuffix(confPath, ".conf") {
		log.Warn(confPath, "not end with .conf, skipped")
		return
	}
	log.Info("try add", confPath)
	confPathAbs, _, err := utils.GetRealPath(confPath)
	if err != nil {
		log.Warnf("filepath.Abs(%s) failed: %v", confPath)
		return
	}
	confPath = confPathAbs
	if m.isRunning(confPath) {
		log.Errorf("%s already added", confPath)
		return
	}
	var conf RunnerConfig
	err = config.LoadEx(&conf, confPath)
	if err != nil {
		log.Warnf("config.LoadEx %s failed:%v", confPath, err)
		return
	}

	forkRunner := func(confPath string) {
		var runner Runner
		i := 0
		for {
			if m.isRunning(confPath) {
				log.Errorf("%s already added - ", confPath)
				return
			}

			if runner, err = NewCustomRunner(conf, m.cleanChan, m.pregistry, m.sregistry); err != nil {
				errVal, ok := err.(*os.PathError)
				if !ok {
					log.Errorf("NewRunner(%v) failed: %v", conf.RunnerName, err)
					return
				}
				i++
				log.Warnf("LogDir(%v) does not exsit after %d rounds, sleep 5 minute and try again...", errVal.Path, i)
				time.Sleep(time.Duration(DIR_NOT_EXIST_SLEEP_TIME) * time.Second)
				continue
			}
			break
		}
		m.lock.Lock()
		defer m.lock.Unlock()
		// double check
		if _, ok := m.runners[confPath]; ok {
			return
		}
		m.addCleanQueue(runner.Cleaner())
		log.Infof("%s added: %#v", conf.RunnerName, confPath)
		go runner.Run()
		m.runners[confPath] = runner
		log.Infof("new runner %s is added, total %d", conf.RunnerName, len(m.runners))
		return
	}

	log.Infof("Start to try add: %#v", conf.RunnerName)
	go forkRunner(confPath)

	return
}
func (m *Manager) isRunning(confPath string) bool {
	m.lock.RLock()
	defer m.lock.RUnlock()
	if _, ok := m.runners[confPath]; ok {
		return true
	}
	return false
}

// 创建文件会触发 create和modify事件
// 重命名会触发 rename和create事件
// 删除会触发 delete事件
// 修改会触发 delete事件、create事件、modify事件以及modify|ATTRIB事件
func (m *Manager) handle(inode uint64, path string, watcher *fsnotify.Watcher) {
	for {
		select {
		case ev, ok := <-watcher.Event:
			if !ok {
				log.Info("Manager watcher chan was closed")
				return
			}
			log.Info("event:", ev)
			if ev.IsDelete() || ev.IsRename() {
				_, err := os.Stat(path)
				if os.IsNotExist(err) {
					// 如果当前监听文件被删除，则不再监听，退出
					log.Warnf("close file watcher path %v inode %v", path, inode)
					watcher.Close()
					delete(m.watchers, inode)
					return
				}
				m.Remove(ev.Name)
			}
			if ev.IsCreate() {
				m.Add(ev.Name)
			}
			if ev.IsModify() && !ev.IsCreate() {
				m.Remove(ev.Name)
				m.Add(ev.Name)
			}
		case err := <-watcher.Error:
			if err != nil {
				log.Error("error:", err)
			}
		}
	}
}

func (m *Manager) doClean(sig cleaner.CleanSignal) {
	m.cleanlock.Lock()
	defer m.cleanlock.Unlock()

	dir := sig.Logdir
	dir, err := filepath.Abs(dir)
	if err != nil {
		log.Errorf("get abs for %v error %v", dir, err)
		return
	}
	file := sig.Filename
	q, ok := m.cleanQueues[dir]
	if !ok {
		log.Errorf("%v cleaner dir %v not exist but got clean signal for delete file %v", sig.Cleaner, dir, file)
		return
	}
	count := q.filecount[file] + 1
	if count >= q.cleanerCount {
		catdir := filepath.Join(dir, file)
		err := os.Remove(catdir)
		if err != nil {
			if os.IsNotExist(err) {
				log.Warnf("clean %v failed as logfile is not exist: %v", catdir, err)
			} else {
				log.Errorf("clean %v failed: %v", catdir, err)
			}
		} else {
			log.Infof("log <%v> was successfully cleaned by cleaner", catdir)
		}
		delete(q.filecount, file)
	} else {
		q.filecount[file] = count
		m.cleanQueues[dir] = q
	}
	return
}

func (m *Manager) clean() {
	for sig := range m.cleanChan {
		m.doClean(sig)
	}
}

func (m *Manager) detectMoreWatchers(confsPath []string) {
	ticker := time.NewTicker(time.Second * 10)
	for {
		select {
		case <-ticker.C:
			log.Debug("try to detect more watchers ")
			m.addWatchers(confsPath)
		}
	}
}

func (m *Manager) addWatchers(confsPath []string) (err error) {
	for _, dir := range confsPath {
		paths, err := filepath.Glob(dir)
		if err != nil {
			log.Errorf("filepath.Glob(%s): %v, err:%v", dir, paths, err)
			continue
		}
		for _, path := range paths {
			inode, err := utils.GetIdentifyIDByPath(path)
			if err != nil {
				log.Errorf("GetIdentifyID(%v) error %v", path, err)
				continue
			}
			_, exist := m.watchers[inode]
			if exist {
				// 如果文件已经被监听，则不再重复监听
				continue
			}
			files, err := ioutil.ReadDir(path)
			if err != nil {
				log.Errorf("ioutil.ReadDir(%s): %v, err:%v", path, files, err)
				continue
			}
			log.Warnf("start to add watcher of conf path %v", path)
			for _, f := range files {
				if f.IsDir() {
					log.Warn("skipped dir", f.Name)
					continue
				}
				m.Add(filepath.Join(path, f.Name()))
			}
			watcher, err := fsnotify.NewWatcher()
			if err != nil {
				log.Errorf("fsnotify.NewWatcher: %v", err)
				continue
			}
			m.watchers[inode] = watcher
			go m.handle(inode, path, watcher)
			if err = watcher.Watch(path); err != nil {
				log.Errorf("watch %v error %v", path, err)
			}
		}
	}
	return nil
}

func (m *Manager) Watch(confsPath []string) (err error) {
	err = m.addWatchers(confsPath)
	if err != nil {
		log.Errorf("addWatchers error : %v", err)
	}
	go m.detectMoreWatchers(confsPath)
	go m.clean()
	return
}

func (m *Manager) Status() (rss map[string]RunnerStatus) {
	rss = make(map[string]RunnerStatus)
	for _, r := range m.runners {
		rss[r.Name()] = r.Status()
	}
	return
}
<commit_msg>change watchers use path instead of inode<commit_after>package mgr

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/qiniu/logkit/cleaner"
	config "github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/parser"
	"github.com/qiniu/logkit/sender"
	"github.com/qiniu/logkit/utils"

	"github.com/howeyc/fsnotify"
	"github.com/qiniu/log"
)

var DIR_NOT_EXIST_SLEEP_TIME = 300 //300 s

type ManagerConfig struct {
	BindHost string `json:"bind_host"`
	Idc      string `json:"idc"`
	Zone     string `json:"zone"`
}

type cleanQueue struct {
	cleanerCount int
	filecount    map[string]int
}

type Manager struct {
	ManagerConfig
	lock        sync.RWMutex
	cleanlock   sync.Mutex
	cleanChan   chan cleaner.CleanSignal
	cleanQueues map[string]*cleanQueue
	runners     map[string]Runner
	watchers    map[string]*fsnotify.Watcher // inode到watcher的映射表
	pregistry   *parser.ParserRegistry
	sregistry   *sender.SenderRegistry
}

func NewManager(conf ManagerConfig) (*Manager, error) {
	ps := parser.NewParserRegistry()
	sr := sender.NewSenderRegistry()
	return NewCustomManager(conf, ps, sr)
}

func NewCustomManager(conf ManagerConfig, pr *parser.ParserRegistry, sr *sender.SenderRegistry) (*Manager, error) {
	m := &Manager{
		ManagerConfig: conf,
		cleanChan:     make(chan cleaner.CleanSignal),
		cleanQueues:   make(map[string]*cleanQueue),
		runners:       make(map[string]Runner),
		watchers:      make(map[string]*fsnotify.Watcher),
		pregistry:     pr,
		sregistry:     sr,
	}
	return m, nil
}

func (m *Manager) Stop() error {
	m.lock.Lock()
	defer m.lock.Unlock()
	for _, runner := range m.runners {
		runner.Stop()
	}
	for _, w := range m.watchers {
		if w != nil {
			w.Close()
		}
	}
	close(m.cleanChan)
	return nil
}

func (m *Manager) Remove(confPath string) {
	if !strings.HasSuffix(confPath, ".conf") {
		log.Warn(confPath, "not end with .conf, skipped")
		return
	}
	log.Info("try remove", confPath)
	confPathAbs, err := filepath.Abs(confPath)
	if err != nil {
		log.Warnf("filepath.Abs(%s) failed: %v", confPath)
		return
	}
	confPath = confPathAbs
	m.lock.Lock()
	defer m.lock.Unlock()

	runner, ok := m.runners[confPath]
	if !ok {
		log.Warnf("%s not added, nothing to do", confPath)
		return
	}

	m.removeCleanQueue(runner.Cleaner())
	runner.Stop()
	delete(m.runners, confPath)
	log.Infof("runner %s be removed, total %d", runner.Name(), len(m.runners))
	return
}

func (m *Manager) addCleanQueue(info CleanInfo) {
	if !info.enable {
		return
	}
	m.cleanlock.Lock()
	defer m.cleanlock.Unlock()
	cq, ok := m.cleanQueues[info.logdir]
	if ok {
		cq.cleanerCount++
	} else {
		cq = &cleanQueue{
			cleanerCount: 1,
			filecount:    make(map[string]int),
		}
	}
	log.Info(">>>>>>>>>>>> add clean queue", cq.cleanerCount, info.logdir)
	m.cleanQueues[info.logdir] = cq
	return
}

func (m *Manager) removeCleanQueue(info CleanInfo) {
	if !info.enable {
		return
	}
	m.cleanlock.Lock()
	defer m.cleanlock.Unlock()
	cq, ok := m.cleanQueues[info.logdir]
	if !ok {
		log.Errorf("can't find clean queue %v to remove", info.logdir)
		return
	}
	cq.cleanerCount--
	if cq.cleanerCount <= 0 {
		delete(m.cleanQueues, info.logdir)
	}
	log.Info(">>>>>>>>>>>> remove clean queue", cq.cleanerCount, info.logdir)
	return
}

func (m *Manager) Add(confPath string) {
	if !strings.HasSuffix(confPath, ".conf") {
		log.Warn(confPath, "not end with .conf, skipped")
		return
	}
	log.Info("try add", confPath)
	confPathAbs, _, err := utils.GetRealPath(confPath)
	if err != nil {
		log.Warnf("filepath.Abs(%s) failed: %v", confPath)
		return
	}
	confPath = confPathAbs
	if m.isRunning(confPath) {
		log.Errorf("%s already added", confPath)
		return
	}
	var conf RunnerConfig
	err = config.LoadEx(&conf, confPath)
	if err != nil {
		log.Warnf("config.LoadEx %s failed:%v", confPath, err)
		return
	}

	forkRunner := func(confPath string) {
		var runner Runner
		i := 0
		for {
			if m.isRunning(confPath) {
				log.Errorf("%s already added - ", confPath)
				return
			}

			if runner, err = NewCustomRunner(conf, m.cleanChan, m.pregistry, m.sregistry); err != nil {
				errVal, ok := err.(*os.PathError)
				if !ok {
					log.Errorf("NewRunner(%v) failed: %v", conf.RunnerName, err)
					return
				}
				i++
				log.Warnf("LogDir(%v) does not exsit after %d rounds, sleep 5 minute and try again...", errVal.Path, i)
				time.Sleep(time.Duration(DIR_NOT_EXIST_SLEEP_TIME) * time.Second)
				continue
			}
			break
		}
		m.lock.Lock()
		defer m.lock.Unlock()
		// double check
		if _, ok := m.runners[confPath]; ok {
			return
		}
		m.addCleanQueue(runner.Cleaner())
		log.Infof("%s added: %#v", conf.RunnerName, confPath)
		go runner.Run()
		m.runners[confPath] = runner
		log.Infof("new runner %s is added, total %d", conf.RunnerName, len(m.runners))
		return
	}

	log.Infof("Start to try add: %#v", conf.RunnerName)
	go forkRunner(confPath)

	return
}
func (m *Manager) isRunning(confPath string) bool {
	m.lock.RLock()
	defer m.lock.RUnlock()
	if _, ok := m.runners[confPath]; ok {
		return true
	}
	return false
}

// 创建文件会触发 create和modify事件
// 重命名会触发 rename和create事件
// 删除会触发 delete事件
// 修改会触发 delete事件、create事件、modify事件以及modify|ATTRIB事件
func (m *Manager) handle(path string, watcher *fsnotify.Watcher) {
	for {
		select {
		case ev, ok := <-watcher.Event:
			if !ok {
				log.Info("Manager watcher chan was closed")
				return
			}
			log.Info("event:", ev)
			if ev.IsDelete() || ev.IsRename() {
				_, err := os.Stat(path)
				if os.IsNotExist(err) {
					// 如果当前监听文件被删除，则不再监听，退出
					log.Warnf("close file watcher path %v", path)
					watcher.Close()
					delete(m.watchers, path)
					return
				}
				m.Remove(ev.Name)
			}
			if ev.IsCreate() {
				m.Add(ev.Name)
			}
			if ev.IsModify() && !ev.IsCreate() {
				m.Remove(ev.Name)
				m.Add(ev.Name)
			}
		case err := <-watcher.Error:
			if err != nil {
				log.Error("error:", err)
			}
		}
	}
}

func (m *Manager) doClean(sig cleaner.CleanSignal) {
	m.cleanlock.Lock()
	defer m.cleanlock.Unlock()

	dir := sig.Logdir
	dir, err := filepath.Abs(dir)
	if err != nil {
		log.Errorf("get abs for %v error %v", dir, err)
		return
	}
	file := sig.Filename
	q, ok := m.cleanQueues[dir]
	if !ok {
		log.Errorf("%v cleaner dir %v not exist but got clean signal for delete file %v", sig.Cleaner, dir, file)
		return
	}
	count := q.filecount[file] + 1
	if count >= q.cleanerCount {
		catdir := filepath.Join(dir, file)
		err := os.Remove(catdir)
		if err != nil {
			if os.IsNotExist(err) {
				log.Warnf("clean %v failed as logfile is not exist: %v", catdir, err)
			} else {
				log.Errorf("clean %v failed: %v", catdir, err)
			}
		} else {
			log.Infof("log <%v> was successfully cleaned by cleaner", catdir)
		}
		delete(q.filecount, file)
	} else {
		q.filecount[file] = count
		m.cleanQueues[dir] = q
	}
	return
}

func (m *Manager) clean() {
	for sig := range m.cleanChan {
		m.doClean(sig)
	}
}

func (m *Manager) detectMoreWatchers(confsPath []string) {
	ticker := time.NewTicker(time.Second * 10)
	for {
		select {
		case <-ticker.C:
			log.Debug("try to detect more watchers ")
			m.addWatchers(confsPath)
		}
	}
}

func (m *Manager) addWatchers(confsPath []string) (err error) {
	for _, dir := range confsPath {
		paths, err := filepath.Glob(dir)
		if err != nil {
			log.Errorf("filepath.Glob(%s): %v, err:%v", dir, paths, err)
			continue
		}
		for _, path := range paths {
			_, exist := m.watchers[path]
			if exist {
				// 如果文件已经被监听，则不再重复监听
				continue
			}
			files, err := ioutil.ReadDir(path)
			if err != nil {
				log.Errorf("ioutil.ReadDir(%s): %v, err:%v", path, files, err)
				continue
			}
			log.Warnf("start to add watcher of conf path %v", path)
			for _, f := range files {
				if f.IsDir() {
					log.Warn("skipped dir", f.Name)
					continue
				}
				m.Add(filepath.Join(path, f.Name()))
			}
			watcher, err := fsnotify.NewWatcher()
			if err != nil {
				log.Errorf("fsnotify.NewWatcher: %v", err)
				continue
			}
			m.watchers[path] = watcher
			go m.handle(path, watcher)
			if err = watcher.Watch(path); err != nil {
				log.Errorf("watch %v error %v", path, err)
			}
		}
	}
	return nil
}

func (m *Manager) Watch(confsPath []string) (err error) {
	err = m.addWatchers(confsPath)
	if err != nil {
		log.Errorf("addWatchers error : %v", err)
	}
	go m.detectMoreWatchers(confsPath)
	go m.clean()
	return
}

func (m *Manager) Status() (rss map[string]RunnerStatus) {
	rss = make(map[string]RunnerStatus)
	for _, r := range m.runners {
		rss[r.Name()] = r.Status()
	}
	return
}
<|endoftext|>


<commit_before>package migrate

import (
	"bytes"
	"database/sql"
	"io"
	"os"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/coopernurse/gorp"
	"github.com/rubenv/gorp-migrate/sqlparse"
)

type MigrationDirection int

const (
	Up MigrationDirection = iota
	Down
)

type Migration struct {
	Id   string
	Up   []string
	Down []string
}

type byId []*Migration

func (b byId) Len() int           { return len(b) }
func (b byId) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b byId) Less(i, j int) bool { return b[i].Id < b[j].Id }

type MigrationRecord struct {
	Id        string    `db:"id"`
	AppliedAt time.Time `db:"applied_at"`
}

type MigrationSource interface {
	FindMigrations() ([]*Migration, error)
}

// A hardcoded set of migrations, in-memory.
type MemoryMigrationSource struct {
	Migrations []*Migration
}

var _ MigrationSource = (*MemoryMigrationSource)(nil)

func (m MemoryMigrationSource) FindMigrations() ([]*Migration, error) {
	return m.Migrations, nil
}

// A set of migrations loaded from a directory.
type FileMigrationSource struct {
	Dir string
}

var _ MigrationSource = (*FileMigrationSource)(nil)

func (f FileMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	file, err := os.Open(f.Dir)
	if err != nil {
		return nil, err
	}

	files, err := file.Readdir(0)
	if err != nil {
		return nil, err
	}

	for _, info := range files {
		if strings.HasSuffix(info.Name(), ".sql") {
			file, err := os.Open(path.Join(f.Dir, info.Name()))
			if err != nil {
				return nil, err
			}

			migration, err := ParseMigration(info.Name(), file)
			if err != nil {
				return nil, err
			}

			migrations = append(migrations, migration)
		}
	}

	return migrations, nil
}

// Migrations from a bindata asset set.
type AssetMigrationSource struct {
	// Asset should return content of file in path if exists
	Asset func(path string) ([]byte, error)

	// AssetDir should return list of files in the path
	AssetDir func(path string) ([]string, error)

	// Path in the bindata to use.
	Dir string
}

var _ MigrationSource = (*FileMigrationSource)(nil)

func (a AssetMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	files, err := a.AssetDir(a.Dir)
	if err != nil {
		return nil, err
	}

	for _, name := range files {
		if strings.HasSuffix(name, ".sql") {
			file, err := a.Asset(path.Join(a.Dir, name))
			if err != nil {
				return nil, err
			}

			migration, err := ParseMigration(name, bytes.NewReader(file))
			if err != nil {
				return nil, err
			}

			migrations = append(migrations, migration)
		}
	}

	return migrations, nil
}

// Migration parsing
func ParseMigration(id string, r io.ReadSeeker) (*Migration, error) {
	m := &Migration{
		Id: id,
	}

	up, err := sqlparse.SplitSQLStatements(r, true)
	if err != nil {
		return nil, err
	}

	down, err := sqlparse.SplitSQLStatements(r, false)
	if err != nil {
		return nil, err
	}

	m.Up = up
	m.Down = down

	return m, nil
}

// Execute a set of migrations
//
// Returns the number of applied migrations.
func Exec(db *gorp.DbMap, m MigrationSource) (int, error) {
	dbMap := &gorp.DbMap{Db: db.Db, Dialect: db.Dialect}
	dbMap.AddTableWithName(MigrationRecord{}, "gorp_migrations").SetKeys(false, "Id")
	//dbMap.TraceOn("", log.New(os.Stdout, "migrate: ", log.Lmicroseconds))

	// Make sure we have the migrations table
	err := dbMap.CreateTablesIfNotExists()
	if err != nil {
		return 0, err
	}

	migrations, err := m.FindMigrations()
	if err != nil {
		return 0, err
	}

	// Make sure migrations are sorted
	sort.Sort(byId(migrations))

	// Find the newest applied migration
	var record MigrationRecord
	err = dbMap.SelectOne(&record, "SELECT * FROM gorp_migrations ORDER BY id DESC LIMIT 1")
	if err != nil && err != sql.ErrNoRows {
		return 0, err
	}

	// Figure out which of the supplied migrations has been applied.
	toApply := ToApply(migrations, record.Id, Up)

	// Apply migrations
	applied := 0
	for _, migration := range toApply {
		for _, stmt := range migration.Up {
			_, err := dbMap.Exec(stmt)
			if err != nil {
				return applied, err
			}
		}

		err = dbMap.Insert(&MigrationRecord{
			Id:        migration.Id,
			AppliedAt: time.Now(),
		})
		if err != nil {
			return applied, err
		}

		applied++
	}

	return applied, nil
}

// Filter a slice of migrations into ones that should be applied.
func ToApply(migrations []*Migration, current string, direction MigrationDirection) []*Migration {
	var index = -1
	for index < len(migrations)-1 && migrations[index+1].Id <= current {
		index++
	}
	toApply := migrations[index+1:]
	return toApply
}

// TODO: Run migration + record insert in transaction.
<commit_msg>Correct type check.<commit_after>package migrate

import (
	"bytes"
	"database/sql"
	"io"
	"os"
	"path"
	"sort"
	"strings"
	"time"

	"github.com/coopernurse/gorp"
	"github.com/rubenv/gorp-migrate/sqlparse"
)

type MigrationDirection int

const (
	Up MigrationDirection = iota
	Down
)

type Migration struct {
	Id   string
	Up   []string
	Down []string
}

type byId []*Migration

func (b byId) Len() int           { return len(b) }
func (b byId) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
func (b byId) Less(i, j int) bool { return b[i].Id < b[j].Id }

type MigrationRecord struct {
	Id        string    `db:"id"`
	AppliedAt time.Time `db:"applied_at"`
}

type MigrationSource interface {
	FindMigrations() ([]*Migration, error)
}

// A hardcoded set of migrations, in-memory.
type MemoryMigrationSource struct {
	Migrations []*Migration
}

var _ MigrationSource = (*MemoryMigrationSource)(nil)

func (m MemoryMigrationSource) FindMigrations() ([]*Migration, error) {
	return m.Migrations, nil
}

// A set of migrations loaded from a directory.
type FileMigrationSource struct {
	Dir string
}

var _ MigrationSource = (*FileMigrationSource)(nil)

func (f FileMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	file, err := os.Open(f.Dir)
	if err != nil {
		return nil, err
	}

	files, err := file.Readdir(0)
	if err != nil {
		return nil, err
	}

	for _, info := range files {
		if strings.HasSuffix(info.Name(), ".sql") {
			file, err := os.Open(path.Join(f.Dir, info.Name()))
			if err != nil {
				return nil, err
			}

			migration, err := ParseMigration(info.Name(), file)
			if err != nil {
				return nil, err
			}

			migrations = append(migrations, migration)
		}
	}

	return migrations, nil
}

// Migrations from a bindata asset set.
type AssetMigrationSource struct {
	// Asset should return content of file in path if exists
	Asset func(path string) ([]byte, error)

	// AssetDir should return list of files in the path
	AssetDir func(path string) ([]string, error)

	// Path in the bindata to use.
	Dir string
}

var _ MigrationSource = (*AssetMigrationSource)(nil)

func (a AssetMigrationSource) FindMigrations() ([]*Migration, error) {
	migrations := make([]*Migration, 0)

	files, err := a.AssetDir(a.Dir)
	if err != nil {
		return nil, err
	}

	for _, name := range files {
		if strings.HasSuffix(name, ".sql") {
			file, err := a.Asset(path.Join(a.Dir, name))
			if err != nil {
				return nil, err
			}

			migration, err := ParseMigration(name, bytes.NewReader(file))
			if err != nil {
				return nil, err
			}

			migrations = append(migrations, migration)
		}
	}

	return migrations, nil
}

// Migration parsing
func ParseMigration(id string, r io.ReadSeeker) (*Migration, error) {
	m := &Migration{
		Id: id,
	}

	up, err := sqlparse.SplitSQLStatements(r, true)
	if err != nil {
		return nil, err
	}

	down, err := sqlparse.SplitSQLStatements(r, false)
	if err != nil {
		return nil, err
	}

	m.Up = up
	m.Down = down

	return m, nil
}

// Execute a set of migrations
//
// Returns the number of applied migrations.
func Exec(db *gorp.DbMap, m MigrationSource) (int, error) {
	dbMap := &gorp.DbMap{Db: db.Db, Dialect: db.Dialect}
	dbMap.AddTableWithName(MigrationRecord{}, "gorp_migrations").SetKeys(false, "Id")
	//dbMap.TraceOn("", log.New(os.Stdout, "migrate: ", log.Lmicroseconds))

	// Make sure we have the migrations table
	err := dbMap.CreateTablesIfNotExists()
	if err != nil {
		return 0, err
	}

	migrations, err := m.FindMigrations()
	if err != nil {
		return 0, err
	}

	// Make sure migrations are sorted
	sort.Sort(byId(migrations))

	// Find the newest applied migration
	var record MigrationRecord
	err = dbMap.SelectOne(&record, "SELECT * FROM gorp_migrations ORDER BY id DESC LIMIT 1")
	if err != nil && err != sql.ErrNoRows {
		return 0, err
	}

	// Figure out which of the supplied migrations has been applied.
	toApply := ToApply(migrations, record.Id, Up)

	// Apply migrations
	applied := 0
	for _, migration := range toApply {
		for _, stmt := range migration.Up {
			_, err := dbMap.Exec(stmt)
			if err != nil {
				return applied, err
			}
		}

		err = dbMap.Insert(&MigrationRecord{
			Id:        migration.Id,
			AppliedAt: time.Now(),
		})
		if err != nil {
			return applied, err
		}

		applied++
	}

	return applied, nil
}

// Filter a slice of migrations into ones that should be applied.
func ToApply(migrations []*Migration, current string, direction MigrationDirection) []*Migration {
	var index = -1
	for index < len(migrations)-1 && migrations[index+1].Id <= current {
		index++
	}
	toApply := migrations[index+1:]
	return toApply
}

// TODO: Run migration + record insert in transaction.
<|endoftext|>


<commit_before>// Package migrate provides a dead simple Go package for performing sql
// migrations using database/sql.
package migrate

import (
	"database/sql"
	"fmt"
	"hash/crc32"
	"sort"
	"sync"
)

type MigrationDirection int

const (
	Up MigrationDirection = iota
	Down
)

type TransactionMode int

const (
	// In this mode, each migration is run in it's own isolated transaction.
	// If a migration fails, only that migration will be rolled back.
	IndividualTransactions TransactionMode = iota

	// In this mode, all migrations are run inside a single transaction. If
	// one migration fails, all migrations are rolled back.
	SingleTransaction
)

// MigrationError is an error that gets returned when an individual migration
// fails.
type MigrationError struct {
	Migration

	// The underlying error.
	Err error
}

// Error implements the error interface.
func (e *MigrationError) Error() string {
	return fmt.Sprintf("migration %d failed: %v", e.ID, e.Err)
}

// The default table to store what migrations have been run.
const DefaultTable = "schema_migrations"

// Migration represents a sql migration that can be migrated up or down.
type Migration struct {
	// ID is a unique, numeric, identifier for this migration.
	ID int

	// Up is a function that gets called when this migration should go up.
	Up func(tx *sql.Tx) error

	// Down is a function that gets called when this migration should go
	// down.
	Down func(tx *sql.Tx) error
}

// byID implements the sort.Interface interface for sorting migrations by
// ID.
type byID []Migration

func (m byID) Len() int           { return len(m) }
func (m byID) Less(i, j int) bool { return m[i].ID < m[j].ID }
func (m byID) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }

// Migrator performs migrations.
type Migrator struct {
	// Table is the table to store what migrations have been run. The zero
	// value is DefaultTable.
	Table string

	// Locker is a sync.Locker to use to ensure that only 1 process is
	// running migrations.
	sync.Locker

	// The TransactionMode to use. The zero value is IndividualTransactions,
	// which runs each migration in it's own transaction.
	TransactionMode TransactionMode

	db *sql.DB
}

// postgresLocker implements the sync.Locker interface using pg_advisory_lock.
type postgresLocker struct {
	key uint32
	db  *sql.DB
}

// NewPostgresLocker returns a new sync.Locker that obtains locks with
// pg_advisory_lock.
func newPostgresLocker(db *sql.DB) sync.Locker {
	key := crc32.ChecksumIEEE([]byte("migrations"))
	return &postgresLocker{
		key: key,
		db:  db,
	}
}

// Lock obtains the advisory lock.
func (l *postgresLocker) Lock() {
	l.do("lock")
}

// Unlock removes the advisory Lock
func (l *postgresLocker) Unlock() {
	l.do("unlock")
}

func (l *postgresLocker) do(m string) {
	_, err := l.db.Exec(fmt.Sprintf("SELECT pg_advisory_%s(%d)", m, l.key))
	if err != nil {
		panic(fmt.Sprintf("migrate: %v", err))
	}
}

// NewMigrator returns a new Migrator instance that will use the sql.DB to
// perform the migrations.
func NewMigrator(db *sql.DB) *Migrator {
	return &Migrator{
		db:     db,
		Locker: new(sync.Mutex),
	}
}

// NewPostgresMigrator returns a new Migrator instance that uses the underlying
// sql.DB connection to a postgres database to perform migrations. It will use
// Postgres's advisory locks to ensure that only 1 migration is run at a time.
func NewPostgresMigrator(db *sql.DB) *Migrator {
	m := NewMigrator(db)
	m.Locker = newPostgresLocker(db)
	return m
}

// Exec runs the migrations in the given direction.
func (m *Migrator) Exec(dir MigrationDirection, migrations ...Migration) error {
	m.Lock()
	defer m.Unlock()

	_, err := m.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version integer primary key not null)", m.table()))
	if err != nil {
		return err
	}

	var tx *sql.Tx
	if m.TransactionMode == SingleTransaction {
		tx, err = m.db.Begin()
		if err != nil {
			return err
		}
	}

	for _, migration := range sortMigrations(dir, migrations) {
		if m.TransactionMode == IndividualTransactions {
			tx, err = m.db.Begin()
			if err != nil {
				return err
			}
		}

		if err := m.runMigration(tx, dir, migration); err != nil {
			tx.Rollback()
			return err
		}

		if m.TransactionMode == IndividualTransactions {
			if err := tx.Commit(); err != nil {
				return err
			}
		}
	}

	if m.TransactionMode == SingleTransaction {
		if err := tx.Commit(); err != nil {
			return err
		}
	}

	return nil
}

// runMigration runs the given Migration in the given direction using the given
// transaction. This function does not commit or rollback the transaction,
// that's the responsibility of the consumer dependending on whether an error
// gets returned.
func (m *Migrator) runMigration(tx *sql.Tx, dir MigrationDirection, migration Migration) error {
	shouldMigrate, err := m.shouldMigrate(tx, migration.ID, dir)
	if err != nil {
		return err
	}

	if !shouldMigrate {
		return nil
	}

	var migrate func(tx *sql.Tx) error
	switch dir {
	case Up:
		migrate = migration.Up
	default:
		migrate = migration.Down
	}

	if err := migrate(tx); err != nil {
		return &MigrationError{Migration: migration, Err: err}
	}

	var query string
	switch dir {
	case Up:
		// Yes. This is a sql injection vulnerability. This gets around
		// the different bindings for sqlite3/postgres.
		//
		// If you're running migrations from user input, you're doing
		// something wrong.
		query = fmt.Sprintf("INSERT INTO %s (version) VALUES (%d)", m.table(), migration.ID)
	default:
		query = fmt.Sprintf("DELETE FROM %s WHERE version = %d", m.table(), migration.ID)
	}

	_, err = tx.Exec(query)
	return err
}

func (m *Migrator) shouldMigrate(tx *sql.Tx, id int, dir MigrationDirection) (bool, error) {
	// Check if this migration has already ran
	var _id int
	err := tx.QueryRow(fmt.Sprintf("SELECT version FROM %s WHERE version = %d", m.table(), id)).Scan(&_id)
	if err != nil && err != sql.ErrNoRows {
		return false, err
	}

	switch dir {
	case Up:
		// If the migration doesn't exist, then we need to run it.
		return err == sql.ErrNoRows, nil
	default:
		// If the migration exists, then we need to remove it.
		return err != sql.ErrNoRows, nil
	}
}

// table returns the name of the table to use to track the migrations.
func (m *Migrator) table() string {
	if m.Table == "" {
		return DefaultTable
	}

	return m.Table
}

// Exec is a convenience method that runs the migrations against the default
// table.
func Exec(db *sql.DB, dir MigrationDirection, migrations ...Migration) error {
	return NewMigrator(db).Exec(dir, migrations...)
}

// Queries returns a func(tx *sql.Tx) error function that performs the given sql
// queries in multiple Exec calls.
func Queries(queries []string) func(*sql.Tx) error {
	return func(tx *sql.Tx) error {
		for _, query := range queries {
			if _, err := tx.Exec(query); err != nil {
				return err
			}
		}

		return nil
	}
}

// sortMigrations sorts the migrations by id.
//
// When the direction is "Up", the migrations will be sorted by ID ascending.
// When the direction is "Down", the migrations will be sorted by ID descending.
func sortMigrations(dir MigrationDirection, migrations []Migration) []Migration {
	var m byID
	for _, migration := range migrations {
		m = append(m, migration)
	}

	switch dir {
	case Up:
		sort.Sort(byID(m))
	default:
		sort.Sort(sort.Reverse(byID(m)))
	}

	return m
}
<commit_msg>Publicize byID as ByID<commit_after>// Package migrate provides a dead simple Go package for performing sql
// migrations using database/sql.
package migrate

import (
	"database/sql"
	"fmt"
	"hash/crc32"
	"sort"
	"sync"
)

type MigrationDirection int

const (
	Up MigrationDirection = iota
	Down
)

type TransactionMode int

const (
	// In this mode, each migration is run in it's own isolated transaction.
	// If a migration fails, only that migration will be rolled back.
	IndividualTransactions TransactionMode = iota

	// In this mode, all migrations are run inside a single transaction. If
	// one migration fails, all migrations are rolled back.
	SingleTransaction
)

// MigrationError is an error that gets returned when an individual migration
// fails.
type MigrationError struct {
	Migration

	// The underlying error.
	Err error
}

// Error implements the error interface.
func (e *MigrationError) Error() string {
	return fmt.Sprintf("migration %d failed: %v", e.ID, e.Err)
}

// The default table to store what migrations have been run.
const DefaultTable = "schema_migrations"

// Migration represents a sql migration that can be migrated up or down.
type Migration struct {
	// ID is a unique, numeric, identifier for this migration.
	ID int

	// Up is a function that gets called when this migration should go up.
	Up func(tx *sql.Tx) error

	// Down is a function that gets called when this migration should go
	// down.
	Down func(tx *sql.Tx) error
}

// ByID implements the sort.Interface interface for sorting migrations by
// ID.
type ByID []Migration

func (m ByID) Len() int           { return len(m) }
func (m ByID) Less(i, j int) bool { return m[i].ID < m[j].ID }
func (m ByID) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }

// Migrator performs migrations.
type Migrator struct {
	// Table is the table to store what migrations have been run. The zero
	// value is DefaultTable.
	Table string

	// Locker is a sync.Locker to use to ensure that only 1 process is
	// running migrations.
	sync.Locker

	// The TransactionMode to use. The zero value is IndividualTransactions,
	// which runs each migration in it's own transaction.
	TransactionMode TransactionMode

	db *sql.DB
}

// postgresLocker implements the sync.Locker interface using pg_advisory_lock.
type postgresLocker struct {
	key uint32
	db  *sql.DB
}

// NewPostgresLocker returns a new sync.Locker that obtains locks with
// pg_advisory_lock.
func newPostgresLocker(db *sql.DB) sync.Locker {
	key := crc32.ChecksumIEEE([]byte("migrations"))
	return &postgresLocker{
		key: key,
		db:  db,
	}
}

// Lock obtains the advisory lock.
func (l *postgresLocker) Lock() {
	l.do("lock")
}

// Unlock removes the advisory Lock
func (l *postgresLocker) Unlock() {
	l.do("unlock")
}

func (l *postgresLocker) do(m string) {
	_, err := l.db.Exec(fmt.Sprintf("SELECT pg_advisory_%s(%d)", m, l.key))
	if err != nil {
		panic(fmt.Sprintf("migrate: %v", err))
	}
}

// NewMigrator returns a new Migrator instance that will use the sql.DB to
// perform the migrations.
func NewMigrator(db *sql.DB) *Migrator {
	return &Migrator{
		db:     db,
		Locker: new(sync.Mutex),
	}
}

// NewPostgresMigrator returns a new Migrator instance that uses the underlying
// sql.DB connection to a postgres database to perform migrations. It will use
// Postgres's advisory locks to ensure that only 1 migration is run at a time.
func NewPostgresMigrator(db *sql.DB) *Migrator {
	m := NewMigrator(db)
	m.Locker = newPostgresLocker(db)
	return m
}

// Exec runs the migrations in the given direction.
func (m *Migrator) Exec(dir MigrationDirection, migrations ...Migration) error {
	m.Lock()
	defer m.Unlock()

	_, err := m.db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version integer primary key not null)", m.table()))
	if err != nil {
		return err
	}

	var tx *sql.Tx
	if m.TransactionMode == SingleTransaction {
		tx, err = m.db.Begin()
		if err != nil {
			return err
		}
	}

	for _, migration := range sortMigrations(dir, migrations) {
		if m.TransactionMode == IndividualTransactions {
			tx, err = m.db.Begin()
			if err != nil {
				return err
			}
		}

		if err := m.runMigration(tx, dir, migration); err != nil {
			tx.Rollback()
			return err
		}

		if m.TransactionMode == IndividualTransactions {
			if err := tx.Commit(); err != nil {
				return err
			}
		}
	}

	if m.TransactionMode == SingleTransaction {
		if err := tx.Commit(); err != nil {
			return err
		}
	}

	return nil
}

// runMigration runs the given Migration in the given direction using the given
// transaction. This function does not commit or rollback the transaction,
// that's the responsibility of the consumer dependending on whether an error
// gets returned.
func (m *Migrator) runMigration(tx *sql.Tx, dir MigrationDirection, migration Migration) error {
	shouldMigrate, err := m.shouldMigrate(tx, migration.ID, dir)
	if err != nil {
		return err
	}

	if !shouldMigrate {
		return nil
	}

	var migrate func(tx *sql.Tx) error
	switch dir {
	case Up:
		migrate = migration.Up
	default:
		migrate = migration.Down
	}

	if err := migrate(tx); err != nil {
		return &MigrationError{Migration: migration, Err: err}
	}

	var query string
	switch dir {
	case Up:
		// Yes. This is a sql injection vulnerability. This gets around
		// the different bindings for sqlite3/postgres.
		//
		// If you're running migrations from user input, you're doing
		// something wrong.
		query = fmt.Sprintf("INSERT INTO %s (version) VALUES (%d)", m.table(), migration.ID)
	default:
		query = fmt.Sprintf("DELETE FROM %s WHERE version = %d", m.table(), migration.ID)
	}

	_, err = tx.Exec(query)
	return err
}

func (m *Migrator) shouldMigrate(tx *sql.Tx, id int, dir MigrationDirection) (bool, error) {
	// Check if this migration has already ran
	var _id int
	err := tx.QueryRow(fmt.Sprintf("SELECT version FROM %s WHERE version = %d", m.table(), id)).Scan(&_id)
	if err != nil && err != sql.ErrNoRows {
		return false, err
	}

	switch dir {
	case Up:
		// If the migration doesn't exist, then we need to run it.
		return err == sql.ErrNoRows, nil
	default:
		// If the migration exists, then we need to remove it.
		return err != sql.ErrNoRows, nil
	}
}

// table returns the name of the table to use to track the migrations.
func (m *Migrator) table() string {
	if m.Table == "" {
		return DefaultTable
	}

	return m.Table
}

// Exec is a convenience method that runs the migrations against the default
// table.
func Exec(db *sql.DB, dir MigrationDirection, migrations ...Migration) error {
	return NewMigrator(db).Exec(dir, migrations...)
}

// Queries returns a func(tx *sql.Tx) error function that performs the given sql
// queries in multiple Exec calls.
func Queries(queries []string) func(*sql.Tx) error {
	return func(tx *sql.Tx) error {
		for _, query := range queries {
			if _, err := tx.Exec(query); err != nil {
				return err
			}
		}

		return nil
	}
}

// sortMigrations sorts the migrations by id.
//
// When the direction is "Up", the migrations will be sorted by ID ascending.
// When the direction is "Down", the migrations will be sorted by ID descending.
func sortMigrations(dir MigrationDirection, migrations []Migration) []Migration {
	var m ByID
	for _, migration := range migrations {
		m = append(m, migration)
	}

	switch dir {
	case Up:
		sort.Sort(ByID(m))
	default:
		sort.Sort(sort.Reverse(ByID(m)))
	}

	return m
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	"github.com/bitly/go-simplejson"
	"github.com/grafana/grafana-plugin-model/go/datasource"
	plugin "github.com/hashicorp/go-plugin"
	"golang.org/x/net/context/ctxhttp"
)

var httpClient = &http.Client{}

type ClickhouseDatasource struct {
	plugin.NetRPCUnsupportedPlugin
}

func (t *ClickhouseDatasource) Query(ctx context.Context, req *datasource.DatasourceRequest) (r *datasource.DatasourceResponse, err error) {
	// catch all panics and override err return value
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("clickhouse plugin panicked: %#w", r)
		}
	}()

	refId := req.Queries[0].RefId
	modelJson, err := simplejson.NewJson([]byte(req.Queries[0].ModelJson))
	if err != nil {
		return nil, err
	}

	query := modelJson.Get("rawQuery").MustString()
	request, err := createRequest(req, query)
	if err != nil {
		return nil, err
	}

	response, err := ctxhttp.Do(ctx, httpClient, request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("invalid status code. status: %v", response.Status)
	}

	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return parseResponse(body, refId)
}

func createRequest(req *datasource.DatasourceRequest, query string) (*http.Request, error) {
	body := ""
	method := http.MethodGet
	headers := http.Header{}
	url, err := url.Parse(req.Datasource.Url)
	if err != nil {
		return nil, fmt.Errorf("unable to parse clickhouse url: %w", err)
	}

	params := url.Query()
	params.Add("query", query+" FORMAT JSON")

	// TODO Fix basic authorization. We have access to basicAuthPassword but not
	// the basic auth name. Users will have to use the useYandexCloudAuthorization
	// option instead for clickhouse auth.
	secureOptions := req.Datasource.DecryptedSecureJsonData
	options := make(map[string]interface{})
	err = json.Unmarshal([]byte(req.Datasource.JsonData), &options)
	if err != nil {
		return nil, fmt.Errorf("unable to parse clickhouse options: %w", err)
	}

	for k, v := range options {
		switch k {
		case "usePOST":
			method = http.MethodPost
			params.Del("query")
			body = query
			break
		case "defaultDatabase":
			db, _ := v.(string)
			params.Add("database", db)
			break
		case "addCorsHeaders":
			params.Add("add_http_cors_header", "1")
			break
		case "useYandexCloudAuthorization":
			chUser := ""
			chKey := ""

			if user, ok := options["xHeaderUser"]; ok {
				chUser, _ = user.(string)
			}

			if key, ok := secureOptions["xHeaderKey"]; ok {
				chKey = key
			}

			headers.Add("X-ClickHouse-User", chUser)
			headers.Add("X-ClickHouse-Key", chKey)
			break
		default:
			if strings.HasPrefix(k, "httpHeaderName") {
				headerKey := strings.Replace(k, "Name", "Value", 1)
				value := ""
				name, _ := v.(string)
				if hv, ok := secureOptions[headerKey]; ok {
					value = hv
				}

				return nil, fmt.Errorf("parse key %s", k)

				headers.Add(name, value)
			}
			break
		}
	}

	url.RawQuery = params.Encode()
	request, err := http.NewRequest(method, url.String(), bytes.NewBufferString(body))
	if err != nil {
		return nil, err
	}

	request.Header = headers
	return request, nil
}

func parseResponse(body []byte, refId string) (*datasource.DatasourceResponse, error) {
	parsedBody := ClickHouseResponse{}
	err := json.Unmarshal(body, &parsedBody)
	if err != nil {
		return nil, fmt.Errorf("unable to parse response json: %s: %w", body, err)
	}

	seriesMap := map[string]*datasource.TimeSeries{}
	for _, meta := range parsedBody.Meta {
		if meta.Name != "t" {
			seriesMap[meta.Name] = &datasource.TimeSeries{Name: meta.Name, Points: []*datasource.Point{}}
		}
	}

	for _, dataPoint := range parsedBody.Data {
		for k, v := range dataPoint {
			if k != "t" {
				timestamp, err := strconv.ParseInt(dataPoint["t"], 10, 64)
				if err != nil {
					return nil, fmt.Errorf("unable to parse timestamp with alias t: %w", err)
				}

				point, err := strconv.ParseFloat(v, 64)
				if err != nil {
					return nil, fmt.Errorf("unable to parse value for '%s': %w", k, err)
				}

				seriesMap[k].Points = append(seriesMap[k].Points, &datasource.Point{
					Timestamp: timestamp,
					Value:     point,
				})
			}
		}
	}

	series := []*datasource.TimeSeries{}
	for _, timeSeries := range seriesMap {
		series = append(series, timeSeries)
	}

	metaJSON, _ := json.Marshal(parsedBody.Meta)
	return &datasource.DatasourceResponse{
		Results: []*datasource.QueryResult{
			&datasource.QueryResult{
				Series:   series,
				RefId:    refId,
				MetaJson: string(metaJSON),
			},
		},
	}, nil
}

type ClickHouseResponse struct {
	Meta []ClickHouseMeta
	Data []map[string]string
	Rows int
}

type ClickHouseMeta struct {
	Name string
	Type string
}
<commit_msg>Aggregate third party imports.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/bitly/go-simplejson"
	"github.com/grafana/grafana-plugin-model/go/datasource"
	plugin "github.com/hashicorp/go-plugin"
	"golang.org/x/net/context"
	"golang.org/x/net/context/ctxhttp"
)

var httpClient = &http.Client{}

type ClickhouseDatasource struct {
	plugin.NetRPCUnsupportedPlugin
}

func (t *ClickhouseDatasource) Query(ctx context.Context, req *datasource.DatasourceRequest) (r *datasource.DatasourceResponse, err error) {
	// catch all panics and override err return value
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("clickhouse plugin panicked: %#w", r)
		}
	}()

	refId := req.Queries[0].RefId
	modelJson, err := simplejson.NewJson([]byte(req.Queries[0].ModelJson))
	if err != nil {
		return nil, err
	}

	query := modelJson.Get("rawQuery").MustString()
	request, err := createRequest(req, query)
	if err != nil {
		return nil, err
	}

	response, err := ctxhttp.Do(ctx, httpClient, request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("invalid status code. status: %v", response.Status)
	}

	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return parseResponse(body, refId)
}

func createRequest(req *datasource.DatasourceRequest, query string) (*http.Request, error) {
	body := ""
	method := http.MethodGet
	headers := http.Header{}
	url, err := url.Parse(req.Datasource.Url)
	if err != nil {
		return nil, fmt.Errorf("unable to parse clickhouse url: %w", err)
	}

	params := url.Query()
	params.Add("query", query+" FORMAT JSON")

	// TODO Fix basic authorization. We have access to basicAuthPassword but not
	// the basic auth name. Users will have to use the useYandexCloudAuthorization
	// option instead for clickhouse auth.
	secureOptions := req.Datasource.DecryptedSecureJsonData
	options := make(map[string]interface{})
	err = json.Unmarshal([]byte(req.Datasource.JsonData), &options)
	if err != nil {
		return nil, fmt.Errorf("unable to parse clickhouse options: %w", err)
	}

	for k, v := range options {
		switch k {
		case "usePOST":
			method = http.MethodPost
			params.Del("query")
			body = query
			break
		case "defaultDatabase":
			db, _ := v.(string)
			params.Add("database", db)
			break
		case "addCorsHeaders":
			params.Add("add_http_cors_header", "1")
			break
		case "useYandexCloudAuthorization":
			chUser := ""
			chKey := ""

			if user, ok := options["xHeaderUser"]; ok {
				chUser, _ = user.(string)
			}

			if key, ok := secureOptions["xHeaderKey"]; ok {
				chKey = key
			}

			headers.Add("X-ClickHouse-User", chUser)
			headers.Add("X-ClickHouse-Key", chKey)
			break
		default:
			if strings.HasPrefix(k, "httpHeaderName") {
				headerKey := strings.Replace(k, "Name", "Value", 1)
				value := ""
				name, _ := v.(string)
				if hv, ok := secureOptions[headerKey]; ok {
					value = hv
				}

				return nil, fmt.Errorf("parse key %s", k)

				headers.Add(name, value)
			}
			break
		}
	}

	url.RawQuery = params.Encode()
	request, err := http.NewRequest(method, url.String(), bytes.NewBufferString(body))
	if err != nil {
		return nil, err
	}

	request.Header = headers
	return request, nil
}

func parseResponse(body []byte, refId string) (*datasource.DatasourceResponse, error) {
	parsedBody := ClickHouseResponse{}
	err := json.Unmarshal(body, &parsedBody)
	if err != nil {
		return nil, fmt.Errorf("unable to parse response json: %s: %w", body, err)
	}

	seriesMap := map[string]*datasource.TimeSeries{}
	for _, meta := range parsedBody.Meta {
		if meta.Name != "t" {
			seriesMap[meta.Name] = &datasource.TimeSeries{Name: meta.Name, Points: []*datasource.Point{}}
		}
	}

	for _, dataPoint := range parsedBody.Data {
		for k, v := range dataPoint {
			if k != "t" {
				timestamp, err := strconv.ParseInt(dataPoint["t"], 10, 64)
				if err != nil {
					return nil, fmt.Errorf("unable to parse timestamp with alias t: %w", err)
				}

				point, err := strconv.ParseFloat(v, 64)
				if err != nil {
					return nil, fmt.Errorf("unable to parse value for '%s': %w", k, err)
				}

				seriesMap[k].Points = append(seriesMap[k].Points, &datasource.Point{
					Timestamp: timestamp,
					Value:     point,
				})
			}
		}
	}

	series := []*datasource.TimeSeries{}
	for _, timeSeries := range seriesMap {
		series = append(series, timeSeries)
	}

	metaJSON, _ := json.Marshal(parsedBody.Meta)
	return &datasource.DatasourceResponse{
		Results: []*datasource.QueryResult{
			&datasource.QueryResult{
				Series:   series,
				RefId:    refId,
				MetaJson: string(metaJSON),
			},
		},
	}, nil
}

type ClickHouseResponse struct {
	Meta []ClickHouseMeta
	Data []map[string]string
	Rows int
}

type ClickHouseMeta struct {
	Name string
	Type string
}
<|endoftext|>


<commit_before>package db

import (
	"crypto/rand"
	"database/sql"
	"encoding/base64"
	"fmt"
	"log"
	"strings"
)

const SessionTimeout = 10
const SessionTimeoutString = "'10 minutes'"

type QueryOptions struct {
	Title    string // filter by title
	Nsfm     bool   // show NSFM sessions
	Protocol string // filter by protocol version (comma separated list accepted)
}

// Get a list of sessions that match the given query parameters
// Parameters:
func QuerySessionList(opts QueryOptions, db *sql.DB) ([]SessionInfo, error) {
	querySql := `
	SELECT host, port, session_id, protocol, title, users, usernames, password, nsfm, owner,
	to_char(started at time zone 'UTC', 'YYYY-MM-DD HH24:MI:ss') as started
	FROM sessions
	WHERE last_active >= current_timestamp - $1::interval AND unlisted=false
	`

	params := []interface{}{SessionTimeoutString}

	if len(opts.Title) > 0 {
		querySql += fmt.Sprintf(" AND title ILIKE '%%' || $%d || '%%'", len(params)+1)
		params = append(params, opts.Title)
	}

	if !opts.Nsfm {
		querySql += ` AND nsfm=false`
	}

	if len(opts.Protocol) > 0 {
		protocols := strings.Split(opts.Protocol, ",")
		placeholders := make([]string, len(protocols))
		for i, v := range protocols {
			placeholders[i] = fmt.Sprintf("$%d", len(params)+1)
			params = append(params, v)
		}
		querySql += ` AND protocol IN (` + strings.Join(placeholders, ",") + `)`
	}

	querySql += ` ORDER BY started DESC`
	rows, err := db.Query(querySql, params...)

	if err != nil {
		log.Println("Sesion list query error:", err)
		return []SessionInfo{}, err
	}
	defer rows.Close()

	sessions := []SessionInfo{}

	for rows.Next() {
		ses := SessionInfo{}
		var usernames string
		err := rows.Scan(&ses.Host, &ses.Port, &ses.Id, &ses.Protocol, &ses.Title, &ses.Users, &usernames, &ses.Password, &ses.Nsfm, &ses.Owner, &ses.Started)
		if len(usernames) > 0 {
			ses.Usernames = strings.Split(usernames, ",") // TODO user array type when pq driver supports it
		} else {
			ses.Usernames = []string{}
		}

		if err != nil {
			log.Println("Session list row error:", err)
			return []SessionInfo{}, err
		}

		sessions = append(sessions, ses)
	}

	return sessions, nil
}

// Is there an active announcement for this session
func IsActiveSession(host, id string, port int, db *sql.DB) (bool, error) {
	querySql := `
	SELECT exists(SELECT 1
	FROM sessions
	WHERE host=$1 AND port=$2 AND session_id=$3 AND last_active >= current_timestamp - $4::interval
	AND unlisted=false
	)`

	params := []interface{}{host, port, id, SessionTimeoutString}

	var exists bool
	if err := db.QueryRow(querySql, params...).Scan(&exists); err != nil {
		log.Println("Active session query error:", err)
		return false, err
	}

	return exists, nil
}

// Get the number of active announcements on this server (all ports)
func GetHostSessionCount(host string, db *sql.DB) (int, error) {
	querySql := `
	SELECT COUNT(*) FROM sessions
	WHERE host=$1 AND last_active >= current_timestamp - $2::interval AND unlisted=false
	`

	var count int
	if err := db.QueryRow(querySql, host, SessionTimeoutString).Scan(&count); err != nil {
		log.Println("Session count query error:", err)
		return 0, err
	}

	return count, nil
}

// Generate a secure random string
func generateUpdateKey() (string, error) {
	keybytes := make([]byte, 128/8)
	_, err := rand.Read(keybytes)
	if err != nil {
		return "", err
	}
	return base64.URLEncoding.EncodeToString(keybytes), nil
}

// Insert a new session to the database
// Note: this function does not validate the data;
// that must be done before calling this
func InsertSession(session SessionInfo, clientIp string, db *sql.DB) (NewSessionInfo, error) {
	var updateKey string
	updateKey, err := generateUpdateKey()
	if err != nil {
		log.Println("Secure random generator failed:", err)
		return NewSessionInfo{}, err
	}

	insertSql := `
	INSERT INTO sessions
	(host, port, session_id, protocol, title, users, usernames, password, nsfm, owner, started, last_active, update_key, client_ip)
	VALUES
	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, current_timestamp, current_timestamp, $11, $12)
	RETURNING id
	`
	var listingId int
	usernames := strings.Join(session.Usernames, ",")
	err = db.QueryRow(insertSql,
		&session.Host,
		&session.Port,
		&session.Id,
		&session.Protocol,
		&session.Title,
		&session.Users,
		&usernames,
		&session.Password,
		&session.Nsfm,
		&session.Owner,
		&updateKey,
		&clientIp,
	).Scan(&listingId)

	if err != nil {
		log.Println("Session insert error:", err)
		return NewSessionInfo{}, err
	}

	return NewSessionInfo{listingId, updateKey}, nil
}

type RefreshError struct {
	message string
}

func (e RefreshError) Error() string {
	return e.message
}

func optString(fields map[string]interface{}, name string) (string, bool) {
	if value, ok := fields[name]; ok {
		val, ok := value.(string)
		return val, ok
	}
	return "", false
}

func optInt(fields map[string]interface{}, name string) (int, bool) {
	if value, ok := fields[name]; ok {
		val, ok := value.(float64)
		return int(val), ok
	}
	return 0, false
}

func optBool(fields map[string]interface{}, name string) (bool, bool) {
	if value, ok := fields[name]; ok {
		val, ok := value.(bool)
		return val, ok
	}
	return false, false
}

// Refresh an announcement
func RefreshSession(refreshFields map[string]interface{}, listingId int, updateKey string, db *sql.DB) error {
	// First, make sure an active listing exists
	querySql := `
	SELECT exists(SELECT 1
	FROM sessions
	WHERE id=$1 AND update_key=$2 AND last_active >= current_timestamp - $3::interval
	AND unlisted=false
	)`

	params := []interface{}{listingId, updateKey, SessionTimeoutString}

	var exists bool
	if err := db.QueryRow(querySql, params...).Scan(&exists); err != nil {
		log.Println("Session refresh query error:", err)
		return err
	}

	if !exists {
		return RefreshError{"not found"}
	}

	// Construct update query
	querySql = `
	UPDATE sessions
	SET last_active=current_timestamp`
	params = []interface{}{}

	if val, ok := optString(refreshFields, "title"); ok {
		querySql += fmt.Sprintf(", title=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optInt(refreshFields, "users"); ok {
		querySql += fmt.Sprintf(", users=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optString(refreshFields, "usernames"); ok {
		querySql += fmt.Sprintf(", usernames=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optBool(refreshFields, "password"); ok {
		querySql += fmt.Sprintf(", password=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optBool(refreshFields, "nsfm"); ok {
		querySql += fmt.Sprintf(", nsfm=$%d", len(params)+1)
		params = append(params, val)
	}

	querySql += fmt.Sprintf(` WHERE id=$%d`, len(params)+1)
	params = append(params, listingId)

	// Execute update
	_, err := db.Exec(querySql, params...)
	if err != nil {
		log.Println("Session refresh error:", err)
	}
	return err
}

// Delete an announcement
func DeleteSession(listingId int, updateKey string, db *sql.DB) (bool, error) {
	querySql := `
	UPDATE sessions
	SET unlisted=true
	WHERE id=$1 AND update_key=$2 AND unlisted=false`

	res, err := db.Exec(querySql, listingId, updateKey)
	if err != nil {
		log.Println("Session delete error:", err)
		return false, err
	}

	rows, err := res.RowsAffected()
	if err != nil {
		log.Println("Session delete error:", err)
		return false, err
	}
	return rows > 0, nil
}
<commit_msg>Sort session list by title<commit_after>package db

import (
	"crypto/rand"
	"database/sql"
	"encoding/base64"
	"fmt"
	"log"
	"strings"
)

const SessionTimeout = 10
const SessionTimeoutString = "'10 minutes'"

type QueryOptions struct {
	Title    string // filter by title
	Nsfm     bool   // show NSFM sessions
	Protocol string // filter by protocol version (comma separated list accepted)
}

// Get a list of sessions that match the given query parameters
// Parameters:
func QuerySessionList(opts QueryOptions, db *sql.DB) ([]SessionInfo, error) {
	querySql := `
	SELECT host, port, session_id, protocol, title, users, usernames, password, nsfm, owner,
	to_char(started at time zone 'UTC', 'YYYY-MM-DD HH24:MI:ss') as started
	FROM sessions
	WHERE last_active >= current_timestamp - $1::interval AND unlisted=false
	`

	params := []interface{}{SessionTimeoutString}

	if len(opts.Title) > 0 {
		querySql += fmt.Sprintf(" AND title ILIKE '%%' || $%d || '%%'", len(params)+1)
		params = append(params, opts.Title)
	}

	if !opts.Nsfm {
		querySql += ` AND nsfm=false`
	}

	if len(opts.Protocol) > 0 {
		protocols := strings.Split(opts.Protocol, ",")
		placeholders := make([]string, len(protocols))
		for i, v := range protocols {
			placeholders[i] = fmt.Sprintf("$%d", len(params)+1)
			params = append(params, v)
		}
		querySql += ` AND protocol IN (` + strings.Join(placeholders, ",") + `)`
	}

	querySql += ` ORDER BY title, users ASC`
	rows, err := db.Query(querySql, params...)

	if err != nil {
		log.Println("Sesion list query error:", err)
		return []SessionInfo{}, err
	}
	defer rows.Close()

	sessions := []SessionInfo{}

	for rows.Next() {
		ses := SessionInfo{}
		var usernames string
		err := rows.Scan(&ses.Host, &ses.Port, &ses.Id, &ses.Protocol, &ses.Title, &ses.Users, &usernames, &ses.Password, &ses.Nsfm, &ses.Owner, &ses.Started)
		if len(usernames) > 0 {
			ses.Usernames = strings.Split(usernames, ",") // TODO user array type when pq driver supports it
		} else {
			ses.Usernames = []string{}
		}

		if err != nil {
			log.Println("Session list row error:", err)
			return []SessionInfo{}, err
		}

		sessions = append(sessions, ses)
	}

	return sessions, nil
}

// Is there an active announcement for this session
func IsActiveSession(host, id string, port int, db *sql.DB) (bool, error) {
	querySql := `
	SELECT exists(SELECT 1
	FROM sessions
	WHERE host=$1 AND port=$2 AND session_id=$3 AND last_active >= current_timestamp - $4::interval
	AND unlisted=false
	)`

	params := []interface{}{host, port, id, SessionTimeoutString}

	var exists bool
	if err := db.QueryRow(querySql, params...).Scan(&exists); err != nil {
		log.Println("Active session query error:", err)
		return false, err
	}

	return exists, nil
}

// Get the number of active announcements on this server (all ports)
func GetHostSessionCount(host string, db *sql.DB) (int, error) {
	querySql := `
	SELECT COUNT(*) FROM sessions
	WHERE host=$1 AND last_active >= current_timestamp - $2::interval AND unlisted=false
	`

	var count int
	if err := db.QueryRow(querySql, host, SessionTimeoutString).Scan(&count); err != nil {
		log.Println("Session count query error:", err)
		return 0, err
	}

	return count, nil
}

// Generate a secure random string
func generateUpdateKey() (string, error) {
	keybytes := make([]byte, 128/8)
	_, err := rand.Read(keybytes)
	if err != nil {
		return "", err
	}
	return base64.URLEncoding.EncodeToString(keybytes), nil
}

// Insert a new session to the database
// Note: this function does not validate the data;
// that must be done before calling this
func InsertSession(session SessionInfo, clientIp string, db *sql.DB) (NewSessionInfo, error) {
	var updateKey string
	updateKey, err := generateUpdateKey()
	if err != nil {
		log.Println("Secure random generator failed:", err)
		return NewSessionInfo{}, err
	}

	insertSql := `
	INSERT INTO sessions
	(host, port, session_id, protocol, title, users, usernames, password, nsfm, owner, started, last_active, update_key, client_ip)
	VALUES
	($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, current_timestamp, current_timestamp, $11, $12)
	RETURNING id
	`
	var listingId int
	usernames := strings.Join(session.Usernames, ",")
	err = db.QueryRow(insertSql,
		&session.Host,
		&session.Port,
		&session.Id,
		&session.Protocol,
		&session.Title,
		&session.Users,
		&usernames,
		&session.Password,
		&session.Nsfm,
		&session.Owner,
		&updateKey,
		&clientIp,
	).Scan(&listingId)

	if err != nil {
		log.Println("Session insert error:", err)
		return NewSessionInfo{}, err
	}

	return NewSessionInfo{listingId, updateKey}, nil
}

type RefreshError struct {
	message string
}

func (e RefreshError) Error() string {
	return e.message
}

func optString(fields map[string]interface{}, name string) (string, bool) {
	if value, ok := fields[name]; ok {
		val, ok := value.(string)
		return val, ok
	}
	return "", false
}

func optInt(fields map[string]interface{}, name string) (int, bool) {
	if value, ok := fields[name]; ok {
		val, ok := value.(float64)
		return int(val), ok
	}
	return 0, false
}

func optBool(fields map[string]interface{}, name string) (bool, bool) {
	if value, ok := fields[name]; ok {
		val, ok := value.(bool)
		return val, ok
	}
	return false, false
}

// Refresh an announcement
func RefreshSession(refreshFields map[string]interface{}, listingId int, updateKey string, db *sql.DB) error {
	// First, make sure an active listing exists
	querySql := `
	SELECT exists(SELECT 1
	FROM sessions
	WHERE id=$1 AND update_key=$2 AND last_active >= current_timestamp - $3::interval
	AND unlisted=false
	)`

	params := []interface{}{listingId, updateKey, SessionTimeoutString}

	var exists bool
	if err := db.QueryRow(querySql, params...).Scan(&exists); err != nil {
		log.Println("Session refresh query error:", err)
		return err
	}

	if !exists {
		return RefreshError{"not found"}
	}

	// Construct update query
	querySql = `
	UPDATE sessions
	SET last_active=current_timestamp`
	params = []interface{}{}

	if val, ok := optString(refreshFields, "title"); ok {
		querySql += fmt.Sprintf(", title=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optInt(refreshFields, "users"); ok {
		querySql += fmt.Sprintf(", users=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optString(refreshFields, "usernames"); ok {
		querySql += fmt.Sprintf(", usernames=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optBool(refreshFields, "password"); ok {
		querySql += fmt.Sprintf(", password=$%d", len(params)+1)
		params = append(params, val)
	}

	if val, ok := optBool(refreshFields, "nsfm"); ok {
		querySql += fmt.Sprintf(", nsfm=$%d", len(params)+1)
		params = append(params, val)
	}

	querySql += fmt.Sprintf(` WHERE id=$%d`, len(params)+1)
	params = append(params, listingId)

	// Execute update
	_, err := db.Exec(querySql, params...)
	if err != nil {
		log.Println("Session refresh error:", err)
	}
	return err
}

// Delete an announcement
func DeleteSession(listingId int, updateKey string, db *sql.DB) (bool, error) {
	querySql := `
	UPDATE sessions
	SET unlisted=true
	WHERE id=$1 AND update_key=$2 AND unlisted=false`

	res, err := db.Exec(querySql, listingId, updateKey)
	if err != nil {
		log.Println("Session delete error:", err)
		return false, err
	}

	rows, err := res.RowsAffected()
	if err != nil {
		log.Println("Session delete error:", err)
		return false, err
	}
	return rows > 0, nil
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"net/http"

	"ratings/models"
	"ratings/parser"

	"github.com/labstack/echo"
)

func Create(context echo.Context) error {
	request, err := parser.Parse(context)

	if err != nil {
		return err
	}

	readDb := GetReadDB()
	defer readDb.Close()

	brand, err := models.GetBrand("Google", readDb)

	fmt.Println(request.Rating)

	return context.JSON(http.StatusOK, brand)
}
<commit_msg>Added getBaseFields()<commit_after>package controller

import (
	"fmt"
	"net/http"

	"ratings/models"
	"ratings/parser"

	"github.com/jinzhu/gorm"
	"github.com/labstack/echo"
)

// Create saves a new rating to the database
func Create(context echo.Context) error {
	request, err := parser.Parse(context)

	if err != nil {
		return err
	}

	readDb := GetReadDB()
	defer readDb.Close()
	writeDb := GetWriteDB()
	defer writeDb.Close()

	appRecord, rangeRecord, platformRecord := getBaseFields(request, readDb)

	fmt.Println("appRecord:", appRecord)
	fmt.Println("rangeRecord:", rangeRecord)

	return context.JSON(http.StatusOK, platformRecord)
}

func getBaseFields(request *parser.Request, db *gorm.DB) (models.App, models.Range, models.Platform) {
	appRecord, appErr := models.GetApp(request.App, db)

	if appErr != nil {
		// TODO: Dispatch error response
		fmt.Println("Error getting app:", appErr)
	}

	rangeRecord, rangeErr := models.GetRange(request.Range, db)

	if rangeErr != nil {
		// TODO: Dispatch error response
		fmt.Println("Error getting range:", rangeErr)
	}

	platformRecord, platformErr := models.GetPlatform(request.Platform.Key, db)

	if platformErr != nil {
		// TODO: Dispatch error response
		fmt.Println("Error getting platform:", platformErr)
	}

	return appRecord, rangeRecord, platformRecord
}

func hasAppUser(request *parser.Request) bool {
	appuser := request.User

	if appuser.Name == "" || appuser.Email == "" || appuser.MiBAID == "" {
		return false
	}

	return true
}

func hasDevice(request *parser.Request) bool {
	device := request.Device

	if device.Name == "" || device.Brand == "" {
		return false
	}

	return true
}

func hasBrowser(request *parser.Request) bool {
	browser := request.Browser

	if browser.Name == "" {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package ctxcopy

import (
	"errors"
	"golang.org/x/net/context"
	"io"
)

const (
	// Zero buffer length error string.
	ZeroBufferLenError string = "Buffer length is 0."
	// Copy canceled string.
	CanceledStr string = "Copy operation is canceled."
)

// copyWithCancelation() reads bytes to buffer from source and write to destination from buffer with cancelation signal.
//
//   Params:
//     dst: Destination.
//     src: Source.
//     buf: Buffer(length should >= 0).
//     cancel: cancelation flag. It'll stop copy while the cancelation signal is set to true.
func copyWithCancelation(dst io.Writer, src io.Reader, buf []byte, cancel *bool) (err error) {
	if len(buf) == 0 {
		return errors.New(ZeroBufferLenError)
	}

	for {
		if cancel != nil && (*cancel) {
			return errors.New(CanceledStr)
		}

		n, err := src.Read(buf)
		if err != nil && err != io.EOF {
			return err
		} else {
			if n == 0 {
				break
			}
		}

		if _, err := dst.Write(buf[:n]); err != nil {
			return err
		}
	}
	return nil
}

// Copy() reads bytes to buffer from source and write to destination from buffer with cancelation signal.
//
//   Params:
//     ctx:
//       Google's Context type which carries deadlines, cacelation signals,
//       and other request-scoped values across API boundaries and between processes.
//       See https://godoc.org/golang.org/x/net/context for more.
//     dst: Destination.
//     src: Source.
//     buf: Buffer(length should >= 0).
func Copy(ctx context.Context, dst io.Writer, src io.Reader, buf []byte) (err error) {
	cancel := false
	c := make(chan error)

	go func() {
		c <- copyWithCancelation(dst, src, buf, &cancel)
	}()

	select {
	case err = <-c:
		return err

	case <-ctx.Done():
		// Set cancelation flag to true
		cancel = true
		// Wait for copyWithCancelation() return.
		<-c
		return ctx.Err()
	}
}
<commit_msg>Import non-standard packages after standard ones.<commit_after>package ctxcopy

import (
	"errors"
	"io"

	"golang.org/x/net/context"
)

const (
	// Zero buffer length error string.
	ZeroBufferLenError string = "Buffer length is 0."
	// Copy canceled string.
	CanceledStr string = "Copy operation is canceled."
)

// copyWithCancelation() reads bytes to buffer from source and write to destination from buffer with cancelation signal.
//
//   Params:
//     dst: Destination.
//     src: Source.
//     buf: Buffer(length should >= 0).
//     cancel: cancelation flag. It'll stop copy while the cancelation signal is set to true.
func copyWithCancelation(dst io.Writer, src io.Reader, buf []byte, cancel *bool) (err error) {
	if len(buf) == 0 {
		return errors.New(ZeroBufferLenError)
	}

	for {
		if cancel != nil && (*cancel) {
			return errors.New(CanceledStr)
		}

		n, err := src.Read(buf)
		if err != nil && err != io.EOF {
			return err
		} else {
			if n == 0 {
				break
			}
		}

		if _, err := dst.Write(buf[:n]); err != nil {
			return err
		}
	}
	return nil
}

// Copy() reads bytes to buffer from source and write to destination from buffer with cancelation signal.
//
//   Params:
//     ctx:
//       Google's Context type which carries deadlines, cacelation signals,
//       and other request-scoped values across API boundaries and between processes.
//       See https://godoc.org/golang.org/x/net/context for more.
//     dst: Destination.
//     src: Source.
//     buf: Buffer(length should >= 0).
func Copy(ctx context.Context, dst io.Writer, src io.Reader, buf []byte) (err error) {
	cancel := false
	c := make(chan error)

	go func() {
		c <- copyWithCancelation(dst, src, buf, &cancel)
	}()

	select {
	case err = <-c:
		return err

	case <-ctx.Done():
		// Set cancelation flag to true
		cancel = true
		// Wait for copyWithCancelation() return.
		<-c
		return ctx.Err()
	}
}
<|endoftext|>


<commit_before>package webHandlers

import (
	"context"
	"math/rand"
	"net/http"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/molecul/qa_portal/checker"
	"github.com/molecul/qa_portal/model"
	"github.com/molecul/qa_portal/web/middleware"
	"github.com/zalando/gin-oauth2/google"
)

func doError(ctx *gin.Context, err error) {
	ctx.JSON(http.StatusInternalServerError, gin.H{"Internal error": err.Error()})
	panic(err)
}

func UserLoginHandler(ctx *gin.Context) {
	gu := ctx.MustGet("user").(google.User)
	usr, err := model.GetUserByEmail(gu.Email)
	if err != nil {
		doError(ctx, err)
		return
	}
	if usr == nil {
		if err = model.CreateUser(new(model.User).FillFromGoogle(&gu)); err != nil {
			doError(ctx, err)
			return
		}
		// TODO ONLY FOR DEBUG. NEED REMOVE
		usr, err = model.GetUserByEmail(gu.Email)
		if err != nil {
			doError(ctx, err)
			return
		}
		if usr == nil {
			doError(ctx, nil)
		}
		// ================================
	} else {
		if err = usr.FillFromGoogle(&gu).Update(); err != nil {
			doError(ctx, err)
			return
		}
	}
	middleware.UserSessionSet(ctx, usr.ID)
	ctx.JSON(http.StatusOK, gin.H{"Hello": "from private", "user": gu, "internal_user": usr})
}

func MainPageHandler(c *gin.Context) {
	c.HTML(http.StatusOK, "index.html", gin.H{
		"title": "Main website",
	})
}

func DockerHealthCheckHandler(c *gin.Context) {
	task := checker.Get().NewTask(&model.Challenge{
		ID:           rand.Int63(),
		Image:        "python:2.7",
		TargetPath:   "/tmp/task.py",
		Cmd:          "echo \"Inside $CHECKER_FILE:\"; cat $CHECKER_FILE",
		InternalName: "test",
	}, &model.Test{
		ID:          rand.Int63(),
		ChallengeID: rand.Int63(),
		InputFile:   "healthcheck",
	})

	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
	if err := task.Do(ctx); err != nil {
		c.JSON(http.StatusBadGateway, err)
		logrus.Print(err)
	} else {
		c.JSON(http.StatusOK, task.Result)
	}
	cancel()
}
<commit_msg>Change random id to timestamp in DockerHealthCheckHandler<commit_after>package webHandlers

import (
	"context"
	"net/http"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gin-gonic/gin"
	"github.com/molecul/qa_portal/checker"
	"github.com/molecul/qa_portal/model"
	"github.com/molecul/qa_portal/web/middleware"
	"github.com/zalando/gin-oauth2/google"
)

func doError(ctx *gin.Context, err error) {
	ctx.JSON(http.StatusInternalServerError, gin.H{"Internal error": err.Error()})
	panic(err)
}

func UserLoginHandler(ctx *gin.Context) {
	gu := ctx.MustGet("user").(google.User)
	usr, err := model.GetUserByEmail(gu.Email)
	if err != nil {
		doError(ctx, err)
		return
	}
	if usr == nil {
		if err = model.CreateUser(new(model.User).FillFromGoogle(&gu)); err != nil {
			doError(ctx, err)
			return
		}
		// TODO ONLY FOR DEBUG. NEED REMOVE
		usr, err = model.GetUserByEmail(gu.Email)
		if err != nil {
			doError(ctx, err)
			return
		}
		if usr == nil {
			doError(ctx, nil)
		}
		// ================================
	} else {
		if err = usr.FillFromGoogle(&gu).Update(); err != nil {
			doError(ctx, err)
			return
		}
	}
	middleware.UserSessionSet(ctx, usr.ID)
	ctx.JSON(http.StatusOK, gin.H{"Hello": "from private", "user": gu, "internal_user": usr})
}

func MainPageHandler(c *gin.Context) {
	c.HTML(http.StatusOK, "index.html", gin.H{
		"title": "Main website",
	})
}

func DockerHealthCheckHandler(c *gin.Context) {
	temp_id, _ := strconv.Atoi(time.Now().Format("20060102150405"))
	task := checker.Get().NewTask(&model.Challenge{
		ID: 	    int64(temp_id),
		Image:      "python:2.7",
		TargetPath: "/tmp/task.py",
		Cmd:        "echo \"Inside $CHECKER_FILE:\"; cat $CHECKER_FILE",
		InternalName: "test",
	}, &model.Test{
		ID:          int64(temp_id),
		ChallengeID: int64(temp_id),
		InputFile:   "healthcheck_"+strconv.FormatInt(int64(temp_id), 10),
	})

	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
	if err := task.Do(ctx); err != nil {
		c.JSON(http.StatusBadGateway, err)
		logrus.Print(err)
	} else {

		c.JSON(http.StatusOK,
			gin.H{
				"exitcode": task.Result.ExitCode,
				"stdout": task.Result.Stdout,
				"stderr": task.Result.Stderr})
	}
	cancel()
}<|endoftext|>