<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/jlaffaye/ftp"
)

// Loggerer ...
type Loggerer struct {
	logger ILogger
	err    error
}

// SetLogger sets objects logger
func (l *Loggerer) SetLogger(logger ILogger) {
	l.logger = logger
}

// Log logs the input text
func (l *Loggerer) Log(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(text...)
	}
}

// Error ...
func (l *Loggerer) Error(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log("ERROR: " + fmt.Sprint(text...))
	}
	if l.err == nil {
		l.err = errors.New(fmt.Sprint(text...))
	}
}

// ResetError ...
func (l *Loggerer) ResetError() {
	l.err = nil
}

// GetError ...
func (l *Loggerer) GetError() error {
	return l.err
}

type ftpConn struct {
	Loggerer
	conn      *ftp.ServerConn
	connected bool
}

func newFtpConn() *ftpConn {
	return &ftpConn{}
}

func (f *ftpConn) dial(addr string) {
	f.ResetError()
	conn, err := ftp.Dial(addr)
	if err != nil {
		f.Error(err)
	}
	f.conn = conn
	f.connected = true
	f.Log("Connected to " + addr)
}

func (f *ftpConn) quit() {
	if !f.connected {
		return
	}
	f.conn.Quit()
	f.connected = false
	f.Log("Connection closed correctly")
}

func (f *ftpConn) login(user string, pass string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.Login(user, pass)
	if err != nil {
		f.Error(err)
	}
	f.Log("Logged in as " + user)
}

func (f *ftpConn) cwd() string {
	if f.GetError() != nil {
		return ""
	}
	cwd, err := f.conn.CurrentDir()
	if err != nil {
		f.Error(err)
	}
	return cwd
}

func (f *ftpConn) cd(path string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.ChangeDir(path)
	if err != nil {
		f.Error(err)
	}
}

func (f *ftpConn) cdup() {
	if f.GetError() != nil {
		return
	}
	f.conn.ChangeDirToParent()
}

func (f *ftpConn) ls(path string) (entries []*ftp.Entry) {
	if f.GetError() != nil {
		return
	}
	entries, err := f.conn.List(path)
	if err != nil {
		f.Error(err)
	}
	return entries
}

func (f *ftpConn) walk(fl map[string]fileEntry) {
	if f.GetError() != nil {
		return
	}
	entries := f.ls("")
	cwd := f.cwd()
	newLine := pad(cwd, len(lastLine))
	fmt.Print(newLine + "\r")
	lastLine = cwd
	for _, element := range entries {
		switch element.Type {
		case ftp.EntryTypeFile:
			if acceptFileName(element.Name) {
				key := cwd + "/" + element.Name
				entry, fileExists := fl[key]
				if fileExists {
					// Old file with new date
					if !entry.Time.Equal(element.Time) {
						f.Log("~ " + truncPad(key, 40, 'l') + " datetime changed")
						fl[key] = newFileEntry(element)
					} else if entry.Size != element.Size {
						f.Log("~ " + truncPad(key, 40, 'l') + " size changed")
						fl[key] = newFileEntry(element)
					} else {
						entry.Found = true
						fl[key] = entry
					}
				} else {
					// New file
					f.Log("+ " + truncPad(key, 40, 'l') + " new file")
					fl[key] = newFileEntry(element)
				}
			}
		case ftp.EntryTypeFolder:
			f.cd(element.Name)
			f.walk(fl)
			f.cdup()
		}
	}
}

func pad(s string, n int) string {
	if n > len(s) {
		return s + strings.Repeat(" ", n-len(s))
	}
	return s
}

func truncPad(s string, n int, side byte) string {
	if len(s) > n {
		if n >= 3 {
			return "..." + s[0+n:len(s)]
		}
		return s[0:n]
	}
	if side == 'r' {
		return strings.Repeat(" ", n-len(s)) + s
	}
	return s + strings.Repeat(" ", n-len(s))
}

func acceptFileName(fileName string) bool {
	if fileMask.MatchString(fileName) {
		return true
	}
	return false
}

func newFileEntry(entry *ftp.Entry) fileEntry {
	file := fileEntry{}
	file.Name = entry.Name
	file.Size = entry.Size
	file.Time = entry.Time
	file.Found = true
	return file
}

type fileEntry struct {
	Name  string
	Size  uint64
	Time  time.Time
	Found bool
}

func (fe *fileEntry) pack() string {
	return fe.Name + "?|" + fmt.Sprintf("%v", fe.Size) + "?|" + fe.Time.String()
}

type tFileList struct {
	Loggerer
	file map[string]fileEntry
}

func newFileList() *tFileList {
	return &tFileList{file: map[string]fileEntry{}}
}

func (fl *tFileList) pack() string {
	output := []string{}
	for key, value := range fl.file {
		output = append(output, "?{"+key+"?}"+value.pack()+"\n")
	}
	sort.Strings(output)
	return strings.Join(output, "")
}

func (fl *tFileList) clean() {
	for key, value := range fl.file {
		if !value.Found {
			delete(fl.file, key)
			fl.Log("- " + truncPad(key, 40, 'l') + " deleted")
		}
	}
}

func (fl tFileList) String() string {
	return fl.pack()
}

func (fl *tFileList) save(filepath string) {
	file, err := os.Create(filepath)
	if err != nil {
		log.Panicln(err)
	}
	defer file.Close()

	_, err = io.Copy(file, strings.NewReader(fl.pack()))
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) load(filepath string) {
	fl.Log("Loading \"" + filepath + "\"...")
	file, err := os.Open(filepath)
	if err != nil {
		fl.Log("\"" + filepath + "\" not found")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		key, entry := fl.parseLine(scanner.Text())
		if key != "" {
			fl.file[key] = entry
		}
	}

	err = scanner.Err()
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) parseLine(line string) (string, fileEntry) {
	// "?{/AMEDIATEKA/ANIMALS_2/SER_05620.mxf?}SER_05620.mxf?|13114515508?|2017-03-17 14:39:39 +0000 UTC"
	if !regExpLine.MatchString(line) {
		fl.Log("ERROR: Wrong input in file list (" + line + ")")
		return "", fileEntry{}
	}
	matches := regExpLine.FindStringSubmatch(line)
	key := matches[1]
	entry := fileEntry{}
	entry.Name = matches[2]
	entrySize, err := strconv.Atoi(matches[3])
	entry.Size = uint64(entrySize)
	entry.Time, err = time.Parse("2006-01-02 15:04:05 +0000 UTC", matches[4])
	if err != nil {
		fl.Log(err)
		return "", fileEntry{}
	}
	return key, entry
}

// ILogger ...
type ILogger interface {
	Log(text ...interface{})
}

type logger struct {
	file   *os.File
	logger *log.Logger
}

func newLogger() *logger {
	l := &logger{}
	file, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
	if err != nil {
		log.Panicln(err)
	}
	l.file = file
	l.logger = log.New(file, "", log.LstdFlags)
	return l
}

func (l *logger) close() {
	l.file.Close()
}

func (l *logger) Log(text ...interface{}) {
	log.Println(text...)
	l.logger.Print(text...)
}

// Global variables are set in private file.
// Ftp server address with port.
// var addr = ""
// var user = ""
// var pass = ""

var regExpLine = regexp.MustCompile(`\?\{(.*)\?\}(.*)\?\|(\d+)\?\|(.*)$`)
var logFilePath = "shuher.log"
var fileListPath = "shuherFileList.txt"
var watcherRootPath = "/AMEDIATEKA"
var fileMask = regexp.MustCompile(`^.*\.mxf$`)
var lastLine string
var longSleepTime = 15 * time.Minute
var shortSleepTime = 1 * time.Minute

func main() {
	// Create objects.
	logger := newLogger()
	defer logger.close()
	ftpConn := newFtpConn()
	ftpConn.SetLogger(logger)
	fileList := newFileList()
	fileList.SetLogger(logger)
	// Load file list.
	fileList.load(fileListPath)
	// Properly close the connection on exit.
	defer ftpConn.quit()

	for {
		// Initialize the connection to the specified ftp server address.
		ftpConn.dial(addr)
		// Authenticate the client with specified user and password.
		ftpConn.login(user, pass)
		// Change directory watcherRootPath.
		ftpConn.cd(watcherRootPath)
		// Walk the directory tree.
		logger.Log("Looking for new files...")
		ftpConn.walk(fileList.file)
		// Terminate the FTP connection.
		ftpConn.quit()
		// Remove deleted files from the fileList.
		fileList.clean()
		// Save new fileList.
		if ftpConn.GetError() == nil {
			fileList.save(fileListPath)
			// Wait for sleepTime before checking again.
			time.Sleep(longSleepTime)
		} else {
			// Wait for sleepTime before checking again.
			time.Sleep(shortSleepTime)
		}
	}
}
<commit_msg>Fix errors.<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/jlaffaye/ftp"
)

// Loggerer ...
type Loggerer struct {
	logger ILogger
	err    error
}

// SetLogger sets objects logger
func (l *Loggerer) SetLogger(logger ILogger) {
	l.logger = logger
}

// Log logs the input text
func (l *Loggerer) Log(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log(text...)
	}
}

// Error ...
func (l *Loggerer) Error(text ...interface{}) {
	if l.logger != nil {
		l.logger.Log("ERROR: " + fmt.Sprint(text...))
	}
	if l.err == nil {
		l.err = errors.New(fmt.Sprint(text...))
	}
}

// ResetError ...
func (l *Loggerer) ResetError() {
	l.err = nil
}

// GetError ...
func (l *Loggerer) GetError() error {
	return l.err
}

type ftpConn struct {
	Loggerer
	conn      *ftp.ServerConn
	connected bool
}

func newFtpConn() *ftpConn {
	return &ftpConn{}
}

func (f *ftpConn) dial(addr string) {
	f.ResetError()
	conn, err := ftp.Dial(addr)
	if err != nil {
		f.Error(err)
		return
	}
	f.conn = conn
	f.connected = true
	f.Log("Connected to " + addr)
}

func (f *ftpConn) quit() {
	if !f.connected {
		return
	}
	f.conn.Quit()
	f.connected = false
	f.Log("Connection closed correctly")
}

func (f *ftpConn) login(user string, pass string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.Login(user, pass)
	if err != nil {
		f.Error(err)
		return
	}
	f.Log("Logged in as " + user)
}

func (f *ftpConn) cwd() string {
	if f.GetError() != nil {
		return ""
	}
	cwd, err := f.conn.CurrentDir()
	if err != nil {
		f.Error(err)
		return ""
	}
	return cwd
}

func (f *ftpConn) cd(path string) {
	if f.GetError() != nil {
		return
	}
	err := f.conn.ChangeDir(path)
	if err != nil {
		f.Error(err)
		return
	}
}

func (f *ftpConn) cdup() {
	if f.GetError() != nil {
		return
	}
	f.conn.ChangeDirToParent()
}

func (f *ftpConn) ls(path string) (entries []*ftp.Entry) {
	if f.GetError() != nil {
		return
	}
	entries, err := f.conn.List(path)
	if err != nil {
		f.Error(err)
		return
	}
	return entries
}

func (f *ftpConn) walk(fl map[string]fileEntry) {
	if f.GetError() != nil {
		return
	}
	entries := f.ls("")
	cwd := f.cwd()
	newLine := pad(cwd, len(lastLine))
	fmt.Print(newLine + "\r")
	lastLine = cwd
	for _, element := range entries {
		switch element.Type {
		case ftp.EntryTypeFile:
			if acceptFileName(element.Name) {
				key := cwd + "/" + element.Name
				entry, fileExists := fl[key]
				if fileExists {
					// Old file with new date
					if !entry.Time.Equal(element.Time) {
						f.Log("~ " + truncPad(key, 40, 'l') + " datetime changed")
						fl[key] = newFileEntry(element)
					} else if entry.Size != element.Size {
						f.Log("~ " + truncPad(key, 40, 'l') + " size changed")
						fl[key] = newFileEntry(element)
					} else {
						entry.Found = true
						fl[key] = entry
					}
				} else {
					// New file
					f.Log("+ " + truncPad(key, 40, 'l') + " new file")
					fl[key] = newFileEntry(element)
				}
			}
		case ftp.EntryTypeFolder:
			f.cd(element.Name)
			f.walk(fl)
			f.cdup()
		}
	}
}

func pad(s string, n int) string {
	if n > len(s) {
		return s + strings.Repeat(" ", n-len(s))
	}
	return s
}

func truncPad(s string, n int, side byte) string {
	if len(s) > n {
		if n >= 3 {
			return "..." + s[len(s)-n+3:len(s)]
		}
		return s[len(s)-n : len(s)]
	}
	if side == 'r' {
		return strings.Repeat(" ", n-len(s)) + s
	}
	return s + strings.Repeat(" ", n-len(s))
}

func acceptFileName(fileName string) bool {
	if fileMask.MatchString(fileName) {
		return true
	}
	return false
}

func newFileEntry(entry *ftp.Entry) fileEntry {
	file := fileEntry{}
	file.Name = entry.Name
	file.Size = entry.Size
	file.Time = entry.Time
	file.Found = true
	return file
}

type fileEntry struct {
	Name  string
	Size  uint64
	Time  time.Time
	Found bool
}

func (fe *fileEntry) pack() string {
	return fe.Name + "?|" + fmt.Sprintf("%v", fe.Size) + "?|" + fe.Time.String()
}

type tFileList struct {
	Loggerer
	file map[string]fileEntry
}

func newFileList() *tFileList {
	return &tFileList{file: map[string]fileEntry{}}
}

func (fl *tFileList) pack() string {
	output := []string{}
	for key, value := range fl.file {
		output = append(output, "?{"+key+"?}"+value.pack()+"\n")
	}
	sort.Strings(output)
	return strings.Join(output, "")
}

func (fl *tFileList) clean() {
	for key, value := range fl.file {
		if !value.Found {
			delete(fl.file, key)
			fl.Log("- " + truncPad(key, 40, 'l') + " deleted")
		} else {
			value.Found = false
		}
	}
}

func (fl tFileList) String() string {
	return fl.pack()
}

func (fl *tFileList) save(filepath string) {
	file, err := os.Create(filepath)
	if err != nil {
		log.Panicln(err)
	}
	defer file.Close()

	_, err = io.Copy(file, strings.NewReader(fl.pack()))
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) load(filepath string) {
	fl.Log("Loading \"" + filepath + "\"...")
	file, err := os.Open(filepath)
	if err != nil {
		fl.Log("\"" + filepath + "\" not found")
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		key, entry := fl.parseLine(scanner.Text())
		if key != "" {
			fl.file[key] = entry
		}
	}

	err = scanner.Err()
	if err != nil {
		log.Panicln(err)
	}
}

func (fl *tFileList) parseLine(line string) (string, fileEntry) {
	// "?{/AMEDIATEKA/ANIMALS_2/SER_05620.mxf?}SER_05620.mxf?|13114515508?|2017-03-17 14:39:39 +0000 UTC"
	if !regExpLine.MatchString(line) {
		fl.Log("ERROR: Wrong input in file list (" + line + ")")
		return "", fileEntry{}
	}
	matches := regExpLine.FindStringSubmatch(line)
	key := matches[1]
	entry := fileEntry{}
	entry.Name = matches[2]
	entrySize, err := strconv.Atoi(matches[3])
	entry.Size = uint64(entrySize)
	entry.Time, err = time.Parse("2006-01-02 15:04:05 +0000 UTC", matches[4])
	if err != nil {
		fl.Log(err)
		return "", fileEntry{}
	}
	return key, entry
}

// ILogger ...
type ILogger interface {
	Log(text ...interface{})
}

type logger struct {
	file   *os.File
	logger *log.Logger
}

func newLogger() *logger {
	l := &logger{}
	file, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0755)
	if err != nil {
		log.Panicln(err)
	}
	l.file = file
	l.logger = log.New(file, "", log.LstdFlags)
	return l
}

func (l *logger) close() {
	l.file.Close()
}

func (l *logger) Log(text ...interface{}) {
	log.Println(text...)
	l.logger.Print(text...)
}

// Global variables are set in private file.
// Ftp server address with port.
// var addr = ""
// var user = ""
// var pass = ""

var regExpLine = regexp.MustCompile(`\?\{(.*)\?\}(.*)\?\|(\d+)\?\|(.*)$`)
var logFilePath = "shuher.log"
var fileListPath = "shuherFileList.txt"
var watcherRootPath = "/AMEDIATEKA"
var fileMask = regexp.MustCompile(`^.*\.mxf$`)
var lastLine string
var longSleepTime = 15 * time.Minute
var shortSleepTime = 1 * time.Minute

func main() {
	// Create objects.
	logger := newLogger()
	defer logger.close()
	ftpConn := newFtpConn()
	ftpConn.SetLogger(logger)
	fileList := newFileList()
	fileList.SetLogger(logger)
	// Load file list.
	fileList.load(fileListPath)
	// Properly close the connection on exit.
	defer ftpConn.quit()

	for {
		// Initialize the connection to the specified ftp server address.
		ftpConn.dial(addr)
		// Authenticate the client with specified user and password.
		ftpConn.login(user, pass)
		// Change directory to watcherRootPath.
		ftpConn.cd(watcherRootPath)
		// Walk the directory tree.
		if ftpConn.GetError() == nil {
			logger.Log("Looking for new files...")
			ftpConn.walk(fileList.file)
		}
		// Terminate the FTP connection.
		ftpConn.quit()
		// Remove deleted files from the fileList.
		fileList.clean()
		// Save new fileList.
		if ftpConn.GetError() == nil {
			fileList.save(fileListPath)
			// Wait for sleepTime before checking again.
			time.Sleep(longSleepTime)
		} else {
			// Wait for sleepTime before checking again.
			time.Sleep(shortSleepTime)
		}
	}
}
<|endoftext|>


<commit_before>package html

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"io"
	"runtime"
	"syscall"
	"time"
)

var startTime time.Time = time.Now()

func writeHeader(writer io.Writer) {
	fmt.Fprintf(writer, "Start time: %s
\n", startTime)
	uptime := time.Since(startTime)
	fmt.Fprintf(writer, "Uptime: %s
\n", uptime)
	var rusage syscall.Rusage
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
	userCpuTime := time.Duration(rusage.Utime.Sec)*time.Second +
		time.Duration(rusage.Utime.Usec)*time.Microsecond
	sysCpuTime := time.Duration(rusage.Stime.Sec)*time.Second +
		time.Duration(rusage.Stime.Usec)*time.Microsecond
	cpuTime := rusage.Utime.Sec + rusage.Stime.Sec
	fmt.Fprintf(writer, "CPU Time: %d%% (User: %s Sys: %s)
\n",
		cpuTime*100/int64(uptime.Seconds()), userCpuTime, sysCpuTime)
	var memStatsBeforeGC, memStatsAfterGC runtime.MemStats
	runtime.ReadMemStats(&memStatsBeforeGC)
	runtime.GC()
	runtime.ReadMemStats(&memStatsAfterGC)
	fmt.Fprintf(writer, "Allocated memory: %s (%s after GC)
\n",
		format.FormatBytes(memStatsBeforeGC.Alloc),
		format.FormatBytes(memStatsAfterGC.Alloc))
	fmt.Fprintf(writer, "System memory: %s (%s after GC)
\n",
		format.FormatBytes(memStatsBeforeGC.Sys),
		format.FormatBytes(memStatsAfterGC.Sys))
}
<commit_msg>Use shorter time format to show application start time.<commit_after>package html

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"io"
	"runtime"
	"syscall"
	"time"
)

var timeFormat string = "02 Jan 2006 15:04:05.99 MST"

var startTime time.Time = time.Now()

func writeHeader(writer io.Writer) {
	fmt.Fprintf(writer, "Start time: %s
\n", startTime.Format(timeFormat))
	uptime := time.Since(startTime)
	fmt.Fprintf(writer, "Uptime: %s
\n", uptime)
	var rusage syscall.Rusage
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
	userCpuTime := time.Duration(rusage.Utime.Sec)*time.Second +
		time.Duration(rusage.Utime.Usec)*time.Microsecond
	sysCpuTime := time.Duration(rusage.Stime.Sec)*time.Second +
		time.Duration(rusage.Stime.Usec)*time.Microsecond
	cpuTime := rusage.Utime.Sec + rusage.Stime.Sec
	fmt.Fprintf(writer, "CPU Time: %d%% (User: %s Sys: %s)
\n",
		cpuTime*100/int64(uptime.Seconds()), userCpuTime, sysCpuTime)
	var memStatsBeforeGC, memStatsAfterGC runtime.MemStats
	runtime.ReadMemStats(&memStatsBeforeGC)
	runtime.GC()
	runtime.ReadMemStats(&memStatsAfterGC)
	fmt.Fprintf(writer, "Allocated memory: %s (%s after GC)
\n",
		format.FormatBytes(memStatsBeforeGC.Alloc),
		format.FormatBytes(memStatsAfterGC.Alloc))
	fmt.Fprintf(writer, "System memory: %s (%s after GC)
\n",
		format.FormatBytes(memStatsBeforeGC.Sys),
		format.FormatBytes(memStatsAfterGC.Sys))
}
<|endoftext|>


<commit_before>package nico

import (
	"bufio"
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
)

// Client is a API client for niconico.
type Client struct {
	http.Client
	loginRawurl    string
	liveBaseRawurl string
	userSession    string
}

// NewClient return new niconico client.
func NewClient() *Client {
	return &Client{
		loginRawurl:    "https://secure.nicovideo.jp/secure/login",
		liveBaseRawurl: "http://live.nicovideo.jp",
	}
}

// Login is login to niconico and get user session.
func (c *Client) Login(ctx context.Context, mail, password string) error {
	v := url.Values{}
	v.Set("mail", mail)
	v.Set("password", password)

	req, err := http.NewRequest(http.MethodPost, c.loginRawurl, strings.NewReader(v.Encode()))
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	cr := c.CheckRedirect
	defer func() { c.CheckRedirect = cr }()
	c.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusFound {
		return errors.New(resp.Status)
	}

	cookies := resp.Cookies()
	for i := len(cookies) - 1; i >= 0; i-- {
		if cookies[i].Name == "user_session" {
			c.userSession = cookies[i].Value
			return nil
		}
	}

	return errors.New("user session not found")
}

// PlayerStatus is niconico live player status.
type PlayerStatus struct {
	Status string `xml:"status,attr"`
	Time   int64  `xml:"time,attr"`
	Stream Stream `xml:"stream"`
	User   User   `xml:"user"`
	Rtmp   Rtmp   `xml:"rtmp"`
	Ms     Ms     `xml:"ms"`

	// TODO
	TidList interface{} `xml:"tid_list"`

	Twitter Twitter `xml:"twitter"`
	Player  Player  `xml:"player"`
	Marquee Marquee `xml:"marquee"`
	Error   Error   `xml:"error"`
}

// Stream is niconico live player status in player status.
type Stream struct {
	ID                       string `xml:"id"`
	Title                    string `xml:"title"`
	Description              string `xml:"description"`
	ProviderType             string `xml:"provider_type"`
	DefaultCommunity         string `xml:"default_community"`
	International            int64  `xml:"international"`
	IsOwner                  int64  `xml:"is_owner"`
	OwnerID                  int64  `xml:"owner_id"`
	OwnerName                string `xml:"owner_name"`
	IsReserved               int64  `xml:"is_reserved"`
	IsNiconicoEnqueteEnabled int64  `xml:"is_niconico_enquete_enabled"`
	WatchCount               int64  `xml:"watch_count"`
	CommentCount             int64  `xml:"comment_count"`
	BaseTime                 int64  `xml:"base_time"`
	OpenTime                 int64  `xml:"open_time"`
	StartTime                int64  `xml:"start_time"`
	EndTime                  int64  `xml:"end_time"`
	IsRerunStream            int64  `xml:"is_rerun_stream"`

	// TODO
	BourbonURL   interface{} `xml:"bourbon_url"`
	FullVideo    interface{} `xml:"full_video"`
	AfterVideo   interface{} `xml:"after_video"`
	BeforeVideo  interface{} `xml:"before_video"`
	KickoutVideo interface{} `xml:"kickout_video"`

	TwitterTag       string `xml:"twitter_tag"`
	DanjoCommentMode int64  `xml:"danjo_comment_mode"`
	InfinityMode     int64  `xml:"infinity_mode"`
	Archive          int64  `xml:"archive"`
	Press            Press  `xml:"press"`

	// TODO
	PluginDelay interface{} `xml:"plugin_delay"`
	PluginURL   interface{} `xml:"plugin_url"`
	PluginURLs  interface{} `xml:"plugin_urls"`

	AllowNetduetto               int64 `xml:"allow_netduetto"`
	NgScoring                    int64 `xml:"ng_scoring"`
	IsNonarchiveTimeshiftEnabled int64 `xml:"is_nonarchive_timeshift_enabled"`
	IsTimeshiftReserved          int64 `xml:"is_timeshift_reserved"`
	HeaderComment                int64 `xml:"header_comment"`
	FooterComment                int64 `xml:"footer_comment"`
	SplitBottom                  int64 `xml:"split_bottom"`
	SplitTop                     int64 `xml:"split_top"`
	BackgroundComment            int64 `xml:"background_comment"`

	// TODO
	FontScale interface{} `xml:"font_scale"`

	CommentLock  int64        `xml:"comment_lock"`
	Telop        Telop        `xml:"telop"`
	ContentsList ContentsList `xml:"contents_list"`
	PictureURL   string       `xml:"picture_url"`
	ThumbURL     string       `xml:"thumb_url"`

	// TODO
	IsPriorityPrefecture interface{} `xml:"is_priority_prefecture"`
}

type Press struct {
	DisplayLines int64 `xml:"display_lines"`
	DisplayTime  int64 `xml:"display_time"`

	// TODO
	StyleConf interface{} `xml:"style_conf"`
}

type Telop struct {
	Enable int64 `xml:"enable"`
}

type ContentsList struct {
	// TODO slice?
	Contents Contents `xml:"contents"`
}

type Contents struct {
	ID           string `xml:"id,attr"`
	DisableAudio int64  `xml:"disableAudio,attr"`
	DisableVideo int64  `xml:"disableVideo,attr"`
	StartTime    int64  `xml:"start_time,attr"`
	Contents     string `xml:",chardata"`
}

// User is niconico user data in player status.
type User struct {
	UserID         int64  `xml:"user_id"`
	Nickname       string `xml:"nickname"`
	IsPremium      int64  `xml:"is_premium"`
	UserAge        int64  `xml:"userAge"`
	UserSex        int64  `xml:"userSex"`
	UserDomain     string `xml:"userDomain"`
	UserPrefecture int64  `xml:"userPrefecture"`
	UserLanguage   string `xml:"userLanguage"`
	RoomLabel      string `xml:"room_label"`
	RoomSeetno     int64  `xml:"room_seetno"`

	// TODO
	IsJoin interface{} `xml:"is_join"`

	TwitterInfo TwitterInfo `xml:"twitter_info"`
}

// TwitterInfo is user's twitter info in user.
type TwitterInfo struct {
	Status          string      `xml:"status"`
	ScreenName      interface{} `xml:"screen_name"`
	FollowersCount  int64       `xml:"followers_count"`
	IsVip           int64       `xml:"is_vip"`
	ProfileImageURL string      `xml:"profile_image_url"`
	AfterAuth       int64       `xml:"after_auth"`
	TweetToken      string      `xml:"tweet_token"`
}

type Rtmp struct {
	IsFms     int64  `xml:"is_fms,attr"`
	RtmptPort int64  `xml:"rtmpt_port,attr"`
	URL       string `xml:"url"`
	Ticket    string `xml:"ticket"`
}

type Ms struct {
	Addr   string `xml:"addr"`
	Port   int64  `xml:"port"`
	Thread int64  `xml:"thread"`
}

func (m *Ms) StreamingComment(ctx context.Context, resFrom int64) (chan Comment, error) {
	var d net.Dialer
	conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", m.Addr, m.Port))
	if err != nil {
		return nil, err
	}

	b, err := xml.Marshal(SendThread{Thread: m.Thread, Version: 20061206, ResFrom: resFrom})
	if err != nil {
		return nil, err
	}
	b = append(b, 0)
	io.WriteString(conn, string(b))

	r := bufio.NewReader(conn)
	ch := make(chan Comment)
	go func() {
		defer close(ch)
		for {
			select {
			case <-ctx.Done():
				return
			default:
			}
			rb, err := r.ReadSlice(0)
			if err != nil {
				ch <- &CommentError{err}
			}
			var rt ReceiveThread
			if err := xml.Unmarshal(rb, &rt); err == nil {
				ch <- &rt
			}
			var chat Chat
			if err := xml.Unmarshal(rb, &chat); err == nil {
				ch <- &chat
			}
		}
	}()
	return ch, nil
}

type Twitter struct {
	LiveEnabled  int64  `xml:"live_enabled"`
	VipModeCount int64  `xml:"vip_mode_count"`
	LiveApiUrl   string `xml:"live_api_url"`
}

type Player struct {
	QosAnalytics                 int64       `xml:"qos_analytics"`
	DialogImage                  DialogImage `xml:"dialog_image"`
	IsNoticeViewerBalloonEnabled int64       `xml:"is_notice_viewer_balloon_enabled"`
	ErrorReport                  int64       `xml:"error_report"`
}

type DialogImage struct {
	Oidashi string `xml:"oidashi"`
}

type Marquee struct {
	Category         string `xml:"category"`
	GameKey          string `xml:"game_key"`
	GameTime         int64  `xml:"game_time"`
	ForceNicowariOff int64  `xml:"force_nicowari_off"`
}

type Error struct {
	Code string `xml:"code"`
}

func (c *Client) GetPlayerStatus(ctx context.Context, liveID string) (*PlayerStatus, error) {
	u, err := url.Parse(c.liveBaseRawurl)
	if err != nil {
		return nil, err
	}
	u.Path = "api/getplayerstatus"

	v := url.Values{}
	v.Set("v", liveID)
	u.RawQuery = v.Encode()

	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)
	req.AddCookie(&http.Cookie{Name: "user_session", Value: c.userSession})

	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New(resp.Status)
	}

	ps := PlayerStatus{}
	if err := xml.NewDecoder(resp.Body).Decode(&ps); err != nil {
		return nil, err
	}
	if ps.Status != "ok" {
		return nil, fmt.Errorf("%s: %s", ps.Status, ps.Error.Code)
	}

	return &ps, nil
}

type SendThread struct {
	XMLName xml.Name `xml:"thread"`
	Thread  int64    `xml:"thread,attr"`
	Version int64    `xml:"version,attr"`
	ResFrom int64    `xml:"res_from,attr"`
}

type Comment interface {
	comment()
}

type ReceiveThread struct {
	XMLName    xml.Name `xml:"thread"`
	Resultcode int64    `xml:"resultcode,attr"`
	Thread     int64    `xml:"thread,attr"`
	LastRes    int64    `xml:"last_res,attr"`
	Ticket     string   `xml:"ticket,attr"`
	Revision   int64    `xml:"revision,attr"`
	ServerTime int64    `xml:"server_time,attr"`
}

func (t *ReceiveThread) comment() {}

type Chat struct {
	XMLName   xml.Name `xml:"chat"`
	Thread    int64    `xml:"thread,attr"`
	No        int64    `xml:"no,attr"`
	Vpos      int64    `xml:"vpos,attr"`
	Date      int64    `xml:"date,attr"`
	DateUsec  int64    `xml:"date_usec,attr"`
	Mail      string   `xml:"mail,attr"`
	UserID    string   `xml:"user_id,attr"`
	Premium   int64    `xml:"premium,attr"`
	Anonymity int64    `xml:"anonymity,attr"`
	Locale    string   `xml:"locale,attr"`
	Comment   string   `xml:",chardata"`
}

func (c *Chat) comment() {}

type CommentError struct{ error }

func (e *CommentError) comment() {}

func (c *Client) GetPostkey(ctx context.Context, thread int64) (string, error) {
	u, err := url.Parse(c.liveBaseRawurl)
	if err != nil {
		return "", err
	}
	u.Path = "api/getpostkey"

	v := url.Values{}
	v.Set("thread", fmt.Sprint(thread))
	u.RawQuery = v.Encode()

	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return "", err
	}
	req = req.WithContext(ctx)
	req.AddCookie(&http.Cookie{Name: "user_session", Value: c.userSession})

	resp, err := c.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", errors.New(resp.Status)
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	rv, err := url.ParseQuery(string(b))
	if err != nil {
		return "", err
	}
	postkey := rv.Get("postkey")
	if postkey == "" {
		return "", errors.New("postkey is empty")
	}

	return postkey, nil
}
<commit_msg>Add comment server connection close at context cancel<commit_after>package nico

import (
	"bufio"
	"context"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
)

// Client is a API client for niconico.
type Client struct {
	http.Client
	loginRawurl    string
	liveBaseRawurl string
	userSession    string
}

// NewClient return new niconico client.
func NewClient() *Client {
	return &Client{
		loginRawurl:    "https://secure.nicovideo.jp/secure/login",
		liveBaseRawurl: "http://live.nicovideo.jp",
	}
}

// Login is login to niconico and get user session.
func (c *Client) Login(ctx context.Context, mail, password string) error {
	v := url.Values{}
	v.Set("mail", mail)
	v.Set("password", password)

	req, err := http.NewRequest(http.MethodPost, c.loginRawurl, strings.NewReader(v.Encode()))
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	cr := c.CheckRedirect
	defer func() { c.CheckRedirect = cr }()
	c.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusFound {
		return errors.New(resp.Status)
	}

	cookies := resp.Cookies()
	for i := len(cookies) - 1; i >= 0; i-- {
		if cookies[i].Name == "user_session" {
			c.userSession = cookies[i].Value
			return nil
		}
	}

	return errors.New("user session not found")
}

// PlayerStatus is niconico live player status.
type PlayerStatus struct {
	Status string `xml:"status,attr"`
	Time   int64  `xml:"time,attr"`
	Stream Stream `xml:"stream"`
	User   User   `xml:"user"`
	Rtmp   Rtmp   `xml:"rtmp"`
	Ms     Ms     `xml:"ms"`

	// TODO
	TidList interface{} `xml:"tid_list"`

	Twitter Twitter `xml:"twitter"`
	Player  Player  `xml:"player"`
	Marquee Marquee `xml:"marquee"`
	Error   Error   `xml:"error"`
}

// Stream is niconico live player status in player status.
type Stream struct {
	ID                       string `xml:"id"`
	Title                    string `xml:"title"`
	Description              string `xml:"description"`
	ProviderType             string `xml:"provider_type"`
	DefaultCommunity         string `xml:"default_community"`
	International            int64  `xml:"international"`
	IsOwner                  int64  `xml:"is_owner"`
	OwnerID                  int64  `xml:"owner_id"`
	OwnerName                string `xml:"owner_name"`
	IsReserved               int64  `xml:"is_reserved"`
	IsNiconicoEnqueteEnabled int64  `xml:"is_niconico_enquete_enabled"`
	WatchCount               int64  `xml:"watch_count"`
	CommentCount             int64  `xml:"comment_count"`
	BaseTime                 int64  `xml:"base_time"`
	OpenTime                 int64  `xml:"open_time"`
	StartTime                int64  `xml:"start_time"`
	EndTime                  int64  `xml:"end_time"`
	IsRerunStream            int64  `xml:"is_rerun_stream"`

	// TODO
	BourbonURL   interface{} `xml:"bourbon_url"`
	FullVideo    interface{} `xml:"full_video"`
	AfterVideo   interface{} `xml:"after_video"`
	BeforeVideo  interface{} `xml:"before_video"`
	KickoutVideo interface{} `xml:"kickout_video"`

	TwitterTag       string `xml:"twitter_tag"`
	DanjoCommentMode int64  `xml:"danjo_comment_mode"`
	InfinityMode     int64  `xml:"infinity_mode"`
	Archive          int64  `xml:"archive"`
	Press            Press  `xml:"press"`

	// TODO
	PluginDelay interface{} `xml:"plugin_delay"`
	PluginURL   interface{} `xml:"plugin_url"`
	PluginURLs  interface{} `xml:"plugin_urls"`

	AllowNetduetto               int64 `xml:"allow_netduetto"`
	NgScoring                    int64 `xml:"ng_scoring"`
	IsNonarchiveTimeshiftEnabled int64 `xml:"is_nonarchive_timeshift_enabled"`
	IsTimeshiftReserved          int64 `xml:"is_timeshift_reserved"`
	HeaderComment                int64 `xml:"header_comment"`
	FooterComment                int64 `xml:"footer_comment"`
	SplitBottom                  int64 `xml:"split_bottom"`
	SplitTop                     int64 `xml:"split_top"`
	BackgroundComment            int64 `xml:"background_comment"`

	// TODO
	FontScale interface{} `xml:"font_scale"`

	CommentLock  int64        `xml:"comment_lock"`
	Telop        Telop        `xml:"telop"`
	ContentsList ContentsList `xml:"contents_list"`
	PictureURL   string       `xml:"picture_url"`
	ThumbURL     string       `xml:"thumb_url"`

	// TODO
	IsPriorityPrefecture interface{} `xml:"is_priority_prefecture"`
}

type Press struct {
	DisplayLines int64 `xml:"display_lines"`
	DisplayTime  int64 `xml:"display_time"`

	// TODO
	StyleConf interface{} `xml:"style_conf"`
}

type Telop struct {
	Enable int64 `xml:"enable"`
}

type ContentsList struct {
	// TODO slice?
	Contents Contents `xml:"contents"`
}

type Contents struct {
	ID           string `xml:"id,attr"`
	DisableAudio int64  `xml:"disableAudio,attr"`
	DisableVideo int64  `xml:"disableVideo,attr"`
	StartTime    int64  `xml:"start_time,attr"`
	Contents     string `xml:",chardata"`
}

// User is niconico user data in player status.
type User struct {
	UserID         int64  `xml:"user_id"`
	Nickname       string `xml:"nickname"`
	IsPremium      int64  `xml:"is_premium"`
	UserAge        int64  `xml:"userAge"`
	UserSex        int64  `xml:"userSex"`
	UserDomain     string `xml:"userDomain"`
	UserPrefecture int64  `xml:"userPrefecture"`
	UserLanguage   string `xml:"userLanguage"`
	RoomLabel      string `xml:"room_label"`
	RoomSeetno     int64  `xml:"room_seetno"`

	// TODO
	IsJoin interface{} `xml:"is_join"`

	TwitterInfo TwitterInfo `xml:"twitter_info"`
}

// TwitterInfo is user's twitter info in user.
type TwitterInfo struct {
	Status          string      `xml:"status"`
	ScreenName      interface{} `xml:"screen_name"`
	FollowersCount  int64       `xml:"followers_count"`
	IsVip           int64       `xml:"is_vip"`
	ProfileImageURL string      `xml:"profile_image_url"`
	AfterAuth       int64       `xml:"after_auth"`
	TweetToken      string      `xml:"tweet_token"`
}

type Rtmp struct {
	IsFms     int64  `xml:"is_fms,attr"`
	RtmptPort int64  `xml:"rtmpt_port,attr"`
	URL       string `xml:"url"`
	Ticket    string `xml:"ticket"`
}

type Ms struct {
	Addr   string `xml:"addr"`
	Port   int64  `xml:"port"`
	Thread int64  `xml:"thread"`
}

func (m *Ms) StreamingComment(ctx context.Context, resFrom int64) (chan Comment, error) {
	var d net.Dialer
	conn, err := d.DialContext(ctx, "tcp", fmt.Sprintf("%s:%d", m.Addr, m.Port))
	if err != nil {
		return nil, err
	}
	go func() {
		<-ctx.Done()
		conn.Close()
	}()

	b, err := xml.Marshal(SendThread{Thread: m.Thread, Version: 20061206, ResFrom: resFrom})
	if err != nil {
		return nil, err
	}
	b = append(b, 0)
	io.WriteString(conn, string(b))

	r := bufio.NewReader(conn)
	ch := make(chan Comment)
	go func() {
		defer close(ch)
		for {
			select {
			case <-ctx.Done():
				return
			default:
			}
			rb, err := r.ReadSlice(0)
			if err != nil {
				ch <- &CommentError{err}
			}
			var rt ReceiveThread
			if err := xml.Unmarshal(rb, &rt); err == nil {
				ch <- &rt
			}
			var chat Chat
			if err := xml.Unmarshal(rb, &chat); err == nil {
				ch <- &chat
			}
		}
	}()
	return ch, nil
}

type Twitter struct {
	LiveEnabled  int64  `xml:"live_enabled"`
	VipModeCount int64  `xml:"vip_mode_count"`
	LiveApiUrl   string `xml:"live_api_url"`
}

type Player struct {
	QosAnalytics                 int64       `xml:"qos_analytics"`
	DialogImage                  DialogImage `xml:"dialog_image"`
	IsNoticeViewerBalloonEnabled int64       `xml:"is_notice_viewer_balloon_enabled"`
	ErrorReport                  int64       `xml:"error_report"`
}

type DialogImage struct {
	Oidashi string `xml:"oidashi"`
}

type Marquee struct {
	Category         string `xml:"category"`
	GameKey          string `xml:"game_key"`
	GameTime         int64  `xml:"game_time"`
	ForceNicowariOff int64  `xml:"force_nicowari_off"`
}

type Error struct {
	Code string `xml:"code"`
}

func (c *Client) GetPlayerStatus(ctx context.Context, liveID string) (*PlayerStatus, error) {
	u, err := url.Parse(c.liveBaseRawurl)
	if err != nil {
		return nil, err
	}
	u.Path = "api/getplayerstatus"

	v := url.Values{}
	v.Set("v", liveID)
	u.RawQuery = v.Encode()

	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(ctx)
	req.AddCookie(&http.Cookie{Name: "user_session", Value: c.userSession})

	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New(resp.Status)
	}

	ps := PlayerStatus{}
	if err := xml.NewDecoder(resp.Body).Decode(&ps); err != nil {
		return nil, err
	}
	if ps.Status != "ok" {
		return nil, fmt.Errorf("%s: %s", ps.Status, ps.Error.Code)
	}

	return &ps, nil
}

type SendThread struct {
	XMLName xml.Name `xml:"thread"`
	Thread  int64    `xml:"thread,attr"`
	Version int64    `xml:"version,attr"`
	ResFrom int64    `xml:"res_from,attr"`
}

type Comment interface {
	comment()
}

type ReceiveThread struct {
	XMLName    xml.Name `xml:"thread"`
	Resultcode int64    `xml:"resultcode,attr"`
	Thread     int64    `xml:"thread,attr"`
	LastRes    int64    `xml:"last_res,attr"`
	Ticket     string   `xml:"ticket,attr"`
	Revision   int64    `xml:"revision,attr"`
	ServerTime int64    `xml:"server_time,attr"`
}

func (t *ReceiveThread) comment() {}

type Chat struct {
	XMLName   xml.Name `xml:"chat"`
	Thread    int64    `xml:"thread,attr"`
	No        int64    `xml:"no,attr"`
	Vpos      int64    `xml:"vpos,attr"`
	Date      int64    `xml:"date,attr"`
	DateUsec  int64    `xml:"date_usec,attr"`
	Mail      string   `xml:"mail,attr"`
	UserID    string   `xml:"user_id,attr"`
	Premium   int64    `xml:"premium,attr"`
	Anonymity int64    `xml:"anonymity,attr"`
	Locale    string   `xml:"locale,attr"`
	Comment   string   `xml:",chardata"`
}

func (c *Chat) comment() {}

type CommentError struct{ error }

func (e *CommentError) comment() {}

func (c *Client) GetPostkey(ctx context.Context, thread int64) (string, error) {
	u, err := url.Parse(c.liveBaseRawurl)
	if err != nil {
		return "", err
	}
	u.Path = "api/getpostkey"

	v := url.Values{}
	v.Set("thread", fmt.Sprint(thread))
	u.RawQuery = v.Encode()

	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
	if err != nil {
		return "", err
	}
	req = req.WithContext(ctx)
	req.AddCookie(&http.Cookie{Name: "user_session", Value: c.userSession})

	resp, err := c.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", errors.New(resp.Status)
	}

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	rv, err := url.ParseQuery(string(b))
	if err != nil {
		return "", err
	}
	postkey := rv.Get("postkey")
	if postkey == "" {
		return "", errors.New("postkey is empty")
	}

	return postkey, nil
}
<|endoftext|>


<commit_before>package io

import (
	"fmt"
	"github.com/ivan-iver/diaspora/lib/time"
	"text/template"
)

var migrationTpl = template.Must(template.New("migration").Parse(`
-- @Up
-- SQL in section 'Up' is executed when this migration is applied

-- @Down
-- SQL section 'Down' is executed when this migration is rolled back

`))

// Template models a new file with basic template data
type Template struct {
	ID   string
	Name string
	Path string
	Data *template.Template
	*File
}

// NewTemplate returns an empty template with basic tags.
func NewTemplate() (tmp *Template, err error) {
	var serie = time.NewSerie()
	tmp = &Template{
		ID:   serie.Formated,
		Data: migrationTpl,
		File: &File{},
	}
	return
}

// Save persist the information template
func (t *Template) Save() (err error) {
	t.File.Name = fmt.Sprintf("%s_%s", t.ID, t.Name)
	t.File.Extension = ".sql"
	path, err := t.File.Pwd()
	t.File.Path = path
	if err != nil {
		return
	}
	err = t.File.Create(t)
	return
}
<commit_msg>Fix missing extension file<commit_after>package io

import (
	"fmt"
	"github.com/ivan-iver/diaspora/lib/time"
	"text/template"
)

var migrationTpl = template.Must(template.New("migration").Parse(`
-- @Up
-- SQL in section 'Up' is executed when this migration is applied

-- @Down
-- SQL section 'Down' is executed when this migration is rolled back

`))

// Template models a new file with basic template data
type Template struct {
	ID   string
	Name string
	Path string
	Data *template.Template
	*File
}

// NewTemplate returns an empty template with basic tags.
func NewTemplate() (tmp *Template, err error) {
	var serie = time.NewSerie()
	tmp = &Template{
		ID:   serie.Formated,
		Data: migrationTpl,
		File: &File{},
	}
	return
}

// Save persist the information template
func (t *Template) Save() (err error) {
	t.File.Name = fmt.Sprintf("%s_%s.sql", t.ID, t.Name)
	path, err := t.File.Pwd()
	t.File.Path = path
	if err != nil {
		return
	}
	err = t.File.Create(t)
	return
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"
	"github.com/MG-RAST/AWE/lib/conf"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"os"
	"strings"
	"time"
)

type CQMgr struct {
	clientMap map[string]*Client
	workQueue *WQueue
	reminder  chan bool
	coReq     chan CoReq  //workunit checkout request (WorkController -> qmgr.Handler)
	coAck     chan CoAck  //workunit checkout item including data and err (qmgr.Handler -> WorkController)
	feedback  chan Notice //workunit execution feedback (WorkController -> qmgr.Handler)
	coSem     chan int    //semaphore for checkout (mutual exclusion between different clients)
}

func NewCQMgr() *CQMgr {
	return &CQMgr{
		clientMap: map[string]*Client{},
		workQueue: NewWQueue(),
		reminder:  make(chan bool),
		coReq:     make(chan CoReq),
		coAck:     make(chan CoAck),
		feedback:  make(chan Notice),
		coSem:     make(chan int, 1), //non-blocking buffered channel
	}
}

//to-do: consider separate some independent tasks into another goroutine to handle

func (qm *CQMgr) Handle() {
	for {
		select {
		case coReq := <-qm.coReq:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit checkout request received, Req=%v\n", coReq))
			works, err := qm.popWorks(coReq)
			ack := CoAck{workunits: works, err: err}
			qm.coAck <- ack

		case notice := <-qm.feedback:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit feedback received, workid=%s, status=%s, clientid=%s\n", notice.WorkId, notice.Status, notice.ClientId))
			if err := qm.handleWorkStatusChange(notice); err != nil {
				logger.Error("handleWorkStatusChange(): " + err.Error())
			}

		case <-qm.reminder:
			logger.Debug(3, "time to update workunit queue....\n")
		}
	}
}

func (qm *CQMgr) Timer() {
	for {
		time.Sleep(10 * time.Second)
		qm.reminder <- true
	}
}

// show functions used in debug
func (qm *CQMgr) ShowWorkQueue() {
	fmt.Printf("current queuing workunits (%d):\n", qm.workQueue.Len())
	for key, _ := range qm.workQueue.workMap {
		fmt.Printf("workunit id: %s\n", key)
	}
	return
}

//---end of mgr methods

//--------client methods-------

func (qm *CQMgr) ClientChecker() {
	for {
		time.Sleep(30 * time.Second)
		for clientid, client := range qm.clientMap {
			if client.Tag == true {
				client.Tag = false
				total_minutes := int(time.Now().Sub(client.RegTime).Minutes())
				hours := total_minutes / 60
				minutes := total_minutes % 60
				client.Serve_time = fmt.Sprintf("%dh%dm", hours, minutes)
				if len(client.Current_work) > 0 {
					client.Idle_time = 0
				} else {
					client.Idle_time += 30
				}
			} else {
				//now client must be gone as tag set to false 30 seconds ago and no heartbeat received thereafter
				logger.Event(event.CLIENT_UNREGISTER, "clientid="+clientid+";name="+qm.clientMap[clientid].Name)

				//requeue unfinished workunits associated with the failed client
				qm.ReQueueWorkunitByClient(clientid)
				//delete the client from client map
				delete(qm.clientMap, clientid)
			}
		}
	}
}

func (qm *CQMgr) ClientHeartBeat(id string) (hbmsg HBmsg, err error) {
	hbmsg = make(map[string]string, 1)
	if _, ok := qm.clientMap[id]; ok {
		qm.clientMap[id].Tag = true
		logger.Debug(3, "HeartBeatFrom:"+"clientid="+id+",name="+qm.clientMap[id].Name)

		//get suspended workunit that need the client to discard
		workids := qm.getWorkByClient(id)
		suspended := []string{}
		for _, workid := range workids {
			if work, ok := qm.workQueue.workMap[workid]; ok {
				if work.State == WORK_STAT_SUSPEND {
					suspended = append(suspended, workid)
				}
			}
		}
		if len(suspended) > 0 {
			hbmsg["discard"] = strings.Join(suspended, ",")
		}
		if qm.clientMap[id].Status == CLIENT_STAT_DELETED {
			hbmsg["stop"] = id
		}
		//hbmsg["discard"] = strings.Join(workids, ",")
		return hbmsg, nil
	}
	return hbmsg, errors.New(e.ClientNotFound)
}

func (qm *CQMgr) RegisterNewClient(files FormFiles) (client *Client, err error) {
	if _, ok := files["profile"]; ok {
		client, err = NewProfileClient(files["profile"].Path)
		os.Remove(files["profile"].Path)
	} else {
		client = NewClient()
	}
	if err != nil {
		return nil, err
	}
	qm.clientMap[client.Id] = client

	if len(client.Current_work) > 0 { //re-registered client
		// move already checked-out workunit from waiting queue (workMap) to checked-out list (coWorkMap)
		for workid, _ := range client.Current_work {
			if qm.workQueue.Has(workid) {
				qm.workQueue.StatusChange(workid, WORK_STAT_CHECKOUT)
			}
		}
	}
	return
}

func (qm *CQMgr) GetClient(id string) (client *Client, err error) {
	if client, ok := qm.clientMap[id]; ok {
		return client, nil
	}
	return nil, errors.New(e.ClientNotFound)
}

func (qm *CQMgr) GetAllClients() []*Client {
	clients := []*Client{}
	for _, client := range qm.clientMap {
		clients = append(clients, client)
	}
	return clients
}

func (qm *CQMgr) DeleteClient(id string) (err error) {
	if client, ok := qm.clientMap[id]; ok {
		client.Status = CLIENT_STAT_DELETED
		return
	}
	return errors.New("client not found")
}

func (qm *CQMgr) SuspendClient(id string) (err error) {
	if client, ok := qm.clientMap[id]; ok {
		if client.Status == CLIENT_STAT_ACTIVE_IDLE || client.Status == CLIENT_STAT_ACTIVE_BUSY {
			client.Status = CLIENT_STAT_SUSPEND
			qm.ReQueueWorkunitByClient(id)
			return
		}
		return errors.New("client is not in active state")
	}
	return errors.New("client not found")
}

func (qm *CQMgr) ResumeClient(id string) (err error) {
	if client, ok := qm.clientMap[id]; ok {
		if client.Status == CLIENT_STAT_SUSPEND {
			client.Status = CLIENT_STAT_ACTIVE_IDLE
			return
		}
		return errors.New("client is not in suspended state")
	}
	return errors.New("client not found")
}

func (qm *CQMgr) ResumeSuspendedClients() (count int) {
	for _, client := range qm.clientMap {
		if client.Status == CLIENT_STAT_SUSPEND {
			client.Status = CLIENT_STAT_ACTIVE_IDLE
			count += 1
		}
	}
	return count
}

func (qm *CQMgr) SuspendAllClients() (count int) {
	for _, client := range qm.clientMap {
		if client.Status == CLIENT_STAT_ACTIVE_IDLE || client.Status == CLIENT_STAT_ACTIVE_BUSY {
			qm.SuspendClient(client.Id)
			count += 1
		}
	}
	return count
}

func (qm *CQMgr) UpdateSubClients(id string, count int) {
	if _, ok := qm.clientMap[id]; ok {
		qm.clientMap[id].SubClients = count
	}
}

//--------end client methods-------

//-------start of workunit methods---

func (qm *CQMgr) CheckoutWorkunits(req_policy string, client_id string, num int) (workunits []*Workunit, err error) {
	//precheck if the client is registered
	if _, hasClient := qm.clientMap[client_id]; !hasClient {
		return nil, errors.New(e.ClientNotFound)
	}
	if qm.clientMap[client_id].Status == CLIENT_STAT_SUSPEND {
		return nil, errors.New(e.ClientSuspended)
	}
	if qm.clientMap[client_id].Status == CLIENT_STAT_DELETED {
		delete(qm.clientMap, client_id)
		return nil, errors.New(e.ClientDeleted)
	}

	//lock semephore, at one time only one client's checkout request can be served
	qm.coSem <- 1

	req := CoReq{policy: req_policy, fromclient: client_id, count: num}
	qm.coReq <- req
	ack := <-qm.coAck

	if ack.err == nil {
		for _, work := range ack.workunits {
			qm.clientMap[client_id].Total_checkout += 1
			qm.clientMap[client_id].Current_work[work.Id] = true
		}
		if qm.clientMap[client_id].Status == CLIENT_STAT_ACTIVE_IDLE {
			qm.clientMap[client_id].Status = CLIENT_STAT_ACTIVE_BUSY
		}
	}

	//unlock
	<-qm.coSem

	return ack.workunits, ack.err
}

func (qm *CQMgr) GetWorkById(id string) (workunit *Workunit, err error) {
	if workunit, hasid := qm.workQueue.workMap[id]; hasid {
		return workunit, nil
	}
	return nil, errors.New(fmt.Sprintf("no workunit found with id %s", id))
}

func (qm *CQMgr) NotifyWorkStatus(notice Notice) {
	qm.feedback <- notice
	return
}

func (qm *CQMgr) popWorks(req CoReq) (works []*Workunit, err error) {
	filtered := qm.filterWorkByClient(req.fromclient)
	if len(filtered) == 0 {
		return nil, errors.New(e.NoEligibleWorkunitFound)
	}
	works, err = qm.workQueue.selectWorkunits(filtered, req.policy, req.count)
	if err == nil { //get workunits successfully, put them into coWorkMap
		for _, work := range works {
			if _, ok := qm.workQueue.workMap[work.Id]; ok {
				qm.workQueue.workMap[work.Id].Client = req.fromclient
				qm.workQueue.workMap[work.Id].CheckoutTime = time.Now()
			}
			qm.workQueue.StatusChange(work.Id, WORK_STAT_CHECKOUT)
		}
	}
	return
}

func (qm *CQMgr) filterWorkByClient(clientid string) (ids []string) {
	client := qm.clientMap[clientid]
	for id, _ := range qm.workQueue.wait {
		if _, ok := qm.workQueue.workMap[id]; !ok {
			continue
		}
		work := qm.workQueue.workMap[id]
		//skip works that are in the client's skip-list
		if contains(client.Skip_work, work.Id) {
			continue
		}
		//skip works that have dedicate client groups which this client doesn't belong to
		if len(work.Info.ClientGroups) > 0 {
			eligible_groups := strings.Split(work.Info.ClientGroups, ",")
			if !contains(eligible_groups, client.Group) {
				continue
			}
		}
		//append works whos apps are supported by the client
		if contains(client.Apps, work.Cmd.Name) || contains(client.Apps, conf.ALL_APP) {
			ids = append(ids, id)
		}
	}
	return ids
}

func (qm *CQMgr) getWorkByClient(clientid string) (ids []string) {
	if client, ok := qm.clientMap[clientid]; ok {
		for id, _ := range client.Current_work {
			ids = append(ids, id)
		}
	}
	return
}

//handle feedback from a client about the execution of a workunit
func (qm *CQMgr) handleWorkStatusChange(notice Notice) (err error) {
	//to be implemented for proxy or server
	return
}

func (qm *CQMgr) FetchDataToken(workid string, clientid string) (token string, err error) {
	//to be implemented for proxy or server
	return
}

func (qm *CQMgr) ShowWorkunits(status string) (workunits []*Workunit) {
	for _, work := range qm.workQueue.workMap {
		if work.State == status || status == "" {
			workunits = append(workunits, work)
		}
	}
	return workunits
}

func (qm *CQMgr) EnqueueWorkunit(work *Workunit) (err error) {
	err = qm.workQueue.Add(work)
	return
}

func (qm *CQMgr) ReQueueWorkunitByClient(clientid string) (err error) {
	workids := qm.getWorkByClient(clientid)
	for _, workid := range workids {
		if qm.workQueue.Has(workid) {
			qm.workQueue.StatusChange(workid, WORK_STAT_QUEUED)
			logger.Event(event.WORK_REQUEUE, "workid="+workid)
		}
	}
	return
}

//---end of workunit methods
<commit_msg>when a client is gone, only requeue its workunits belonging to active jobs (rule out suspended jobs)<commit_after>package core

import (
	"errors"
	"fmt"
	"github.com/MG-RAST/AWE/lib/conf"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"os"
	"strings"
	"time"
)

type CQMgr struct {
	clientMap map[string]*Client
	workQueue *WQueue
	reminder  chan bool
	coReq     chan CoReq  //workunit checkout request (WorkController -> qmgr.Handler)
	coAck     chan CoAck  //workunit checkout item including data and err (qmgr.Handler -> WorkController)
	feedback  chan Notice //workunit execution feedback (WorkController -> qmgr.Handler)
	coSem     chan int    //semaphore for checkout (mutual exclusion between different clients)
}

func NewCQMgr() *CQMgr {
	return &CQMgr{
		clientMap: map[string]*Client{},
		workQueue: NewWQueue(),
		reminder:  make(chan bool),
		coReq:     make(chan CoReq),
		coAck:     make(chan CoAck),
		feedback:  make(chan Notice),
		coSem:     make(chan int, 1), //non-blocking buffered channel
	}
}

//to-do: consider separate some independent tasks into another goroutine to handle

func (qm *CQMgr) Handle() {
	for {
		select {
		case coReq := <-qm.coReq:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit checkout request received, Req=%v\n", coReq))
			works, err := qm.popWorks(coReq)
			ack := CoAck{workunits: works, err: err}
			qm.coAck <- ack

		case notice := <-qm.feedback:
			logger.Debug(2, fmt.Sprintf("qmgr: workunit feedback received, workid=%s, status=%s, clientid=%s\n", notice.WorkId, notice.Status, notice.ClientId))
			if err := qm.handleWorkStatusChange(notice); err != nil {
				logger.Error("handleWorkStatusChange(): " + err.Error())
			}

		case <-qm.reminder:
			logger.Debug(3, "time to update workunit queue....\n")
		}
	}
}

func (qm *CQMgr) Timer() {
	for {
		time.Sleep(10 * time.Second)
		qm.reminder <- true
	}
}

// show functions used in debug
func (qm *CQMgr) ShowWorkQueue() {
	fmt.Printf("current queuing workunits (%d):\n", qm.workQueue.Len())
	for key, _ := range qm.workQueue.workMap {
		fmt.Printf("workunit id: %s\n", key)
	}
	return
}

//---end of mgr methods

//--------client methods-------

func (qm *CQMgr) ClientChecker() {
	for {
		time.Sleep(30 * time.Second)
		for clientid, client := range qm.clientMap {
			if client.Tag == true {
				client.Tag = false
				total_minutes := int(time.Now().Sub(client.RegTime).Minutes())
				hours := total_minutes / 60
				minutes := total_minutes % 60
				client.Serve_time = fmt.Sprintf("%dh%dm", hours, minutes)
				if len(client.Current_work) > 0 {
					client.Idle_time = 0
				} else {
					client.Idle_time += 30
				}
			} else {
				//now client must be gone as tag set to false 30 seconds ago and no heartbeat received thereafter
				logger.Event(event.CLIENT_UNREGISTER, "clientid="+clientid+";name="+qm.clientMap[clientid].Name)

				//requeue unfinished workunits associated with the failed client
				qm.ReQueueWorkunitByClient(clientid)
				//delete the client from client map
				delete(qm.clientMap, clientid)
			}
		}
	}
}

func (qm *CQMgr) ClientHeartBeat(id string) (hbmsg HBmsg, err error) {
	hbmsg = make(map[string]string, 1)
	if _, ok := qm.clientMap[id]; ok {
		qm.clientMap[id].Tag = true
		logger.Debug(3, "HeartBeatFrom:"+"clientid="+id+",name="+qm.clientMap[id].Name)

		//get suspended workunit that need the client to discard
		workids := qm.getWorkByClient(id)
		suspended := []string{}
		for _, workid := range workids {
			if work, ok := qm.workQueue.workMap[workid]; ok {
				if work.State == WORK_STAT_SUSPEND {
					suspended = append(suspended, workid)
				}
			}
		}
		if len(suspended) > 0 {
			hbmsg["discard"] = strings.Join(suspended, ",")
		}
		if qm.clientMap[id].Status == CLIENT_STAT_DELETED {
			hbmsg["stop"] = id
		}
		//hbmsg["discard"] = strings.Join(workids, ",")
		return hbmsg, nil
	}
	return hbmsg, errors.New(e.ClientNotFound)
}

func (qm *CQMgr) RegisterNewClient(files FormFiles) (client *Client, err error) {
	if _, ok := files["profile"]; ok {
		client, err = NewProfileClient(files["profile"].Path)
		os.Remove(files["profile"].Path)
	} else {
		client = NewClient()
	}
	if err != nil {
		return nil, err
	}
	qm.clientMap[client.Id] = client

	if len(client.Current_work) > 0 { //re-registered client
		// move already checked-out workunit from waiting queue (workMap) to checked-out list (coWorkMap)
		for workid, _ := range client.Current_work {
			if qm.workQueue.Has(workid) {
				qm.workQueue.StatusChange(workid, WORK_STAT_CHECKOUT)
			}
		}
	}
	return
}

func (qm *CQMgr) GetClient(id string) (client *Client, err error) {
	if client, ok := qm.clientMap[id]; ok {
		return client, nil
	}
	return nil, errors.New(e.ClientNotFound)
}

func (qm *CQMgr) GetAllClients() []*Client {
	clients := []*Client{}
	for _, client := range qm.clientMap {
		clients = append(clients, client)
	}
	return clients
}

func (qm *CQMgr) DeleteClient(id string) (err error) {
	if client, ok := qm.clientMap[id]; ok {
		client.Status = CLIENT_STAT_DELETED
		return
	}
	return errors.New("client not found")
}

func (qm *CQMgr) SuspendClient(id string) (err error) {
	if client, ok := qm.clientMap[id]; ok {
		if client.Status == CLIENT_STAT_ACTIVE_IDLE || client.Status == CLIENT_STAT_ACTIVE_BUSY {
			client.Status = CLIENT_STAT_SUSPEND
			qm.ReQueueWorkunitByClient(id)
			return
		}
		return errors.New("client is not in active state")
	}
	return errors.New("client not found")
}

func (qm *CQMgr) ResumeClient(id string) (err error) {
	if client, ok := qm.clientMap[id]; ok {
		if client.Status == CLIENT_STAT_SUSPEND {
			client.Status = CLIENT_STAT_ACTIVE_IDLE
			return
		}
		return errors.New("client is not in suspended state")
	}
	return errors.New("client not found")
}

func (qm *CQMgr) ResumeSuspendedClients() (count int) {
	for _, client := range qm.clientMap {
		if client.Status == CLIENT_STAT_SUSPEND {
			client.Status = CLIENT_STAT_ACTIVE_IDLE
			count += 1
		}
	}
	return count
}

func (qm *CQMgr) SuspendAllClients() (count int) {
	for _, client := range qm.clientMap {
		if client.Status == CLIENT_STAT_ACTIVE_IDLE || client.Status == CLIENT_STAT_ACTIVE_BUSY {
			qm.SuspendClient(client.Id)
			count += 1
		}
	}
	return count
}

func (qm *CQMgr) UpdateSubClients(id string, count int) {
	if _, ok := qm.clientMap[id]; ok {
		qm.clientMap[id].SubClients = count
	}
}

//--------end client methods-------

//-------start of workunit methods---

func (qm *CQMgr) CheckoutWorkunits(req_policy string, client_id string, num int) (workunits []*Workunit, err error) {
	//precheck if the client is registered
	if _, hasClient := qm.clientMap[client_id]; !hasClient {
		return nil, errors.New(e.ClientNotFound)
	}
	if qm.clientMap[client_id].Status == CLIENT_STAT_SUSPEND {
		return nil, errors.New(e.ClientSuspended)
	}
	if qm.clientMap[client_id].Status == CLIENT_STAT_DELETED {
		delete(qm.clientMap, client_id)
		return nil, errors.New(e.ClientDeleted)
	}

	//lock semephore, at one time only one client's checkout request can be served
	qm.coSem <- 1

	req := CoReq{policy: req_policy, fromclient: client_id, count: num}
	qm.coReq <- req
	ack := <-qm.coAck

	if ack.err == nil {
		for _, work := range ack.workunits {
			qm.clientMap[client_id].Total_checkout += 1
			qm.clientMap[client_id].Current_work[work.Id] = true
		}
		if qm.clientMap[client_id].Status == CLIENT_STAT_ACTIVE_IDLE {
			qm.clientMap[client_id].Status = CLIENT_STAT_ACTIVE_BUSY
		}
	}

	//unlock
	<-qm.coSem

	return ack.workunits, ack.err
}

func (qm *CQMgr) GetWorkById(id string) (workunit *Workunit, err error) {
	if workunit, hasid := qm.workQueue.workMap[id]; hasid {
		return workunit, nil
	}
	return nil, errors.New(fmt.Sprintf("no workunit found with id %s", id))
}

func (qm *CQMgr) NotifyWorkStatus(notice Notice) {
	qm.feedback <- notice
	return
}

func (qm *CQMgr) popWorks(req CoReq) (works []*Workunit, err error) {
	filtered := qm.filterWorkByClient(req.fromclient)
	if len(filtered) == 0 {
		return nil, errors.New(e.NoEligibleWorkunitFound)
	}
	works, err = qm.workQueue.selectWorkunits(filtered, req.policy, req.count)
	if err == nil { //get workunits successfully, put them into coWorkMap
		for _, work := range works {
			if _, ok := qm.workQueue.workMap[work.Id]; ok {
				qm.workQueue.workMap[work.Id].Client = req.fromclient
				qm.workQueue.workMap[work.Id].CheckoutTime = time.Now()
			}
			qm.workQueue.StatusChange(work.Id, WORK_STAT_CHECKOUT)
		}
	}
	return
}

func (qm *CQMgr) filterWorkByClient(clientid string) (ids []string) {
	client := qm.clientMap[clientid]
	for id, _ := range qm.workQueue.wait {
		if _, ok := qm.workQueue.workMap[id]; !ok {
			continue
		}
		work := qm.workQueue.workMap[id]
		//skip works that are in the client's skip-list
		if contains(client.Skip_work, work.Id) {
			continue
		}
		//skip works that have dedicate client groups which this client doesn't belong to
		if len(work.Info.ClientGroups) > 0 {
			eligible_groups := strings.Split(work.Info.ClientGroups, ",")
			if !contains(eligible_groups, client.Group) {
				continue
			}
		}
		//append works whos apps are supported by the client
		if contains(client.Apps, work.Cmd.Name) || contains(client.Apps, conf.ALL_APP) {
			ids = append(ids, id)
		}
	}
	return ids
}

func (qm *CQMgr) getWorkByClient(clientid string) (ids []string) {
	if client, ok := qm.clientMap[clientid]; ok {
		for id, _ := range client.Current_work {
			ids = append(ids, id)
		}
	}
	return
}

//handle feedback from a client about the execution of a workunit
func (qm *CQMgr) handleWorkStatusChange(notice Notice) (err error) {
	//to be implemented for proxy or server
	return
}

func (qm *CQMgr) FetchDataToken(workid string, clientid string) (token string, err error) {
	//to be implemented for proxy or server
	return
}

func (qm *CQMgr) ShowWorkunits(status string) (workunits []*Workunit) {
	for _, work := range qm.workQueue.workMap {
		if work.State == status || status == "" {
			workunits = append(workunits, work)
		}
	}
	return workunits
}

func (qm *CQMgr) EnqueueWorkunit(work *Workunit) (err error) {
	err = qm.workQueue.Add(work)
	return
}

func (qm *CQMgr) ReQueueWorkunitByClient(clientid string) (err error) {
	workids := qm.getWorkByClient(clientid)
	for _, workid := range workids {
		if qm.workQueue.Has(workid) {
			jobid, _ := GetJobIdByWorkId(workid)
			if job, err := LoadJob(jobid); err == nil {
				if contains(JOB_STATS_ACTIVE, job.State) { //only requeue workunits belonging to active jobs (rule out suspended jobs)
					qm.workQueue.StatusChange(workid, WORK_STAT_QUEUED)
					logger.Event(event.WORK_REQUEUE, "workid="+workid)
				}
			}
		}
	}
	return
}

//---end of workunit methods
<|endoftext|>


<commit_before>package collector

import (
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	subsystem = "replset"
	myState   = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "my_state",
		Help:      "An integer between 0 and 10 that represents the replica state of the current member",
	}, []string{"set"})

	myReplicaLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "my_replica_lag",
		Help:      "An integer shows the replication lag in seconds, -1 if no master found",
	}, []string{"set"})

	term = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "term",
		Help:      "The election count for the replica set, as known to this replica set member",
	}, []string{"set"})
	numberOfMembers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "number_of_members",
		Help:      "The number of replica set mebers",
	}, []string{"set"})
	heartbeatIntervalMillis = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "heartbeat_interval_millis",
		Help:      "The frequency in milliseconds of the heartbeats",
	}, []string{"set"})
	memberHealth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_health",
		Help:      "This field conveys if the member is up (1) or down (0).",
	}, []string{"set", "name"})
	memberState = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_state",
		Help:      "The value of state is an integer between 0 and 10 that represents the replica state of the member.",
	}, []string{"set", "name"})
	memberUptime = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_uptime",
		Help:      "The uptime field holds a value that reflects the number of seconds that this member has been online.",
	}, []string{"set", "name"})
	memberOptimeDate = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_optime_date",
		Help:      "The last entry from the oplog that this member applied.",
	}, []string{"set", "name"})
	memberElectionDate = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_election_date",
		Help:      "The timestamp the node was elected as replica leader",
	}, []string{"set", "name"})
	memberLastHeartbeat = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_last_heartbeat",
		Help:      "The lastHeartbeat value provides an ISODate formatted date and time of the transmission time of last heartbeat received from this member",
	}, []string{"set", "name"})
	memberLastHeartbeatRecv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_last_heartbeat_recv",
		Help:      "The lastHeartbeatRecv value provides an ISODate formatted date and time that the last heartbeat was received from this member",
	}, []string{"set", "name"})
	memberPingMs = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_ping_ms",
		Help:      "The pingMs represents the number of milliseconds (ms) that a round-trip packet takes to travel between the remote member and the local instance.",
	}, []string{"set", "name"})
	memberConfigVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_config_version",
		Help:      "The configVersion value is the replica set configuration version.",
	}, []string{"set", "name"})
	memberOptime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_optime",
		Help:      "Information regarding the last operation from the operation log that this member has applied.",
	}, []string{"set", "name"})
)

// ReplSetStatus keeps the data returned by the GetReplSetStatus method
type ReplSetStatus struct {
	Set                     string    `bson:"set"`
	Date                    time.Time `bson:"date"`
	MyState                 int32     `bson:"myState"`
	Term                    *int32    `bson:"term,omitempty"`
	HeartbeatIntervalMillis *float64  `bson:"heartbeatIntervalMillis,omitempty"`
	Members                 []Member  `bson:"members"`
}

// Member represents an array element of ReplSetStatus.Members
type Member struct {
	Name                 string      `bson:"name"`
	Self                 *bool       `bson:"self,omitempty"`
	Health               *int32      `bson:"health,omitempty"`
	State                int32       `bson:"state"`
	StateStr             string      `bson:"stateStr"`
	Uptime               float64     `bson:"uptime"`
	Optime               interface{} `bson:"optime"`
	OptimeDate           time.Time   `bson:"optimeDate"`
	ElectionTime         *time.Time  `bson:"electionTime,omitempty"`
	ElectionDate         *time.Time  `bson:"electionDate,omitempty"`
	LastHeartbeat        *time.Time  `bson:"lastHeartbeat,omitempty"`
	LastHeartbeatRecv    *time.Time  `bson:"lastHeartbeatRecv,omitempty"`
	LastHeartbeatMessage *string     `bson:"lastHeartbeatMessage,omitempty"`
	PingMs               *float64    `bson:"pingMs,omitempty"`
	SyncingTo            *string     `bson:"syncingTo,omitempty"`
	ConfigVersion        *int32      `bson:"configVersion,omitempty"`
}

// Export exports the replSetGetStatus stati to be consumed by prometheus
func (replStatus *ReplSetStatus) Export(ch chan<- prometheus.Metric) {
	myState.Reset()
	myReplicaLag.Reset()
	term.Reset()
	numberOfMembers.Reset()
	heartbeatIntervalMillis.Reset()
	memberState.Reset()
	memberHealth.Reset()
	memberUptime.Reset()
	memberOptimeDate.Reset()
	memberElectionDate.Reset()
	memberLastHeartbeat.Reset()
	memberLastHeartbeatRecv.Reset()
	memberPingMs.Reset()
	memberConfigVersion.Reset()

	myState.WithLabelValues(replStatus.Set).Set(float64(replStatus.MyState))

	// new in version 3.2
	if replStatus.Term != nil {
		term.WithLabelValues(replStatus.Set).Set(float64(*replStatus.Term))
	}
	numberOfMembers.WithLabelValues(replStatus.Set).Set(float64(len(replStatus.Members)))

	// new in version 3.2
	if replStatus.HeartbeatIntervalMillis != nil {
		heartbeatIntervalMillis.WithLabelValues(replStatus.Set).Set(*replStatus.HeartbeatIntervalMillis)
	}

	var (
		primaryOpTime time.Time
		myOpTime      time.Time
	)
	for _, member := range replStatus.Members {
		ls := prometheus.Labels{
			"set":  replStatus.Set,
			"name": member.Name,
		}
		if member.State == 1 {
			primaryOpTime = member.OptimeDate
		}
		if member.Self != nil && *member.Self {
			myOpTime = member.OptimeDate
		}
		memberState.With(ls).Set(float64(member.State))

		// ReplSetStatus.Member.Health is not available on the node you're connected to
		if member.Health != nil {
			memberHealth.With(ls).Set(float64(*member.Health))
		}

		memberUptime.With(ls).Set(member.Uptime)

		memberOptimeDate.With(ls).Set(float64(member.OptimeDate.Unix()))

		// ReplSetGetStatus.Member.ElectionTime is only available on the PRIMARY
		if member.ElectionDate != nil {
			memberElectionDate.With(ls).Set(float64((*member.ElectionDate).Unix()))
		}
		if member.LastHeartbeat != nil {
			memberLastHeartbeat.With(ls).Set(float64((*member.LastHeartbeat).Unix()))
		}
		if member.LastHeartbeatRecv != nil {
			memberLastHeartbeatRecv.With(ls).Set(float64((*member.LastHeartbeatRecv).Unix()))
		}
		if member.PingMs != nil {
			memberPingMs.With(ls).Set(*member.PingMs)
		}
		if member.ConfigVersion != nil {
			memberConfigVersion.With(ls).Set(float64(*member.ConfigVersion))
		}
	}
	if !primaryOpTime.IsZero() && !myOpTime.IsZero() {
		myReplicaLag.WithLabelValues(replStatus.Set).Set(float64(primaryOpTime.Unix() - myOpTime.Unix()))
	} else {
		myReplicaLag.WithLabelValues(replStatus.Set).Set(-1.0)
	}
	// collect metrics
	myState.Collect(ch)
	myReplicaLag.Collect(ch)
	term.Collect(ch)
	numberOfMembers.Collect(ch)
	heartbeatIntervalMillis.Collect(ch)
	memberState.Collect(ch)
	memberHealth.Collect(ch)
	memberUptime.Collect(ch)
	memberOptimeDate.Collect(ch)
	memberElectionDate.Collect(ch)
	memberLastHeartbeat.Collect(ch)
	memberLastHeartbeatRecv.Collect(ch)
	memberPingMs.Collect(ch)
	memberConfigVersion.Collect(ch)
}

// Describe describes the replSetGetStatus metrics for prometheus
func (replStatus *ReplSetStatus) Describe(ch chan<- *prometheus.Desc) {
	myState.Describe(ch)
	myReplicaLag.Describe(ch)
	term.Describe(ch)
	numberOfMembers.Describe(ch)
	heartbeatIntervalMillis.Describe(ch)
	memberState.Describe(ch)
	memberHealth.Describe(ch)
	memberUptime.Describe(ch)
	memberOptimeDate.Describe(ch)
	memberElectionDate.Describe(ch)
	memberLastHeartbeatRecv.Describe(ch)
	memberPingMs.Describe(ch)
	memberConfigVersion.Describe(ch)
}

// GetReplSetStatus returns the replica status info
func GetReplSetStatus(session *mgo.Session) *ReplSetStatus {
	result := &ReplSetStatus{}
	err := session.DB("admin").Run(bson.D{{"replSetGetStatus", 1}}, result)
	if err != nil {
		glog.Error("Failed to get replSet status.")
		return nil
	}
	return result
}
<commit_msg>Add simple metric to detect no master issue<commit_after>package collector

import (
	"time"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	subsystem = "replset"
	myState   = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "my_state",
		Help:      "An integer between 0 and 10 that represents the replica state of the current member",
	}, []string{"set"})

	myReplicaLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "my_replica_lag",
		Help:      "An integer shows the replication lag in seconds, -1 if no master found",
	}, []string{"set"})

	masterCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "master_count",
		Help:      "The number of master, any value except 1 means something wrong",
	}, []string{})

	term = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "term",
		Help:      "The election count for the replica set, as known to this replica set member",
	}, []string{"set"})
	numberOfMembers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "number_of_members",
		Help:      "The number of replica set mebers",
	}, []string{"set"})
	heartbeatIntervalMillis = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "heartbeat_interval_millis",
		Help:      "The frequency in milliseconds of the heartbeats",
	}, []string{"set"})
	memberHealth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_health",
		Help:      "This field conveys if the member is up (1) or down (0).",
	}, []string{"set", "name"})
	memberState = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_state",
		Help:      "The value of state is an integer between 0 and 10 that represents the replica state of the member.",
	}, []string{"set", "name"})
	memberUptime = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_uptime",
		Help:      "The uptime field holds a value that reflects the number of seconds that this member has been online.",
	}, []string{"set", "name"})
	memberOptimeDate = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_optime_date",
		Help:      "The last entry from the oplog that this member applied.",
	}, []string{"set", "name"})
	memberElectionDate = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_election_date",
		Help:      "The timestamp the node was elected as replica leader",
	}, []string{"set", "name"})
	memberLastHeartbeat = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_last_heartbeat",
		Help:      "The lastHeartbeat value provides an ISODate formatted date and time of the transmission time of last heartbeat received from this member",
	}, []string{"set", "name"})
	memberLastHeartbeatRecv = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_last_heartbeat_recv",
		Help:      "The lastHeartbeatRecv value provides an ISODate formatted date and time that the last heartbeat was received from this member",
	}, []string{"set", "name"})
	memberPingMs = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_ping_ms",
		Help:      "The pingMs represents the number of milliseconds (ms) that a round-trip packet takes to travel between the remote member and the local instance.",
	}, []string{"set", "name"})
	memberConfigVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_config_version",
		Help:      "The configVersion value is the replica set configuration version.",
	}, []string{"set", "name"})
	memberOptime = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: Namespace,
		Subsystem: subsystem,
		Name:      "member_optime",
		Help:      "Information regarding the last operation from the operation log that this member has applied.",
	}, []string{"set", "name"})
)

// ReplSetStatus keeps the data returned by the GetReplSetStatus method
type ReplSetStatus struct {
	Set                     string    `bson:"set"`
	Date                    time.Time `bson:"date"`
	MyState                 int32     `bson:"myState"`
	Term                    *int32    `bson:"term,omitempty"`
	HeartbeatIntervalMillis *float64  `bson:"heartbeatIntervalMillis,omitempty"`
	Members                 []Member  `bson:"members"`
}

// Member represents an array element of ReplSetStatus.Members
type Member struct {
	Name                 string      `bson:"name"`
	Self                 *bool       `bson:"self,omitempty"`
	Health               *int32      `bson:"health,omitempty"`
	State                int32       `bson:"state"`
	StateStr             string      `bson:"stateStr"`
	Uptime               float64     `bson:"uptime"`
	Optime               interface{} `bson:"optime"`
	OptimeDate           time.Time   `bson:"optimeDate"`
	ElectionTime         *time.Time  `bson:"electionTime,omitempty"`
	ElectionDate         *time.Time  `bson:"electionDate,omitempty"`
	LastHeartbeat        *time.Time  `bson:"lastHeartbeat,omitempty"`
	LastHeartbeatRecv    *time.Time  `bson:"lastHeartbeatRecv,omitempty"`
	LastHeartbeatMessage *string     `bson:"lastHeartbeatMessage,omitempty"`
	PingMs               *float64    `bson:"pingMs,omitempty"`
	SyncingTo            *string     `bson:"syncingTo,omitempty"`
	ConfigVersion        *int32      `bson:"configVersion,omitempty"`
}

// Export exports the replSetGetStatus stati to be consumed by prometheus
func (replStatus *ReplSetStatus) Export(ch chan<- prometheus.Metric) {
	myState.Reset()
	myReplicaLag.Reset()
	term.Reset()
	numberOfMembers.Reset()
	heartbeatIntervalMillis.Reset()
	memberState.Reset()
	memberHealth.Reset()
	memberUptime.Reset()
	memberOptimeDate.Reset()
	memberElectionDate.Reset()
	memberLastHeartbeat.Reset()
	memberLastHeartbeatRecv.Reset()
	memberPingMs.Reset()
	memberConfigVersion.Reset()
	masterCount.Reset()

	myState.WithLabelValues(replStatus.Set).Set(float64(replStatus.MyState))

	// new in version 3.2
	if replStatus.Term != nil {
		term.WithLabelValues(replStatus.Set).Set(float64(*replStatus.Term))
	}
	numberOfMembers.WithLabelValues(replStatus.Set).Set(float64(len(replStatus.Members)))

	// new in version 3.2
	if replStatus.HeartbeatIntervalMillis != nil {
		heartbeatIntervalMillis.WithLabelValues(replStatus.Set).Set(*replStatus.HeartbeatIntervalMillis)
	}

	var (
		primaryOpTime time.Time
		myOpTime      time.Time
	)
	mCount := 0
	for _, member := range replStatus.Members {
		ls := prometheus.Labels{
			"set":  replStatus.Set,
			"name": member.Name,
		}
		if member.State == 1 {
			primaryOpTime = member.OptimeDate
		}
		if member.Self != nil && *member.Self {
			myOpTime = member.OptimeDate
		}
		memberState.With(ls).Set(float64(member.State))
		if member.State == 1 {
			mCount += 1
		}

		// ReplSetStatus.Member.Health is not available on the node you're connected to
		if member.Health != nil {
			memberHealth.With(ls).Set(float64(*member.Health))
		}

		memberUptime.With(ls).Set(member.Uptime)

		memberOptimeDate.With(ls).Set(float64(member.OptimeDate.Unix()))

		// ReplSetGetStatus.Member.ElectionTime is only available on the PRIMARY
		if member.ElectionDate != nil {
			memberElectionDate.With(ls).Set(float64((*member.ElectionDate).Unix()))
		}
		if member.LastHeartbeat != nil {
			memberLastHeartbeat.With(ls).Set(float64((*member.LastHeartbeat).Unix()))
		}
		if member.LastHeartbeatRecv != nil {
			memberLastHeartbeatRecv.With(ls).Set(float64((*member.LastHeartbeatRecv).Unix()))
		}
		if member.PingMs != nil {
			memberPingMs.With(ls).Set(*member.PingMs)
		}
		if member.ConfigVersion != nil {
			memberConfigVersion.With(ls).Set(float64(*member.ConfigVersion))
		}
	}
	if !primaryOpTime.IsZero() && !myOpTime.IsZero() {
		myReplicaLag.WithLabelValues(replStatus.Set).Set(float64(primaryOpTime.Unix() - myOpTime.Unix()))
	} else {
		myReplicaLag.WithLabelValues(replStatus.Set).Set(-1.0)
	}
	masterCount.WithLabelValues().Set(float64(mCount))
	// collect metrics
	myState.Collect(ch)
	myReplicaLag.Collect(ch)
	term.Collect(ch)
	numberOfMembers.Collect(ch)
	heartbeatIntervalMillis.Collect(ch)
	memberState.Collect(ch)
	masterCount.Collect(ch)
	memberHealth.Collect(ch)
	memberUptime.Collect(ch)
	memberOptimeDate.Collect(ch)
	memberElectionDate.Collect(ch)
	memberLastHeartbeat.Collect(ch)
	memberLastHeartbeatRecv.Collect(ch)
	memberPingMs.Collect(ch)
	memberConfigVersion.Collect(ch)
}

// Describe describes the replSetGetStatus metrics for prometheus
func (replStatus *ReplSetStatus) Describe(ch chan<- *prometheus.Desc) {
	myState.Describe(ch)
	myReplicaLag.Describe(ch)
	term.Describe(ch)
	numberOfMembers.Describe(ch)
	heartbeatIntervalMillis.Describe(ch)
	memberState.Describe(ch)
	memberHealth.Describe(ch)
	memberUptime.Describe(ch)
	memberOptimeDate.Describe(ch)
	memberElectionDate.Describe(ch)
	memberLastHeartbeatRecv.Describe(ch)
	memberPingMs.Describe(ch)
	memberConfigVersion.Describe(ch)
}

// GetReplSetStatus returns the replica status info
func GetReplSetStatus(session *mgo.Session) *ReplSetStatus {
	result := &ReplSetStatus{}
	err := session.DB("admin").Run(bson.D{{"replSetGetStatus", 1}}, result)
	if err != nil {
		glog.Error("Failed to get replSet status.")
		return nil
	}
	return result
}
<|endoftext|>