<commit_before>package logsend

import (
	"errors"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/Unknwon/com"
	"github.com/howeyc/fsnotify"
	"os"
	"path/filepath"
	"strings"
	"study2016/logshot/logger"
	"time"
)

//监听的路径-文件映射
var WatcherMap map[string]*File = make(map[string]*File)

//监听文件
func WatchFiles(configFile string) {
	rule, err := LoadConfigFromFile(configFile)
	if err != nil {
		logger.GetLogger().Errorln("Can't load config", err)
	}
	files := make([]string, 0)
	files = append(files, rule.watchDir)
	assignedFiles, err := assignFiles(files, rule)
	//统计分配文件的数量
	//当每个文件完成任务后,这个数目减少1
	//主要用于判断doneCh的过程中,不至于因为doneCh的阻塞导致deadlock
	assignedFilesCount := len(assignedFiles)

	if err != nil {
		logger.GetLogger().Errorln("can't assign file per rule", err)
	}

	doneCh := make(chan string)
	for _, file := range assignedFiles {
		file.doneCh = doneCh
		//并行处理文件采集
		go file.tail()
	}

	//如果监听对象为目录,开启异步监听
	if com.IsDir(rule.watchDir) {
		go continueWatch(&rule.watchDir, rule, &assignedFilesCount, doneCh)
	}

	for {
		select {
		case fpath := <-doneCh:
			assignedFilesCount = assignedFilesCount - 1
			if assignedFilesCount == 0 {
				logger.GetLogger().Errorln("finished reading file %+v", fpath)
				return
			}
		}
	}
}

//单文件分配
func assignSingleFile(filepath string, rule *Rule) (*File, error) {
	is_dir := com.IsDir(filepath)
	if !is_dir {
		file, err := NewFile(filepath)
		if err != nil {
			return nil, err
		}
		file.rule = rule
		return file, nil
	}
	return nil, errors.New("file not found : " + filepath)
}

//为文件分配规则
func assignFiles(allFiles []string, rule *Rule) ([]*File, error) {
	files := make([]*File, 0)
	for _, f := range allFiles {
		is_dir := com.IsDir(f)
		//watch-dir是目录的形式
		if is_dir {
			//遍历文件目录
			filepath.Walk(f, func(pth string, info os.FileInfo, err error) error {
				if err != nil {
					panic(err)
				}
				if !info.IsDir() && !strings.Contains(info.Name(), ".DS_Store") {
					file, err := NewFile(pth)
					if err != nil {
						return err
					}
					file.rule = rule
					files = appendWatch(files, file)
				}
				return nil
			})
		} else {
			//分配单文件
			file, err := assignSingleFile(f, rule)
			if err != nil {
				return files, err
			}
			files = appendWatch(files, file)
		}
	}
	return files, nil
}

func appendWatch(files []*File, file *File) []*File {
	WatcherMap[file.Tail.Filename] = file
	if files != nil {
		files = append(files, file)
	}
	return files
}

func continueWatch(dir *string, rule *Rule, totalFileCount *int, doneCh chan string) {
	//判断dir是否是目录结构
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.GetLogger().Errorln(err.Error())
	}
	done := make(chan bool)
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if ev.IsCreate() {
					*totalFileCount = *totalFileCount + 1
					println("发现文件创建:", ev.Name, "当前监控文件数:", *totalFileCount)
					file, err := assignSingleFile(ev.Name, rule)
					appendWatch(nil, file)
					if err == nil {
						file.doneCh = doneCh
						go file.tail() //异步传输
					}
				} else if ev.IsDelete() { //文件被删除的情况,需要进行资源回收
					//获取被删除的file对象,并发完成消息到其doneCh
					logger.GetLogger().Infof("tailing file is deleted : %s \n ", ev.Name)
					if delete_file, ok := WatcherMap[ev.Name]; ok {
						delete(WatcherMap, ev.Name)
						delete_file.Tail.Stop()// stop the line tail
					} else {
						logger.GetLogger().Errorf("get delete file fail : %s", ev.Name)
					}
				}
			case err := <-watcher.Error:
				logger.GetLogger().Errorln("error:", err)
			}
		}
	}()
	//监听目录
	err = watcher.Watch(*dir)
	if err != nil {
		logger.GetLogger().Errorln(err.Error())
	}
	<-done

	/* ... do stuff ... */
	watcher.Close()
}

//监听文件结构
type File struct {
	Tail   *tail.Tail
	rule   *Rule
	doneCh chan string
}

//创建监听文件
func NewFile(fpath string) (*File, error) {
	file := &File{}
	var err error

	//是否采用低版本的poll监听方式(darwin除外)
	//Linux2.6.32以下无法使用inotity,需要把Poll打开,采用 Polling的方式
	isPoll := Conf.IsPoll
	if Conf.ReadWholeLog && Conf.ReadAlway { //全量并持续采集
		file.Tail, err = tail.TailFile(fpath, tail.Config{
			Follow: true,
			ReOpen: true,
			Poll:   isPoll,
			Logger: logger.GetLogger(), //使用自定义的日志器
		})
	} else if Conf.ReadWholeLog { //全量但只采集一次
		file.Tail, err = tail.TailFile(fpath, tail.Config{
			Poll:   isPoll,
			Logger: logger.GetLogger(), //使用自定义的日志器
		})
	} else {
		//从当前文件最尾端开始采集
		seekInfo := &tail.SeekInfo{Offset: 0, Whence: 2}
		file.Tail, err = tail.TailFile(fpath, tail.Config{
			Follow:   true,
			ReOpen:   true,
			Poll:     isPoll,
			Location: seekInfo,
			Logger:   logger.GetLogger(), //使用自定义的日志器
		})
	}
	return file, err
}

func (self *File) tail() {
	logger.GetLogger().Infoln(fmt.Sprintf("start tailing %s", self.Tail.Filename))

	//功能收尾
	defer func() {
		//关闭Sender初始化过程中建立的通讯
		closeRule(self.rule)
		//通知结束通道,让主调用方结束
		logger.GetLogger().Infof("file-watching has done : %s", self.Tail.Filename)
		self.doneCh <- self.Tail.Filename
	}()
	for line := range self.Tail.Lines {
		checkLineRule(&line.Text, self.rule)
	}
}

//检查并进行发送
func checkLineRule(line *string, rule *Rule) {
	//日志行对象
	logline := &LogLine{
		Ts:   time.Now().UTC().UnixNano(),
		Line: []byte(*line),
	}
	rule.SendLogLine(logline)
}

//关闭规则
func closeRule(rule *Rule) {
	rule.CloseSender()
}
<commit_msg>support some functions<commit_after>package logsend

import (
	"errors"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/Unknwon/com"
	"github.com/howeyc/fsnotify"
	"os"
	"path/filepath"
	"strings"
	"study2016/logshot/logger"
	"time"
)

//监听的路径-文件映射
var WatcherMap map[string]*File = make(map[string]*File)

//监听文件
func WatchFiles(configFile string) {
	rule, err := LoadConfigFromFile(configFile)
	if err != nil {
		logger.GetLogger().Errorln("Can't load config", err)
	}
	files := make([]string, 0)
	files = append(files, rule.watchDir)
	assignedFiles, err := assignFiles(files, rule)
	//统计分配文件的数量
	//当每个文件完成任务后,这个数目减少1
	//主要用于判断doneCh的过程中,不至于因为doneCh的阻塞导致deadlock
	assignedFilesCount := len(assignedFiles)

	if err != nil {
		logger.GetLogger().Errorln("can't assign file per rule", err)
	}

	doneCh := make(chan string)
	for _, file := range assignedFiles {
		file.doneCh = doneCh
		//并行处理文件采集
		go file.tail()
	}

	//如果监听对象为目录,开启异步监听
	if com.IsDir(rule.watchDir) {
		go continueWatch(&rule.watchDir, rule, &assignedFilesCount, doneCh)
	}

	for {
		select {
		case fpath := <-doneCh:
			assignedFilesCount = assignedFilesCount - 1
			if assignedFilesCount == 0 {
				logger.GetLogger().Infof("finished reading file %+v", fpath)
				return
			}
		}
	}
}

//单文件分配
func assignSingleFile(filepath string, rule *Rule) (*File, error) {
	is_dir := com.IsDir(filepath)
	if !is_dir {
		file, err := NewFile(filepath)
		if err != nil {
			return nil, err
		}
		file.rule = rule
		return file, nil
	}
	return nil, errors.New("file not found : " + filepath)
}

//为文件分配规则
func assignFiles(allFiles []string, rule *Rule) ([]*File, error) {
	files := make([]*File, 0)
	for _, f := range allFiles {
		is_dir := com.IsDir(f)
		//watch-dir是目录的形式
		if is_dir {
			//遍历文件目录
			filepath.Walk(f, func(pth string, info os.FileInfo, err error) error {
				if err != nil {
					panic(err)
				}
				if !info.IsDir() && !strings.Contains(info.Name(), ".DS_Store") {
					file, err := NewFile(pth)
					if err != nil {
						return err
					}
					file.rule = rule
					files = appendWatch(files, file)
				}
				return nil
			})
		} else {
			//分配单文件
			file, err := assignSingleFile(f, rule)
			if err != nil {
				return files, err
			}
			files = appendWatch(files, file)
		}
	}
	return files, nil
}

func appendWatch(files []*File, file *File) []*File {
	WatcherMap[file.Tail.Filename] = file
	if files != nil {
		files = append(files, file)
	}
	return files
}

func continueWatch(dir *string, rule *Rule, totalFileCount *int, doneCh chan string) {
	//判断dir是否是目录结构
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		logger.GetLogger().Errorln(err.Error())
	}
	done := make(chan bool)
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if ev.IsCreate() {
					*totalFileCount = *totalFileCount + 1
					println("发现文件创建:", ev.Name, "当前监控文件数:", *totalFileCount)
					file, err := assignSingleFile(ev.Name, rule)
					appendWatch(nil, file)
					if err == nil {
						file.doneCh = doneCh
						go file.tail() //异步传输
					}
				} else if ev.IsDelete() { //文件被删除的情况,需要进行资源回收
					//获取被删除的file对象,并发完成消息到其doneCh
					logger.GetLogger().Infof("tailing file is deleted : %s ", ev.Name)
					if delete_file, ok := WatcherMap[ev.Name]; ok {
						delete(WatcherMap, ev.Name)
						if len(WatcherMap) == 0 {
							closeRule(rule)
						}
						delete_file.Tail.Stop() // stop the line tail
					} else {
						logger.GetLogger().Errorf("get delete file fail : %s", ev.Name)
					}
				}
			case err := <-watcher.Error:
				logger.GetLogger().Errorln("error:", err)
			}
		}
	}()
	//监听目录
	err = watcher.Watch(*dir)
	if err != nil {
		logger.GetLogger().Errorln(err.Error())
	}
	<-done

	/* ... do stuff ... */
	watcher.Close()
}

//监听文件结构
type File struct {
	Tail   *tail.Tail
	rule   *Rule
	doneCh chan string
}

//创建监听文件
func NewFile(fpath string) (*File, error) {
	file := &File{}
	var err error

	//是否采用低版本的poll监听方式(darwin除外)
	//Linux2.6.32以下无法使用inotity,需要把Poll打开,采用 Polling的方式
	isPoll := Conf.IsPoll
	if Conf.ReadWholeLog && Conf.ReadAlway { //全量并持续采集
		file.Tail, err = tail.TailFile(fpath, tail.Config{
			Follow: true,
			ReOpen: true,
			Poll:   isPoll,
			Logger: logger.GetLogger(), //使用自定义的日志器
		})
	} else if Conf.ReadWholeLog { //全量但只采集一次
		file.Tail, err = tail.TailFile(fpath, tail.Config{
			Poll:   isPoll,
			Logger: logger.GetLogger(), //使用自定义的日志器
		})
	} else {
		//从当前文件最尾端开始采集
		seekInfo := &tail.SeekInfo{Offset: 0, Whence: 2}
		file.Tail, err = tail.TailFile(fpath, tail.Config{
			Follow:   true,
			ReOpen:   true,
			Poll:     isPoll,
			Location: seekInfo,
			Logger:   logger.GetLogger(), //使用自定义的日志器
		})
	}
	return file, err
}

func (self *File) tail() {
	logger.GetLogger().Infoln(fmt.Sprintf("start tailing %s", self.Tail.Filename))

	//功能收尾
	defer func() {
		//通知结束通道,让主调用方结束
		logger.GetLogger().Infof("file-watching has done : %s", self.Tail.Filename)
		self.doneCh <- self.Tail.Filename
	}()
	for line := range self.Tail.Lines {
		checkLineRule(&line.Text, self.rule)
	}
}

//检查并进行发送
func checkLineRule(line *string, rule *Rule) {
	//日志行对象
	logline := &LogLine{
		Ts:   time.Now().UTC().UnixNano(),
		Line: []byte(*line),
	}
	rule.SendLogLine(logline)
}

//关闭规则
func closeRule(rule *Rule) {
	logger.GetLogger().Infoln("stop rule send channel ")
	rule.CloseSender()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type logEntry struct {
	Node string
	Time int64
	Msg  string
}

var (
	tsFormat = "2006/01/02 15:04:05"
	addr     string
	dbFile   string
	logChan  chan *logEntry
)

var logSplit = regexp.MustCompile("^([\\w-:]+) (\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2}) (.+)$")
var responseCheck = regexp.MustCompile("response time: (\\d+)us$")

func init() {
	flLogBuffer := flag.Uint("b", 16, "log entries to buffer")
	flDbFile := flag.String("f", "logs.db", "database file")
	port := flag.Uint("p", 5988, "port to listen on")
	flag.Parse()

	addr = fmt.Sprintf(":%d", *port)
	dbFile = *flDbFile
	logChan = make(chan *logEntry, *flLogBuffer)
}

func main() {
	dbSetup()
	go log()
	go listen()

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Kill, os.Interrupt, syscall.SIGTERM)
	<-sigc
	close(logChan)
	<-time.After(100 * time.Millisecond)

	os.Exit(1)
}

func listen() {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to resolve TCP address:", err.Error())
		os.Exit(1)
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to set up TCP listener:", err.Error())
	}
	for {
		conn, err := listener.Accept()
		if err != nil {
			fmt.Println("[!] TCP error:", err.Error())
			continue
		}
		go processMessage(conn)
	}

}

func processMessage(conn net.Conn) {
	fmt.Println("[+] client connected:", conn.RemoteAddr())
	defer conn.Close()
	r := bufio.NewReader(conn)

	for {
		msg, err := r.ReadString(0x0a)
		if err != nil {
			if err != io.EOF {
				fmt.Println("[!] error reading from client:",
					err.Error())
			}
			return
		} else if msg == "" {
			return
		}
		msg = strings.Trim(string(msg), "\n \t")
		fmt.Println("-- ", msg)

		nodeID := logSplit.ReplaceAllString(msg, "$1")
		dateString := logSplit.ReplaceAllString(msg, "$2")
		logMsg := logSplit.ReplaceAllString(msg, "$3")
		tm, err := time.Parse(tsFormat, dateString)
		if err != nil {
			fmt.Fprintf(os.Stderr, "[!] error parsing time %s: %s\n",
				dateString, err.Error())
			return
		}
		le := &logEntry{nodeID, tm.UTC().Unix(), logMsg}
		logChan <- le
	}
}

func log() {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	for {
		le, ok := <-logChan
		if !ok {
			fmt.Println("[+] shutting down database listener")
			return
		}
		_, err := db.Exec("insert into entries values (?, ?, ?)",
			le.Node, le.Time, le.Msg)
		if err != nil {
			fmt.Println("[!] database error:", err.Error())
			continue
		}

		if responseCheck.MatchString(le.Msg) {
			respString := responseCheck.ReplaceAllString(le.Msg, "$1")
			rTime, err := strconv.Atoi(respString)
			if err != nil {
				fmt.Println("[!] error reading response time:", err.Error())
				return
			}
			_, err = db.Exec("insert into response_times values (?, ?, ?)",
				le.Node, le.Time, rTime)
			if err != nil {
				fmt.Println("[!] error writing to database:", err.Error())
			}
		}

	}
}

func logResponseTime(db *sql.DB, le *logEntry) {
}

func dbSetup() {
	entryTable()
	respTable()
}

func entryTable() {
	const createSql = `CREATE TABLE entries (node text, timestamp integer, message string)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='entries'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table entries`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}

func respTable() {
	const createSql = `CREATE TABLE response_times (node text, timestamp integer, microsec integer)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='response_times'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table response_times`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}
<commit_msg>Fix 'database is locked' error.<commit_after>package main

import (
	"bufio"
	"database/sql"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

type logEntry struct {
	Node string
	Time int64
	Msg  string
}

var (
	tsFormat = "2006/01/02 15:04:05"
	addr     string
	dbFile   string
	logChan  chan *logEntry
)

var logSplit = regexp.MustCompile("^([\\w-:]+) (\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2}) (.+)$")
var responseCheck = regexp.MustCompile("response time: (\\d+)us$")

func init() {
	flLogBuffer := flag.Uint("b", 16, "log entries to buffer")
	flDbFile := flag.String("f", "logs.db", "database file")
	port := flag.Uint("p", 5988, "port to listen on")
	flag.Parse()

	addr = fmt.Sprintf(":%d", *port)
	dbFile = *flDbFile
	logChan = make(chan *logEntry, *flLogBuffer)
}

func main() {
	dbSetup()
	go log()
	go listen()

	sigc := make(chan os.Signal, 1)
	signal.Notify(sigc, os.Kill, os.Interrupt, syscall.SIGTERM)
	<-sigc
	close(logChan)
	<-time.After(100 * time.Millisecond)

	os.Exit(1)
}

func listen() {
	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to resolve TCP address:", err.Error())
		os.Exit(1)
	}

	listener, err := net.ListenTCP("tcp", tcpAddr)
	if err != nil {
		fmt.Fprintln(os.Stderr, "[!] failed to set up TCP listener:", err.Error())
	}
	for {
		conn, err := listener.Accept()
		if err != nil {
			fmt.Println("[!] TCP error:", err.Error())
			continue
		}
		go processMessage(conn)
	}

}

func processMessage(conn net.Conn) {
	fmt.Println("[+] client connected:", conn.RemoteAddr())
	defer conn.Close()
	r := bufio.NewReader(conn)

	for {
		msg, err := r.ReadString(0x0a)
		if err != nil {
			if err != io.EOF {
				fmt.Println("[!] error reading from client:",
					err.Error())
			}
			return
		} else if msg == "" {
			return
		}
		msg = strings.Trim(string(msg), "\n \t")
		fmt.Println("-- ", msg)

		nodeID := logSplit.ReplaceAllString(msg, "$1")
		dateString := logSplit.ReplaceAllString(msg, "$2")
		logMsg := logSplit.ReplaceAllString(msg, "$3")
		tm, err := time.Parse(tsFormat, dateString)
		if err != nil {
			fmt.Fprintf(os.Stderr, "[!] error parsing time %s: %s\n",
				dateString, err.Error())
			return
		}
		le := &logEntry{nodeID, tm.UTC().Unix(), logMsg}
		logChan <- le
	}
}

func log() {
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	for {
		le, ok := <-logChan
		if !ok {
			fmt.Println("[+] shutting down database listener")
			return
		}
		_, err := db.Exec("insert into entries values (?, ?, ?)",
			le.Node, le.Time, le.Msg)
		if err != nil {
			fmt.Println("[!] database error:", err.Error())
			continue
		}

		if responseCheck.MatchString(le.Msg) {
			respString := responseCheck.ReplaceAllString(le.Msg, "$1")
			rTime, err := strconv.Atoi(respString)
			if err != nil {
				fmt.Println("[!] error reading response time:", err.Error())
				return
			}
			_, err = db.Exec("insert into response_times values (?, ?, ?)",
				le.Node, le.Time, rTime)
			if err != nil {
				fmt.Println("[!] error writing to database:", err.Error())
			}
		}

	}
}

func logResponseTime(db *sql.DB, le *logEntry) {
}

func dbSetup() {
	entryTable()
	respTable()
}

func entryTable() {
	const createSql = `CREATE TABLE entries (node text, timestamp integer, message string)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='entries'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}
	rows.Close()

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table entries`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}

func respTable() {
	const createSql = `CREATE TABLE response_times (node text, timestamp integer, microsec integer)`
	db, err := sql.Open("sqlite3", dbFile)
	if err != nil {
		fmt.Println("[!] failed to open DB file:", err.Error())
		os.Exit(1)
	}
	defer db.Close()

	rows, err := db.Query(`select sql from sqlite_master
                               where type='table' and name='response_times'`)
	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	}

	var tblSql string
	for rows.Next() {
		err = rows.Scan(&tblSql)
		break
	}
	rows.Close()

	if err != nil {
		fmt.Println("[!] error reading database:", err.Error())
		os.Exit(1)
	} else if tblSql == "" {
		fmt.Println("[+] creating table")
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	} else if tblSql != createSql {
		fmt.Println("[+] schema out of sync")
		_, err = db.Exec(`drop table response_times`)
		if err != nil {
			fmt.Println("[!] error dropping table:", err.Error())
			os.Exit(1)
		}
		_, err = db.Exec(createSql)
		if err != nil {
			fmt.Println("[!] error creating table:", err.Error())
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
)

func main() {
	sound := RandomSound()
	fmt.Println(sound)
}

func RandomSound() Sound {
	sound := Sound{}
	sound.randomiseSound()
	for !sound.IsValid() {
		sound.randomiseSound()
	}
	return sound
}

func (sound *Sound) randomiseSound() {
	sound.Point = ArticulationPoint(rand.Intn(int(ArticulationPointCount)))
	sound.Manner = ArticulationManner(rand.Intn(int(ArticulationMannerCount)))
	sound.Shape = TongueShape(rand.Intn(int(TongueShapeCount)))
	sound.Voice = Voice(rand.Intn(int(VoiceCount)))
	sound.Rounded = rand.Intn(2) == 0
	sound.Nasal = rand.Intn(2) == 0
}

type Sound struct {
	Point   ArticulationPoint
	Manner  ArticulationManner
	Shape   TongueShape
	Rounded bool
	Nasal   bool
	Voice   Voice
}

type ArticulationPoint int

const (
	LabialLabial ArticulationPoint = iota
	LabialDental
	CoronalLabial
	CoronalDental
	CoronalAlveolar
	CoronalPostAlveolar
	CoronalRetroflex
	DorsalPostAlveolar
	DorsalPalatal
	DorsalPalVel
	DorsalVelar
	DorsalVelUlu
	DorsalUvular
	RadicalPharyngeal
	RadicalEpiglottal
	Glottal
	ArticulationPointCount
)

type ArticulationManner int

const (
	Closed ArticulationManner = iota
	Stop
	Flap
	Trill
	Fricative
	Approximant
	Close
	NearClose
	CloseMid
	Mid
	OpenMid
	NearOpen
	Open
	ArticulationMannerCount
)

type TongueShape int

const (
	Central TongueShape = iota
	Lateral
	Sibilant
	TongueShapeCount
)

type Voice int

const (
	Aspirated Voice = iota
	Voiceless
	Breathy
	Modal
	Creaky
	VoiceCount
)

func (sound Sound) IsValid() bool {
	if sound.Rounded {
		switch sound.Point {
		case LabialLabial, LabialDental, CoronalLabial:
			return false
		}
	}

	if sound.Voice == Voiceless {
		//this isn't really invalid, just very unusual
		switch sound.Manner {
		case Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return false
		}
	}

	if sound.Voice == Aspirated {
		switch sound.Manner {
		case Stop:
			break
		default:
			return false
		}
	}

	if sound.Manner == Closed && !sound.Nasal {
		return false
	}

	switch sound.Point {
	case LabialLabial:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case LabialDental:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalLabial:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalDental:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalAlveolar:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalPostAlveolar:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalRetroflex:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case DorsalPostAlveolar:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant:
			return true
		default:
			return false
		}
	case DorsalPalatal:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalPalVel:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalVelar:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalVelUlu:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalUvular:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case RadicalPharyngeal:
		switch sound.Manner {
		case Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case RadicalEpiglottal:
		switch sound.Manner {
		case Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case Glottal:
		switch sound.Manner {
		case Stop, Fricative:
			return true
		default:
			return false
		}
	default:
		panic("Weird")
	}
}
<commit_msg>Implemented Standardise and encode/decode<commit_after>package main

import (
	"fmt"
	"math/rand"
)

func main() {
	sound := RandomSound()
	fmt.Println(sound)
}

func RandomSound() Sound {
	sound := Sound{}
	sound.randomiseSound()
	for !sound.IsValid() {
		sound.randomiseSound()
	}
	sound.Standardise()
	return sound
}

func (sound *Sound) randomiseSound() {
	sound.Point = ArticulationPoint(rand.Intn(int(ArticulationPointCount)))
	sound.Manner = ArticulationManner(rand.Intn(int(ArticulationMannerCount)))
	sound.Shape = TongueShape(rand.Intn(int(TongueShapeCount)))
	sound.Voice = Voice(rand.Intn(int(VoiceCount)))
	sound.Rounded = rand.Intn(2) == 0
	sound.Nasal = rand.Intn(2) == 0
}

type Sound struct {
	Point   ArticulationPoint
	Manner  ArticulationManner
	Shape   TongueShape
	Rounded bool
	Nasal   bool
	Voice   Voice
}

type ArticulationPoint int

const (
	LabialLabial ArticulationPoint = iota
	LabialDental
	CoronalLabial
	CoronalDental
	CoronalAlveolar
	CoronalPostAlveolar
	CoronalRetroflex
	DorsalPostAlveolar
	DorsalPalatal
	DorsalPalVel
	DorsalVelar
	DorsalVelUlu
	DorsalUvular
	RadicalPharyngeal
	RadicalEpiglottal
	Glottal
	ArticulationPointCount
)

type ArticulationManner int

const (
	Closed ArticulationManner = iota
	Stop
	Flap
	Trill
	Fricative
	Approximant
	Close
	NearClose
	CloseMid
	Mid
	OpenMid
	NearOpen
	Open
	ArticulationMannerCount
)

type TongueShape int

const (
	Central TongueShape = iota
	Lateral
	Sibilant
	TongueShapeCount
)

type Voice int

const (
	Aspirated Voice = iota
	Voiceless
	Breathy
	Modal
	Creaky
	VoiceCount
)

func (sound *Sound) IsValid() bool {
	if sound.Rounded {
		switch sound.Point {
		case LabialLabial, LabialDental, CoronalLabial:
			return false
		}
	}

	if sound.Voice == Voiceless {
		//this isn't really invalid, just very unusual
		switch sound.Manner {
		case Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return false
		}
	}

	if sound.Voice == Aspirated {
		switch sound.Manner {
		case Stop:
			break
		default:
			return false
		}
	}

	if sound.Manner == Closed && !sound.Nasal {
		return false
	}

	switch sound.Point {
	case LabialLabial:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case LabialDental:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalLabial:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalDental:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalAlveolar:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalPostAlveolar:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case CoronalRetroflex:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case DorsalPostAlveolar:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant:
			return true
		default:
			return false
		}
	case DorsalPalatal:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalPalVel:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalVelar:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalVelUlu:
		switch sound.Manner {
		case Closed, Stop, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case DorsalUvular:
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			return true
		default:
			return false
		}
	case RadicalPharyngeal:
		switch sound.Manner {
		case Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case RadicalEpiglottal:
		switch sound.Manner {
		case Stop, Flap, Trill, Fricative, Approximant:
			return true
		default:
			return false
		}
	case Glottal:
		switch sound.Manner {
		case Stop, Fricative:
			return true
		default:
			return false
		}
	default:
		panic("Weird")
	}
}

func (sound *Sound) Standardise() {
	if sound.Shape == Sibilant {
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			sound.Shape = Central
		}
		switch sound.Point {
		case LabialLabial, LabialDental, CoronalLabial, DorsalPalatal, DorsalPalVel, DorsalVelar, DorsalVelUlu, DorsalUvular, RadicalPharyngeal, RadicalEpiglottal, Glottal:
			sound.Shape = Central
		}
	}

	if sound.Shape == Lateral {
		switch sound.Manner {
		case Closed, Stop, Trill, Close, NearClose, CloseMid, Mid, OpenMid, NearOpen, Open:
			sound.Shape = Central
		}
		switch sound.Point {
		case LabialLabial, LabialDental, RadicalPharyngeal, RadicalEpiglottal, Glottal:
			sound.Shape = Central
		}
	}

	if sound.Point == DorsalPalVel {
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			sound.Point = DorsalPalatal
		}
	}

	if sound.Point == DorsalVelUlu {
		switch sound.Manner {
		case Closed, Stop, Flap, Trill, Fricative, Approximant:
			sound.Point = DorsalUvular
		}
	}
}

func (sound *Sound) Encode() int {
	enc := int(sound.Point)
	enc += 16 * int(sound.Manner)
	enc += 16 * 13 * int(sound.Shape)
	enc += 16 * 13 * 3 * int(sound.Voice)
	value := 0
	if sound.Rounded {
		value = 1
	}
	enc += 16 * 13 * 3 * 5 * value
	value = 0
	if sound.Nasal {
		value = 1
	}
	enc += 16 * 13 * 3 * 5 * 2 * value
	return enc
}

func Decode(enc int) Sound {
	sound := new(Sound)
	res := enc / (16 * 13 * 3 * 5 * 2)
	sound.Nasal = res == 1
	enc -= res * (16 * 13 * 3 * 5 * 2)
	res = enc / (16 * 13 * 3 * 5)
	sound.Rounded = res == 1
	enc -= res * (16 * 13 * 3 * 5)
	res = enc / (16 * 13 * 3)
	sound.Voice = Voice(res)
	enc -= res * (16 * 13 * 3)
	res = enc / (16 * 13)
	sound.Shape = TongueShape(res)
	enc -= res * (16 * 13)
	res = enc / 16
	sound.Manner = ArticulationManner(res)
	enc -= res * 16
	sound.Point = ArticulationPoint(enc)
	return *sound
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/xiaonanln/goworld"
	"github.com/xiaonanln/goworld/engine/common"
	"github.com/xiaonanln/goworld/engine/consts"
	"github.com/xiaonanln/goworld/engine/entity"
	"github.com/xiaonanln/goworld/engine/gwlog"
)

// Player 对象代表一名玩家
type Player struct {
	entity.Entity
}

func (a *Player) DescribeEntityType(desc *entity.EntityTypeDesc) {
	desc.SetPersistent(true).SetUseAOI(true)
	desc.DefineAttr("name", "AllClients", "Persistent")
	desc.DefineAttr("lv", "AllClients", "Persistent")
	desc.DefineAttr("hp", "AllClients")
	desc.DefineAttr("hpmax", "AllClients")
	desc.DefineAttr("action", "AllClients")
	desc.DefineAttr("spaceKind", "Persistent")
}

// OnCreated 在Player对象创建后被调用
func (a *Player) OnCreated() {
	a.Entity.OnCreated()
	a.setDefaultAttrs()
}

// setDefaultAttrs 设置玩家的一些默认属性
func (a *Player) setDefaultAttrs() {
	a.Attrs.SetDefaultInt("spaceKind", 1)
	a.Attrs.SetDefaultStr("name", "noname")
	a.Attrs.SetDefaultInt("lv", 1)
	a.Attrs.SetDefaultInt("hp", 100)
	a.Attrs.SetDefaultInt("hpmax", 100)
	a.Attrs.SetDefaultStr("action", "idle")

	a.SetClientSyncing(true)
}

// GetSpaceID 获得玩家的场景ID并发给调用者
func (a *Player) GetSpaceID(callerID common.EntityID) {
	a.Call(callerID, "OnGetPlayerSpaceID", a.ID, a.Space.ID)
}

func (p *Player) enterSpace(spaceKind int) {
	if p.Space.Kind == spaceKind {
		return
	}
	if consts.DEBUG_SPACES {
		gwlog.Infof("%s enter space from %d => %d", p, p.Space.Kind, spaceKind)
	}
	goworld.CallService("SpaceService", "EnterSpace", p.ID, spaceKind)
}

// OnClientConnected is called when client is connected
func (a *Player) OnClientConnected() {
	gwlog.Infof("%s client connected", a)
	a.enterSpace(int(a.GetInt("spaceKind")))
}

// OnClientDisconnected is called when client is lost
func (a *Player) OnClientDisconnected() {
	gwlog.Infof("%s client disconnected", a)
	a.Destroy()
}

// EnterSpace_Client is enter space RPC for client
func (a *Player) EnterSpace_Client(kind int) {
	a.enterSpace(kind)
}

// DoEnterSpace is called by SpaceService to notify avatar entering specified space
func (a *Player) DoEnterSpace(kind int, spaceID common.EntityID) {
	// let the avatar enter space with spaceID
	a.EnterSpace(spaceID, entity.Vector3{})
}

//func (a *Player) randomPosition() entity.Vector3 {
//	minCoord, maxCoord := -400, 400
//	return entity.Vector3{
//		X: entity.Coord(minCoord + rand.Intn(maxCoord-minCoord)),
//		Y: 0,
//		Z: entity.Coord(minCoord + rand.Intn(maxCoord-minCoord)),
//	}
//}

// OnEnterSpace is called when avatar enters a space
func (a *Player) OnEnterSpace() {
	gwlog.Infof("%s ENTER SPACE %s", a, a.Space)
	a.SetClientSyncing(true)
}

func (a *Player) SetAction_Client(action string) {
	if a.GetInt("hp") <= 0 { // dead already
		return
	}

	a.Attrs.SetStr("action", action)
}

func (a *Player) ShootMiss_Client() {
	a.CallAllClients("Shoot")
}

func (a *Player) ShootHit_Client(victimID common.EntityID) {
	a.CallAllClients("Shoot")
	victim := a.Space.GetEntity(victimID)
	if victim == nil {
		gwlog.Warnf("Shoot %s, but monster not found", victimID)
		return
	}

	if victim.Attrs.GetInt("hp") <= 0 {
		return
	}

	monster := victim.I.(*Monster)
	monster.TakeDamage(10)
}

func (player *Player) TakeDamage(damage int64) {
	hp := player.GetInt("hp")
	if hp <= 0 {
		return
	}

	hp = hp - damage
	if hp < 0 {
		hp = 0
	}

	player.Attrs.SetInt("hp", hp)

	if hp <= 0 {
		// now player dead ...
		player.Attrs.SetStr("action", "death")
		player.SetClientSyncing(false)
	}
}
<commit_msg>tune goworld<commit_after>package main

import (
	"github.com/xiaonanln/goworld"
	"github.com/xiaonanln/goworld/engine/common"
	"github.com/xiaonanln/goworld/engine/consts"
	"github.com/xiaonanln/goworld/engine/entity"
	"github.com/xiaonanln/goworld/engine/gwlog"
)

// Player 对象代表一名玩家
type Player struct {
	entity.Entity
}

func (a *Player) DescribeEntityType(desc *entity.EntityTypeDesc) {
	desc.SetPersistent(true).SetUseAOI(true)
	desc.DefineAttr("name", "AllClients", "Persistent")
	desc.DefineAttr("lv", "AllClients", "Persistent")
	desc.DefineAttr("hp", "AllClients")
	desc.DefineAttr("hpmax", "AllClients")
	desc.DefineAttr("action", "AllClients")
	desc.DefineAttr("spaceKind", "Persistent")
}

// OnCreated 在Player对象创建后被调用
func (a *Player) OnCreated() {
	a.Entity.OnCreated()
	a.setDefaultAttrs()
}

// setDefaultAttrs 设置玩家的一些默认属性
func (a *Player) setDefaultAttrs() {
	a.Attrs.SetDefaultInt("spaceKind", 1)
	a.Attrs.SetDefaultStr("name", "noname")
	a.Attrs.SetDefaultInt("lv", 1)
	a.Attrs.SetDefaultInt("hp", 100)
	a.Attrs.SetDefaultInt("hpmax", 100)
	a.Attrs.SetDefaultStr("action", "idle")

	a.SetClientSyncing(true)
}

// GetSpaceID 获得玩家的场景ID并发给调用者
func (a *Player) GetSpaceID(callerID common.EntityID) {
	a.Call(callerID, "OnGetPlayerSpaceID", a.ID, a.Space.ID)
}

func (p *Player) enterSpace(spaceKind int) {
	if p.Space.Kind == spaceKind {
		return
	}
	if consts.DEBUG_SPACES {
		gwlog.Infof("%s enter space from %d => %d", p, p.Space.Kind, spaceKind)
	}
	goworld.CallService("SpaceService", "EnterSpace", p.ID, spaceKind)
}

// OnClientConnected is called when client is connected
func (a *Player) OnClientConnected() {
	gwlog.Infof("%s client connected", a)
	a.enterSpace(int(a.GetInt("spaceKind")))
}

// OnClientDisconnected is called when client is lost
func (a *Player) OnClientDisconnected() {
	gwlog.Infof("%s client disconnected", a)
	a.Destroy()
}

// EnterSpace_Client is enter space RPC for client
func (a *Player) EnterSpace_Client(kind int) {
	a.enterSpace(kind)
}

// DoEnterSpace is called by SpaceService to notify avatar entering specified space
func (a *Player) DoEnterSpace(kind int, spaceID common.EntityID) {
	// let the avatar enter space with spaceID
	a.EnterSpace(spaceID, entity.Vector3{})
}

//func (a *Player) randomPosition() entity.Vector3 {
//	minCoord, maxCoord := -400, 400
//	return entity.Vector3{
//		X: entity.Coord(minCoord + rand.Intn(maxCoord-minCoord)),
//		Y: 0,
//		Z: entity.Coord(minCoord + rand.Intn(maxCoord-minCoord)),
//	}
//}

// OnEnterSpace is called when avatar enters a space
func (a *Player) OnEnterSpace() {
	gwlog.Infof("%s ENTER SPACE %s", a, a.Space)
	a.SetClientSyncing(true)
}

func (a *Player) SetAction_Client(action string) {
	if a.GetInt("hp") <= 0 { // dead already
		return
	}

	a.Attrs.SetStr("action", action)
}

func (a *Player) ShootMiss_Client() {
	a.CallAllClients("Shoot")
}

func (a *Player) ShootHit_Client(victimID common.EntityID) {
	a.CallAllClients("Shoot")
	victim := a.Space.GetEntity(victimID)
	if victim == nil {
		gwlog.Warnf("Shoot %s, but monster not found", victimID)
		return
	}

	if victim.Attrs.GetInt("hp") <= 0 {
		return
	}

	monster := victim.I.(*Monster)
	monster.TakeDamage(50)
}

func (player *Player) TakeDamage(damage int64) {
	hp := player.GetInt("hp")
	if hp <= 0 {
		return
	}

	hp = hp - damage
	if hp < 0 {
		hp = 0
	}

	player.Attrs.SetInt("hp", hp)

	if hp <= 0 {
		// now player dead ...
		player.Attrs.SetStr("action", "death")
		player.SetClientSyncing(false)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"path"

	"github.com/coreos/container-linux-config-transpiler/config/astyaml"
	"github.com/coreos/container-linux-config-transpiler/internal/util"

	ignTypes "github.com/coreos/ignition/config/v2_2/types"
	"github.com/coreos/ignition/config/validate/astnode"
	"github.com/coreos/ignition/config/validate/report"
	"github.com/vincent-petithory/dataurl"
)

var (
	DefaultFileMode = 0644
	DefaultDirMode  = 0755

	WarningUnsetFileMode = fmt.Errorf("mode unspecified for file, defaulting to %#o", DefaultFileMode)
	WarningUnsetDirMode  = fmt.Errorf("mode unspecified for directory, defaulting to %#o", DefaultDirMode)

	ErrTooManyFileSources = errors.New("only one of the following can be set: local, inline, remote.url")
)

type FileUser struct {
	Id   *int   `yaml:"id"`
	Name string `yaml:"name"`
}

type FileGroup struct {
	Id   *int   `yaml:"id"`
	Name string `yaml:"name"`
}

type File struct {
	Filesystem string       `yaml:"filesystem"`
	Path       string       `yaml:"path"`
	User       *FileUser    `yaml:"user"`
	Group      *FileGroup   `yaml:"group"`
	Mode       *int         `yaml:"mode"`
	Contents   FileContents `yaml:"contents"`
	Overwrite  *bool        `yaml:"overwrite"`
	Append     bool         `yaml:"append"`
}

type FileContents struct {
	Remote Remote `yaml:"remote"`
	Inline string `yaml:"inline"`
	Local  string `yaml:"local"`
}

type Remote struct {
	Url          string       `yaml:"url"`
	Compression  string       `yaml:"compression"`
	Verification Verification `yaml:"verification"`
}

type Directory struct {
	Filesystem string     `yaml:"filesystem"`
	Path       string     `yaml:"path"`
	User       *FileUser  `yaml:"user"`
	Group      *FileGroup `yaml:"group"`
	Mode       *int       `yaml:"mode"`
	Overwrite  *bool      `yaml:"overwrite"`
}

type Link struct {
	Filesystem string     `yaml:"filesystem"`
	Path       string     `yaml:"path"`
	User       *FileUser  `yaml:"user"`
	Group      *FileGroup `yaml:"group"`
	Hard       bool       `yaml:"hard"`
	Target     string     `yaml:"target"`
	Overwrite  *bool      `yaml:"overwrite"`
}

func (f File) ValidateMode() report.Report {
	if f.Mode == nil {
		return report.ReportFromError(WarningUnsetFileMode, report.EntryWarning)
	}
	return report.Report{}
}

func (d Directory) ValidateMode() report.Report {
	if d.Mode == nil {
		return report.ReportFromError(WarningUnsetDirMode, report.EntryWarning)
	}
	return report.Report{}
}

func (fc FileContents) Validate() report.Report {
	i := 0
	if fc.Remote.Url != "" {
		i++
	}
	if fc.Inline != "" {
		i++
	}
	if fc.Local != "" {
		i++
	}
	if i > 1 {
		return report.ReportFromError(ErrTooManyFileSources, report.EntryError)
	}
	return report.Report{}
}

func init() {
	register(func(in Config, ast astnode.AstNode, out ignTypes.Config, platform string) (ignTypes.Config, report.Report, astnode.AstNode) {
		r := report.Report{}
		files_node, _ := getNodeChildPath(ast, "storage", "files")
		for i, file := range in.Storage.Files {
			if file.Mode == nil {
				file.Mode = util.IntToPtr(DefaultFileMode)
			}
			file_node, _ := getNodeChild(files_node, i)
			newFile := ignTypes.File{
				Node: ignTypes.Node{
					Filesystem: file.Filesystem,
					Path:       file.Path,
					Overwrite:  file.Overwrite,
				},
				FileEmbedded1: ignTypes.FileEmbedded1{
					Mode:   file.Mode,
					Append: file.Append,
				},
			}
			if file.User != nil {
				newFile.User = &ignTypes.NodeUser{
					ID:   file.User.Id,
					Name: file.User.Name,
				}
			}
			if file.Group != nil {
				newFile.Group = &ignTypes.NodeGroup{
					ID:   file.Group.Id,
					Name: file.Group.Name,
				}
			}

			if file.Contents.Inline != "" {
				newFile.Contents = ignTypes.FileContents{
					Source: (&url.URL{
						Scheme: "data",
						Opaque: "," + dataurl.EscapeString(file.Contents.Inline),
					}).String(),
				}
			}

			if file.Contents.Local != "" {
				// The provided local file path is relative to the value of the
				// --files-dir flag.
				filesDir := flag.Lookup("files-dir")
				if filesDir == nil || filesDir.Value.String() == "" {
					err := errors.New("local files require setting the --files-dir flag to the directory that contains the file")
					flagReport := report.ReportFromError(err, report.EntryError)
					if n, err := getNodeChildPath(file_node, "contents", "local"); err == nil {
						line, col, _ := n.ValueLineCol(nil)
						flagReport.AddPosition(line, col, "")
					}
					r.Merge(flagReport)
					continue
				}
				localPath := path.Join(filesDir.Value.String(), file.Contents.Local)
				contents, err := ioutil.ReadFile(localPath)
				if err != nil {
					// If the file could not be read, record error and continue.
					convertReport := report.ReportFromError(err, report.EntryError)
					if n, err := getNodeChildPath(file_node, "contents", "local"); err == nil {
						line, col, _ := n.ValueLineCol(nil)
						convertReport.AddPosition(line, col, "")
					}
					r.Merge(convertReport)
					continue
				}

				// Include the contents of the local file as if it were provided inline.
				newFile.Contents = ignTypes.FileContents{
					Source: (&url.URL{
						Scheme: "data",
						Opaque: "," + dataurl.Escape(contents),
					}).String(),
				}
			}

			if file.Contents.Remote.Url != "" {
				source, err := url.Parse(file.Contents.Remote.Url)
				if err != nil {
					// if invalid, record error and continue
					convertReport := report.ReportFromError(err, report.EntryError)
					if n, err := getNodeChildPath(file_node, "contents", "remote", "url"); err == nil {
						line, col, _ := n.ValueLineCol(nil)
						convertReport.AddPosition(line, col, "")
					}
					r.Merge(convertReport)
					continue
				}

				// patch the yaml tree to look like the ignition tree by making contents
				// the remote section and changing the name from url -> source
				asYamlNode, ok := file_node.(astyaml.YamlNode)
				if ok {
					newContents, _ := getNodeChildPath(file_node, "contents", "remote")
					newContentsAsYaml := newContents.(astyaml.YamlNode)
					asYamlNode.ChangeKey("contents", "contents", newContentsAsYaml)

					url, _ := getNodeChild(newContents.(astyaml.YamlNode), "url")
					newContentsAsYaml.ChangeKey("url", "source", url.(astyaml.YamlNode))
				}

				newFile.Contents = ignTypes.FileContents{Source: source.String()}

			}

			if newFile.Contents == (ignTypes.FileContents{}) {
				newFile.Contents = ignTypes.FileContents{
					Source: "data:,",
				}
			}

			newFile.Contents.Compression = file.Contents.Remote.Compression
			newFile.Contents.Verification = convertVerification(file.Contents.Remote.Verification)

			out.Storage.Files = append(out.Storage.Files, newFile)
		}
		for _, dir := range in.Storage.Directories {
			if dir.Mode == nil {
				dir.Mode = util.IntToPtr(DefaultDirMode)
			}
			newDir := ignTypes.Directory{
				Node: ignTypes.Node{
					Filesystem: dir.Filesystem,
					Path:       dir.Path,
					Overwrite:  dir.Overwrite,
				},
				DirectoryEmbedded1: ignTypes.DirectoryEmbedded1{
					Mode: dir.Mode,
				},
			}
			if dir.User != nil {
				newDir.User = &ignTypes.NodeUser{
					ID:   dir.User.Id,
					Name: dir.User.Name,
				}
			}
			if dir.Group != nil {
				newDir.Group = &ignTypes.NodeGroup{
					ID:   dir.Group.Id,
					Name: dir.Group.Name,
				}
			}
			out.Storage.Directories = append(out.Storage.Directories, newDir)
		}
		for _, link := range in.Storage.Links {
			newLink := ignTypes.Link{
				Node: ignTypes.Node{
					Filesystem: link.Filesystem,
					Path:       link.Path,
					Overwrite:  link.Overwrite,
				},
				LinkEmbedded1: ignTypes.LinkEmbedded1{
					Hard:   link.Hard,
					Target: link.Target,
				},
			}
			if link.User != nil {
				newLink.User = &ignTypes.NodeUser{
					ID:   link.User.Id,
					Name: link.User.Name,
				}
			}
			if link.Group != nil {
				newLink.Group = &ignTypes.NodeGroup{
					ID:   link.Group.Id,
					Name: link.Group.Name,
				}
			}
			out.Storage.Links = append(out.Storage.Links, newLink)
		}
		return out, r, ast
	})
}
<commit_msg>types/{files/dirs/links}: default to root fs<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"path"

	"github.com/coreos/container-linux-config-transpiler/config/astyaml"
	"github.com/coreos/container-linux-config-transpiler/internal/util"

	ignTypes "github.com/coreos/ignition/config/v2_2/types"
	"github.com/coreos/ignition/config/validate/astnode"
	"github.com/coreos/ignition/config/validate/report"
	"github.com/vincent-petithory/dataurl"
)

var (
	DefaultFileMode = 0644
	DefaultDirMode  = 0755

	WarningUnsetFileMode = fmt.Errorf("mode unspecified for file, defaulting to %#o", DefaultFileMode)
	WarningUnsetDirMode  = fmt.Errorf("mode unspecified for directory, defaulting to %#o", DefaultDirMode)

	ErrTooManyFileSources = errors.New("only one of the following can be set: local, inline, remote.url")
)

type FileUser struct {
	Id   *int   `yaml:"id"`
	Name string `yaml:"name"`
}

type FileGroup struct {
	Id   *int   `yaml:"id"`
	Name string `yaml:"name"`
}

type File struct {
	Filesystem string       `yaml:"filesystem"`
	Path       string       `yaml:"path"`
	User       *FileUser    `yaml:"user"`
	Group      *FileGroup   `yaml:"group"`
	Mode       *int         `yaml:"mode"`
	Contents   FileContents `yaml:"contents"`
	Overwrite  *bool        `yaml:"overwrite"`
	Append     bool         `yaml:"append"`
}

type FileContents struct {
	Remote Remote `yaml:"remote"`
	Inline string `yaml:"inline"`
	Local  string `yaml:"local"`
}

type Remote struct {
	Url          string       `yaml:"url"`
	Compression  string       `yaml:"compression"`
	Verification Verification `yaml:"verification"`
}

type Directory struct {
	Filesystem string     `yaml:"filesystem"`
	Path       string     `yaml:"path"`
	User       *FileUser  `yaml:"user"`
	Group      *FileGroup `yaml:"group"`
	Mode       *int       `yaml:"mode"`
	Overwrite  *bool      `yaml:"overwrite"`
}

type Link struct {
	Filesystem string     `yaml:"filesystem"`
	Path       string     `yaml:"path"`
	User       *FileUser  `yaml:"user"`
	Group      *FileGroup `yaml:"group"`
	Hard       bool       `yaml:"hard"`
	Target     string     `yaml:"target"`
	Overwrite  *bool      `yaml:"overwrite"`
}

func (f File) ValidateMode() report.Report {
	if f.Mode == nil {
		return report.ReportFromError(WarningUnsetFileMode, report.EntryWarning)
	}
	return report.Report{}
}

func (d Directory) ValidateMode() report.Report {
	if d.Mode == nil {
		return report.ReportFromError(WarningUnsetDirMode, report.EntryWarning)
	}
	return report.Report{}
}

func (fc FileContents) Validate() report.Report {
	i := 0
	if fc.Remote.Url != "" {
		i++
	}
	if fc.Inline != "" {
		i++
	}
	if fc.Local != "" {
		i++
	}
	if i > 1 {
		return report.ReportFromError(ErrTooManyFileSources, report.EntryError)
	}
	return report.Report{}
}

func init() {
	register(func(in Config, ast astnode.AstNode, out ignTypes.Config, platform string) (ignTypes.Config, report.Report, astnode.AstNode) {
		r := report.Report{}
		files_node, _ := getNodeChildPath(ast, "storage", "files")
		for i, file := range in.Storage.Files {
			if file.Mode == nil {
				file.Mode = util.IntToPtr(DefaultFileMode)
			}
			if file.Filesystem == "" {
				file.Filesystem = "root"
			}
			file_node, _ := getNodeChild(files_node, i)
			newFile := ignTypes.File{
				Node: ignTypes.Node{
					Filesystem: file.Filesystem,
					Path:       file.Path,
					Overwrite:  file.Overwrite,
				},
				FileEmbedded1: ignTypes.FileEmbedded1{
					Mode:   file.Mode,
					Append: file.Append,
				},
			}
			if file.User != nil {
				newFile.User = &ignTypes.NodeUser{
					ID:   file.User.Id,
					Name: file.User.Name,
				}
			}
			if file.Group != nil {
				newFile.Group = &ignTypes.NodeGroup{
					ID:   file.Group.Id,
					Name: file.Group.Name,
				}
			}

			if file.Contents.Inline != "" {
				newFile.Contents = ignTypes.FileContents{
					Source: (&url.URL{
						Scheme: "data",
						Opaque: "," + dataurl.EscapeString(file.Contents.Inline),
					}).String(),
				}
			}

			if file.Contents.Local != "" {
				// The provided local file path is relative to the value of the
				// --files-dir flag.
				filesDir := flag.Lookup("files-dir")
				if filesDir == nil || filesDir.Value.String() == "" {
					err := errors.New("local files require setting the --files-dir flag to the directory that contains the file")
					flagReport := report.ReportFromError(err, report.EntryError)
					if n, err := getNodeChildPath(file_node, "contents", "local"); err == nil {
						line, col, _ := n.ValueLineCol(nil)
						flagReport.AddPosition(line, col, "")
					}
					r.Merge(flagReport)
					continue
				}
				localPath := path.Join(filesDir.Value.String(), file.Contents.Local)
				contents, err := ioutil.ReadFile(localPath)
				if err != nil {
					// If the file could not be read, record error and continue.
					convertReport := report.ReportFromError(err, report.EntryError)
					if n, err := getNodeChildPath(file_node, "contents", "local"); err == nil {
						line, col, _ := n.ValueLineCol(nil)
						convertReport.AddPosition(line, col, "")
					}
					r.Merge(convertReport)
					continue
				}

				// Include the contents of the local file as if it were provided inline.
				newFile.Contents = ignTypes.FileContents{
					Source: (&url.URL{
						Scheme: "data",
						Opaque: "," + dataurl.Escape(contents),
					}).String(),
				}
			}

			if file.Contents.Remote.Url != "" {
				source, err := url.Parse(file.Contents.Remote.Url)
				if err != nil {
					// if invalid, record error and continue
					convertReport := report.ReportFromError(err, report.EntryError)
					if n, err := getNodeChildPath(file_node, "contents", "remote", "url"); err == nil {
						line, col, _ := n.ValueLineCol(nil)
						convertReport.AddPosition(line, col, "")
					}
					r.Merge(convertReport)
					continue
				}

				// patch the yaml tree to look like the ignition tree by making contents
				// the remote section and changing the name from url -> source
				asYamlNode, ok := file_node.(astyaml.YamlNode)
				if ok {
					newContents, _ := getNodeChildPath(file_node, "contents", "remote")
					newContentsAsYaml := newContents.(astyaml.YamlNode)
					asYamlNode.ChangeKey("contents", "contents", newContentsAsYaml)

					url, _ := getNodeChild(newContents.(astyaml.YamlNode), "url")
					newContentsAsYaml.ChangeKey("url", "source", url.(astyaml.YamlNode))
				}

				newFile.Contents = ignTypes.FileContents{Source: source.String()}

			}

			if newFile.Contents == (ignTypes.FileContents{}) {
				newFile.Contents = ignTypes.FileContents{
					Source: "data:,",
				}
			}

			newFile.Contents.Compression = file.Contents.Remote.Compression
			newFile.Contents.Verification = convertVerification(file.Contents.Remote.Verification)

			out.Storage.Files = append(out.Storage.Files, newFile)
		}
		for _, dir := range in.Storage.Directories {
			if dir.Mode == nil {
				dir.Mode = util.IntToPtr(DefaultDirMode)
			}
			if dir.Filesystem == "" {
				dir.Filesystem = "root"
			}
			newDir := ignTypes.Directory{
				Node: ignTypes.Node{
					Filesystem: dir.Filesystem,
					Path:       dir.Path,
					Overwrite:  dir.Overwrite,
				},
				DirectoryEmbedded1: ignTypes.DirectoryEmbedded1{
					Mode: dir.Mode,
				},
			}
			if dir.User != nil {
				newDir.User = &ignTypes.NodeUser{
					ID:   dir.User.Id,
					Name: dir.User.Name,
				}
			}
			if dir.Group != nil {
				newDir.Group = &ignTypes.NodeGroup{
					ID:   dir.Group.Id,
					Name: dir.Group.Name,
				}
			}
			out.Storage.Directories = append(out.Storage.Directories, newDir)
		}
		for _, link := range in.Storage.Links {
			if link.Filesystem == "" {
				link.Filesystem = "root"
			}
			newLink := ignTypes.Link{
				Node: ignTypes.Node{
					Filesystem: link.Filesystem,
					Path:       link.Path,
					Overwrite:  link.Overwrite,
				},
				LinkEmbedded1: ignTypes.LinkEmbedded1{
					Hard:   link.Hard,
					Target: link.Target,
				},
			}
			if link.User != nil {
				newLink.User = &ignTypes.NodeUser{
					ID:   link.User.Id,
					Name: link.User.Name,
				}
			}
			if link.Group != nil {
				newLink.Group = &ignTypes.NodeGroup{
					ID:   link.Group.Id,
					Name: link.Group.Name,
				}
			}
			out.Storage.Links = append(out.Storage.Links, newLink)
		}
		return out, r, ast
	})
}
<|endoftext|>


<commit_before>package filesystem

import (
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
)

func (objSrv *FileSystemObjectServer) getObjectReader(hash hash.Hash) (uint64,
	io.Reader, error) {
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hash))
	file, err := os.Open(filename)
	if err != nil {
		return 0, nil, err
	}
	defer file.Close()
	fi, err := file.Stat()
	if err != nil {
		return 0, nil, err
	}
	return uint64(fi.Size()), file, nil
}
<commit_msg>Fix bug in objectserverver.filesystem.getObjectReader(): do not close reader.<commit_after>package filesystem

import (
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
)

func (objSrv *FileSystemObjectServer) getObjectReader(hash hash.Hash) (uint64,
	io.Reader, error) {
	filename := path.Join(objSrv.baseDir, objectcache.HashToFilename(hash))
	file, err := os.Open(filename)
	if err != nil {
		return 0, nil, err
	}
	fi, err := file.Stat()
	if err != nil {
		file.Close()
		return 0, nil, err
	}
	return uint64(fi.Size()), file, nil
}
<|endoftext|>


<commit_before>package manager

import (
	"fmt"
	"path/filepath"
	"sort"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
)

type Leadership uint8

const (
	ZKFlagNone         = 0
	LeaderElectionPath = "/leader-election"

	LeadershipUnknown  Leadership = 1
	LeadershipLeader   Leadership = 2
	LeadershipFollower Leadership = 3
)

var (
	ZKDefaultACL = zk.WorldACL(zk.PermAll)
)

func connect(addrs []string) (*zk.Conn, error) {
	conn, connChan, err := zk.Connect(addrs, 5*time.Second)
	if err != nil {
		return nil, fmt.Errorf("zk connection to [%v] error: %v", addrs, err)
	}

	for {
		select {
		case connEvent := <-connChan:
			if connEvent.State == zk.StateConnected {
				log.Info("connect to zookeeper server success!")
				return conn, nil
			}
			// TODO(nmg) should be re-connect.
			if connEvent.State == zk.StateDisconnected {
				log.Info("lost connection from zookeeper")
				return nil, nil
			}
			// TODO(nmg) currently not work.
		case _ = <-time.After(time.Second * 5):
			conn.Close()
			return nil, nil
		}
	}
}

func (m *Manager) setLeader(path string) {
	p := filepath.Join(m.electRootPath, path)
	_, err := m.ZKClient.Set(p, []byte(m.cfg.Advertise), -1)
	if err != nil {
		log.Infof("Update leader address error %s", err.Error())
	}
}

func (m *Manager) getLeader(path string) (string, error) {
	p := filepath.Join(m.electRootPath, path)
	for {
		b, _, err := m.ZKClient.Get(p)
		if err != nil {
			log.Infof("Get leader address error %s", err.Error())
			return "", err
		}

		if len(b) > 0 {
			return string(b), nil
		}

		time.Sleep(1 * time.Second)
	}
}

func (m *Manager) isLeader(path string) (bool, error, string) {
	children, _, err := m.ZKClient.Children(m.electRootPath)
	if err != nil {
		return false, err, ""
	}

	sort.Strings(children)

	p := children[0]

	return path == p, nil, p
}

func (m *Manager) elect() (string, error) {
	leader, err, p := m.isLeader(m.myid)
	if err != nil {
		return "", err
	}
	if leader {
		log.Info("Electing leader success.")
		m.leader = m.cfg.Advertise
		m.setLeader(p)
		m.leadershipChangeCh <- LeadershipLeader

		return p, nil
	}

	log.Infof("Leader manager has been elected.")

	l, err := m.getLeader(p)
	if err != nil {
		if err == zk.ErrNoNode {
			log.Errorf("Leader lost again. start new electing...")
			return m.elect()
		}
		log.Errorf("Detect new leader error %s", err.Error())
		return "", err
	}
	log.Infof("Detect new leader at %s", l)

	m.leader = l

	m.leadershipChangeCh <- LeadershipFollower

	return p, nil

}

func (m *Manager) electLeader() (string, error) {
	p := filepath.Join(m.electRootPath, "0")
	path, err := m.ZKClient.Create(p, nil, zk.FlagEphemeral|zk.FlagSequence, ZKDefaultACL)
	if err != nil {
		return "", err
	}

	m.myid = filepath.Base(path)

	return m.elect()
}

func (m *Manager) watchLeader(path string) error {
	p := filepath.Join(m.electRootPath, path)
	_, _, childCh, err := m.ZKClient.ChildrenW(p)
	if err != nil {
		log.Infof("Watch children error %s", err)
		return err
	}

	for {
		childEvent := <-childCh
		if childEvent.Type == zk.EventNodeDeleted {
			log.Info("Lost leading manager. Start electing new leader...")
			// If it is better to run following steps in a seprated goroutine?
			// (memory leak maybe)
			p, err := m.elect()
			if err != nil {
				log.Infof("Electing new leader error %s", err.Error())
				return err
			}
			m.watchLeader(p)
		}
	}
}
<commit_msg>temporarily fix bug on potential dead loop for zk  HA<commit_after>package manager

import (
	"fmt"
	"path/filepath"
	"sort"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/samuel/go-zookeeper/zk"
)

type Leadership uint8

const (
	ZKFlagNone         = 0
	LeaderElectionPath = "/leader-election"

	LeadershipUnknown  Leadership = 1
	LeadershipLeader   Leadership = 2
	LeadershipFollower Leadership = 3
)

var (
	ZKDefaultACL = zk.WorldACL(zk.PermAll)
)

func connect(addrs []string) (*zk.Conn, error) {
	conn, connChan, err := zk.Connect(addrs, 5*time.Second)
	if err != nil {
		return nil, fmt.Errorf("zk connection to [%v] error: %v", addrs, err)
	}

	for {
		select {
		case connEvent := <-connChan:
			if connEvent.State == zk.StateConnected {
				log.Info("connect to zookeeper server success!")
				return conn, nil
			}
			// TODO(nmg) should be re-connect.
			if connEvent.State == zk.StateDisconnected {
				log.Info("lost connection from zookeeper")
				return nil, nil
			}
			// TODO(nmg) currently not work.
		case _ = <-time.After(time.Second * 5):
			conn.Close()
			return nil, nil
		}
	}
}

func (m *Manager) setLeader(path string) {
	p := filepath.Join(m.electRootPath, path)
	_, err := m.ZKClient.Set(p, []byte(m.cfg.Advertise), -1)
	if err != nil {
		log.Infof("Update leader address error %s", err.Error())
	}
}

func (m *Manager) getLeader(path string) (string, error) {
	p := filepath.Join(m.electRootPath, path)
	for {
		b, _, err := m.ZKClient.Get(p)
		if err != nil {
			log.Infof("Get leader address error %s", err.Error())
			return "", err
		}

		if len(b) > 0 {
			return string(b), nil
		}

		time.Sleep(1 * time.Second)
	}
}

func (m *Manager) isLeader(path string) (bool, error, string) {
	children, _, err := m.ZKClient.Children(m.electRootPath)
	if err != nil {
		return false, err, ""
	}

	sort.Strings(children)

	p := children[0]

	return path == p, nil, p
}

func (m *Manager) elect() (string, error) {
	leader, err, p := m.isLeader(m.myid)
	if err != nil {
		return "", err
	}
	if leader {
		log.Info("Electing leader success.")
		m.leader = m.cfg.Advertise
		m.setLeader(p)
		m.leadershipChangeCh <- LeadershipLeader

		return p, nil
	}

	log.Infof("Leader manager has been elected.")

	l, err := m.getLeader(p)
	if err != nil {
		if err == zk.ErrNoNode {
			log.Errorf("Leader lost again. start new electing...")
			return m.elect()
		}
		log.Errorf("Detect new leader error %s", err.Error())
		return "", err
	}
	log.Infof("Detect new leader at %s", l)

	m.leader = l

	m.leadershipChangeCh <- LeadershipFollower

	return p, nil

}

func (m *Manager) electLeader() (string, error) {
	p := filepath.Join(m.electRootPath, "0")
	path, err := m.ZKClient.Create(p, nil, zk.FlagEphemeral|zk.FlagSequence, ZKDefaultACL)
	if err != nil {
		return "", err
	}

	m.myid = filepath.Base(path)

	return m.elect()
}

func (m *Manager) watchLeader(path string) error {
	p := filepath.Join(m.electRootPath, path)
	_, _, childCh, err := m.ZKClient.ChildrenW(p)
	if err != nil {
		log.Infof("Watch children error %s", err)
		return err
	}

	for {
		childEvent := <-childCh
		if childEvent.Type == zk.EventNodeDeleted {
			log.Info("Lost leading manager. Start electing new leader...")
			// If it is better to run following steps in a seprated goroutine?
			// (memory leak maybe)
			p, err := m.elect()
			if err != nil {
				log.Infof("Electing new leader error %s", err.Error())
				return err
			}
			m.watchLeader(p)
		}
		time.Sleep(time.Millisecond * 100)
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	"errors"

	. "github.com/bluemixgaragelondon/cf-blue-green-deploy"
	"github.com/cloudfoundry-incubator/candiedyaml"
	"code.cloudfoundry.org/cli/cf/manifest"
	"code.cloudfoundry.org/cli/cf/i18n"
	"code.cloudfoundry.org/cli/utils/generic"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type localeGetter struct{}

func (l localeGetter) Locale() string {
	return "en-us"
}

var _ = Describe("Manifest reader", func() {

	// testing code that calls into cf cli requires T to point to a translate func
	i18n.T = i18n.Init(localeGetter{})

	Context("When the manifest file is present", func() {
		Context("when the manifest contain a host but no app name", func() {
			repo := FakeRepo{yaml: `---
        host: foo`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			It("Returns params that contain the host", func() {
				Expect(manifestAppFinder.AppParams().Hosts).To(ContainElement("foo"))
			})
		})

		Context("when the manifest contains a different app name", func() {
			repo := FakeRepo{yaml: `---
        name: bar
        host: foo`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			It("Returns nil", func() {
				Expect(manifestAppFinder.AppParams()).To(BeNil())
			})
		})

		Context("when the manifest contains multiple apps with 1 matching", func() {
			repo := FakeRepo{yaml: `---
        applications:
        - name: bar
          host: barhost
        - name: foo
          hosts:
          - host1
          - host2
          domains:
          - example1.com
          - example2.com`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			It("Returns the correct app", func() {
				Expect(*manifestAppFinder.AppParams().Name).To(Equal("foo"))
				Expect(manifestAppFinder.AppParams().Hosts).To(ConsistOf("host1", "host2"))
				Expect(manifestAppFinder.AppParams().Domains).To(ConsistOf("example1.com", "example2.com"))
			})
		})
	})

	Context("When no manifest file is present", func() {
		repo := FakeRepo{err: errors.New("Error finding manifest")}
		manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

		It("Returns nil", func() {
			Expect(manifestAppFinder.AppParams()).To(BeNil())
		})
	})

	Context("When manifest file is empty", func() {
		repo := FakeRepo{yaml: ``}
		manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

		It("Returns nil", func() {
			Expect(manifestAppFinder.AppParams()).To(BeNil())
		})
	})

	Describe("Route Lister", func() {
		It("returns a list of Routes from the manifest", func() {
			repo := FakeRepo{yaml: `---
          name: foo
          hosts:
          - host1
          - host2
          domains:
          - example.com
          - example.net`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			routes := manifestAppFinder.RoutesFromManifest("example.com")

			Expect(routes).To(ConsistOf(
				Route{Host: "host1", Domain: Domain{Name: "example.com"}},
				Route{Host: "host1", Domain: Domain{Name: "example.net"}},
				Route{Host: "host2", Domain: Domain{Name: "example.com"}},
				Route{Host: "host2", Domain: Domain{Name: "example.net"}},
			))
		})

		Context("when app has just hosts, no domains", func() {
			It("returns Application", func() {
				repo := FakeRepo{yaml: `---
          name: foo
          hosts:
          - host1
          - host2`,
				}
				manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}
				routes := manifestAppFinder.RoutesFromManifest("example.com")

				Expect(routes).To(ConsistOf(
					Route{Host: "host1", Domain: Domain{Name: "example.com"}},
					Route{Host: "host2", Domain: Domain{Name: "example.com"}},
				))
			})
		})

		Context("when no matching application", func() {
			It("returns nil", func() {
				repo := FakeRepo{yaml: ``}
				manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

				Expect(manifestAppFinder.RoutesFromManifest("example.com")).To(BeNil())
			})
		})
	})
})

type FakeRepo struct {
	yaml string
	err  error
}

func (r *FakeRepo) ReadManifest(path string) (*manifest.Manifest, error) {
	yamlMap := generic.NewMap()
	candiedyaml.Unmarshal([]byte(r.yaml), yamlMap)
	return &manifest.Manifest{Data: yamlMap}, r.err
}
<commit_msg>Test to reproduce problem with route support<commit_after>package main_test

import (
	"errors"

	"code.cloudfoundry.org/cli/cf/i18n"
	"code.cloudfoundry.org/cli/cf/manifest"
	"code.cloudfoundry.org/cli/utils/generic"
	. "github.com/bluemixgaragelondon/cf-blue-green-deploy"
	"github.com/cloudfoundry-incubator/candiedyaml"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

type localeGetter struct{}

func (l localeGetter) Locale() string {
	return "en-us"
}

var _ = Describe("Manifest reader", func() {

	// testing code that calls into cf cli requires T to point to a translate func
	i18n.T = i18n.Init(localeGetter{})

	Context("When the manifest file is present", func() {
		Context("when the manifest contain a host but no app name", func() {
			repo := FakeRepo{yaml: `---
        host: foo`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			It("Returns params that contain the host", func() {
				Expect(manifestAppFinder.AppParams().Hosts).To(ContainElement("foo"))
			})
		})

		Context("when the manifest contains a different app name", func() {
			repo := FakeRepo{yaml: `---
        name: bar
        host: foo`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			It("Returns nil", func() {
				Expect(manifestAppFinder.AppParams()).To(BeNil())
			})
		})

		Context("when the manifest contains multiple apps with 1 matching", func() {
			repo := FakeRepo{yaml: `---
        applications:
        - name: bar
          host: barhost
        - name: foo
          hosts:
          - host1
          - host2
          domains:
          - example1.com
          - example2.com`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			It("Returns the correct app", func() {
				Expect(*manifestAppFinder.AppParams().Name).To(Equal("foo"))
				Expect(manifestAppFinder.AppParams().Hosts).To(ConsistOf("host1", "host2"))
				Expect(manifestAppFinder.AppParams().Domains).To(ConsistOf("example1.com", "example2.com"))
			})
		})
	})

	Context("When no manifest file is present", func() {
		repo := FakeRepo{err: errors.New("Error finding manifest")}
		manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

		It("Returns nil", func() {
			Expect(manifestAppFinder.AppParams()).To(BeNil())
		})
	})

	Context("When manifest file is empty", func() {
		repo := FakeRepo{yaml: ``}
		manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

		It("Returns nil", func() {
			Expect(manifestAppFinder.AppParams()).To(BeNil())
		})
	})

	Describe("Route Lister", func() {
		It("returns a list of Routes from the manifest", func() {
			repo := FakeRepo{yaml: `---
          name: foo
          hosts:
          - host1
          - host2
          domains:
          - example.com
          - example.net`,
			}
			manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

			routes := manifestAppFinder.RoutesFromManifest("example.com")

			Expect(routes).To(ConsistOf(
				Route{Host: "host1", Domain: Domain{Name: "example.com"}},
				Route{Host: "host1", Domain: Domain{Name: "example.net"}},
				Route{Host: "host2", Domain: Domain{Name: "example.com"}},
				Route{Host: "host2", Domain: Domain{Name: "example.net"}},
			))
		})

		Context("when app has just hosts, no domains", func() {
			It("returns Application", func() {
				repo := FakeRepo{yaml: `---
          name: foo
          hosts:
          - host1
          - host2`,
				}
				manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}
				routes := manifestAppFinder.RoutesFromManifest("example.com")

				Expect(routes).To(ConsistOf(
					Route{Host: "host1", Domain: Domain{Name: "example.com"}},
					Route{Host: "host2", Domain: Domain{Name: "example.com"}},
				))
			})
		})

		PContext("when app has just routes, no hosts or domains", func() {
			It("returns those routes", func() {
				repo := FakeRepo{yaml: `---
          name: foo
          routes:
          - route1.domain1
          - route2.domain2`,
				}
				manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}
				routes := manifestAppFinder.RoutesFromManifest("example.com")

				Expect(routes).To(ConsistOf(
					Route{Host: "route1", Domain: Domain{Name: "domain1"}},
					Route{Host: "route2", Domain: Domain{Name: "domain2"}},
				))
			})
		})

		Context("when no matching application", func() {
			It("returns nil", func() {
				repo := FakeRepo{yaml: ``}
				manifestAppFinder := ManifestAppFinder{AppName: "foo", Repo: &repo}

				Expect(manifestAppFinder.RoutesFromManifest("example.com")).To(BeNil())
			})
		})
	})
})

type FakeRepo struct {
	yaml string
	err  error
}

func (r *FakeRepo) ReadManifest(path string) (*manifest.Manifest, error) {
	yamlMap := generic.NewMap()
	candiedyaml.Unmarshal([]byte(r.yaml), yamlMap)
	return &manifest.Manifest{Data: yamlMap}, r.err
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"runtime"
	"sync"
	"time"
)

type M map[string]interface{}

type Request struct {
	Method string
	Token  string
}

type Response struct {
	Token string      `json:"token"`
	Error string      `json:"error"`
	Data  interface{} `json:"data"`
}

type Broker struct {
	served uint
	start  time.Time
	rLck   sync.Mutex
	wLck   sync.Mutex
	r      io.Reader
	w      io.Writer
	in     *bufio.Reader
	out    *json.Encoder
	Wg     *sync.WaitGroup
}

func NewBroker(r io.Reader, w io.Writer) *Broker {
	return &Broker{
		r:   r,
		w:   w,
		in:  bufio.NewReader(r),
		out: json.NewEncoder(w),
		Wg:  &sync.WaitGroup{},
	}
}

func (b *Broker) Send(resp Response) error {
	err := b.SendNoLog(resp)
	if err != nil {
		logger.Println("Cannot send result", err)
	}
	return err
}

func (b *Broker) SendNoLog(resp Response) error {
	b.wLck.Lock()
	defer b.wLck.Unlock()

	if resp.Data == nil {
		resp.Data = M{}
	}

	s, err := json.Marshal(resp)
	if err != nil {
		return err
	}

	// the only expected write failure are due to broken pipes
	// which usually means the client has gone away so just ignore the error
	b.w.Write(s)
	b.w.Write([]byte{'\n'})
	return nil
}

func (b *Broker) call(req *Request, cl Caller) {
	defer b.Wg.Done()
	b.served++

	defer func() {
		err := recover()
		if err != nil {
			logger.Printf("%v#%v PANIC: %v\n", req.Method, req.Token, err)
			b.Send(Response{
				Token: req.Token,
				Error: "broker: " + req.Method + "#" + req.Token + " PANIC",
			})
		}
	}()

	res, err := cl.Call()
	b.Send(Response{
		Token: req.Token,
		Error: err,
		Data:  res,
	})
}

func (b *Broker) accept() (stopLooping bool) {
	line, err := b.in.ReadBytes('\n')

	if err == io.EOF {
		stopLooping = true
	} else if err != nil {
		logger.Println("Cannot read input", err)
		b.Send(Response{
			Error: err.Error(),
		})
		return
	}

	req := &Request{}
	dec := json.NewDecoder(bytes.NewReader(line))
	// if this fails, we are unable to return a useful error(no token to send it to)
	// so we'll simply/implicitly drop the request since it has no method
	// we can safely assume that all such cases will be empty lines and not an actual request
	dec.Decode(&req)

	if req.Method == "" {
		return
	}

	if req.Method == "bye-ni" {
		return true
	}

	m := registry.Lookup(req.Method)
	if m == nil {
		e := "Invalid method " + req.Method
		logger.Println(e)
		b.Send(Response{
			Token: req.Token,
			Error: e,
		})
		return
	}

	cl := m(b)
	err = dec.Decode(cl)
	if err != nil {
		logger.Println("Cannot decode arg", err)
		b.Send(Response{
			Token: req.Token,
			Error: err.Error(),
		})
		return
	}

	b.Wg.Add(1)
	go b.call(req, cl)

	return
}

func (b *Broker) Loop(decorate bool) {
	b.start = time.Now()

	if decorate {
		go b.SendNoLog(Response{
			Token: "margo.hello",
			Data: M{
				"time": b.start.String(),
			},
		})
	}

	for {
		stopLooping := b.accept()
		if stopLooping {
			break
		}
		runtime.Gosched()
	}

	if decorate {
		b.SendNoLog(Response{
			Token: "margo.bye-ni",
			Data: M{
				"served": b.served,
				"uptime": time.Now().Sub(b.start).String(),
			},
		})
	}
}
<commit_msg>* try not to return null results to the client<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"runtime"
	"sync"
	"time"
)

type M map[string]interface{}

type Request struct {
	Method string
	Token  string
}

type Response struct {
	Token string      `json:"token"`
	Error string      `json:"error"`
	Data  interface{} `json:"data"`
}

type Broker struct {
	served uint
	start  time.Time
	rLck   sync.Mutex
	wLck   sync.Mutex
	r      io.Reader
	w      io.Writer
	in     *bufio.Reader
	out    *json.Encoder
	Wg     *sync.WaitGroup
}

func NewBroker(r io.Reader, w io.Writer) *Broker {
	return &Broker{
		r:   r,
		w:   w,
		in:  bufio.NewReader(r),
		out: json.NewEncoder(w),
		Wg:  &sync.WaitGroup{},
	}
}

func (b *Broker) Send(resp Response) error {
	err := b.SendNoLog(resp)
	if err != nil {
		logger.Println("Cannot send result", err)
	}
	return err
}

func (b *Broker) SendNoLog(resp Response) error {
	b.wLck.Lock()
	defer b.wLck.Unlock()

	if resp.Data == nil {
		resp.Data = M{}
	}

	s, err := json.Marshal(resp)
	if err != nil {
		return err
	}

	// the only expected write failure are due to broken pipes
	// which usually means the client has gone away so just ignore the error
	b.w.Write(s)
	b.w.Write([]byte{'\n'})
	return nil
}

func (b *Broker) call(req *Request, cl Caller) {
	defer b.Wg.Done()
	b.served++

	defer func() {
		err := recover()
		if err != nil {
			logger.Printf("%v#%v PANIC: %v\n", req.Method, req.Token, err)
			b.Send(Response{
				Token: req.Token,
				Error: "broker: " + req.Method + "#" + req.Token + " PANIC",
			})
		}
	}()

	res, err := cl.Call()
	if res == nil {
		res = M{}
	} else if v, ok := res.(M); ok && v == nil {
		res = M{}
	}

	b.Send(Response{
		Token: req.Token,
		Error: err,
		Data:  res,
	})
}

func (b *Broker) accept() (stopLooping bool) {
	line, err := b.in.ReadBytes('\n')

	if err == io.EOF {
		stopLooping = true
	} else if err != nil {
		logger.Println("Cannot read input", err)
		b.Send(Response{
			Error: err.Error(),
		})
		return
	}

	req := &Request{}
	dec := json.NewDecoder(bytes.NewReader(line))
	// if this fails, we are unable to return a useful error(no token to send it to)
	// so we'll simply/implicitly drop the request since it has no method
	// we can safely assume that all such cases will be empty lines and not an actual request
	dec.Decode(&req)

	if req.Method == "" {
		return
	}

	if req.Method == "bye-ni" {
		return true
	}

	m := registry.Lookup(req.Method)
	if m == nil {
		e := "Invalid method " + req.Method
		logger.Println(e)
		b.Send(Response{
			Token: req.Token,
			Error: e,
		})
		return
	}

	cl := m(b)
	err = dec.Decode(cl)
	if err != nil {
		logger.Println("Cannot decode arg", err)
		b.Send(Response{
			Token: req.Token,
			Error: err.Error(),
		})
		return
	}

	b.Wg.Add(1)
	go b.call(req, cl)

	return
}

func (b *Broker) Loop(decorate bool) {
	b.start = time.Now()

	if decorate {
		go b.SendNoLog(Response{
			Token: "margo.hello",
			Data: M{
				"time": b.start.String(),
			},
		})
	}

	for {
		stopLooping := b.accept()
		if stopLooping {
			break
		}
		runtime.Gosched()
	}

	if decorate {
		b.SendNoLog(Response{
			Token: "margo.bye-ni",
			Data: M{
				"served": b.served,
				"uptime": time.Now().Sub(b.start).String(),
			},
		})
	}
}
<|endoftext|>