<commit_before>package logger

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strconv"
	"sync"
	"time"
)

const (
	_VER string = "1.0.0"
)

type UNIT int64

const (
	_       = iota
	KB UNIT = 1 << (iota * 10)
	MB
	GB
	TB
)

const (
	LOG int = iota
	DEBUG
	INFO
	WARN
	ERROR
	FATAL
)

const (
	OS_LINUX = iota
	OS_X
	OS_WIN
	OS_OTHERS
)

//日志结构对象
var logObj *LogFile
var logLevel = 1
var maxFileSize int64
var maxFileCount int32
var dailyFlag bool
var consoleAppender = false

const (
	//TimeDayFormat 日期格式化到日
	TimeDayFormat = "2006-01-02"
	//TimeFormat 日期格式化到秒
	TimeFormat = "2006-01-02 15:04:05"
)

var logFormat = "%s %s:%d %s %s"
var logObjFormat = "%s %s:%d %s %s %s %s"
var consoleFormat = "%s:%d %s %s"

//SetConsole 设置终端是否显示
func SetConsole(isConsole bool) {
	consoleAppender = isConsole
}

//SetLevel 设置日子级别
func SetLevel(_level int) {
	logLevel = _level
}

//RollingLogger 生成按文件大小及数量分割日子类
func RollingLogger(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
	rollingLogger(fileDir, fileName, maxNumber, maxSize, _unit)
}

//SetRollingFile 生成按文件大小及数量分割日子类
func SetRollingFile(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
	rollingLogger(fileDir, fileName, maxNumber, maxSize, _unit)
}

func rollingLogger(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
	maxFileCount = maxNumber
	maxFileSize = maxSize * int64(_unit)
	dailyFlag = false
	logObj = &LogFile{dir: fileDir, filename: fileName, mu: new(sync.Mutex)}
	logObj.mu.Lock()
	defer logObj.mu.Unlock()

	logObj.logfile, _ = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0777)
	fi, err := logObj.logfile.Stat()
	if err != nil {
		log.Println(err.Error())
		return
	}
	logObj.filesize = fi.Size()
}

//DailyLogger new按日期分割日子类
func DailyLogger(fileDir, filename string) {
	dailyLogger(fileDir, filename)
}

func dailyLogger(fileDir, fileName string) {
	dailyFlag = true
	t, _ := time.Parse(TimeDayFormat, time.Now().Format(TimeDayFormat))
	logObj = &LogFile{dir: fileDir, filename: fileName, _date: &t, mu: new(sync.Mutex)}
	logObj.mu.Lock()
	defer logObj.mu.Unlock()

	if !logObj.isMustRename() {
		var err error
		logObj.logfile, err = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0777)
		if err != nil {
			log.Println(err.Error())
		}
	} else {
		logObj.rename()
	}
}

func concat(delimiter string, input ...interface{}) string {
	buffer := bytes.Buffer{}
	l := len(input)
	for i := 0; i < l; i++ {
		buffer.WriteString(fmt.Sprint(input[i]))
		if i < l-1 {
			buffer.WriteString(delimiter)
		}
	}
	return buffer.String()
}

func console(msg string) {
	if logObj == nil || logObj.logfile == nil || consoleAppender {
		log.Print(msg)
	}
}

func buildJSONMessage(level int, l *LogObj, msg string) string {
	file, line := getTraceFileLine()
	logInfo := map[string]interface{}{"atime": time.Now().Format(TimeFormat), "bfile": file + " " + strconv.Itoa(line), "clevel": getTraceLevelName(level)}

	if l != nil {
		logInfo["dlogid"] = l.logid
		logInfo["etag"] = l.tag
	}
	logInfo["msg"] = msg
	resb, err := json.Marshal(logInfo)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	return string(resb) + getOsEol()
}

func buildLogMessage(level int, l *LogObj, msg string) string {
	file, line := getTraceFileLine()
	logInfo := ""
	if l == nil {
		logInfo = fmt.Sprintf(logFormat+getOsEol(), time.Now().Format(TimeFormat), file, line, getTraceLevelName(level), msg)
	} else {
		logInfo = fmt.Sprintf(logObjFormat+getOsEol(), time.Now().Format(TimeFormat), file, line, l.logid, l.tag, getTraceLevelName(level), msg)
	}
	return logInfo
}

func catchError() {
	if err := recover(); err != nil {
		log.Println("err", err)
	}
}

//JSON 设置日志格式
func JSON(js bool) {
	if js {
		logObj.SetJSON()
	} else {
		logObj.UnSetJSON()
	}
}

//Trace write
func Trace(level int, l *LogObj, v ...interface{}) bool {
	defer catchError()
	if logObj != nil {
		logObj.mu.Lock()
		defer logObj.mu.Unlock()
	}
	msg := concat(" ", v...)
	logStr := ""
	if (l != nil && l.json) || logObj.json {
		logStr = buildJSONMessage(level, l, msg)
	} else {
		logStr = buildLogMessage(level, l, msg)
	}
	console(logStr)

	if v[0] != nil {
		if remote, ok := v[0].(string); ok && remote == "remote" {
			remoteMsg := concat(" ", v[1:]...)
			go httpLog(remoteMsg)
		}
	}
	if level >= logLevel {
		if logObj != nil {
			_, err := logObj.write([]byte(logStr))
			if err != nil {
				log.Println(err.Error())
				return false
			}
		}
	}
	return true
}

//Log LOG
func Log(v ...interface{}) bool {
	return Trace(LOG, nil, v...)
}

//Debug DEBUG
func Debug(v ...interface{}) bool {
	return Trace(DEBUG, nil, v...)
}

//Info INFO
func Info(v ...interface{}) bool {
	return Trace(INFO, nil, v...)
}

//Warn WARN
func Warn(v ...interface{}) bool {
	return Trace(WARN, nil, v...)
}

//Error ERROR
func Error(v ...interface{}) bool {
	return Trace(ERROR, nil, v...)
}

//Fatal FATAL
func Fatal(v ...interface{}) bool {
	return Trace(FATAL, nil, v...)
}
<commit_msg>修改json时默认输出错误<commit_after>package logger

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"strconv"
	"sync"
	"time"
)

const (
	_VER string = "1.0.0"
)

type UNIT int64

const (
	_       = iota
	KB UNIT = 1 << (iota * 10)
	MB
	GB
	TB
)

const (
	LOG int = iota
	DEBUG
	INFO
	WARN
	ERROR
	FATAL
)

const (
	OS_LINUX = iota
	OS_X
	OS_WIN
	OS_OTHERS
)

//日志结构对象
var logObj *LogFile
var logLevel = 1
var maxFileSize int64
var maxFileCount int32
var dailyFlag bool
var consoleAppender = false

const (
	//TimeDayFormat 日期格式化到日
	TimeDayFormat = "2006-01-02"
	//TimeFormat 日期格式化到秒
	TimeFormat = "2006-01-02 15:04:05"
)

var logFormat = "%s %s:%d %s %s"
var logObjFormat = "%s %s:%d %s %s %s %s"
var consoleFormat = "%s:%d %s %s"

//SetConsole 设置终端是否显示
func SetConsole(isConsole bool) {
	consoleAppender = isConsole
}

//SetLevel 设置日子级别
func SetLevel(_level int) {
	logLevel = _level
}

//RollingLogger 生成按文件大小及数量分割日子类
func RollingLogger(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
	rollingLogger(fileDir, fileName, maxNumber, maxSize, _unit)
}

//SetRollingFile 生成按文件大小及数量分割日子类
func SetRollingFile(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
	rollingLogger(fileDir, fileName, maxNumber, maxSize, _unit)
}

func rollingLogger(fileDir, fileName string, maxNumber int32, maxSize int64, _unit UNIT) {
	maxFileCount = maxNumber
	maxFileSize = maxSize * int64(_unit)
	dailyFlag = false
	logObj = &LogFile{dir: fileDir, filename: fileName, mu: new(sync.Mutex)}
	logObj.mu.Lock()
	defer logObj.mu.Unlock()

	logObj.logfile, _ = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0777)
	fi, err := logObj.logfile.Stat()
	if err != nil {
		log.Println(err.Error())
		return
	}
	logObj.filesize = fi.Size()
}

//DailyLogger new按日期分割日子类
func DailyLogger(fileDir, filename string) {
	dailyLogger(fileDir, filename)
}

func dailyLogger(fileDir, fileName string) {
	dailyFlag = true
	t, _ := time.Parse(TimeDayFormat, time.Now().Format(TimeDayFormat))
	logObj = &LogFile{dir: fileDir, filename: fileName, _date: &t, mu: new(sync.Mutex)}
	logObj.mu.Lock()
	defer logObj.mu.Unlock()

	if !logObj.isMustRename() {
		var err error
		logObj.logfile, err = os.OpenFile(fileDir+"/"+fileName, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0777)
		if err != nil {
			log.Println(err.Error())
		}
	} else {
		logObj.rename()
	}
}

func concat(delimiter string, input ...interface{}) string {
	buffer := bytes.Buffer{}
	l := len(input)
	for i := 0; i < l; i++ {
		buffer.WriteString(fmt.Sprint(input[i]))
		if i < l-1 {
			buffer.WriteString(delimiter)
		}
	}
	return buffer.String()
}

func console(msg string) {
	if logObj == nil || logObj.logfile == nil || consoleAppender {
		log.Print(msg)
	}
}

func buildJSONMessage(level int, l *LogObj, msg string) string {
	file, line := getTraceFileLine()
	logInfo := map[string]interface{}{"atime": time.Now().Format(TimeFormat), "bfile": file + " " + strconv.Itoa(line), "clevel": getTraceLevelName(level)}

	if l != nil {
		logInfo["dlogid"] = l.logid
		logInfo["etag"] = l.tag
	}
	logInfo["msg"] = msg
	resb, err := json.Marshal(logInfo)
	if err != nil {
		log.Println(err.Error())
		return ""
	}
	return string(resb) + getOsEol()
}

func buildLogMessage(level int, l *LogObj, msg string) string {
	file, line := getTraceFileLine()
	logInfo := ""
	if l == nil {
		logInfo = fmt.Sprintf(logFormat+getOsEol(), time.Now().Format(TimeFormat), file, line, getTraceLevelName(level), msg)
	} else {
		logInfo = fmt.Sprintf(logObjFormat+getOsEol(), time.Now().Format(TimeFormat), file, line, l.logid, l.tag, getTraceLevelName(level), msg)
	}
	return logInfo
}

func catchError() {
	if err := recover(); err != nil {
		log.Println("err", err)
	}
}

//JSON 设置日志格式
func JSON(js bool) {
	if js {
		logObj.SetJSON()
	} else {
		logObj.UnSetJSON()
	}
}

//Trace write
func Trace(level int, l *LogObj, v ...interface{}) bool {
	defer catchError()
	if logObj != nil {
		logObj.mu.Lock()
		defer logObj.mu.Unlock()
	}
	msg := concat(" ", v...)
	logStr := ""
	if l != nil && (l.json || logObj.json) {
		logStr = buildJSONMessage(level, l, msg)
	} else {
		logStr = buildLogMessage(level, l, msg)
	}
	console(logStr)

	if v[0] != nil {
		if remote, ok := v[0].(string); ok && remote == "remote" {
			remoteMsg := concat(" ", v[1:]...)
			go httpLog(remoteMsg)
		}
	}
	if level >= logLevel {
		if logObj != nil {
			_, err := logObj.write([]byte(logStr))
			if err != nil {
				log.Println(err.Error())
				return false
			}
		}
	}
	return true
}

//Log LOG
func Log(v ...interface{}) bool {
	return Trace(LOG, nil, v...)
}

//Debug DEBUG
func Debug(v ...interface{}) bool {
	return Trace(DEBUG, nil, v...)
}

//Info INFO
func Info(v ...interface{}) bool {
	return Trace(INFO, nil, v...)
}

//Warn WARN
func Warn(v ...interface{}) bool {
	return Trace(WARN, nil, v...)
}

//Error ERROR
func Error(v ...interface{}) bool {
	return Trace(ERROR, nil, v...)
}

//Fatal FATAL
func Fatal(v ...interface{}) bool {
	return Trace(FATAL, nil, v...)
}
<|endoftext|>


<commit_before>package rtime

import (
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"math/rand"
	"time"

	"io"

	"github.com/boltdb/bolt"
	"github.com/juju/errors"
)

var (
	boltdb *bolt.DB
)

const (
	TFormat = "2006-01-02T15:04:05.999999"
)

func MustInitWriter(pth string) {
	var err error
	boltdb, err = bolt.Open(pth, 0600, nil)

	if err != nil {
		LOGGER.Error("db_open_failed", "err", errors.ErrorStack(err))
		panic(err)
	}
}

func Write(data []byte, p *packet) {
	err := boltdb.Update(func(tx *bolt.Tx) error {
		app, err := tx.CreateBucketIfNotExists([]byte(p.App))
		if err != nil {
			LOGGER.Error("cant_create_app_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		view, err := app.CreateBucketIfNotExists([]byte(p.Name))
		if err != nil {
			LOGGER.Error("cant_create_view_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		host, err := view.CreateBucketIfNotExists([]byte(p.Host))
		if err != nil {
			LOGGER.Error("cant_create_host_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		timings, err := host.CreateBucketIfNotExists([]byte("timings"))
		if err != nil {
			LOGGER.Error("cant_create_timings_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		jsons, err := host.CreateBucketIfNotExists([]byte("jsons"))
		if err != nil {
			LOGGER.Error("cant_create_jsons_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		ts := []byte(time.Now().Format(TFormat))
		LOGGER.Debug("key", "ts", string(ts))

		b := make([]byte, 8)
		binary.LittleEndian.PutUint64(b, p.OTime)

		err = timings.Put(ts, b)
		if err != nil {
			LOGGER.Error("cant_insert_timing", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		err = jsons.Put(ts, data)
		if err != nil {
			LOGGER.Error("cant_insert_json", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		LOGGER.Debug("inserted", "id", string(ts))
		return nil
	})

	if err != nil {
		LOGGER.Error("boltd_update_failed", "err", errors.ErrorStack(err))
	}
}

func ListViews(appname string) (views []string, err error) {
	err = errors.Trace(
		boltdb.View(func(tx *bolt.Tx) error {
			app := tx.Bucket([]byte(appname))
			if app == nil {
				LOGGER.Error("unknown_app", "app", appname)
				return errors.New("unknown app")
			}

			return errors.Trace(
				app.ForEach(func(name, value []byte) error {
					if value == nil {
						views = append(views, string(name))
					}
					return nil
				}),
			)
		}),
	)
	return
}

func ListApps() (apps []string, err error) {
	err = errors.Trace(
		boltdb.View(func(tx *bolt.Tx) error {
			return errors.Trace(
				tx.ForEach(func(name []byte, _ *bolt.Bucket) error {
					apps = append(apps, string(name))
					return nil
				}),
			)
		}),
	)
	return
}

func UniqueID() string {
	u := make([]byte, 16)
	_, err := rand.Read(u)
	if err != nil {
		LOGGER.Error("rand_failed", "err", errors.ErrorStack(err))
	}
	return hex.EncodeToString(u)
}

type ViewData struct {
	timings []uint16
	id      string
	ceiling uint64
	ids     []string  // not exported to clients
	created time.Time // not exported
}

func (vd *ViewData) writeTo(w io.Writer) error {
	LOGGER.Info("vd.id", "id", vd.id, "len", len(vd.id))
	for _, c := range vd.id {
		w.Write([]byte{byte(c), 0})
	}

	b := make([]byte, 8)
	binary.LittleEndian.PutUint64(b, vd.ceiling)
	w.Write(b)

	for _, v := range vd.timings {
		w.Write([]byte{byte(v % 256), byte(v / 256)})
	}
	return nil
}

func GetViewData(
	app, view, host, starts, ends string, floor, ceiling uint64,
) (*ViewData, error) {
	ids := []string{}
	timings := []uint64{}

	start, err := time.Parse(TFormat, starts)
	if err != nil {
		return nil, errors.Trace(err)
	}

	end, err := time.Parse(TFormat, ends)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if end.Before(start) || end.UnixNano()-start.UnixNano() < 1024 {
		return nil, errors.New(
			fmt.Sprintf(
				"invalid start = %s, end = %s, close=%t",
				start.UnixNano(), end.UnixNano(), !end.Before(start),
			),
		)
	}

	err = errors.Trace(
		boltdb.View(func(tx *bolt.Tx) error {
			appb := tx.Bucket([]byte(app))
			if appb == nil {
				LOGGER.Error("unknown_app", "app", app)
				return errors.New("unknown app")
			}

			viewb := appb.Bucket([]byte(view))
			if viewb == nil {
				LOGGER.Error("unknown_view", "app", app, "view", view)
				return errors.New("unknown view")
			}

			if host == "" {
				err := viewb.ForEach(func(name, value []byte) error {
					if value != nil {
						// should never happen
						return nil
					}

					var err error

					ids, timings, err = process_host(
						viewb.Bucket(name), ids, timings, starts, ends,
					)

					return errors.Trace(err)
				})

				if err != nil {
					return errors.Trace(err)
				}
			} else {
				hostb := viewb.Bucket([]byte(host))
				if hostb == nil {
					LOGGER.Error(
						"unknown_host", "app", app, "view", view, "host", host,
					)
					return errors.New("unknown host")
				}

				var err error
				ids, timings, err = process_host(hostb, ids, timings, starts, ends)

				return errors.Trace(err)
			}

			return nil
		}),
	)

	if err != nil {
		return nil, errors.Trace(err)
	}

	ceiling, tdigest, idigest := diget(start, end, ids, timings, floor, ceiling)
	LOGGER.Debug("digest", "tdigest", tdigest, "idigest", idigest)
	return &ViewData{
		timings: tdigest,
		ids:     idigest,
		ceiling: ceiling,
		id:      UniqueID(),
		created: time.Now(),
	}, nil
}

func d2slot(snano, step uint64, dt string) uint16 {
	ts, err := time.Parse(TFormat, dt)
	if err != nil {
		LOGGER.Error("invalid_ts", "ts", dt)
		return 0
	}

	return uint16((uint64(ts.UnixNano()) - snano) / step)
}

func normalise(v, floor, ceiling uint64) uint8 {
	if v > ceiling {
		return 63
	}
	return uint8(64*(float32(v-floor)/float32(ceiling-floor)) - 1)
}

func pack(slot uint16, v uint8) uint16 {
	return uint16(v)*1024 + (slot % 1024)
}

func diget(
	start, end time.Time, ids []string, timings []uint64, floor, ceiling uint64,
) (uint64, []uint16, []string) {
	LOGGER.Info("digest", "ids", ids, "timings", timings)

	snano := uint64(start.UnixNano())
	step := (uint64(end.UnixNano()) - snano) / 1024

	tdigest := make([]uint16, 1024)
	idigest := make([]string, 1024)

	if ceiling == 0 {
		for _, v := range timings {
			if ceiling < v {
				ceiling = v
			}
		}
	}

	for i := range ids {
		slot := d2slot(snano, step, ids[i])
		idigest[slot] = ids[i]
		tdigest[slot] = pack(slot, normalise(timings[i], floor, ceiling))
	}

	return ceiling, tdigest, idigest
}

func process_host(
	hostb *bolt.Bucket, ids []string, timings []uint64, start, end string,
) ([]string, []uint64, error) {

	timingsb := hostb.Bucket([]byte("timings"))
	if timingsb == nil {
		LOGGER.Warn("no timings bucket")
		return ids, timings, nil
	}

	c := timingsb.Cursor()

	LOGGER.Debug("process_host", "start", start, "end", end)
	for k, v := c.Seek([]byte(start)); true; k, v = c.Next() {
		sk := string(k)
		if sk > end || sk == "" {
			break
		}

		if v == nil {
			// should never happen
			continue
		}

		ids = append(ids, sk)
		timings = append(timings, binary.LittleEndian.Uint64(v))
	}

	return ids, timings, nil
}

func JSON(id string) ([]byte, error) { return nil, nil }
<commit_msg>/apps now returns views and hosts in one go<commit_after>package rtime

import (
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"math/rand"
	"time"

	"io"

	"github.com/boltdb/bolt"
	"github.com/juju/errors"
)

var (
	boltdb *bolt.DB
)

const (
	TFormat = "2006-01-02T15:04:05.999999"
)

func MustInitWriter(pth string) {
	var err error
	boltdb, err = bolt.Open(pth, 0600, nil)

	if err != nil {
		LOGGER.Error("db_open_failed", "err", errors.ErrorStack(err))
		panic(err)
	}
}

func Write(data []byte, p *packet) {
	err := boltdb.Update(func(tx *bolt.Tx) error {
		app, err := tx.CreateBucketIfNotExists([]byte(p.App))
		if err != nil {
			LOGGER.Error("cant_create_app_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		view, err := app.CreateBucketIfNotExists([]byte(p.Name))
		if err != nil {
			LOGGER.Error("cant_create_view_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		host, err := view.CreateBucketIfNotExists([]byte(p.Host))
		if err != nil {
			LOGGER.Error("cant_create_host_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		timings, err := host.CreateBucketIfNotExists([]byte("timings"))
		if err != nil {
			LOGGER.Error("cant_create_timings_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		jsons, err := host.CreateBucketIfNotExists([]byte("jsons"))
		if err != nil {
			LOGGER.Error("cant_create_jsons_bucket", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		ts := []byte(time.Now().Format(TFormat))
		LOGGER.Debug("key", "ts", string(ts))

		b := make([]byte, 8)
		binary.LittleEndian.PutUint64(b, p.OTime)

		err = timings.Put(ts, b)
		if err != nil {
			LOGGER.Error("cant_insert_timing", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		err = jsons.Put(ts, data)
		if err != nil {
			LOGGER.Error("cant_insert_json", "err", errors.ErrorStack(err))
			return errors.Trace(err)
		}

		LOGGER.Debug("inserted", "id", string(ts))
		return nil
	})

	if err != nil {
		LOGGER.Error("boltd_update_failed", "err", errors.ErrorStack(err))
	}
}

func ListViews(appname string) (views []string, err error) {
	err = errors.Trace(
		boltdb.View(func(tx *bolt.Tx) error {
			app := tx.Bucket([]byte(appname))
			if app == nil {
				LOGGER.Error("unknown_app", "app", appname)
				return errors.New("unknown app")
			}

			return errors.Trace(
				app.ForEach(func(name, value []byte) error {
					if value == nil {
						views = append(views, string(name))
					}
					return nil
				}),
			)
		}),
	)
	return
}

type View struct {
	Name  string   `json:"name"`
	Hosts []string `json:"hosts"`
}

type App struct {
	Name  string `json:"name"`
	Views []View `json:"views"`
}

func ListApps() (apps []App, err error) {
	err = boltdb.View(func(tx *bolt.Tx) error {
		err := tx.ForEach(func(name []byte, appb *bolt.Bucket) error {
			app := App{Name: string(name), Views: []View{}}
			apps = append(apps, app)

			err := appb.ForEach(func(name, value []byte) error {
				if value != nil {
					// should never happen
					return nil
				}
				view := View{Name: string(name), Hosts: []string{}}
				app.Views = append(app.Views, view)

				err := appb.Bucket(name).ForEach(func(name, hostb []byte) error {
					if hostb != nil {
						// should never happen
						return nil
					}
					view.Hosts = append(view.Hosts, string(name))
					return nil
				})

				return errors.Trace(err)
			})

			return errors.Trace(err)
		})

		return errors.Trace(err)
	})

	return apps, errors.Trace(err)
}

func UniqueID() string {
	u := make([]byte, 16)
	_, err := rand.Read(u)
	if err != nil {
		LOGGER.Error("rand_failed", "err", errors.ErrorStack(err))
	}
	return hex.EncodeToString(u)
}

type ViewData struct {
	timings []uint16
	id      string
	ceiling uint64
	ids     []string  // not exported to clients
	created time.Time // not exported
}

func (vd *ViewData) writeTo(w io.Writer) error {
	LOGGER.Info("vd.id", "id", vd.id, "len", len(vd.id))
	for _, c := range vd.id {
		w.Write([]byte{byte(c), 0})
	}

	b := make([]byte, 8)
	binary.LittleEndian.PutUint64(b, vd.ceiling)
	w.Write(b)

	for _, v := range vd.timings {
		w.Write([]byte{byte(v % 256), byte(v / 256)})
	}
	return nil
}

func GetViewData(
	app, view, host, starts, ends string, floor, ceiling uint64,
) (*ViewData, error) {
	ids := []string{}
	timings := []uint64{}

	start, err := time.Parse(TFormat, starts)
	if err != nil {
		return nil, errors.Trace(err)
	}

	end, err := time.Parse(TFormat, ends)
	if err != nil {
		return nil, errors.Trace(err)
	}

	if end.Before(start) || end.UnixNano()-start.UnixNano() < 1024 {
		return nil, errors.New(
			fmt.Sprintf(
				"invalid start = %s, end = %s, close=%t",
				start.UnixNano(), end.UnixNano(), !end.Before(start),
			),
		)
	}

	err = errors.Trace(
		boltdb.View(func(tx *bolt.Tx) error {
			appb := tx.Bucket([]byte(app))
			if appb == nil {
				LOGGER.Error("unknown_app", "app", app)
				return errors.New("unknown app")
			}

			viewb := appb.Bucket([]byte(view))
			if viewb == nil {
				LOGGER.Error("unknown_view", "app", app, "view", view)
				return errors.New("unknown view")
			}

			if host == "" {
				err := viewb.ForEach(func(name, value []byte) error {
					if value != nil {
						// should never happen
						return nil
					}

					var err error

					ids, timings, err = process_host(
						viewb.Bucket(name), ids, timings, starts, ends,
					)

					return errors.Trace(err)
				})

				if err != nil {
					return errors.Trace(err)
				}
			} else {
				hostb := viewb.Bucket([]byte(host))
				if hostb == nil {
					LOGGER.Error(
						"unknown_host", "app", app, "view", view, "host", host,
					)
					return errors.New("unknown host")
				}

				var err error
				ids, timings, err = process_host(hostb, ids, timings, starts, ends)

				return errors.Trace(err)
			}

			return nil
		}),
	)

	if err != nil {
		return nil, errors.Trace(err)
	}

	ceiling, tdigest, idigest := diget(start, end, ids, timings, floor, ceiling)
	LOGGER.Debug("digest", "tdigest", tdigest, "idigest", idigest)
	return &ViewData{
		timings: tdigest,
		ids:     idigest,
		ceiling: ceiling,
		id:      UniqueID(),
		created: time.Now(),
	}, nil
}

func d2slot(snano, step uint64, dt string) uint16 {
	ts, err := time.Parse(TFormat, dt)
	if err != nil {
		LOGGER.Error("invalid_ts", "ts", dt)
		return 0
	}

	return uint16((uint64(ts.UnixNano()) - snano) / step)
}

func normalise(v, floor, ceiling uint64) uint8 {
	if v > ceiling {
		return 63
	}
	return uint8(64*(float32(v-floor)/float32(ceiling-floor)) - 1)
}

func pack(slot uint16, v uint8) uint16 {
	return uint16(v)*1024 + (slot % 1024)
}

func diget(
	start, end time.Time, ids []string, timings []uint64, floor, ceiling uint64,
) (uint64, []uint16, []string) {
	LOGGER.Info("digest", "ids", ids, "timings", timings)

	snano := uint64(start.UnixNano())
	step := (uint64(end.UnixNano()) - snano) / 1024

	tdigest := make([]uint16, 1024)
	idigest := make([]string, 1024)

	if ceiling == 0 {
		for _, v := range timings {
			if ceiling < v {
				ceiling = v
			}
		}
	}

	for i := range ids {
		slot := d2slot(snano, step, ids[i])
		idigest[slot] = ids[i]
		tdigest[slot] = pack(slot, normalise(timings[i], floor, ceiling))
	}

	return ceiling, tdigest, idigest
}

func process_host(
	hostb *bolt.Bucket, ids []string, timings []uint64, start, end string,
) ([]string, []uint64, error) {

	timingsb := hostb.Bucket([]byte("timings"))
	if timingsb == nil {
		LOGGER.Warn("no timings bucket")
		return ids, timings, nil
	}

	c := timingsb.Cursor()

	LOGGER.Debug("process_host", "start", start, "end", end)
	for k, v := c.Seek([]byte(start)); true; k, v = c.Next() {
		sk := string(k)
		if sk > end || sk == "" {
			break
		}

		if v == nil {
			// should never happen
			continue
		}

		ids = append(ids, sk)
		timings = append(timings, binary.LittleEndian.Uint64(v))
	}

	return ids, timings, nil
}

func JSON(id string) ([]byte, error) { return nil, nil }
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	"github.com/labstack/echo"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/cli"
	config "github.com/qiniu/logkit/conf"
	_ "github.com/qiniu/logkit/metric/builtin"
	"github.com/qiniu/logkit/mgr"
	"github.com/qiniu/logkit/times"
	_ "github.com/qiniu/logkit/transforms/builtin"
	. "github.com/qiniu/logkit/utils/models"
	utilsos "github.com/qiniu/logkit/utils/os"
)

//Config of logkit
type Config struct {
	MaxProcs          int      `json:"max_procs"`
	DebugLevel        int      `json:"debug_level"`
	ProfileHost       string   `json:"profile_host"`
	ConfsPath         []string `json:"confs_path"`
	LogPath           string   `json:"log"`
	CleanSelfLog      bool     `json:"clean_self_log"`
	CleanSelfDir      string   `json:"clean_self_dir"`
	CleanSelfPattern  string   `json:"clean_self_pattern"`
	CleanSelfDuration string   `json:"clean_self_duration"`
	CleanSelfLogCnt   int      `json:"clean_self_cnt"`
	TimeLayouts       []string `json:"timeformat_layouts"`
	StaticRootPath    string   `json:"static_root_path"`
	mgr.ManagerConfig
}

var conf Config

const (
	NextVersion        = "v1.5.4"
	defaultReserveCnt  = 5
	defaultLogDir      = "./"
	defaultLogPattern  = "*.log-*"
	defaultLogDuration = 10 * time.Minute
	defaultRotateSize  = 100 * 1024 * 1024
)

const usage = `logkit, Very easy-to-use server agent for collecting & sending logs & metrics.

Usage:

  logkit [commands|flags]

The commands & flags are:

  -v                 print the version to stdout.
  -h                 print logkit usage info to stdout.
  -upgrade           check and upgrade version.

  -f <file>          configuration file to load

Examples:

  # start logkit
  logkit -f logkit.conf

  # check version
  logkit -v

  # checking and upgrade version
  logkit -upgrade
`

var (
	fversion = flag.Bool("v", false, "print the version to stdout")
	upgrade  = flag.Bool("upgrade", false, "check and upgrade version")
	confName = flag.String("f", "logkit.conf", "configuration file to load")
)

func getValidPath(confPaths []string) (paths []string) {
	paths = make([]string, 0)
	exits := make(map[string]bool)
	for _, v := range confPaths {
		rp, err := filepath.Abs(v)
		if err != nil {
			log.Errorf("Get real path of ConfsPath %v error %v, ignore it", v, rp)
			continue
		}
		if _, ok := exits[rp]; ok {
			log.Errorf("ConfsPath %v duplicated, ignore", rp)
			continue
		}
		exits[rp] = true
		paths = append(paths, rp)
	}
	return
}

type MatchFile struct {
	Name    string
	ModTime time.Time
}

type MatchFiles []MatchFile

func (f MatchFiles) Len() int           { return len(f) }
func (f MatchFiles) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
func (f MatchFiles) Less(i, j int) bool { return f[i].ModTime.Before(f[j].ModTime) }

func cleanLogkitLog(dir, pattern string, reserveCnt int) {
	var err error
	path := filepath.Join(dir, pattern)
	matches, err := filepath.Glob(path)
	if err != nil {
		log.Errorf("filepath.Glob path %v error %v", path, err)
		return
	}
	var files MatchFiles
	for _, name := range matches {
		info, err := os.Stat(name)
		if err != nil {
			log.Errorf("os.Stat name %v error %v", name, err)
			continue
		}
		files = append(files, MatchFile{
			Name:    name,
			ModTime: info.ModTime(),
		})
	}
	if len(files) <= reserveCnt {
		return
	}
	sort.Sort(files)
	for _, f := range files[0 : len(files)-reserveCnt] {
		err := os.Remove(f.Name)
		if err != nil {
			log.Errorf("Remove %s failed , error: %v", f, err)
			continue
		}
	}
	return
}

func loopCleanLogkitLog(dir, pattern string, reserveCnt int, duration string, exitchan chan struct{}) {
	if len(dir) <= 0 {
		dir = defaultLogDir
	}
	if len(pattern) <= 0 {
		pattern = defaultLogPattern
	}
	if reserveCnt <= 0 {
		reserveCnt = defaultReserveCnt
	}
	var (
		dur time.Duration
		err error
	)
	if duration != "" {
		dur, err = time.ParseDuration(duration)
		if err != nil {
			log.Warnf("clean self duration parse failed: %v, Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'. Use default duration: 10m", err)
			dur = defaultLogDuration
		}
	} else {
		dur = defaultLogDuration
	}
	ticker := time.NewTicker(dur)
	defer ticker.Stop()
	for {
		select {
		case <-exitchan:
			return
		case <-ticker.C:
			cleanLogkitLog(dir, pattern, reserveCnt)
		}
	}
}

func rotateLog(path string) (file *os.File, err error) {
	newfile := path + "-" + time.Now().Format("0102030405")
	file, err = os.OpenFile(newfile, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666)
	if err != nil {
		err = fmt.Errorf("rotateLog open newfile %v err %v", newfile, err)
		return
	}
	log.SetOutput(file)
	return
}

func loopRotateLogs(path string, rotateSize int64, dur time.Duration, exitchan chan struct{}) {
	file, err := rotateLog(path)
	if err != nil {
		log.Fatal(err)
	}
	ticker := time.NewTicker(dur)
	defer ticker.Stop()
	for {
		select {
		case <-exitchan:
			return
		case <-ticker.C:
			info, err := file.Stat()
			if err != nil {
				log.Warnf("stat log error %v", err)
			} else {
				if info.Size() >= rotateSize {
					newfile, err := rotateLog(path)
					if err != nil {
						log.Errorf("rotate log %v error %v, use old log to write logkit log", path, err)
					} else {
						file.Close()
						file = newfile
					}
				}
			}

		}
	}
}

func usageExit(rc int) {
	fmt.Println(usage)
	os.Exit(rc)
}

//！！！注意： 自动生成 grok pattern代码，下述注释请勿删除！！！
//go:generate go run tools/generators/grok_pattern_generator.go
func main() {
	flag.Usage = func() { usageExit(0) }
	flag.Parse()
	switch {
	case *fversion:
		fmt.Println("logkit version: ", NextVersion)
		osInfo := utilsos.GetOSInfo()
		fmt.Println("Hostname: ", osInfo.Hostname)
		fmt.Println("Core: ", osInfo.Core)
		fmt.Println("OS: ", osInfo.OS)
		fmt.Println("Platform: ", osInfo.Platform)
		return
	case *upgrade:
		cli.CheckAndUpgrade(NextVersion)
		return
	}

	if err := config.LoadEx(&conf, *confName); err != nil {
		log.Fatal("config.Load failed:", err)
	}
	if conf.TimeLayouts != nil {
		times.AddLayout(conf.TimeLayouts)
	}
	if conf.MaxProcs <= 0 {
		conf.MaxProcs = NumCPU
	}
	MaxProcs = conf.MaxProcs
	runtime.GOMAXPROCS(conf.MaxProcs)
	log.SetOutputLevel(conf.DebugLevel)

	var (
		stopRotate         = make(chan struct{}, 0)
		logdir, logpattern string
		err                error
	)
	defer close(stopRotate)
	if conf.LogPath != "" {
		logdir, logpattern, err = LogDirAndPattern(conf.LogPath)
		if err != nil {
			log.Fatal(err)
		}
		go loopRotateLogs(filepath.Join(logdir, logpattern), defaultRotateSize, 10*time.Second, stopRotate)
		conf.CleanSelfPattern = logpattern + "-*"
		conf.CleanSelfDir = logdir
		conf.ManagerConfig.CollectLogPath = filepath.Join(logdir, logpattern+"-*")
	}

	log.Infof("Welcome to use Logkit, Version: %v \n\nConfig: %#v", NextVersion, conf)
	m, err := mgr.NewManager(conf.ManagerConfig)
	if err != nil {
		log.Fatalf("NewManager: %v", err)
	}
	m.Version = NextVersion

	if m.CollectLogRunner != nil {
		go m.CollectLogRunner.Run()
		time.Sleep(time.Second) // 等待1秒让收集器启动
	}

	paths := getValidPath(conf.ConfsPath)
	if len(paths) <= 0 {
		log.Warnf("Cannot read or create any ConfsPath %v", conf.ConfsPath)
	}
	if err = m.Watch(paths); err != nil {
		log.Fatalf("watch path error %v", err)
	}
	m.RestoreWebDir()

	stopClean := make(chan struct{}, 0)
	defer close(stopClean)
	if conf.CleanSelfLog {
		if conf.CleanSelfDir == "" && logdir != "" {
			conf.CleanSelfDir = logdir
		}
		if conf.CleanSelfPattern == "" && logpattern != "" {
			conf.CleanSelfPattern = logpattern + "-*"
		}
		go loopCleanLogkitLog(conf.CleanSelfDir, conf.CleanSelfPattern, conf.CleanSelfLogCnt, conf.CleanSelfDuration, stopClean)
	}
	if len(conf.BindHost) > 0 {
		m.BindHost = conf.BindHost
	}
	e := echo.New()
	e.Static("/", conf.StaticRootPath)

	// start rest service
	rs := mgr.NewRestService(m, e)
	if conf.ProfileHost != "" {
		log.Infof("go profile_host was open at %v", conf.ProfileHost)
		go func() {
			log.Fatal(http.ListenAndServe(conf.ProfileHost, nil))
		}()
	}
	if err = rs.Register(); err != nil {
		log.Fatalf("register master error %v", err)
	}
	utilsos.WaitForInterrupt(func() {
		rs.Stop()
		if conf.CleanSelfLog {
			stopClean <- struct{}{}
		}
		m.Stop()
	})
}
<commit_msg>next verion to v1.5.5<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	"github.com/labstack/echo"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/cli"
	config "github.com/qiniu/logkit/conf"
	_ "github.com/qiniu/logkit/metric/builtin"
	"github.com/qiniu/logkit/mgr"
	"github.com/qiniu/logkit/times"
	_ "github.com/qiniu/logkit/transforms/builtin"
	. "github.com/qiniu/logkit/utils/models"
	utilsos "github.com/qiniu/logkit/utils/os"
)

//Config of logkit
type Config struct {
	MaxProcs          int      `json:"max_procs"`
	DebugLevel        int      `json:"debug_level"`
	ProfileHost       string   `json:"profile_host"`
	ConfsPath         []string `json:"confs_path"`
	LogPath           string   `json:"log"`
	CleanSelfLog      bool     `json:"clean_self_log"`
	CleanSelfDir      string   `json:"clean_self_dir"`
	CleanSelfPattern  string   `json:"clean_self_pattern"`
	CleanSelfDuration string   `json:"clean_self_duration"`
	CleanSelfLogCnt   int      `json:"clean_self_cnt"`
	TimeLayouts       []string `json:"timeformat_layouts"`
	StaticRootPath    string   `json:"static_root_path"`
	mgr.ManagerConfig
}

var conf Config

const (
	NextVersion        = "v1.5.5"
	defaultReserveCnt  = 5
	defaultLogDir      = "./"
	defaultLogPattern  = "*.log-*"
	defaultLogDuration = 10 * time.Minute
	defaultRotateSize  = 100 * 1024 * 1024
)

const usage = `logkit, Very easy-to-use server agent for collecting & sending logs & metrics.

Usage:

  logkit [commands|flags]

The commands & flags are:

  -v                 print the version to stdout.
  -h                 print logkit usage info to stdout.
  -upgrade           check and upgrade version.

  -f <file>          configuration file to load

Examples:

  # start logkit
  logkit -f logkit.conf

  # check version
  logkit -v

  # checking and upgrade version
  logkit -upgrade
`

var (
	fversion = flag.Bool("v", false, "print the version to stdout")
	upgrade  = flag.Bool("upgrade", false, "check and upgrade version")
	confName = flag.String("f", "logkit.conf", "configuration file to load")
)

func getValidPath(confPaths []string) (paths []string) {
	paths = make([]string, 0)
	exits := make(map[string]bool)
	for _, v := range confPaths {
		rp, err := filepath.Abs(v)
		if err != nil {
			log.Errorf("Get real path of ConfsPath %v error %v, ignore it", v, rp)
			continue
		}
		if _, ok := exits[rp]; ok {
			log.Errorf("ConfsPath %v duplicated, ignore", rp)
			continue
		}
		exits[rp] = true
		paths = append(paths, rp)
	}
	return
}

type MatchFile struct {
	Name    string
	ModTime time.Time
}

type MatchFiles []MatchFile

func (f MatchFiles) Len() int           { return len(f) }
func (f MatchFiles) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
func (f MatchFiles) Less(i, j int) bool { return f[i].ModTime.Before(f[j].ModTime) }

func cleanLogkitLog(dir, pattern string, reserveCnt int) {
	var err error
	path := filepath.Join(dir, pattern)
	matches, err := filepath.Glob(path)
	if err != nil {
		log.Errorf("filepath.Glob path %v error %v", path, err)
		return
	}
	var files MatchFiles
	for _, name := range matches {
		info, err := os.Stat(name)
		if err != nil {
			log.Errorf("os.Stat name %v error %v", name, err)
			continue
		}
		files = append(files, MatchFile{
			Name:    name,
			ModTime: info.ModTime(),
		})
	}
	if len(files) <= reserveCnt {
		return
	}
	sort.Sort(files)
	for _, f := range files[0 : len(files)-reserveCnt] {
		err := os.Remove(f.Name)
		if err != nil {
			log.Errorf("Remove %s failed , error: %v", f, err)
			continue
		}
	}
	return
}

func loopCleanLogkitLog(dir, pattern string, reserveCnt int, duration string, exitchan chan struct{}) {
	if len(dir) <= 0 {
		dir = defaultLogDir
	}
	if len(pattern) <= 0 {
		pattern = defaultLogPattern
	}
	if reserveCnt <= 0 {
		reserveCnt = defaultReserveCnt
	}
	var (
		dur time.Duration
		err error
	)
	if duration != "" {
		dur, err = time.ParseDuration(duration)
		if err != nil {
			log.Warnf("clean self duration parse failed: %v, Valid time units are 'ns', 'us' (or 'µs'), 'ms', 's', 'm', 'h'. Use default duration: 10m", err)
			dur = defaultLogDuration
		}
	} else {
		dur = defaultLogDuration
	}
	ticker := time.NewTicker(dur)
	defer ticker.Stop()
	for {
		select {
		case <-exitchan:
			return
		case <-ticker.C:
			cleanLogkitLog(dir, pattern, reserveCnt)
		}
	}
}

func rotateLog(path string) (file *os.File, err error) {
	newfile := path + "-" + time.Now().Format("0102030405")
	file, err = os.OpenFile(newfile, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0666)
	if err != nil {
		err = fmt.Errorf("rotateLog open newfile %v err %v", newfile, err)
		return
	}
	log.SetOutput(file)
	return
}

func loopRotateLogs(path string, rotateSize int64, dur time.Duration, exitchan chan struct{}) {
	file, err := rotateLog(path)
	if err != nil {
		log.Fatal(err)
	}
	ticker := time.NewTicker(dur)
	defer ticker.Stop()
	for {
		select {
		case <-exitchan:
			return
		case <-ticker.C:
			info, err := file.Stat()
			if err != nil {
				log.Warnf("stat log error %v", err)
			} else {
				if info.Size() >= rotateSize {
					newfile, err := rotateLog(path)
					if err != nil {
						log.Errorf("rotate log %v error %v, use old log to write logkit log", path, err)
					} else {
						file.Close()
						file = newfile
					}
				}
			}

		}
	}
}

func usageExit(rc int) {
	fmt.Println(usage)
	os.Exit(rc)
}

//！！！注意： 自动生成 grok pattern代码，下述注释请勿删除！！！
//go:generate go run tools/generators/grok_pattern_generator.go
func main() {
	flag.Usage = func() { usageExit(0) }
	flag.Parse()
	switch {
	case *fversion:
		fmt.Println("logkit version: ", NextVersion)
		osInfo := utilsos.GetOSInfo()
		fmt.Println("Hostname: ", osInfo.Hostname)
		fmt.Println("Core: ", osInfo.Core)
		fmt.Println("OS: ", osInfo.OS)
		fmt.Println("Platform: ", osInfo.Platform)
		return
	case *upgrade:
		cli.CheckAndUpgrade(NextVersion)
		return
	}

	if err := config.LoadEx(&conf, *confName); err != nil {
		log.Fatal("config.Load failed:", err)
	}
	if conf.TimeLayouts != nil {
		times.AddLayout(conf.TimeLayouts)
	}
	if conf.MaxProcs <= 0 {
		conf.MaxProcs = NumCPU
	}
	MaxProcs = conf.MaxProcs
	runtime.GOMAXPROCS(conf.MaxProcs)
	log.SetOutputLevel(conf.DebugLevel)

	var (
		stopRotate         = make(chan struct{}, 0)
		logdir, logpattern string
		err                error
	)
	defer close(stopRotate)
	if conf.LogPath != "" {
		logdir, logpattern, err = LogDirAndPattern(conf.LogPath)
		if err != nil {
			log.Fatal(err)
		}
		go loopRotateLogs(filepath.Join(logdir, logpattern), defaultRotateSize, 10*time.Second, stopRotate)
		conf.CleanSelfPattern = logpattern + "-*"
		conf.CleanSelfDir = logdir
		conf.ManagerConfig.CollectLogPath = filepath.Join(logdir, logpattern+"-*")
	}

	log.Infof("Welcome to use Logkit, Version: %v \n\nConfig: %#v", NextVersion, conf)
	m, err := mgr.NewManager(conf.ManagerConfig)
	if err != nil {
		log.Fatalf("NewManager: %v", err)
	}
	m.Version = NextVersion

	if m.CollectLogRunner != nil {
		go m.CollectLogRunner.Run()
		time.Sleep(time.Second) // 等待1秒让收集器启动
	}

	paths := getValidPath(conf.ConfsPath)
	if len(paths) <= 0 {
		log.Warnf("Cannot read or create any ConfsPath %v", conf.ConfsPath)
	}
	if err = m.Watch(paths); err != nil {
		log.Fatalf("watch path error %v", err)
	}
	m.RestoreWebDir()

	stopClean := make(chan struct{}, 0)
	defer close(stopClean)
	if conf.CleanSelfLog {
		if conf.CleanSelfDir == "" && logdir != "" {
			conf.CleanSelfDir = logdir
		}
		if conf.CleanSelfPattern == "" && logpattern != "" {
			conf.CleanSelfPattern = logpattern + "-*"
		}
		go loopCleanLogkitLog(conf.CleanSelfDir, conf.CleanSelfPattern, conf.CleanSelfLogCnt, conf.CleanSelfDuration, stopClean)
	}
	if len(conf.BindHost) > 0 {
		m.BindHost = conf.BindHost
	}
	e := echo.New()
	e.Static("/", conf.StaticRootPath)

	// start rest service
	rs := mgr.NewRestService(m, e)
	if conf.ProfileHost != "" {
		log.Infof("go profile_host was open at %v", conf.ProfileHost)
		go func() {
			log.Fatal(http.ListenAndServe(conf.ProfileHost, nil))
		}()
	}
	if err = rs.Register(); err != nil {
		log.Fatalf("register master error %v", err)
	}
	utilsos.WaitForInterrupt(func() {
		rs.Stop()
		if conf.CleanSelfLog {
			stopClean <- struct{}{}
		}
		m.Stop()
	})
}
<|endoftext|>


<commit_before>package dns

// This file is in flux

import (
	"math/rand"
	"strings"
	"time"
)

type SecurityStatus int

const (
	SECURE SecurityStatus = iota
	INSECURE
	BOGUS
	INDETERMINATE
)

// Check if the returned message has a delegation signer record
// Algo:
// The auth section's owner name (should be all equal)  - seperate check!
// The ownername of the DS records must match the right side of the qname
//
func AssertDelegationSigner(m *Msg, trustdb []*RR_DNSKEY) error {

	// look for the DS(s)
	dss := make([]*RR_DS, 0)
	// If there are ddssen, there should also be a SIG (what if not?)
	var sig *RR_RRSIG
	for _, r := range m.Ns {
		if d, ok := r.(*RR_DS); ok {
			dss = append(dss, d)
			continue
		}
		if s, ok := r.(*RR_RRSIG); ok {
			if s.TypeCovered == TypeDS {
				sig = s
			}
		}
	}
	if len(dss) == 0 {
		// No DSs found ...
		return nil
	}
	println("DSs found", len(dss))
	if sig == nil {
		// No SIG found ...
		return nil
	}
	println("SIG found")

	// Ownername of the DSs should match the qname
	if CompareLabels(dss[0].Header().Name, m.Question[0].Name) == 0 {
		// No match
	}
	// Optionally keep track of these comparison, it should increase
	println("Match found between delegation DS and qname")
	println(dss[0].String())
	println(sig.String())

	return nil

}

// Types of answers (without looking the RFCs)
// len(m.Ns) > 0
// NS records in there? -> delegation (rcode should be rcode.Success)
//	- secure delegation -> DS should be there
//	- insecure delegation -> Proof of no DS (either NSEC or NSEC3)
//	- plain old DNS delegation -> ...
// SOA record in there? -> nxdomain   (rcode should be rcode.Nxdomain)

// Lookup does a (secure) DNS lookup. The message m contains
// the question to be asked. Lookup returns last packet seen
// which is either the answer or a packet somewhere in the
// tree where the error occured.
// TODO: check return code - see unbound for what we want
//
func Lookup(m *Msg) (*Msg, error) {

	// This is actually what needs to be done when parsing packets too
	a, aaaa := primingZone()
	ds := primingTrust()
	for i, r := range a {
		println(i, r)
	}
	for i, r := range aaaa {
		println(i, r)
	}
	for i, r := range ds {
		println(i, r.String())
	}
	c := new(Client)
	a1 := randomAddress(a)
	println(a1)
	n, _, _, e := c.ExchangeRtt(m, a1+":53")
	if e == nil {
		println(n.String())
	} else {
		println(e.Error())
		return nil, e
	}
	// n is our reply, deal with it
	// Check for DS
	// Check for DS absent (NSEC/NSEC3)
	if len(n.Ns) > 0 && len(n.Answer) == 0 && n.Rcode == RcodeSuccess { // Referral
		// the ns name of the nameservers should match the right most labels. Check the answer
		println("Referral")
		for i, j := range addrFromReferral4(n) {
			println(i, j)
		}
	}
	return nil, nil
}

// Parse a referral packet and return two lists (v4 and v6) of
// ip addresses to try next. As the NS records in a referral
// are not signed (the belong to the child), no validation takes
// place at this step.
func parseReferral(m *Msg) ([]*RR_A, []*RR_AAAA) {
	return nil, nil
}

// Parse the builtin root zone and return two lists (v4 and v6) of
// ip addresses to try.
// Glue checking this is -- should be done much nicer/better
func primingZone() (a, aaaa []string) {
	nss := make(map[string]bool) // List of ns names
	// Walk the records, get each NS for . and look for the addresses
	for rr := range ParseZone(strings.NewReader(NamedRoot), "", "named.root") {
		if rr.RR.Header().Name == "." && rr.RR.Header().Rrtype == TypeNS {
			nss[rr.RR.(*RR_NS).Ns] = true
			continue
		}
		if rr.RR.Header().Rrtype == TypeA {
			for n, _ := range nss {
				if rr.RR.Header().Name == n {
					a = append(a, rr.RR.(*RR_A).A.String())
				}
			}
			continue
		}
		if rr.RR.Header().Rrtype == TypeAAAA {
			for n, _ := range nss {
				if rr.RR.Header().Name == n {
					aaaa = append(aaaa, rr.RR.(*RR_AAAA).AAAA.String())
				}
			}
		}
	}
	// Randomize
	return
}

// Validate the root key with the DS records we've gotten offline
func createTrustDB(dss []*RR_DS, a, aaaa []string) *[]RR_DNSKEY {
	// Query a root server, get the DNSKEY, toDS() and check
	return nil

}

// Parse the builtin trust anchor and return the DS records
func primingTrust() []*RR_DS {
	ta, _ := ReadTrustAnchor(strings.NewReader(RootAnchorXML))
	// Don't care about validity just yet
	dss := make([]*RR_DS, 0)
	for _, t := range ta {
		dss = append(dss, t.Anchor)
	}
	return dss
}

func randomAddress(list []string) string {
	rand.Seed(int64(time.Now().Nanosecond()))
	if len(list) == 0 {
		return ""
	}
	return list[rand.Intn(len(list))]
}

// Pull the addresses out of the referral message
// Check the zone
func addrFromReferral4(m *Msg) (addr []string) {
	for _, ns := range m.Ns {
		if ns.Header().Rrtype != TypeNS {
			continue
		}
		for _, a := range m.Extra {
			if a.Header().Rrtype != TypeA {
				continue
			}
			if ns.(*RR_NS).Ns == a.Header().Name {
				addr = append(addr, a.(*RR_A).A.String())
			}
		}
	}
	return
}
<commit_msg>remove lookup<commit_after><|endoftext|>


<commit_before>package safe

func (s *Safe) Replace(name, username, password string) (*Account, error) {
	account, err := s.Get(name)
	if err != nil {
		return nil, err
	}

	account = NewAccount(name, username, password)
	s.Accounts[name] = *account

	if err = s.save(); err != nil {
		return nil, err
	}

	return account, nil
}
<commit_msg>Cleanup variable scope<commit_after>package safe

func (s *Safe) Replace(name, username, password string) (*Account, error) {
	if _, err := s.Get(name); err != nil {
		return nil, err
	}

	account := NewAccount(name, username, password)
	s.Accounts[name] = *account

	if err := s.save(); err != nil {
		return nil, err
	}

	return account, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

/*
type FileInfo interface {
    Name() string       // base name of the file
    Size() int64        // length in bytes for regular files;
    Mode() FileMode     // file mode bits
    ModTime() time.Time // modification time
    IsDir() bool        // abbreviation for Mode().IsDir()
    Sys() interface{}   // underlying data source (can return nil)
}
*/

type File struct {
	Filename string
	Path     string
	Size     int64
	ModTime  time.Time
	IsDir    bool
}

type FileMessage struct {
	ConnId   int                    `json:"id"`
	Function string                 `json:"type"`
	Data     map[string]interface{} `json:"data"`
}

// Echos the $HOME env to find the home directory, returns root if err
func getHomeDir(id int) string {
	// Create a new ssh session to access the remote host's $HOME env variable
	session, err := conns[id].sshClient.NewSession()
	defer session.Close()

	if err != nil {
		fmt.Println("Error creating ssh session:", err)
		return "/"
	}
	output, err := session.Output("echo $HOME")
	if err != nil {
		fmt.Println("Error finding home dir:", err)
		return "/"
	}
	return strings.TrimSpace(string(output))
}

func FileStruct(file os.FileInfo, dir string) File {
	return File{
		Filename: file.Name(),
		Path:     dir + "/" + file.Name(),
		ModTime:  file.ModTime(),
		IsDir:    file.IsDir(),
		Size:     file.Size(),
	}
}

func printFile(id int, filename string, path string) {
	fmt.Println("Fetching ", filename)
	dest, err := os.Create(filename)
	if err != nil {
		fmt.Println(err)
	}

	src, err := conns[id].sftpClient.Open(path + "/" + filename)
	info, err := src.Stat()
	contents := make([]byte, info.Size())

	src.Read(contents)
	dest.Write(contents)
	dest.Close()
}

// Print the target directory's file listing
func printDirectory(id int, dirpath string) {
	fmt.Println("Printing contents of", dirpath)

	listing, err := conns[id].sftpClient.ReadDir(dirpath)

	if err != nil {
		log.Fatal("Could not open directory : ", err)
	}

	var files []File
	for _, file := range listing {
		files = append(files, FileStruct(file, dirpath))
	}

	// conns[id].putDirectory("./TestingDir", ".")

	data := make(map[string]interface{})
	data["files"] = files
	data["path"] = dirpath

	jsonMessage, _ := json.Marshal(FileMessage{id, "FETCH_FILES_SUCCESS", data})
	_, _ = socket.Write(jsonMessage)
}

/// MoveDirectory moves named directory from src -> dest
func MoveDirectory(from, to string, src, dest int) {
	target := path.Join(to, path.Base(from))
	session, _ := conns[dest].sshClient.NewSession()
	session.Run("sh -c `mkdir " + target + " && cd " + target + "`")
	session.Close()

	// Read contents of the directory
	files, err := conns[src].sftpClient.ReadDir(from)
	if err != nil {
		fmt.Println(err)
		session, _ = conns[src].sshClient.NewSession()
		session.Run("cd ..")
		session.Close()
		return
	}

	// Copy each file in that directory
	for _, file := range files {
		destination := path.Join(from, file.Name())
		if file.IsDir() {
			MoveDirectory(destination, target, src, dest)
		} else {
			MoveFile(destination, target, src, dest)
		}
	}

	// Change back to parent directory
	session, _ = conns[src].sshClient.NewSession()
	session.Run("cd ..")
	session.Close()
}

/// MoveFile moves named file from src -> dest
func MoveFile(from, to string, src, dest int) {
	file, err := conns[src].sftpClient.Open(from)
	if err != nil {
		fmt.Println(err)
		return
	}

	contents, err := ioutil.ReadAll(file)
	file.Close()

	file, err = conns[dest].sftpClient.Create(path.Join(to, path.Base(from)))
	if err != nil {
		fmt.Println(err)
		return
	}
	file.Write(contents)
	file.Close()
}

func (c *clients) putDirectory(src, dest string) {
	fmt.Println("Putting directory ", src)
	session, _ := c.sshClient.NewSession()
	session.Run("mkdir " + path.Join(dest, path.Base(src)))
	session.Close()

	files, _ := ioutil.ReadDir(src)
	for _, file := range files {
		if file.IsDir() {
			c.putDirectory(path.Join(src, file.Name()), path.Join(dest, path.Base(src)))
		} else {
			c.putFile(path.Join(src, file.Name()), path.Join(dest, path.Base(src)))
		}
	}
}

func (c *clients) putFile(src, dest string) {
	srcFile, _ := os.Open(src)
	destFile, err := c.sftpClient.Create(path.Join(dest, path.Base(src)))
	if err != nil {
		fmt.Println(err)
		return
	}

	info, _ := srcFile.Stat()
	contents := make([]byte, info.Size())
	srcFile.Read(contents)

	_, err = destFile.Write(contents)
	if err != nil {
		fmt.Println("Problem writing file : ", err)
	}
}

func (c *clients) getDirectory(filepath string) {
	fmt.Println("Fetching directory ", filepath)

	os.Mkdir(path.Base(filepath), os.ModeDir|os.ModePerm)
	os.Chdir(path.Base(filepath))

	files, err := c.sftpClient.ReadDir(filepath)
	if err != nil {
		fmt.Println(err)
		return
	}

	for _, file := range files {
		f := FileStruct(file, filepath)
		if f.IsDir {
			c.getDirectory(f.Path)
		} else {
			c.getFile(f)
		}
	}

	os.Chdir("..")
}

// getFile copies the file specified to the local host
func (c *clients) getFile(file File) {
	fmt.Println("Fetching ", file.Filename)
	dest, err := os.Create(file.Filename)
	if err != nil {
		fmt.Println(err)
	}

	src, err := c.sftpClient.Open(file.Path)
	info, err := src.Stat()
	contents := make([]byte, info.Size())

	src.Read(contents)
	dest.Write(contents)
	dest.Close()
}
<commit_msg>Remove unneeded functions, add error checking<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"time"
)

type File struct {
	Filename string
	Path     string
	Size     int64
	ModTime  time.Time
	IsDir    bool
}

type FileMessage struct {
	ConnId   int                    `json:"id"`
	Function string                 `json:"type"`
	Data     map[string]interface{} `json:"data"`
}

func FileStruct(file os.FileInfo, dir string) File {
	return File{
		Filename: file.Name(),
		Path:     path.Join(dir, file.Name()),
		ModTime:  file.ModTime(),
		IsDir:    file.Isdir(),
		Size:     file.Size(),
	}
}

// Echos the $HOME env to find the home directory, returns root if err
func getHomeDir(id int) string {
	// Create a new ssh session to access the remote host's $HOME env variable
	session, err := conns[id].sshClient.NewSession()
	defer session.Close()

	if err != nil {
		fmt.Println("Error creating ssh session:", err)
		return "/"
	}
	output, err := session.Output("echo $HOME")
	if err != nil {
		fmt.Println("Error finding home dir:", err)
		return "/"
	}
	return strings.TrimSpace(string(output))
}

func printFile(id int, filename, filepath string) {
	dest, err := os.Create(filename)
	if err != nil {
		fmt.Println(err)
	}

	src, err := conns[id].sftpClient.Open(path.Join(filepath, filename))
	info, err := src.Stat()
	contents := make([]byte, info.Size())

	src.Read(contents)
	dest.Write(contents)
	dest.Close()
}

// Print the target directory's file listing
func printDirectory(id int, dirpath string) {
	fmt.Println("Printing contents of", dirpath)

	listing, err := conns[id].sftpClient.ReadDir(dirpath)
	if err != nil {
		log.Fatal("Could not open directory : ", err)
	}

	files := make([]File, len(listing))
	for i, file := range listing {
		files[i] = FileStruct(file, dirpath)
	}

	data := make(map[string]interface{})
	data["files"] = files
	data["path"] = dirpath

	jsonMessage, _ := json.Marshal(FileMessage{id, "FETCH_FILES_SUCCESS", data})
	_, _ = socket.Write(jsonMessage)
}

// MoveDirectory moves named directory from src -> dest
func MoveDirectory(from, to string, src, dest int) {
	target := path.Join(to, path.Base(from))
	session, err := conns[dest].sshClient.NewSession()
	if err != nil {
		log.Fatal(err)
	}
	session.Run("sh -c `mkdir " + target + " && cd " + target + "`")
	session.Close()

	// Read contents of the directory
	files, err := conns[src].sftpClient.ReadDir(from)
	if err != nil {
		log.Println(err)
		session, err = conns[src].sshClient.NewSession()
		if err != nil {
			log.Fatal(err)
		}
		session.Run("cd ..")
		session.Close()
		return
	}

	// Copy each file in that directory
	for _, file := range files {
		destination := path.Join(from, file.Name())
		if file.IsDir() {
			MoveDirectory(destination, target, src, dest)
		} else {
			MoveFile(destination, target, src, dest)
		}
	}

	// Change back to parent directory
	session, err = conns[src].sshClient.NewSession()
	if err != nil {
		log.Fatal(err)
	}
	session.Run("cd ..")
	session.Close()
}

/// MoveFile moves named file from src -> dest
func MoveFile(from, to string, src, dest int) {
	file, err := conns[src].sftpClient.Open(from)
	if err != nil {
		log.Fatal(err)
	}

	contents, err := ioutil.ReadAll(file)
	file.Close()

	file, err = conns[dest].sftpClient.Create(path.Join(to, path.Base(from)))
	if err != nil {
		log.Fatal(err)
	}
	file.Write(contents)
	file.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/mfonda/simhash"
	"strconv"
)

func main() {

	var docs = [][]byte{
		[]byte("34 23 56 567 878"),
		[]byte("33 21 53 560 871"),
		[]byte("34 22 56 567 878"),
	}

	hashes := make([]uint64, len(docs))
	for i, d := range docs {
		hashes[i] = simhash.Simhash(simhash.NewWordFeatureSet(d))
		fmt.Printf("Simhash of %s: %x\n", d, hashes[i])
	}
	q, e := strconv.ParseUint("8329707b4eb870c", 16, 64)
	if e != nil {
		fmt.Println(e.Error())
	}
	q2, e := strconv.ParseUint("8329707b4eb8707", 16, 64)

	if e != nil {
		fmt.Println(e.Error())
	}
	fmt.Printf("qwqw:%d\n", simhash.Compare(q, q2))

	fmt.Printf("Comparison of `%s` and `%s`: %d\n", docs[0], docs[1], simhash.Compare(hashes[0], hashes[1]))
	fmt.Printf("Comparison of `%s` and `%s`: %d\n", docs[0], docs[2], simhash.Compare(hashes[0], hashes[2]))
}
<commit_msg>simhash测试<commit_after>package main

import (
	//"encoding/binary"
	"fmt"
	"github.com/mfonda/simhash"
	"strconv"
)

func main() {

	var docs = [][]byte{
		[]byte("Z2X6629402"),
		[]byte("q2X6629302"),
		[]byte("ZyX6629302"),
	}

	uint64s, _ := strconv.ParseUint("1001100110111100011011010111101101100111100111000110111010010", 2, 64)
	fmt.Printf("64:%x\n", uint64s)

	hashes := make([]uint64, len(docs))
	for i, d := range docs {
		hashes[i] = simhash.Simhash(simhash.NewWordFeatureSet(d))
		fmt.Printf("Simhash of %s: %x b:%b\n", d, hashes[i], hashes[i])
	}

	q, e := strconv.ParseUint("13378daf6cf38dd2", 16, 64)
	if e != nil {
		fmt.Println(e.Error())
	}
	q2, e := strconv.ParseUint("f597e9511bbc518d", 16, 64)
	if e != nil {
		fmt.Println(e.Error())
	}
	fmt.Printf("simhash:%d\n", simhash.Compare(q, q2))

	fmt.Printf("Comparison of `%s` and `%s`: %d\n", docs[0], docs[1], simhash.Compare(hashes[0], hashes[1]))
	fmt.Printf("Comparison of `%s` and `%s`: %d\n", docs[0], docs[2], simhash.Compare(hashes[0], hashes[2]))
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strconv

// FormatUint returns the string representation of i in the given base,
// for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z'
// for digit values >= 10.
func FormatUint(i uint64, base int) string {
	if i < uint64(len(smallints)) && base == 10 {
		return smallints[i]
	}
	_, s := formatBits(nil, i, base, false, false)
	return s
}

// FormatInt returns the string representation of i in the given base,
// for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z'
// for digit values >= 10.
func FormatInt(i int64, base int) string {
	if 0 <= i && i < int64(len(smallints)) && base == 10 {
		return smallints[i]
	}
	_, s := formatBits(nil, uint64(i), base, i < 0, false)
	return s
}

// Itoa is shorthand for FormatInt(int64(i), 10).
func Itoa(i int) string {
	return FormatInt(int64(i), 10)
}

// AppendInt appends the string form of the integer i,
// as generated by FormatInt, to dst and returns the extended buffer.
func AppendInt(dst []byte, i int64, base int) []byte {
	if 0 <= i && i < int64(len(smallints)) && base == 10 {
		return append(dst, smallints[i]...)
	}
	dst, _ = formatBits(dst, uint64(i), base, i < 0, true)
	return dst
}

// AppendUint appends the string form of the unsigned integer i,
// as generated by FormatUint, to dst and returns the extended buffer.
func AppendUint(dst []byte, i uint64, base int) []byte {
	if i < uint64(len(smallints)) && base == 10 {
		return append(dst, smallints[i]...)
	}
	dst, _ = formatBits(dst, i, base, false, true)
	return dst
}

const host32bit = ^uint(0)>>32 == 0

const digits = "0123456789abcdefghijklmnopqrstuvwxyz"

var smallints = [...]string{
	"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
	"10", "11", "12", "13", "14", "15", "16", "17", "18", "19",
	"20", "21", "22", "23", "24", "25", "26", "27", "28", "29",
	"30", "31", "32", "33", "34", "35", "36", "37", "38", "39",
	"40", "41", "42", "43", "44", "45", "46", "47", "48", "49",
	"50", "51", "52", "53", "54", "55", "56", "57", "58", "59",
	"60", "61", "62", "63", "64", "65", "66", "67", "68", "69",
	"70", "71", "72", "73", "74", "75", "76", "77", "78", "79",
	"80", "81", "82", "83", "84", "85", "86", "87", "88", "89",
	"90", "91", "92", "93", "94", "95", "96", "97", "98", "99",
}

var shifts = [len(digits) + 1]uint{
	1 << 1: 1,
	1 << 2: 2,
	1 << 3: 3,
	1 << 4: 4,
	1 << 5: 5,
}

// formatBits computes the string representation of u in the given base.
// If neg is set, u is treated as negative int64 value. If append_ is
// set, the string is appended to dst and the resulting byte slice is
// returned as the first result value; otherwise the string is returned
// as the second result value.
//
func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s string) {
	if base < 2 || base > len(digits) {
		panic("strconv: illegal AppendInt/FormatInt base")
	}
	// 2 <= base && base <= len(digits)

	var a [64 + 1]byte // +1 for sign of 64bit value in base 2
	i := len(a)

	if neg {
		u = -u
	}

	// convert bits
	// We use uint values where we can because those will
	// fit into a single register even on a 32bit machine.
	if base == 10 {
		// common case: use constants for / because
		// the compiler can optimize it into a multiply+shift

		if host32bit {
			// convert the lower digits using 32bit operations
			for u >= 1e9 {
				// Avoid using r = a%b in addition to q = a/b
				// since 64bit division and modulo operations
				// are calculated by runtime functions on 32bit machines.
				q := u / 1e9
				us := uint(u - q*1e9) // u % 1e9 fits into a uint
				for j := 9; j > 0; j-- {
					i--
					a[i] = byte(us%10 + '0')
					us /= 10
				}
				u = q
			}
			// u < 1e9
		}

		// u guaranteed to fit into a uint
		us := uint(u)
		for us >= 10 {
			i--
			a[i] = byte(us%10 + '0')
			us /= 10
		}
		// us < 10
		i--
		a[i] = byte(us + '0')

	} else if s := shifts[base]; s > 0 {
		// base is power of 2: use shifts and masks instead of / and %
		b := uint64(base)
		m := uint(base) - 1 // == 1<<s - 1
		for u >= b {
			i--
			a[i] = digits[uint(u)&m]
			u >>= s
		}
		// u < base
		i--
		a[i] = digits[uint(u)]
	} else {
		// general case
		b := uint64(base)
		for u >= b {
			i--
			// Avoid using r = a%b in addition to q = a/b
			// since 64bit division and modulo operations
			// are calculated by runtime functions on 32bit machines.
			q := u / b
			a[i] = digits[uint(u-q*b)]
			u = q
		}
		// u < base
		i--
		a[i] = digits[uint(u)]
	}

	// add sign, if any
	if neg {
		i--
		a[i] = '-'
	}

	if append_ {
		d = append(dst, a[i:]...)
		return
	}
	s = string(a[i:])
	return
}
<commit_msg>strconv: replace small int string table with constant string<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package strconv

const fastSmalls = true // enable fast path for small integers

// FormatUint returns the string representation of i in the given base,
// for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z'
// for digit values >= 10.
func FormatUint(i uint64, base int) string {
	if fastSmalls && i < nSmalls && base == 10 {
		return small(int(i))
	}
	_, s := formatBits(nil, i, base, false, false)
	return s
}

// FormatInt returns the string representation of i in the given base,
// for 2 <= base <= 36. The result uses the lower-case letters 'a' to 'z'
// for digit values >= 10.
func FormatInt(i int64, base int) string {
	if fastSmalls && 0 <= i && i < nSmalls && base == 10 {
		return small(int(i))
	}
	_, s := formatBits(nil, uint64(i), base, i < 0, false)
	return s
}

// Itoa is shorthand for FormatInt(int64(i), 10).
func Itoa(i int) string {
	return FormatInt(int64(i), 10)
}

// AppendInt appends the string form of the integer i,
// as generated by FormatInt, to dst and returns the extended buffer.
func AppendInt(dst []byte, i int64, base int) []byte {
	if fastSmalls && 0 <= i && i < nSmalls && base == 10 {
		return append(dst, small(int(i))...)
	}
	dst, _ = formatBits(dst, uint64(i), base, i < 0, true)
	return dst
}

// AppendUint appends the string form of the unsigned integer i,
// as generated by FormatUint, to dst and returns the extended buffer.
func AppendUint(dst []byte, i uint64, base int) []byte {
	if fastSmalls && i < nSmalls && base == 10 {
		return append(dst, small(int(i))...)
	}
	dst, _ = formatBits(dst, i, base, false, true)
	return dst
}

// small returns the string for an i with 0 <= i < nSmalls.
func small(i int) string {
	off := 0
	if i < 10 {
		off = 1
	}
	return smallsString[i*2+off : i*2+2]
}

const nSmalls = 100

const smallsString = "00010203040506070809" +
	"10111213141516171819" +
	"20212223242526272829" +
	"30313233343536373839" +
	"40414243444546474849" +
	"50515253545556575859" +
	"60616263646566676869" +
	"70717273747576777879" +
	"80818283848586878889" +
	"90919293949596979899"

const host32bit = ^uint(0)>>32 == 0

const digits = "0123456789abcdefghijklmnopqrstuvwxyz"

var shifts = [len(digits) + 1]uint{
	1 << 1: 1,
	1 << 2: 2,
	1 << 3: 3,
	1 << 4: 4,
	1 << 5: 5,
}

// formatBits computes the string representation of u in the given base.
// If neg is set, u is treated as negative int64 value. If append_ is
// set, the string is appended to dst and the resulting byte slice is
// returned as the first result value; otherwise the string is returned
// as the second result value.
//
func formatBits(dst []byte, u uint64, base int, neg, append_ bool) (d []byte, s string) {
	if base < 2 || base > len(digits) {
		panic("strconv: illegal AppendInt/FormatInt base")
	}
	// 2 <= base && base <= len(digits)

	var a [64 + 1]byte // +1 for sign of 64bit value in base 2
	i := len(a)

	if neg {
		u = -u
	}

	// convert bits
	// We use uint values where we can because those will
	// fit into a single register even on a 32bit machine.
	if base == 10 {
		// common case: use constants for / because
		// the compiler can optimize it into a multiply+shift

		if host32bit {
			// convert the lower digits using 32bit operations
			for u >= 1e9 {
				// Avoid using r = a%b in addition to q = a/b
				// since 64bit division and modulo operations
				// are calculated by runtime functions on 32bit machines.
				q := u / 1e9
				us := uint(u - q*1e9) // u % 1e9 fits into a uint
				for j := 9; j > 0; j-- {
					i--
					a[i] = byte(us%10 + '0')
					us /= 10
				}
				u = q
			}
			// u < 1e9
		}

		// u guaranteed to fit into a uint
		us := uint(u)
		for us >= 10 {
			i--
			a[i] = byte(us%10 + '0')
			us /= 10
		}
		// us < 10
		i--
		a[i] = byte(us + '0')

	} else if s := shifts[base]; s > 0 {
		// base is power of 2: use shifts and masks instead of / and %
		b := uint64(base)
		m := uint(base) - 1 // == 1<<s - 1
		for u >= b {
			i--
			a[i] = digits[uint(u)&m]
			u >>= s
		}
		// u < base
		i--
		a[i] = digits[uint(u)]
	} else {
		// general case
		b := uint64(base)
		for u >= b {
			i--
			// Avoid using r = a%b in addition to q = a/b
			// since 64bit division and modulo operations
			// are calculated by runtime functions on 32bit machines.
			q := u / b
			a[i] = digits[uint(u-q*b)]
			u = q
		}
		// u < base
		i--
		a[i] = digits[uint(u)]
	}

	// add sign, if any
	if neg {
		i--
		a[i] = '-'
	}

	if append_ {
		d = append(dst, a[i:]...)
		return
	}
	s = string(a[i:])
	return
}
<|endoftext|>


<commit_before>// Package taskqueue provides the functionality for receiving, handling and executing tasks.
// In this file are the routines for the taskqueue itself.
package taskqueue

import (
	"fmt"
	"github.com/jpillora/backoff"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/nevsnode/gordon/config"
	"github.com/nevsnode/gordon/output"
	"github.com/nevsnode/gordon/stats"
	"strings"
	"sync"
	"time"
)

const (
	backlog = 1
)

type failedTask struct {
	configTask config.Task
	queueTask  QueueTask
}

var (
	errorNoNewTask          = fmt.Errorf("No new task available")
	errorNoNewTasksAccepted = fmt.Errorf("No new tasks accepted")

	conf            config.Config
	shutdown        bool
	shutdownChan    chan bool
	waitGroup       sync.WaitGroup
	waitGroupFailed sync.WaitGroup
	redisPool       *pool.Pool
	errorBackoff    map[string]*backoff.Backoff
	workerChan      map[string]chan QueueTask
	failedChan      chan failedTask
)

// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	poolSize := 1
	for _, ct := range conf.Tasks {
		if ct.FailedTasksTTL > 0 {
			poolSize++
			break
		}
	}

	var err error
	redisPool, err = pool.New(conf.RedisNetwork, conf.RedisAddress, poolSize)
	if err != nil {
		output.NotifyError("redis pool.New():", err)
	}

	stats.InitTasks(conf.Tasks)

	workerChan = make(map[string]chan QueueTask)
	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	errorBackoff = make(map[string]*backoff.Backoff)
	for _, ct := range conf.Tasks {
		if ct.BackoffEnabled {
			errorBackoff[ct.Type] = &backoff.Backoff{
				Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
				Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
				Factor: ct.BackoffFactor,
				Jitter: true,
			}
		}

		workerChan[ct.Type] = make(chan QueueTask, backlog)

		for i := 0; i < ct.Workers; i++ {
			waitGroup.Add(1)
			go taskWorker(ct)
		}
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}

// Stop will cause the taskqueue to stop accepting new tasks and shutdown the
// worker routines after they've finished their current tasks
func Stop() {
	if shutdown {
		return
	}

	shutdown = true
	shutdownChan <- true

	for taskType := range workerChan {
		close(workerChan[taskType])
	}
}

// Wait waits, to keep the application running as long as there are workers
func Wait() {
	waitGroup.Wait()

	close(failedChan)
	waitGroupFailed.Wait()

	redisPool.Empty()
}

func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	var runningIntervalLoop sync.WaitGroup
	go func() {
		for {
			runningIntervalLoop.Wait()
			time.Sleep(interval.Duration())

			if shutdown {
				break
			}

			runIntervalLoop <- true
			runningIntervalLoop.Add(1)
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if shutdown {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		runningIntervalLoop.Done()
	}

	Stop()
	waitGroup.Done()
}

func taskWorker(ct config.Task) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task:\n%s \"%s\"\n\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff[ct.Type] != nil {
			if err == nil {
				errorBackoff[ct.Type].Reset()
			} else {
				time.Sleep(errorBackoff[ct.Type].Duration())
			}
		}
	}

	waitGroup.Done()
}

func failedTaskWorker() {
	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			return
		}

		rc, err := redisPool.Get()
		if err != nil {
			output.NotifyError("redisPool.Get():", err)
			return
		}
		defer redisPool.Put(rc)

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			return
		}

		// add to list
		reply := rc.Cmd("RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err)
			return
		}

		// set expire
		reply = rc.Cmd("EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err)
			return
		}
	}

	waitGroupFailed.Done()
}

func acceptsTasks(taskType string) bool {
	if shutdown {
		return false
	}

	return len(workerChan[taskType]) < backlog
}
<commit_msg>adjusted message for failed tasks<commit_after>// Package taskqueue provides the functionality for receiving, handling and executing tasks.
// In this file are the routines for the taskqueue itself.
package taskqueue

import (
	"fmt"
	"github.com/jpillora/backoff"
	"github.com/mediocregopher/radix.v2/pool"
	"github.com/nevsnode/gordon/config"
	"github.com/nevsnode/gordon/output"
	"github.com/nevsnode/gordon/stats"
	"strings"
	"sync"
	"time"
)

const (
	backlog = 1
)

type failedTask struct {
	configTask config.Task
	queueTask  QueueTask
}

var (
	errorNoNewTask          = fmt.Errorf("No new task available")
	errorNoNewTasksAccepted = fmt.Errorf("No new tasks accepted")

	conf            config.Config
	shutdown        bool
	shutdownChan    chan bool
	waitGroup       sync.WaitGroup
	waitGroupFailed sync.WaitGroup
	redisPool       *pool.Pool
	errorBackoff    map[string]*backoff.Backoff
	workerChan      map[string]chan QueueTask
	failedChan      chan failedTask
)

// Start initialises several variables and creates necessary go-routines
func Start(c config.Config) {
	conf = c

	poolSize := 1
	for _, ct := range conf.Tasks {
		if ct.FailedTasksTTL > 0 {
			poolSize++
			break
		}
	}

	var err error
	redisPool, err = pool.New(conf.RedisNetwork, conf.RedisAddress, poolSize)
	if err != nil {
		output.NotifyError("redis pool.New():", err)
	}

	stats.InitTasks(conf.Tasks)

	workerChan = make(map[string]chan QueueTask)
	failedChan = make(chan failedTask)
	shutdownChan = make(chan bool, 1)

	errorBackoff = make(map[string]*backoff.Backoff)
	for _, ct := range conf.Tasks {
		if ct.BackoffEnabled {
			errorBackoff[ct.Type] = &backoff.Backoff{
				Min:    time.Duration(ct.BackoffMin) * time.Millisecond,
				Max:    time.Duration(ct.BackoffMax) * time.Millisecond,
				Factor: ct.BackoffFactor,
				Jitter: true,
			}
		}

		workerChan[ct.Type] = make(chan QueueTask, backlog)

		for i := 0; i < ct.Workers; i++ {
			waitGroup.Add(1)
			go taskWorker(ct)
		}
	}

	waitGroupFailed.Add(1)
	go failedTaskWorker()

	waitGroup.Add(1)
	go queueWorker()
}

// Stop will cause the taskqueue to stop accepting new tasks and shutdown the
// worker routines after they've finished their current tasks
func Stop() {
	if shutdown {
		return
	}

	shutdown = true
	shutdownChan <- true

	for taskType := range workerChan {
		close(workerChan[taskType])
	}
}

// Wait waits, to keep the application running as long as there are workers
func Wait() {
	waitGroup.Wait()

	close(failedChan)
	waitGroupFailed.Wait()

	redisPool.Empty()
}

func queueWorker() {
	interval := backoff.Backoff{
		Min:    time.Duration(conf.IntervalMin) * time.Millisecond,
		Max:    time.Duration(conf.IntervalMax) * time.Millisecond,
		Factor: conf.IntervalFactor,
	}

	runIntervalLoop := make(chan bool)
	var runningIntervalLoop sync.WaitGroup
	go func() {
		for {
			runningIntervalLoop.Wait()
			time.Sleep(interval.Duration())

			if shutdown {
				break
			}

			runIntervalLoop <- true
			runningIntervalLoop.Add(1)
		}
	}()

	go func() {
		for {
			select {
			case <-shutdownChan:
				runIntervalLoop <- false
			}
		}
	}()

	for <-runIntervalLoop {
		for taskType, configTask := range conf.Tasks {
			if shutdown {
				break
			}

			output.Debug("Checking for new tasks (" + taskType + ")")

			// check if there are available workers
			if !acceptsTasks(taskType) {
				continue
			}

			queueKey := conf.RedisQueueKey + ":" + taskType

			llen, err := redisPool.Cmd("LLEN", queueKey).Int()
			if err != nil {
				// Errors here are likely redis-connection errors, so we'll
				// need to notify about it
				output.NotifyError("redisPool.Cmd() Error:", err)
				break
			}

			// there are no new tasks in redis
			if llen == 0 {
				continue
			}

			// iterate over all entries in redis, until no more are available,
			// or all workers are busy, for a maximum of 2 * workers
			for i := 0; i < (configTask.Workers * 2); i++ {
				if !acceptsTasks(taskType) {
					break
				}

				value, err := redisPool.Cmd("LPOP", queueKey).Str()
				if err != nil {
					// no more tasks found
					break
				}

				output.Debug("Fetched task for type", taskType, "with payload", value)

				task, err := NewQueueTask(value)
				if err != nil {
					output.NotifyError("NewQueueTask():", err)
					continue
				}

				workerChan[taskType] <- task

				// we've actually are handling new tasks so reset the interval
				interval.Reset()
			}
		}

		runningIntervalLoop.Done()
	}

	Stop()
	waitGroup.Done()
}

func taskWorker(ct config.Task) {
	for task := range workerChan[ct.Type] {
		output.Debug("Executing task type", ct.Type, "with arguments", task.Args)
		stats.IncrTaskCount(ct.Type)

		err := task.Execute(ct.Script)
		if err != nil {
			task.ErrorMessage = fmt.Sprintf("%s", err)
			failedChan <- failedTask{
				configTask: ct,
				queueTask:  task,
			}

			msg := fmt.Sprintf("Failed executing task: %s \"%s\"\n%s", ct.Script, strings.Join(task.Args, "\" \""), err)
			output.NotifyError(msg)
		}

		if errorBackoff[ct.Type] != nil {
			if err == nil {
				errorBackoff[ct.Type].Reset()
			} else {
				time.Sleep(errorBackoff[ct.Type].Duration())
			}
		}
	}

	waitGroup.Done()
}

func failedTaskWorker() {
	for ft := range failedChan {
		ct := ft.configTask
		qt := ft.queueTask

		if ct.FailedTasksTTL == 0 {
			return
		}

		rc, err := redisPool.Get()
		if err != nil {
			output.NotifyError("redisPool.Get():", err)
			return
		}
		defer redisPool.Put(rc)

		queueKey := conf.RedisQueueKey + ":" + ct.Type + ":failed"

		jsonString, err := qt.GetJSONString()
		if err != nil {
			output.NotifyError("failedTaskWorker(), qt.GetJSONString():", err)
			return
		}

		// add to list
		reply := rc.Cmd("RPUSH", queueKey, jsonString)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), RPUSH:", reply.Err)
			return
		}

		// set expire
		reply = rc.Cmd("EXPIRE", queueKey, ct.FailedTasksTTL)
		if reply.Err != nil {
			output.NotifyError("failedTaskWorker(), EXPIRE:", reply.Err)
			return
		}
	}

	waitGroupFailed.Done()
}

func acceptsTasks(taskType string) bool {
	if shutdown {
		return false
	}

	return len(workerChan[taskType]) < backlog
}
<|endoftext|>


<commit_before>package storage

import (
	"errors"
	"log"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	replicaType ReplicationType

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
	v.load()
	return
}
func (v *Volume) load() {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		log.Fatalf("New Volume [ERROR] %s\n", e)
	}
	if v.replicaType == CopyNil {
		v.readSuperBlock()
	} else {
		v.maybeWriteSuperBlock()
	}
	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
	if ie != nil {
		log.Fatalf("Write Volume Index [ERROR] %s\n", ie)
	}
	v.nm = LoadNeedleMap(indexFile)
}
func (v *Volume) Size() int64 {
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	return -1
}
func (v *Volume) Close() {
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() {
	stat, _ := v.dataFile.Stat()
	if stat.Size() == 0 {
		header := make([]byte, SuperBlockSize)
		header[0] = 1
		header[1] = v.replicaType.Byte()
		v.dataFile.Write(header)
	}
}
func (v *Volume) readSuperBlock() {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := v.dataFile.Read(header); error == nil {
		v.replicaType, _ = NewReplicationTypeFromByte(header[1])
	}
}
func (v *Volume) NeedToReplicate() bool{
  return v.replicaType.GetCopyCount()>1
}

func (v *Volume) write(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	offset, _ := v.dataFile.Seek(0, 2)
	ret := n.Append(v.dataFile)
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*8 < offset {
		v.nm.Put(n.Id, uint32(offset/8), n.Size)
	}
	return ret
}
func (v *Volume) delete(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//log.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*8), 0)
		n.Append(v.dataFile)
		return nv.Size
	}
	return 0
}
func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*8, 0)
		return n.Read(v.dataFile, nv.Size)
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat"))
	os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx"))
	v.load()
	return 0, nil
}

func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
	src, err := os.OpenFile(srcName, os.O_RDONLY, 0644)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer dst.Close()

	idx, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer idx.Close()

	src.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := src.Read(header); error == nil {
		dst.Write(header)
	}

	n, rest := ReadNeedle(src)
	nm := NewNeedleMap(idx)
	old_offset := uint32(SuperBlockSize)
  new_offset := uint32(SuperBlockSize)
	for n != nil {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if !ok || nv.Offset*8 != old_offset {
			log.Println("expected offset should be", nv.Offset*8, "skipping", (rest - 16), "key", n.Id, "volume offset", old_offset, "data_size", n.Size, "rest", rest)
			src.Seek(int64(rest), 1)
		} else {
			if nv.Size > 0 {
				nm.Put(n.Id, new_offset/8, n.Size)
				bytes := make([]byte, n.Size+4)
				src.Read(bytes)
				n.Data = bytes[:n.Size]
				n.Checksum = NewCRC(n.Data)
				n.Append(dst)
				new_offset += rest+16
				log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
      src.Seek(int64(rest-n.Size-4), 1)
		}
		old_offset += rest+16
		n, rest = ReadNeedle(src)
	}

	return nil
}
<commit_msg>adjust formatting<commit_after>package storage

import (
	"errors"
	"log"
	"os"
	"path"
	"sync"
)

const (
	SuperBlockSize = 8
)

type Volume struct {
	Id       VolumeId
	dir      string
	dataFile *os.File
	nm       *NeedleMap

	replicaType ReplicationType

	accessLock sync.Mutex
}

func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume) {
	v = &Volume{dir: dirname, Id: id, replicaType: replicationType}
	v.load()
	return
}
func (v *Volume) load() {
	var e error
	fileName := path.Join(v.dir, v.Id.String())
	v.dataFile, e = os.OpenFile(fileName+".dat", os.O_RDWR|os.O_CREATE, 0644)
	if e != nil {
		log.Fatalf("New Volume [ERROR] %s\n", e)
	}
	if v.replicaType == CopyNil {
		v.readSuperBlock()
	} else {
		v.maybeWriteSuperBlock()
	}
	indexFile, ie := os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644)
	if ie != nil {
		log.Fatalf("Write Volume Index [ERROR] %s\n", ie)
	}
	v.nm = LoadNeedleMap(indexFile)
}
func (v *Volume) Size() int64 {
	stat, e := v.dataFile.Stat()
	if e == nil {
		return stat.Size()
	}
	return -1
}
func (v *Volume) Close() {
	v.nm.Close()
	v.dataFile.Close()
}
func (v *Volume) maybeWriteSuperBlock() {
	stat, _ := v.dataFile.Stat()
	if stat.Size() == 0 {
		header := make([]byte, SuperBlockSize)
		header[0] = 1
		header[1] = v.replicaType.Byte()
		v.dataFile.Write(header)
	}
}
func (v *Volume) readSuperBlock() {
	v.dataFile.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := v.dataFile.Read(header); error == nil {
		v.replicaType, _ = NewReplicationTypeFromByte(header[1])
	}
}
func (v *Volume) NeedToReplicate() bool {
	return v.replicaType.GetCopyCount() > 1
}

func (v *Volume) write(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	offset, _ := v.dataFile.Seek(0, 2)
	ret := n.Append(v.dataFile)
	nv, ok := v.nm.Get(n.Id)
	if !ok || int64(nv.Offset)*8 < offset {
		v.nm.Put(n.Id, uint32(offset/8), n.Size)
	}
	return ret
}
func (v *Volume) delete(n *Needle) uint32 {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	//log.Println("key", n.Id, "volume offset", nv.Offset, "data_size", n.Size, "cached size", nv.Size)
	if ok {
		v.nm.Delete(n.Id)
		v.dataFile.Seek(int64(nv.Offset*8), 0)
		n.Append(v.dataFile)
		return nv.Size
	}
	return 0
}
func (v *Volume) read(n *Needle) (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	nv, ok := v.nm.Get(n.Id)
	if ok && nv.Offset > 0 {
		v.dataFile.Seek(int64(nv.Offset)*8, 0)
		return n.Read(v.dataFile, nv.Size)
	}
	return -1, errors.New("Not Found")
}

func (v *Volume) compact() error {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()

	filePath := path.Join(v.dir, v.Id.String())
	return v.copyDataAndGenerateIndexFile(filePath+".dat", filePath+".cpd", filePath+".cpx")
}
func (v *Volume) commitCompact() (int, error) {
	v.accessLock.Lock()
	defer v.accessLock.Unlock()
	v.dataFile.Close()
	os.Rename(path.Join(v.dir, v.Id.String()+".cpd"), path.Join(v.dir, v.Id.String()+".dat"))
	os.Rename(path.Join(v.dir, v.Id.String()+".cpx"), path.Join(v.dir, v.Id.String()+".idx"))
	v.load()
	return 0, nil
}

func (v *Volume) copyDataAndGenerateIndexFile(srcName, dstName, idxName string) (err error) {
	src, err := os.OpenFile(srcName, os.O_RDONLY, 0644)
	if err != nil {
		return err
	}
	defer src.Close()

	dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer dst.Close()

	idx, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return err
	}
	defer idx.Close()

	src.Seek(0, 0)
	header := make([]byte, SuperBlockSize)
	if _, error := src.Read(header); error == nil {
		dst.Write(header)
	}

	n, rest := ReadNeedle(src)
	nm := NewNeedleMap(idx)
	old_offset := uint32(SuperBlockSize)
	new_offset := uint32(SuperBlockSize)
	for n != nil {
		nv, ok := v.nm.Get(n.Id)
		//log.Println("file size is", n.Size, "rest", rest)
		if !ok || nv.Offset*8 != old_offset {
			log.Println("expected offset should be", nv.Offset*8, "skipping", (rest - 16), "key", n.Id, "volume offset", old_offset, "data_size", n.Size, "rest", rest)
			src.Seek(int64(rest), 1)
		} else {
			if nv.Size > 0 {
				nm.Put(n.Id, new_offset/8, n.Size)
				bytes := make([]byte, n.Size+4)
				src.Read(bytes)
				n.Data = bytes[:n.Size]
				n.Checksum = NewCRC(n.Data)
				n.Append(dst)
				new_offset += rest + 16
				log.Println("saving key", n.Id, "volume offset", old_offset, "=>", new_offset, "data_size", n.Size, "rest", rest)
			}
			src.Seek(int64(rest-n.Size-4), 1)
		}
		old_offset += rest + 16
		n, rest = ReadNeedle(src)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// GroupName is the group name use in this package.
const GroupName = "tpr.client-go.k8s.io"

// SchemeGroupVersion is the group version used to register these objects.
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"}

// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

func init() {
	// We only register manually-written functions here. The registration of the
	// generated functions takes place in the generated files. The separation
	// makes the code compile even when the generated files are missing.
	SchemeBuilder.Register(addKnownTypes)
}

// addKnownTypes adds the set of types defined in this package to the supplied scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Example{},
		&ExampleList{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}
<commit_msg>Remove redundant third-party type registration<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// GroupName is the group name use in this package.
const GroupName = "tpr.client-go.k8s.io"

// SchemeGroupVersion is the group version used to register these objects.
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"}

// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

// addKnownTypes adds the set of types defined in this package to the supplied scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Example{},
		&ExampleList{},
	)
	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
	return nil
}
<|endoftext|>