<commit_before>package log

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/ltick/tick-framework/config"
	libLog "github.com/ltick/tick-log"
)

var (
	errInitiate       = "logger: initiate error"
	errStartup        = "logger: startup error"
	errInvalidLogType = "logger: invalid log type '%s'"
)

type LogConfig struct {
	Name            string
	Formatter       string
	Type            string
	FileName        string
	FileRotate      string
	FileBackupCount string
	Writer          string // the writer name of writer (stdout, stderr, discard)
	MaxLevel        string
}

type Log struct {
	Name            string
	Formatter       Formatter
	Type            Type
	FileName        string
	FileRotate      bool
	FileBackupCount int64
	Writer          Writer // the writer name of writer (stdout, stderr, discard)
	MaxLevel        Level
}

// Formatter describes the formatter of a log message.
type Formatter int

const (
	FormatterDefault Formatter = iota
	FormatterRaw
	FormatterSys
)

// LevelNames maps log levels to names
var FormatterNames = map[Formatter]string{
	FormatterDefault: "Default",
	FormatterRaw:     "Raw",
	FormatterSys:     "Sys",
}

// String returns the string representation of the log level
func (w Formatter) String() string {
	if name, ok := FormatterNames[w]; ok {
		return name
	}
	return FormatterNames[FormatterDefault]
}

func StringToFormatter(name string) Formatter {
	for formatter, formatterName := range FormatterNames {
		if strings.ToLower(name) == strings.ToLower(formatterName) {
			return formatter
		}
	}
	return FormatterDefault
}

// Writer describes the writer of a log message.
type Writer int

const (
	WriterUnknown Writer = iota
	WriterStdout
	WriterStderr
	WriterDiscard
)

// LevelNames maps log levels to names
var WriterNames = map[Writer]string{
	WriterUnknown: "unknown",
	WriterStdout:  "stdout",
	WriterStderr:  "stderr",
	WriterDiscard: "discard",
}

// String returns the string representation of the log level
func (w Writer) String() string {
	if name, ok := WriterNames[w]; ok {
		return name
	}
	return WriterNames[WriterUnknown]
}

func StringToWriter(name string) Writer {
	for writer, writerName := range WriterNames {
		if strings.ToLower(name) == strings.ToLower(writerName) {
			return writer
		}
	}
	return WriterUnknown
}

// Type describes the type of a log message.
type Type int

const (
	TypeUnknown Type = iota
	TypeFile
	TypeConsole
)

// LevelNames maps log levels to names
var TypeNames = map[Type]string{
	TypeUnknown: "Unknown",
	TypeFile:    "File",
	TypeConsole: "Console",
}

// String returns the string representation of the log level
func (t Type) String() string {
	if name, ok := TypeNames[t]; ok {
		return name
	}
	return TypeNames[TypeUnknown]
}

func StringToType(name string) Type {
	for typ, typeName := range TypeNames {
		if strings.ToLower(name) == strings.ToLower(typeName) {
			return typ
		}
	}
	return TypeUnknown
}

// Level describes the level of a log message.
type Level int

// RFC5424 log message levels.
const (
	LevelEmergency Level = iota
	LevelAlert
	LevelCritical
	LevelError
	LevelWarning
	LevelNotice
	LevelInfo
	LevelDebug
)

// LevelNames maps log levels to names
var LevelNames = map[Level]string{
	LevelDebug:     "Debug",
	LevelInfo:      "Info",
	LevelNotice:    "Notice",
	LevelWarning:   "Warning",
	LevelError:     "Error",
	LevelCritical:  "Critical",
	LevelAlert:     "Alert",
	LevelEmergency: "Emergency",
}

// String returns the string representation of the log level
func (l Level) String() string {
	if name, ok := LevelNames[l]; ok {
		return name
	}
	return "Unknown"
}

func NewLogger() *Logger {
	logger := &Logger{}
	return logger
}

type Logger struct {
	Config   *config.Config `inject:"true"`
	Provider string
	Logs     []*LogConfig
	handler  Handler
}

func (l *Logger) Initiate(ctx context.Context) (context.Context, error) {
	err := Register("tick", NewTickHandler)
	if err != nil {
		return ctx, errors.Annotatef(err, errInitiate)
	}
	err = l.Use(ctx, "tick")
	if err != nil {
		return ctx, errors.Annotatef(err, errInitiate)
	}
	if l.Logs != nil {
		logs := make([]*Log, 0)
		for _, logConfig := range l.Logs {
			logConfigMaxLevel := LevelDebug
			for level, levelName := range LevelNames {
				if levelName == logConfig.MaxLevel {
					logConfigMaxLevel = level
					break
				}
			}
			logConfigFileRotate, err := strconv.ParseBool(logConfig.FileRotate)
			if err != nil {
				return ctx, errors.Annotatef(err, errInitiate)
			}
			logConfigFileBackupCount, err := strconv.ParseInt(logConfig.FileBackupCount, 10, 64)
			if err != nil {
				return ctx, errors.Annotatef(err, errInitiate)
			}
			switch StringToType(logConfig.Type) {
			case TypeFile:
				logs = append(logs, &Log{
					Name:            logConfig.Name,
					Type:            TypeFile,
					Formatter:       StringToFormatter(logConfig.Formatter),
					FileName:        logConfig.FileName,
					FileRotate:      logConfigFileRotate,
					FileBackupCount: logConfigFileBackupCount,
					MaxLevel:        logConfigMaxLevel,
				})
			case TypeConsole:
				logs = append(logs, &Log{
					Name:      logConfig.Name,
					Type:      TypeConsole,
					Formatter: StringToFormatter(logConfig.Formatter),
					Writer:    StringToWriter(logConfig.Writer),
					MaxLevel:  logConfigMaxLevel,
				})
			default:
				return ctx, errors.Errorf(errInvalidLogType, StringToType(logConfig.Type))
			}
		}
		var logProviders map[string]interface{} = make(map[string]interface{})
		logConfigProviderConfigs := make([]string, len(logs))
		for index, lg := range logs {
			switch lg.Type {
			case TypeFile:
				logFileName, err := filepath.Abs(lg.FileName)
				if err != nil {
					return ctx, err
				}
				_, err = os.Stat(logFileName)
				if err != nil {
					if os.IsNotExist(err) {
						_, err = os.Create(logFileName)
						if err != nil {
							return ctx, err
						}
					} else {
						return ctx, err
					}
				}
				logFileRotate := "true"
				if !lg.FileRotate {
					logFileRotate = "false"
				}
				if lg.FileBackupCount == 0 {
					lg.FileBackupCount = -1
				}
				logConfigProviderName := lg.Name + "FileTarget"
				logProviders[logConfigProviderName] = NewFileTarget
				logConfigProviderConfigs[index] = `
"` + lg.Name + `":{
	"type": "` + logConfigProviderName + `",
	"FileName":"` + logFileName + `",
	"Rotate": ` + logFileRotate + `,,
	"BackupCount": ` + strconv.FormatInt(lg.FileBackupCount, 10) + `,
	"MaxBytes":` + strconv.Itoa(1<<22) + `
}`
			case TypeConsole:
				logWriter := lg.Writer
				logConfigProviderName := lg.Name + "ConsoleTarget"
				logProviders[logConfigProviderName] = NewConsoleTarget
				logConfigProviderConfigs[index] = `
"` + lg.Name + `":{
	"type": "` + logConfigProviderName + `",
	"WriterName":"` + logWriter.String() + `"
}`
				index++
			}
		}
		logConfig := `{`
		if len(logConfigProviderConfigs) > 0 {
			logConfig = logConfig + `"Targets": {` + strings.Join(logConfigProviderConfigs, ",") + `}`
		}
		logConfig = logConfig + `}`
		err := l.Config.ConfigureJsonConfig(l.handler, []byte(logConfig), logProviders)
		if err != nil {
			return ctx, errors.Annotate(err, errInitiate)
		}
		// logger
		for _, lg := range logs {
			l.NewLogger(lg.Name)
			switch lg.Formatter {
			case FormatterRaw:
				err = l.SetLoggerFormatter(lg.Name, RawLogFormatter())
				if err != nil {
					return ctx, errors.Annotate(err, errInitiate)
				}
			case FormatterSys:
				err = l.SetLoggerFormatter(lg.Name, SysLogFormatter())
				if err != nil {
					return ctx, errors.Annotate(err, errInitiate)
				}
			case FormatterDefault:
				err = l.SetLoggerFormatter(lg.Name, DefaultLogFormatter())
				if err != nil {
					return ctx, errors.Annotate(err, errInitiate)
				}
			}
			err = l.SetLoggerTarget(lg.Name, lg.Name)
			if err != nil {
				return ctx, errors.Annotate(err, errInitiate)
			}
			err = l.SetLoggerMaxLevel(lg.Name, lg.MaxLevel)
			if err != nil {
				return ctx, errors.Annotate(err, errInitiate)
			}
			err = l.OpenLogger(lg.Name)
			if err != nil {
				return ctx, errors.Annotate(err, errInitiate)
			}
		}
	}
	return ctx, nil
}
func (l *Logger) OnStartup(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (l *Logger) OnShutdown(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (l *Logger) GetProvider() string {
	return l.Provider
}
func (l *Logger) Use(ctx context.Context, Provider string) error {
	handler, err := Use(Provider)
	if err != nil {
		return err
	}
	l.Provider = Provider
	l.handler = handler()
	err = l.handler.Initiate(ctx)
	if err != nil {
		return errors.Annotatef(err, errInitiate, l.Provider)
	}
	return nil
}

func (l *Logger) NewLogger(name string) *libLog.Logger {
	return l.handler.NewLogger(name)
}
func (l *Logger) GetLogger(name string) (*libLog.Logger, error) {
	return l.handler.GetLogger(name)
}
func (l *Logger) GetLoggerTarget(name string) (libLog.Target, error) {
	return l.handler.GetLoggerTarget(name)
}
func (l *Logger) RegisterLoggerTarget(name string, targetType string, targetConfig string) error {
	return l.handler.RegisterLoggerTarget(name, targetType, targetConfig)
}
func (l *Logger) SetLoggerTarget(name string, targetName string) error {
	return l.handler.SetLoggerTarget(name, targetName)
}
func (l *Logger) SetLoggerMaxLevel(name string, level Level) error {
	switch level {
	case LevelEmergency:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelEmergency)
	case LevelAlert:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelAlert)
	case LevelCritical:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelCritical)
	case LevelError:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelError)
	case LevelWarning:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelWarning)
	case LevelNotice:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelNotice)
	case LevelInfo:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelInfo)
	case LevelDebug:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelDebug)
	default:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelDebug)
	}
	return nil
}
func (l *Logger) SetLoggerCallStackDepth(name string, d int) error {
	return l.handler.SetLoggerCallStackDepth(name, d)
}
func (l *Logger) SetLoggerCallStackFilter(name string, f string) error {
	return l.handler.SetLoggerCallStackFilter(name, f)
}
func (l *Logger) SetLoggerFormatter(name string, f libLog.Formatter) error {
	return l.handler.SetLoggerFormatter(name, f)
}
func (l *Logger) SetLoggerBufferSize(name string, b int) error {
	return l.handler.SetLoggerBufferSize(name, b)
}
func (l *Logger) OpenLogger(name string) error {
	return l.handler.OpenLogger(name)
}
func (l *Logger) CloseLogger(name string) error {
	return l.handler.CloseLogger(name)
}

func DefaultLogFormatter() libLog.Formatter {
	return func(l *libLog.Logger, e *libLog.Entry) string {
		return fmt.Sprintf("%s|%s|%v%v", e.Time.Format(time.RFC3339), e.Level, e.Message, e.CallStack)
	}
}
func RawLogFormatter() libLog.Formatter {
	return func(l *libLog.Logger, e *libLog.Entry) string {
		return fmt.Sprintf("%v%v", e.Message, e.CallStack)
	}
}
func SysLogFormatter() libLog.Formatter {
	return func(l *libLog.Logger, e *libLog.Entry) string {
		return fmt.Sprintf(`%s %s`, e.Time.Format("2006/01/02 15:04:05"), e.Message)
	}
}
func NewConsoleTarget() *libLog.ConsoleTarget {
	return libLog.NewConsoleTarget()
}
func NewFileTarget() *libLog.FileTarget {
	return libLog.NewFileTarget()
}

type logHandler func() Handler

var logHandlers = make(map[string]logHandler)

func Register(name string, logHandler logHandler) error {
	if logHandler == nil {
		return errors.New("logger: Register log is nil")
	}
	if _, ok := logHandlers[name]; !ok {
		logHandlers[name] = logHandler
	}
	return nil
}
func Use(name string) (logHandler, error) {
	if _, exist := logHandlers[name]; !exist {
		return nil, errors.New(fmt.Sprintf("logger: unknown log '%s' (forgotten register?)", name))
	}
	return logHandlers[name], nil
}

type Handler interface {
	Initiate(ctx context.Context) error
	NewLogger(name string) *libLog.Logger
	GetLogger(name string) (*libLog.Logger, error)
	GetLoggerTarget(name string) (libLog.Target, error)
	RegisterLoggerTarget(name string, targetType string, targetConfig string) error
	SetLoggerTarget(name string, targetName string) error
	SetLoggerMaxLevel(name string, level libLog.Level) error
	SetLoggerCallStackDepth(name string, d int) error
	SetLoggerCallStackFilter(name string, f string) error
	SetLoggerFormatter(name string, f libLog.Formatter) error
	SetLoggerBufferSize(name string, b int) error
	OpenLogger(name string) error
	CloseLogger(name string) error
}
<commit_msg>feat: 提升日志模块健壮性<commit_after>package log

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/ltick/tick-framework/config"
	libLog "github.com/ltick/tick-log"
)

var (
	errInitiate       = "logger: initiate error"
	errStartup        = "logger: startup error"
	errInvalidLogType = "logger: invalid log type '%s'"
)

type LogConfig struct {
	Name            string
	Formatter       string
	Type            string
	FileName        string
	FileRotate      string
	FileBackupCount string
	Writer          string // the writer name of writer (stdout, stderr, discard)
	MaxLevel        string
}

type Log struct {
	Name            string
	Formatter       Formatter
	Type            Type
	FileName        string
	FileRotate      bool
	FileBackupCount int64
	Writer          Writer // the writer name of writer (stdout, stderr, discard)
	MaxLevel        Level
}

// Formatter describes the formatter of a log message.
type Formatter int

const (
	FormatterDefault Formatter = iota
	FormatterRaw
	FormatterSys
)

// LevelNames maps log levels to names
var FormatterNames = map[Formatter]string{
	FormatterDefault: "Default",
	FormatterRaw:     "Raw",
	FormatterSys:     "Sys",
}

// String returns the string representation of the log level
func (w Formatter) String() string {
	if name, ok := FormatterNames[w]; ok {
		return name
	}
	return FormatterNames[FormatterDefault]
}

func StringToFormatter(name string) Formatter {
	for formatter, formatterName := range FormatterNames {
		if strings.ToLower(name) == strings.ToLower(formatterName) {
			return formatter
		}
	}
	return FormatterDefault
}

// Writer describes the writer of a log message.
type Writer int

const (
	WriterUnknown Writer = iota
	WriterStdout
	WriterStderr
	WriterDiscard
)

// LevelNames maps log levels to names
var WriterNames = map[Writer]string{
	WriterUnknown: "unknown",
	WriterStdout:  "stdout",
	WriterStderr:  "stderr",
	WriterDiscard: "discard",
}

// String returns the string representation of the log level
func (w Writer) String() string {
	if name, ok := WriterNames[w]; ok {
		return name
	}
	return WriterNames[WriterUnknown]
}

func StringToWriter(name string) Writer {
	for writer, writerName := range WriterNames {
		if strings.ToLower(name) == strings.ToLower(writerName) {
			return writer
		}
	}
	return WriterUnknown
}

// Type describes the type of a log message.
type Type int

const (
	TypeUnknown Type = iota
	TypeFile
	TypeConsole
)

// LevelNames maps log levels to names
var TypeNames = map[Type]string{
	TypeUnknown: "Unknown",
	TypeFile:    "File",
	TypeConsole: "Console",
}

// String returns the string representation of the log level
func (t Type) String() string {
	if name, ok := TypeNames[t]; ok {
		return name
	}
	return TypeNames[TypeUnknown]
}

func StringToType(name string) Type {
	for typ, typeName := range TypeNames {
		if strings.ToLower(name) == strings.ToLower(typeName) {
			return typ
		}
	}
	return TypeUnknown
}

// Level describes the level of a log message.
type Level int

// RFC5424 log message levels.
const (
	LevelEmergency Level = iota
	LevelAlert
	LevelCritical
	LevelError
	LevelWarning
	LevelNotice
	LevelInfo
	LevelDebug
)

// LevelNames maps log levels to names
var LevelNames = map[Level]string{
	LevelDebug:     "Debug",
	LevelInfo:      "Info",
	LevelNotice:    "Notice",
	LevelWarning:   "Warning",
	LevelError:     "Error",
	LevelCritical:  "Critical",
	LevelAlert:     "Alert",
	LevelEmergency: "Emergency",
}

// String returns the string representation of the log level
func (l Level) String() string {
	if name, ok := LevelNames[l]; ok {
		return name
	}
	return "Unknown"
}

func NewLogger() *Logger {
	logger := &Logger{}
	return logger
}

type Logger struct {
	Config   *config.Config `inject:"true"`
	Provider string
	Logs     []*LogConfig
	handler  Handler
}

func (l *Logger) Initiate(ctx context.Context) (context.Context, error) {
	err := Register("tick", NewTickHandler)
	if err != nil {
		return ctx, errors.Annotatef(err, errInitiate)
	}
	err = l.Use(ctx, "tick")
	if err != nil {
		return ctx, errors.Annotatef(err, errInitiate)
	}
	if l.Logs != nil {
		logs := make([]*Log, 0)
		for _, logConfig := range l.Logs {
			logConfigMaxLevel := LevelDebug
			for level, levelName := range LevelNames {
				if levelName == logConfig.MaxLevel {
					logConfigMaxLevel = level
					break
				}
			}
			logConfigFileRotate := false
			if logConfig.FileRotate == "true" || logConfig.FileRotate == "false" {
				logConfigFileRotate, err = strconv.ParseBool(logConfig.FileRotate)
				if err != nil {
					return ctx, errors.Annotatef(err, errInitiate)
				}
			}
			logConfigFileBackupCount, err := strconv.ParseInt(logConfig.FileBackupCount, 10, 64)
			if err != nil {
				return ctx, errors.Annotatef(err, errInitiate)
			}
			switch StringToType(logConfig.Type) {
			case TypeFile:
				logs = append(logs, &Log{
					Name:            logConfig.Name,
					Type:            TypeFile,
					Formatter:       StringToFormatter(logConfig.Formatter),
					FileName:        logConfig.FileName,
					FileRotate:      logConfigFileRotate,
					FileBackupCount: logConfigFileBackupCount,
					MaxLevel:        logConfigMaxLevel,
				})
			case TypeConsole:
				logs = append(logs, &Log{
					Name:      logConfig.Name,
					Type:      TypeConsole,
					Formatter: StringToFormatter(logConfig.Formatter),
					Writer:    StringToWriter(logConfig.Writer),
					MaxLevel:  logConfigMaxLevel,
				})
			default:
				return ctx, errors.Errorf(errInvalidLogType, StringToType(logConfig.Type))
			}
		}
		var logProviders map[string]interface{} = make(map[string]interface{})
		logConfigProviderConfigs := make([]string, len(logs))
		for index, lg := range logs {
			switch lg.Type {
			case TypeFile:
				logFileName, err := filepath.Abs(lg.FileName)
				if err != nil {
					return ctx, err
				}
				_, err = os.Stat(logFileName)
				if err != nil {
					if os.IsNotExist(err) {
						_, err = os.Create(logFileName)
						if err != nil {
							return ctx, err
						}
					} else {
						return ctx, err
					}
				}
				logFileRotate := "true"
				if !lg.FileRotate {
					logFileRotate = "false"
				}
				if lg.FileBackupCount == 0 {
					lg.FileBackupCount = -1
				}
				logConfigProviderName := lg.Name + "FileTarget"
				logProviders[logConfigProviderName] = NewFileTarget
				logConfigProviderConfigs[index] = `
"` + lg.Name + `":{
	"type": "` + logConfigProviderName + `",
	"FileName":"` + logFileName + `",
	"Rotate": ` + logFileRotate + `,,
	"BackupCount": ` + strconv.FormatInt(lg.FileBackupCount, 10) + `,
	"MaxBytes":` + strconv.Itoa(1<<22) + `
}`
			case TypeConsole:
				logWriter := lg.Writer
				logConfigProviderName := lg.Name + "ConsoleTarget"
				logProviders[logConfigProviderName] = NewConsoleTarget
				logConfigProviderConfigs[index] = `
"` + lg.Name + `":{
	"type": "` + logConfigProviderName + `",
	"WriterName":"` + logWriter.String() + `"
}`
				index++
			}
		}
		logConfig := `{`
		if len(logConfigProviderConfigs) > 0 {
			logConfig = logConfig + `"Targets": {` + strings.Join(logConfigProviderConfigs, ",") + `}`
		}
		logConfig = logConfig + `}`
		err := l.Config.ConfigureJsonConfig(l.handler, []byte(logConfig), logProviders)
		if err != nil {
			return ctx, errors.Annotate(err, errInitiate)
		}
		// logger
		for _, lg := range logs {
			l.NewLogger(lg.Name)
			switch lg.Formatter {
			case FormatterRaw:
				err = l.SetLoggerFormatter(lg.Name, RawLogFormatter())
				if err != nil {
					return ctx, errors.Annotate(err, errInitiate)
				}
			case FormatterSys:
				err = l.SetLoggerFormatter(lg.Name, SysLogFormatter())
				if err != nil {
					return ctx, errors.Annotate(err, errInitiate)
				}
			case FormatterDefault:
				err = l.SetLoggerFormatter(lg.Name, DefaultLogFormatter())
				if err != nil {
					return ctx, errors.Annotate(err, errInitiate)
				}
			}
			err = l.SetLoggerTarget(lg.Name, lg.Name)
			if err != nil {
				return ctx, errors.Annotate(err, errInitiate)
			}
			err = l.SetLoggerMaxLevel(lg.Name, lg.MaxLevel)
			if err != nil {
				return ctx, errors.Annotate(err, errInitiate)
			}
			err = l.OpenLogger(lg.Name)
			if err != nil {
				return ctx, errors.Annotate(err, errInitiate)
			}
		}
	}
	return ctx, nil
}
func (l *Logger) OnStartup(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (l *Logger) OnShutdown(ctx context.Context) (context.Context, error) {
	return ctx, nil
}
func (l *Logger) GetProvider() string {
	return l.Provider
}
func (l *Logger) Use(ctx context.Context, Provider string) error {
	handler, err := Use(Provider)
	if err != nil {
		return err
	}
	l.Provider = Provider
	l.handler = handler()
	err = l.handler.Initiate(ctx)
	if err != nil {
		return errors.Annotatef(err, errInitiate, l.Provider)
	}
	return nil
}

func (l *Logger) NewLogger(name string) *libLog.Logger {
	return l.handler.NewLogger(name)
}
func (l *Logger) GetLogger(name string) (*libLog.Logger, error) {
	return l.handler.GetLogger(name)
}
func (l *Logger) GetLoggerTarget(name string) (libLog.Target, error) {
	return l.handler.GetLoggerTarget(name)
}
func (l *Logger) RegisterLoggerTarget(name string, targetType string, targetConfig string) error {
	return l.handler.RegisterLoggerTarget(name, targetType, targetConfig)
}
func (l *Logger) SetLoggerTarget(name string, targetName string) error {
	return l.handler.SetLoggerTarget(name, targetName)
}
func (l *Logger) SetLoggerMaxLevel(name string, level Level) error {
	switch level {
	case LevelEmergency:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelEmergency)
	case LevelAlert:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelAlert)
	case LevelCritical:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelCritical)
	case LevelError:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelError)
	case LevelWarning:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelWarning)
	case LevelNotice:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelNotice)
	case LevelInfo:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelInfo)
	case LevelDebug:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelDebug)
	default:
		return l.handler.SetLoggerMaxLevel(name, libLog.LevelDebug)
	}
	return nil
}
func (l *Logger) SetLoggerCallStackDepth(name string, d int) error {
	return l.handler.SetLoggerCallStackDepth(name, d)
}
func (l *Logger) SetLoggerCallStackFilter(name string, f string) error {
	return l.handler.SetLoggerCallStackFilter(name, f)
}
func (l *Logger) SetLoggerFormatter(name string, f libLog.Formatter) error {
	return l.handler.SetLoggerFormatter(name, f)
}
func (l *Logger) SetLoggerBufferSize(name string, b int) error {
	return l.handler.SetLoggerBufferSize(name, b)
}
func (l *Logger) OpenLogger(name string) error {
	return l.handler.OpenLogger(name)
}
func (l *Logger) CloseLogger(name string) error {
	return l.handler.CloseLogger(name)
}

func DefaultLogFormatter() libLog.Formatter {
	return func(l *libLog.Logger, e *libLog.Entry) string {
		return fmt.Sprintf("%s|%s|%v%v", e.Time.Format(time.RFC3339), e.Level, e.Message, e.CallStack)
	}
}
func RawLogFormatter() libLog.Formatter {
	return func(l *libLog.Logger, e *libLog.Entry) string {
		return fmt.Sprintf("%v%v", e.Message, e.CallStack)
	}
}
func SysLogFormatter() libLog.Formatter {
	return func(l *libLog.Logger, e *libLog.Entry) string {
		return fmt.Sprintf(`%s %s`, e.Time.Format("2006/01/02 15:04:05"), e.Message)
	}
}
func NewConsoleTarget() *libLog.ConsoleTarget {
	return libLog.NewConsoleTarget()
}
func NewFileTarget() *libLog.FileTarget {
	return libLog.NewFileTarget()
}

type logHandler func() Handler

var logHandlers = make(map[string]logHandler)

func Register(name string, logHandler logHandler) error {
	if logHandler == nil {
		return errors.New("logger: Register log is nil")
	}
	if _, ok := logHandlers[name]; !ok {
		logHandlers[name] = logHandler
	}
	return nil
}
func Use(name string) (logHandler, error) {
	if _, exist := logHandlers[name]; !exist {
		return nil, errors.New(fmt.Sprintf("logger: unknown log '%s' (forgotten register?)", name))
	}
	return logHandlers[name], nil
}

type Handler interface {
	Initiate(ctx context.Context) error
	NewLogger(name string) *libLog.Logger
	GetLogger(name string) (*libLog.Logger, error)
	GetLoggerTarget(name string) (libLog.Target, error)
	RegisterLoggerTarget(name string, targetType string, targetConfig string) error
	SetLoggerTarget(name string, targetName string) error
	SetLoggerMaxLevel(name string, level libLog.Level) error
	SetLoggerCallStackDepth(name string, d int) error
	SetLoggerCallStackFilter(name string, f string) error
	SetLoggerFormatter(name string, f libLog.Formatter) error
	SetLoggerBufferSize(name string, b int) error
	OpenLogger(name string) error
	CloseLogger(name string) error
}
<|endoftext|>


<commit_before>package logging

import (
	"compress/gzip"
	"io"
	"os/exec"
	"time"
)

type RemoteLog struct {
	Host          string
	Pattern       string
	Tail          bool
	Time          time.Time
	Compress      bool
	CustomLogRoot string
}

const (
	DEFAULT_LOG_ROOT = "/var/log/hourly"
	HOURLY_PATTERN   = "2006/01/02/2006-01-02T15.log"
)

func NewRemoteLogFromTime(host string, t time.Time, pattern string) *RemoteLog {
	return &RemoteLog{
		Time:    t,
		Host:    host,
		Pattern: pattern,
	}
}

func (rl *RemoteLog) LogRoot() string {
	if rl.CustomLogRoot != "" {
		return rl.CustomLogRoot
	}
	return DEFAULT_LOG_ROOT
}

func (rl *RemoteLog) Current() string {
	return rl.LogRoot() + "/current"
}

func (rl *RemoteLog) Path() string {
	if !rl.Time.IsZero() {
		return rl.Time.UTC().Format(rl.LogRoot() + "/" + HOURLY_PATTERN)
	}
	return rl.Current()
}

func (rl *RemoteLog) GzipPath() string {
	return rl.Path() + ".gz"
}

func (rl *RemoteLog) Command() string {
	cmd := rl.CatCmd()
	if rl.Pattern != "" {
		cmd += " | " + rl.GrepCmd()
	}
	if rl.Compress {
		cmd += " | gzip"
	}
	return cmd
}

func (rl *RemoteLog) GrepCmd() string {
	return "grep " + rl.Pattern
}

func (rl *RemoteLog) CatCmd() string {
	if rl.Tail {
		return "tail -n 0 -F " + rl.Current()
	}
	return "{ test -e " + rl.Path() + " && cat " + rl.Path() + "; test -e " + rl.GzipPath() + " && cat " + rl.GzipPath() + " | gunzip; }"
}

func (rl *RemoteLog) Reader() (reader io.ReadCloser, e error) {
	c := rl.Command()
	var cmd *exec.Cmd
	if rl.Host != "" {
		cmd = exec.Command("ssh", "-t", "-l", "root", rl.Host, c)
	} else {
		cmd = exec.Command("bash", "-c", c)
	}
	reader, e = cmd.StdoutPipe()
	if e != nil {
		return nil, e
	}
	e = cmd.Start()
	if e != nil {
		return nil, e
	}
	if rl.Compress {
		reader, e = gzip.NewReader(reader)
		if e != nil {
			return nil, e
		}
	}
	return reader, nil
}
<commit_msg>allow parsing the current file from beginning<commit_after>package logging

import (
	"compress/gzip"
	"io"
	"os/exec"
	"time"
)

type RemoteLog struct {
	Host          string
	Pattern       string
	Tail          bool
	Time          time.Time
	Compress      bool
	CustomLogRoot string
	FromBegin     bool // to be used with tail
}

const (
	DEFAULT_LOG_ROOT = "/var/log/hourly"
	HOURLY_PATTERN   = "2006/01/02/2006-01-02T15.log"
)

func NewRemoteLogFromTime(host string, t time.Time, pattern string) *RemoteLog {
	return &RemoteLog{
		Time:    t,
		Host:    host,
		Pattern: pattern,
	}
}

func (rl *RemoteLog) LogRoot() string {
	if rl.CustomLogRoot != "" {
		return rl.CustomLogRoot
	}
	return DEFAULT_LOG_ROOT
}

func (rl *RemoteLog) Current() string {
	return rl.LogRoot() + "/current"
}

func (rl *RemoteLog) Path() string {
	if !rl.Time.IsZero() {
		return rl.Time.UTC().Format(rl.LogRoot() + "/" + HOURLY_PATTERN)
	}
	return rl.Current()
}

func (rl *RemoteLog) GzipPath() string {
	return rl.Path() + ".gz"
}

func (rl *RemoteLog) Command() string {
	cmd := rl.CatCmd()
	if rl.Pattern != "" {
		cmd += " | " + rl.GrepCmd()
	}
	if rl.Compress {
		cmd += " | gzip"
	}
	return cmd
}

func (rl *RemoteLog) GrepCmd() string {
	return "grep " + rl.Pattern
}

func (rl *RemoteLog) CatCmd() string {
	if rl.Tail {
		n := "0"
		if rl.FromBegin {
			n = "+0"
		}
		return "tail -n " + n + " -F " + rl.Current()
	}
	return "{ test -e " + rl.Path() + " && cat " + rl.Path() + "; test -e " + rl.GzipPath() + " && cat " + rl.GzipPath() + " | gunzip; }"
}

func (rl *RemoteLog) Open() (reader io.ReadCloser, e error) {
	c := rl.Command()
	var cmd *exec.Cmd
	if rl.Host != "" {
		cmd = exec.Command("ssh", "-t", "-l", "root", rl.Host, c)
	} else {
		cmd = exec.Command("bash", "-c", c)
	}
	reader, e = cmd.StdoutPipe()
	if e != nil {
		return nil, e
	}
	e = cmd.Start()
	if e != nil {
		return nil, e
	}
	if rl.Compress {
		reader, e = gzip.NewReader(reader)
		if e != nil {
			return nil, e
		}
	}
	return reader, nil
}
<|endoftext|>


<commit_before>// Package logging is the logging package used by Neptune.io agent.
// The implementation might route the logging via any other Go logging package but
// should mask those details from remaining agent packages.
package logging

import (
	"fmt"
	"os"

	"github.com/Sirupsen/logrus"

	"gopkg.in/natefinch/lumberjack.v2"
)

const (
	maxLogFileSizeInMB = 10
	maxNumLogFiles     = 10
)

// Fields type, used to pass to key value pairs.
type Fields map[string]interface{}

var log *logrus.Logger

func init() {
	// Create a new instance of the logger. You can have any number of instances.
	log = logrus.New()
}

func convertToLogrusFields(fields Fields) logrus.Fields {
	result := logrus.Fields{}
	for k := range fields {
		result[k] = fields[k]
	}
	return result
}

func Debug(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Debug(msg)
	} else {
		log.Debug(msg)
	}
}

func Info(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Info(msg)
	} else {
		log.Info(msg)
	}
}

func Warn(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Warn(msg)
	} else {
		log.Warn(msg)
	}
}

func Error(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Error(msg)
	} else {
		log.Error(msg)
	}
}

// Function to setup logger for agent.
func SetupLogger(logfile string, debugMode bool, errorsChannel chan string) error {
	f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Printf("Error opening file: %v\n", err)
		return err
	}
	defer f.Close()

	log.Out = &lumberjack.Logger{
		Filename:   logfile,
		MaxSize:    maxLogFileSizeInMB, // megabytes
		MaxBackups: maxNumLogFiles,
		LocalTime:  true,
	}

	hook := NewNeptuneHook(logrus.ErrorLevel, errorsChannel)
	log.Hooks.Add(hook)

	// Only log the info severity or above.
	if debugMode {
		log.Level = logrus.DebugLevel
	} else {
		log.Level = logrus.InfoLevel
	}

	log.Formatter = &logrus.TextFormatter{
		DisableColors: true,
		FullTimestamp: true,
	}
	return nil
}
<commit_msg>Minor code cleanup.<commit_after>// Package logging is the logging package used by Neptune.io agent.
// The implementation might route the logging via any other Go logging package but
// should mask those details from remaining agent packages.
package logging

import (
	"fmt"
	"os"

	"github.com/Sirupsen/logrus"

	"gopkg.in/natefinch/lumberjack.v2"
)

const (
	maxLogFileSizeInMB = 10
	maxNumLogFiles     = 10
)

// Fields type, used to pass to key value pairs.
type Fields map[string]interface{}

var log = logrus.New()

func convertToLogrusFields(fields Fields) logrus.Fields {
	result := logrus.Fields{}
	for k := range fields {
		result[k] = fields[k]
	}
	return result
}

func Debug(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Debug(msg)
	} else {
		log.Debug(msg)
	}
}

func Info(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Info(msg)
	} else {
		log.Info(msg)
	}
}

func Warn(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Warn(msg)
	} else {
		log.Warn(msg)
	}
}

func Error(msg string, fields Fields) {
	if fields != nil {
		log.WithFields(convertToLogrusFields(fields)).Error(msg)
	} else {
		log.Error(msg)
	}
}

// Function to setup logger for agent.
func SetupLogger(logfile string, debugMode bool, errorsChannel chan string) error {
	f, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Printf("Error opening file: %v\n", err)
		return err
	}
	defer f.Close()

	log.Out = &lumberjack.Logger{
		Filename:   logfile,
		MaxSize:    maxLogFileSizeInMB, // megabytes
		MaxBackups: maxNumLogFiles,
		LocalTime:  true,
	}

	hook := NewNeptuneHook(logrus.ErrorLevel, errorsChannel)
	log.Hooks.Add(hook)

	// Only log the info severity or above.
	if debugMode {
		log.Level = logrus.DebugLevel
	} else {
		log.Level = logrus.InfoLevel
	}

	log.Formatter = &logrus.TextFormatter{
		DisableColors: true,
		FullTimestamp: true,
	}
	return nil
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
)

type Node interface{}

type Identifier struct {
	Name string
	Type Type
}

func (i Identifier) EvaluatesTo() Type {
	return i.Type
}

func NewIdentifier(name string) Identifier {
	return Identifier{name, AnyType}
}

type Statement interface {
	Node
}
type Expression interface {
	EvaluatesTo() Type
}

const AnyType = String | Integer | Float | Boolean | Null | Resource | Array | Object

type OperatorExpression struct {
	Operand1 Expression
	Operand2 Expression
	Operand3 Expression
	Type     Type
	Operator string
}

func (o OperatorExpression) String() string {
	if o.Operand2 == nil {
		return fmt.Sprintf("(%s%v~%v)", o.Operand1, o.Operator, o.Type)
	}
	if o.Operand3 == nil {
		return fmt.Sprintf("(%s %v %s~%v)", o.Operand1, o.Operator, o.Operand2, o.Type)
	}
	return fmt.Sprintf("(%s ? %s : %s~%v)", o.Operand1, o.Operand2, o.Operand3, o.Type)
}

func (o OperatorExpression) EvaluatesTo() Type {
	return o.Type
}

func Echo(expr Expression) EchoStmt {
	return EchoStmt{Expression: expr}
}

type EchoStmt struct {
	Expression Expression
}

type ReturnStmt struct {
	Expression
}

type AssignmentStmt struct {
	Assignee Identifier
	Value    Expression
}

type FunctionCallStmt struct {
	FunctionCallExpression
}

type FunctionCallExpression struct {
	FunctionName string
	Arguments    []Expression
}

func (f FunctionCallExpression) EvaluatesTo() Type {
	return String | Integer | Float | Boolean | Null | Resource | Array | Object
}

type Block struct {
	Statements []Statement
	Scope      Scope
}

type FunctionStmt struct {
	FunctionDefinition
	Body Block
}

type FunctionDefinition struct {
	Name      string
	Arguments []FunctionArgument
}

type FunctionArgument struct {
	TypeHint   string
	Identifier Identifier
}

type Class struct {
	Name       string
	Extends    *Class
	Implements []*Interface
	Methods    []Method
}

type Constant struct {
	Identifier
	Value interface{}
}

type Interface struct {
	Methods []FunctionDefinition
}

type Method struct {
	*FunctionStmt
	Visibility Visibility
}

type Visibility int

const (
	Private Visibility = iota
	Protected
	Public
)

type IfStmt struct {
	Condition  Expression
	TrueBlock  Statement
	FalseBlock Statement
}

type ForStmt struct {
	Initialization Expression
	Termination    Expression
	Iteration      Expression
	LoopBlock      Block
}

type WhileStmt struct {
	Termination Expression
	LoopBlock   Block
}

type DoWhileStmt struct {
	Termination Expression
	LoopBlock   Block
}

type TryStmt struct {
	TryBlock     *Block
	FinallyBlock *Block
	CatchStmts   []*CatchStmt
}

type CatchStmt struct {
	CatchBlock Block
	CatchType  Type
}

type Literal struct {
	Type Type
}

func (l Literal) EvaluatesTo() Type {
	return l.Type
}

type ForeachStmt struct {
	Source    Expression
	Key       *Identifier
	Value     Identifier
	LoopBlock Block
}
<commit_msg>Added some comments to the nodes file.<commit_after>package ast

import (
	"fmt"
)

// Node encapsulates every AST node.
type Node interface {

// An Identifier is specifically a variable in code.
type Identifier struct {
	Name string
	Type Type
}

// EvaluatesTo returns the known type of the variable.
func (i Identifier) EvaluatesTo() Type {
	return i.Type
}

// NewIdentifier intializes an identifier node with its type set to AnyType.
func NewIdentifier(name string) Identifier {
	return Identifier{name, AnyType}
}

// A statement is an executable piece of code. It may be as simple as
// a function call or a variable assignment. It also includes things like
// "if".
type Statement interface {
	Node
}

// An Expression is a snippet of code that evaluates to a single value when run
// and does not represent a program instruction.
type Expression interface {
	EvaluatesTo() Type
}

// AnyType is a bitmask of all the valid types.
const AnyType = String | Integer | Float | Boolean | Null | Resource | Array | Object

// OperatorExpression is an expression that applies an operator to one, two, or three
// operands. The operator determines how many operands it should contain.
type OperatorExpression struct {
	Operand1 Expression
	Operand2 Expression
	Operand3 Expression
	Type     Type
	Operator string
}

func (o OperatorExpression) String() string {
	if o.Operand2 == nil {
		return fmt.Sprintf("(%s%v~%v)", o.Operand1, o.Operator, o.Type)
	}
	if o.Operand3 == nil {
		return fmt.Sprintf("(%s %v %s~%v)", o.Operand1, o.Operator, o.Operand2, o.Type)
	}
	return fmt.Sprintf("(%s ? %s : %s~%v)", o.Operand1, o.Operand2, o.Operand3, o.Type)
}

func (o OperatorExpression) EvaluatesTo() Type {
	return o.Type
}

// Echo returns a new echo statement.
func Echo(expr Expression) EchoStmt {
	return EchoStmt{Expression: expr}
}

// Echo represents an echo statement. It may be either a literal statement
// or it may be from data outside PHP-mode, such as "here" in: <? not here ?> here <? not here ?>
type EchoStmt struct {
	Expression Expression
}

// ReturnStmt represents a function return.
type ReturnStmt struct {
	Expression
}

// AssignmentStmt represents an assignment.
type AssignmentStmt struct {
	Assignee Identifier
	Value    Expression
}

type FunctionCallStmt struct {
	FunctionCallExpression
}

type FunctionCallExpression struct {
	FunctionName string
	Arguments    []Expression
}

func (f FunctionCallExpression) EvaluatesTo() Type {
	return String | Integer | Float | Boolean | Null | Resource | Array | Object
}

type Block struct {
	Statements []Statement
	Scope      Scope
}

type FunctionStmt struct {
	FunctionDefinition
	Body Block
}

type FunctionDefinition struct {
	Name      string
	Arguments []FunctionArgument
}

type FunctionArgument struct {
	TypeHint   string
	Identifier Identifier
}

type Class struct {
	Name       string
	Extends    *Class
	Implements []*Interface
	Methods    []Method
}

type Constant struct {
	Identifier
	Value interface{}
}

type Interface struct {
	Methods []FunctionDefinition
}

type Method struct {
	*FunctionStmt
	Visibility Visibility
}

type Visibility int

const (
	Private Visibility = iota
	Protected
	Public
)

type IfStmt struct {
	Condition  Expression
	TrueBlock  Statement
	FalseBlock Statement
}

type ForStmt struct {
	Initialization Expression
	Termination    Expression
	Iteration      Expression
	LoopBlock      Block
}

type WhileStmt struct {
	Termination Expression
	LoopBlock   Block
}

type DoWhileStmt struct {
	Termination Expression
	LoopBlock   Block
}

type TryStmt struct {
	TryBlock     *Block
	FinallyBlock *Block
	CatchStmts   []*CatchStmt
}

type CatchStmt struct {
	CatchBlock Block
	CatchType  Type
}

type Literal struct {
	Type Type
}

func (l Literal) EvaluatesTo() Type {
	return l.Type
}

type ForeachStmt struct {
	Source    Expression
	Key       *Identifier
	Value     Identifier
	LoopBlock Block
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
onEncryptedMessage is called for messages from encrypted peers. Will redestribute
the message according to its type.

TODO describe order of operations (successful lock -> request model -> sync -> push / pull difference)
*/
func (c *chaninterface) onEncryptedMessage(address string, msgType shared.MsgType, message string) {
	// TODO switch and handle messages NOTE FIXME implement
	switch msgType {
	case shared.MsgLock:
		msg := &shared.LockMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncLockMessage(address, *msg)
	case shared.MsgNotify:
		msg := &shared.NotifyMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncNotifyMessage(address, *msg)
	case shared.MsgRequest:
		msg := &shared.RequestMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncRequestMessage(address, *msg)
	default:
		c.warn("Unknown object received:", msgType.String())
	}
}

/*
onEncLockMessage handles lock messages. Notably it requests a model on a
successful lock, starting a synchronization.

FIXME: peer never sends a release request to locked encrypted peers! For now we
just await the timeout... TODO
*/
func (c *chaninterface) onEncLockMessage(address string, msg shared.LockMessage) {
	switch msg.Action {
	case shared.LoAccept:
		// remember that this peer is locked
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not set peer to locked as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(true)
		// if LOCKED request model file to begin sync
		rm := shared.CreateRequestMessage(shared.OtModel, shared.IDMODEL)
		c.requestFile(address, rm, c.encModelReceived)
	case shared.LoRelease:
		// unset lock of this peer
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not release peer as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(false)
	default:
		c.warn("Unknown lock action received:", msg.Action.String())
	}
}

/*
onEncNotifyMessage handles the reception of notification messages.
*/
func (c *chaninterface) onEncNotifyMessage(address string, msg shared.NotifyMessage) {
	switch msg.Notify {
	case shared.NoMissing:
		// remove transfer as no file will come
		delete(c.inTransfers, c.buildKey(address, msg.Identification))
		// if model --> create it
		if msg.Identification == shared.IDMODEL {
			// log that encrypted was empty and that we'll just upload our current state
			c.log("Encrypted is empty, nothing to merge, uploading directly.")
			// so send push for model and all objects
			c.sendCompletePushes(address)
			return
		}
		// TODO: if object --> error... maybe "reset" the encrypted peer?
		log.Println("DEBUG: object missing!", msg.Identification, msg.Notify)
		log.Println("DEBUG: TODO: reset encrypted peer?")
	default:
		c.warn("Unknown notify type received:", msg.Notify.String())
	}
}

/*
onEncRequestMessage handles the reception of a request message following a push
message. Triggers the sending of the requested file.
*/
func (c *chaninterface) onEncRequestMessage(address string, msg shared.RequestMessage) {
	var path string
	// if model has been requested --> path is different as not tracked itself
	if msg.Identification == shared.IDMODEL {
		path = c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + shared.MODELJSON
		// get model info
		model, err := c.tin.model.Read()
		if err != nil {
			c.warn("Failed to read model:", err.Error())
			return
		}
		// convert to json
		data, err := json.Marshal(model)
		if err != nil {
			c.warn("Failed to marshal model to JSON:", err.Error())
			return
		}
		// write json to temp file
		err = ioutil.WriteFile(path, data, shared.FILEPERMISSIONMODE)
		if err != nil {
			c.warn("Failed to write model info to temp file:", err.Error())
			return
		}
	} else {
		// get subPath for file
		subPath, err := c.tin.model.GetSubPath(msg.Identification)
		if err != nil {
			c.warn("Failed to locate subpath for request message!", msg.Identification)
			return
		}
		path = c.tin.Path + "/" + subPath
	}
	// and send file (concurrent because of encryption)
	go c.encSendFile(address, msg.Identification, path, msg.ObjType)
	// TODO: shouldn't we reread the msg.ObjType from disk too?
}

/*
encSendFile handles uploading a file to the encrypted peer. This function is MADE
to run concurrently. Path is the path where the file CURRENTLY resides: the method
will copy all its data to SENDINGDIR, encrypt it there, and then send it.
*/
func (c *chaninterface) encSendFile(address, identification, path string, ot shared.ObjectType) {
	// read file data
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.warn("Failed to read data:", err.Error())
		return
	}
	// TODO encrypt here? FIXME
	log.Println("DEBUG: encrypt here")
	// write to temp file
	sendPath := c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + identification
	err = ioutil.WriteFile(sendPath, data, shared.FILEPERMISSIONMODE)
	if err != nil {
		c.warn("Failed to write (encrypted) data to sending file:", err.Error())
		return
	}
	// send file
	err = c.tin.channel.SendFile(address, sendPath, identification, func(success bool) {
		if !success {
			c.log("Failed to upload file!", ot.String(), identification)
		}
		// remove sending temp file always
		err := os.Remove(sendPath)
		if err != nil {
			c.warn("Failed to remove sending file!", err.Error())
		}
	})
	if err != nil {
		c.warn("Failed to send file:", err.Error())
		return
	}
	// done
}

/*
sendCompletePushes sends push models for everything, starting with the model.
This will result in the encrypted peer requesting all objects.
*/
func (c *chaninterface) sendCompletePushes(address string) {
	// vars we'll use
	var pm shared.PushMessage
	// start by sending push for model
	pm = shared.CreatePushMessage(shared.IDMODEL, shared.OtModel)
	c.tin.channel.Send(address, pm.JSON())
	for path, stin := range c.tin.model.StaticInfos {
		// if directory, skip
		if stin.Directory {
			continue
		}
		c.encSendPush(address, path, stin.Identification)
	}
	// and done
}

/*
encModelReceived is called when a model is received from an encrypted peer. It
triggers the complete sync with the encrypted state, concluding with updating
the encrypted peer to be up to date with this peer.
*/
func (c *chaninterface) encModelReceived(address, path string) {
	// no matter what: remove temp file
	defer func() {
		err := os.Remove(path)
		if err != nil {
			c.warn("encModelReceived: failed to remove received temp model file!", err.Error())
		}
	}()
	// read model
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.log("encModelReceived: failed to read received model file:", err.Error())
		return
	}
	// TODO decrypt file!
	log.Println("DEBUG: TODO: decrypt model here!")
	// unmarshal
	foreignModel := &shared.ObjectInfo{}
	err = json.Unmarshal(data, foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to parse JSON:", err.Error())
		return
	}
	// get updates we have to apply to ourselves before updating encrypted
	updateLists, err := c.tin.model.Sync(foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to sync models:", err.Error())
		return
	}
	// apply updates to bring our model to a merged state
	log.Println("DEBUG: encrypted is", len(updateLists), "possible operations ahead.")
	var wg sync.WaitGroup
	for _, um := range updateLists {
		wg.Add(1) // wait for each subroutine
		go func(um *shared.UpdateMessage) {
			defer func() { wg.Done() }() // no matter what unlock sync
			err := c.handleEncryptedMessage(address, um)
			if err != nil {
				c.log("encModelReceived: handleMessage failed with:", err.Error())
			}
		}(um) // pass UpdateMessage because otherwise the pointer will change on each for iteration
	}
	wg.Wait() // wait for all updates to have been applied
	// when completely done, start uploading current state
	log.Println("DEBUG: now upload my current state to encrypted!")
	// build path maps and associated object maps of foreign model
	foreignPaths := make(map[string]bool)
	foreignObjs := make(map[string]*shared.ObjectInfo)
	foreignModel.ForEach(func(obj shared.ObjectInfo) {
		foreignPaths[obj.Path] = true
		// strip of children and write to objects
		obj.Objects = nil
		foreignObjs[obj.Path] = &obj
	})
	// get what must be changed
	created, remained, removed := shared.Difference(foreignPaths, c.tin.model.TrackedPaths)
	// for each path: check and create messages accordingly
	for _, create := range created {
		stin, exists := c.tin.model.StaticInfos[create]
		if !exists {
			// continue because this means it doesn't actually exist
			c.warn("encModelReceived: missing stin for locally created object!")
			continue
		}
		if stin.Directory {
			continue
		}
		log.Println("Send push for", create, stin.Identification)
		c.encSendPush(address, create, stin.Identification)
	}
	for _, remains := range remained {
		stin, exists := c.tin.model.StaticInfos[remains]
		if !exists {
			// continue because this means it doesn't actually exist
			c.warn("encModelReceived: missing stin for locally held object!")
			continue
		}
		if stin.Directory {
			continue
		}
		// TODO check if something changed since last push...
		log.Println("Send push for", remains, "after checking modify!", stin.Identification)
		c.encSendPush(address, remains, stin.Identification)
	}
	// removed objects: use notify to have encrypted delete them
	for _, remove := range removed {
		stin, exists := foreignObjs[remove]
		if !exists {
			c.warn("encModelReceived: missing stin from foreign objects!")
			continue
		}
		if stin.Directory {
			continue
		}
		log.Println("Send notify for", remove, stin.Identification)
		nm := shared.CreateNotifyMessage(shared.NoRemoved, stin.Identification)
		c.tin.channel.Send(address, nm.JSON())
	}
	// TODO wg wait here FIXME: waiting to unlock means waiting for requests. Abstract messages away into channels?
	log.Println("DEBUG: done encrypted sync")
	return
	// finally unlock when done
	lm := shared.CreateLockMessage(shared.LoRelease)
	c.tin.channel.Send(address, lm.JSON())
}

/*
handleEncryptedMessage looks at the message, fetches files if required, and correctly
applies it to the model.

TODO handle how we update the encrypted peer – upload created and modified files, remove deleted files.
*/
func (c *chaninterface) handleEncryptedMessage(address string, msg *shared.UpdateMessage) error {
	// use check message to prepare message and check for special cases
	msg, err := c.tin.model.CheckMessage(msg)
	// if update known --> ignore it
	if err == model.ErrIgnoreUpdate {
		return nil
	}
	// if encrypted has a removal that we have registered as done, remove it
	if err == model.ErrObjectRemovalDone {
		log.Println("DEBUG: remove removal from encrypted.", msg.String())
		// TODO what exactly do I have to tell encrypted to remove? FIXME
		nm := shared.CreateNotifyMessage(shared.NoRemoved, msg.Object.Identification)
		c.tin.channel.Send(address, nm.JSON())
		// done
		return nil
	}
	// if still error, return it
	if err != nil {
		return err
	}
	// --> IF CheckMessage was ok, we can now handle applying the message
	// apply directories directly
	if msg.Object.Directory {
		// no merge because it should never happen for directories
		return c.tin.model.ApplyUpdateMessage(msg)
	}
	op := msg.Operation
	// create and modify must first fetch the file
	if op == shared.OpCreate || op == shared.OpModify {
		rm := shared.CreateRequestMessage(shared.OtObject, msg.Object.Identification)
		var wg sync.WaitGroup
		wg.Add(1)
		c.requestFile(address, rm, func(address, path string) {
			// force calling function to wait until this has been handled
			defer func() { wg.Done() }()
			// rename to correct name for model
			err := os.Rename(path, c.temppath+"/"+rm.Identification)
			if err != nil {
				c.log("Failed to move file to temp: " + err.Error())
				return
			}
			// TODO decrypt file!
			log.Println("DEBUG: TODO: decrypt file here!")
			// apply
			err = c.mergeUpdate(*msg)
			if err != nil {
				c.log("File application error: " + err.Error())
			}
			// done
		})
		// wait for file to be received before returning
		wg.Wait()
		// errors may turn up but only when the file has been received, so done here
		return nil
	} else if op == shared.OpRemove {
		// remove is without file transfer, so directly apply
		return c.mergeUpdate(*msg)
	}
	c.warn("Unknown operation received, ignoring update message!")
	return shared.ErrIllegalParameters
}

/*
encSendPush sends a PushMessage for the given parameters, setting the ObjType
according to the path.
*/
func (c *chaninterface) encSendPush(address, path, identification string) {
	peerDir := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.PEERSDIR
	authPath := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.AUTHJSON
	// default object type is OtObject
	objectType := shared.OtObject
	// if peer --> update objectType
	if strings.HasPrefix(path, peerDir) {
		objectType = shared.OtPeer
	}
	// if auth file --> update objectType
	if path == authPath {
		objectType = shared.OtAuth
	}
	pm := shared.CreatePushMessage(identification, objectType)
	c.tin.channel.Send(address, pm.JSON())
}
<commit_msg>debug stuff and applied knowledge :P<commit_after>package core

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"sync"

	"github.com/tinzenite/model"
	"github.com/tinzenite/shared"
)

/*
onEncryptedMessage is called for messages from encrypted peers. Will redestribute
the message according to its type.

TODO describe order of operations (successful lock -> request model -> sync -> push / pull difference)
*/
func (c *chaninterface) onEncryptedMessage(address string, msgType shared.MsgType, message string) {
	// TODO switch and handle messages NOTE FIXME implement
	switch msgType {
	case shared.MsgLock:
		msg := &shared.LockMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncLockMessage(address, *msg)
	case shared.MsgNotify:
		msg := &shared.NotifyMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncNotifyMessage(address, *msg)
	case shared.MsgRequest:
		msg := &shared.RequestMessage{}
		err := json.Unmarshal([]byte(message), msg)
		if err != nil {
			log.Println(err.Error())
			return
		}
		c.onEncRequestMessage(address, *msg)
	default:
		c.warn("Unknown object received:", msgType.String())
	}
}

/*
onEncLockMessage handles lock messages. Notably it requests a model on a
successful lock, starting a synchronization.

FIXME: peer never sends a release request to locked encrypted peers! For now we
just await the timeout... TODO
*/
func (c *chaninterface) onEncLockMessage(address string, msg shared.LockMessage) {
	switch msg.Action {
	case shared.LoAccept:
		// remember that this peer is locked
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not set peer to locked as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(true)
		// if LOCKED request model file to begin sync
		rm := shared.CreateRequestMessage(shared.OtModel, shared.IDMODEL)
		c.requestFile(address, rm, c.encModelReceived)
	case shared.LoRelease:
		// unset lock of this peer
		_, exists := c.tin.peers[address]
		if !exists {
			c.warn("Can not release peer as peer doesn't exist!")
			return
		}
		c.tin.peers[address].SetLocked(false)
	default:
		c.warn("Unknown lock action received:", msg.Action.String())
	}
}

/*
onEncNotifyMessage handles the reception of notification messages.
*/
func (c *chaninterface) onEncNotifyMessage(address string, msg shared.NotifyMessage) {
	switch msg.Notify {
	case shared.NoMissing:
		// remove transfer as no file will come
		delete(c.inTransfers, c.buildKey(address, msg.Identification))
		// if model --> create it
		if msg.Identification == shared.IDMODEL {
			// log that encrypted was empty and that we'll just upload our current state
			c.log("Encrypted is empty, nothing to merge, uploading directly.")
			// so send push for model and all objects
			c.sendCompletePushes(address)
			return
		}
		// TODO: if object --> error... maybe "reset" the encrypted peer?
		log.Println("DEBUG: object missing!", msg.Identification, msg.Notify)
		log.Println("DEBUG: TODO: reset encrypted peer?")
	default:
		c.warn("Unknown notify type received:", msg.Notify.String())
	}
}

/*
onEncRequestMessage handles the reception of a request message following a push
message. Triggers the sending of the requested file.
*/
func (c *chaninterface) onEncRequestMessage(address string, msg shared.RequestMessage) {
	var path string
	// if model has been requested --> path is different as not tracked itself
	if msg.Identification == shared.IDMODEL {
		path = c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + shared.MODELJSON
		// get model info
		model, err := c.tin.model.Read()
		if err != nil {
			c.warn("Failed to read model:", err.Error())
			return
		}
		// convert to json
		data, err := json.Marshal(model)
		if err != nil {
			c.warn("Failed to marshal model to JSON:", err.Error())
			return
		}
		// write json to temp file
		err = ioutil.WriteFile(path, data, shared.FILEPERMISSIONMODE)
		if err != nil {
			c.warn("Failed to write model info to temp file:", err.Error())
			return
		}
	} else {
		// get subPath for file
		subPath, err := c.tin.model.GetSubPath(msg.Identification)
		if err != nil {
			c.warn("Failed to locate subpath for request message!", msg.Identification)
			return
		}
		path = c.tin.Path + "/" + subPath
	}
	// and send file (concurrent because of encryption)
	go c.encSendFile(address, msg.Identification, path, msg.ObjType)
	// TODO: shouldn't we reread the msg.ObjType from disk too?
}

/*
encSendFile handles uploading a file to the encrypted peer. This function is MADE
to run concurrently. Path is the path where the file CURRENTLY resides: the method
will copy all its data to SENDINGDIR, encrypt it there, and then send it.
*/
func (c *chaninterface) encSendFile(address, identification, path string, ot shared.ObjectType) {
	// read file data
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.warn("Failed to read data:", err.Error())
		return
	}
	// TODO encrypt here? FIXME
	// log.Println("DEBUG: encrypt here")
	// write to temp file
	sendPath := c.tin.Path + "/" + shared.TINZENITEDIR + "/" + shared.SENDINGDIR + "/" + identification
	err = ioutil.WriteFile(sendPath, data, shared.FILEPERMISSIONMODE)
	if err != nil {
		c.warn("Failed to write (encrypted) data to sending file:", err.Error())
		return
	}
	// send file
	err = c.tin.channel.SendFile(address, sendPath, identification, func(success bool) {
		if !success {
			c.log("Failed to upload file!", ot.String(), identification)
		}
		// remove sending temp file always
		err := os.Remove(sendPath)
		if err != nil {
			c.warn("Failed to remove sending file!", err.Error())
		}
	})
	if err != nil {
		c.warn("Failed to send file:", err.Error())
		return
	}
	// done
}

/*
sendCompletePushes sends push models for everything, starting with the model.
This will result in the encrypted peer requesting all objects.
*/
func (c *chaninterface) sendCompletePushes(address string) {
	// vars we'll use
	var pm shared.PushMessage
	// start by sending push for model
	pm = shared.CreatePushMessage(shared.IDMODEL, shared.OtModel)
	c.tin.channel.Send(address, pm.JSON())
	for path, stin := range c.tin.model.StaticInfos {
		// if directory, skip
		if stin.Directory {
			continue
		}
		c.encSendPush(address, path, stin.Identification)
	}
	// and done
}

/*
encModelReceived is called when a model is received from an encrypted peer. It
triggers the complete sync with the encrypted state, concluding with updating
the encrypted peer to be up to date with this peer.
*/
func (c *chaninterface) encModelReceived(address, path string) {
	// no matter what: remove temp file
	defer func() {
		err := os.Remove(path)
		if err != nil {
			c.warn("encModelReceived: failed to remove received temp model file!", err.Error())
		}
	}()
	// read model
	data, err := ioutil.ReadFile(path)
	if err != nil {
		c.log("encModelReceived: failed to read received model file:", err.Error())
		return
	}
	// TODO decrypt file!
	// log.Println("DEBUG: TODO: decrypt model here!")
	// unmarshal
	foreignModel := &shared.ObjectInfo{}
	err = json.Unmarshal(data, foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to parse JSON:", err.Error())
		return
	}
	// get updates we have to apply to ourselves before updating encrypted
	updateLists, err := c.tin.model.Sync(foreignModel)
	if err != nil {
		c.log("encModelReceived: failed to sync models:", err.Error())
		return
	}
	// apply updates to bring our model to a merged state
	log.Println("DEBUG: encrypted is", len(updateLists), "possible operations ahead.")
	var wg sync.WaitGroup
	for _, um := range updateLists {
		log.Println("DEBUG: pointer value:", um)
		wg.Add(1) // wait for each subroutine
		// pass by value, not reference
		go func(um shared.UpdateMessage) {
			defer func() { wg.Done() }() // no matter what unlock sync
			err := c.handleEncryptedMessage(address, &um)
			if err != nil {
				c.log("encModelReceived: handleMessage failed with:", err.Error())
			}
		}(*um) // pass UpdateMessage because otherwise the pointer will change on each for iteration
	}
	wg.Wait() // wait for all updates to have been applied
	// when completely done, start uploading current state
	log.Println("DEBUG: now upload my current state to encrypted!")
	// build path maps and associated object maps of foreign model
	foreignPaths := make(map[string]bool)
	foreignObjs := make(map[string]*shared.ObjectInfo)
	foreignModel.ForEach(func(obj shared.ObjectInfo) {
		foreignPaths[obj.Path] = true
		// strip of children and write to objects
		obj.Objects = nil
		foreignObjs[obj.Path] = &obj
	})
	// get what must be changed
	created, remained, removed := shared.Difference(foreignPaths, c.tin.model.TrackedPaths)
	// for each path: check and create messages accordingly
	for _, create := range created {
		stin, exists := c.tin.model.StaticInfos[create]
		if !exists {
			// continue because this means it doesn't actually exist
			c.warn("encModelReceived: missing stin for locally created object!")
			continue
		}
		if stin.Directory {
			continue
		}
		log.Println("Send push for created", create)
		c.encSendPush(address, create, stin.Identification)
	}
	for _, remains := range remained {
		stin, exists := c.tin.model.StaticInfos[remains]
		if !exists {
			// continue because this means it doesn't actually exist
			c.warn("encModelReceived: missing stin for locally held object!")
			continue
		}
		if stin.Directory {
			continue
		}
		// TODO check if something changed since last push...
		log.Println("Send push for modified", remains)
		c.encSendPush(address, remains, stin.Identification)
	}
	// removed objects: use notify to have encrypted delete them
	for _, remove := range removed {
		stin, exists := foreignObjs[remove]
		if !exists {
			c.warn("encModelReceived: missing stin from foreign objects!")
			continue
		}
		if stin.Directory {
			continue
		}
		log.Println("Send notify for removal of", remove)
		nm := shared.CreateNotifyMessage(shared.NoRemoved, stin.Identification)
		c.tin.channel.Send(address, nm.JSON())
	}
	// TODO wg wait here FIXME: waiting to unlock means waiting for requests. Abstract messages away into channels?
	log.Println("DEBUG: done encrypted sync")
	return
	// finally unlock when done
	lm := shared.CreateLockMessage(shared.LoRelease)
	c.tin.channel.Send(address, lm.JSON())
}

/*
handleEncryptedMessage looks at the message, fetches files if required, and correctly
applies it to the model.

TODO handle how we update the encrypted peer – upload created and modified files, remove deleted files.
*/
func (c *chaninterface) handleEncryptedMessage(address string, msg *shared.UpdateMessage) error {
	// use check message to prepare message and check for special cases
	msg, err := c.tin.model.CheckMessage(msg)
	// if update known --> ignore it
	if err == model.ErrIgnoreUpdate {
		return nil
	}
	// if encrypted has a removal that we have registered as done, remove it
	if err == model.ErrObjectRemovalDone {
		log.Println("DEBUG: remove removal from encrypted.", msg.String())
		// TODO what exactly do I have to tell encrypted to remove? FIXME
		nm := shared.CreateNotifyMessage(shared.NoRemoved, msg.Object.Identification)
		c.tin.channel.Send(address, nm.JSON())
		// done
		return nil
	}
	// if still error, return it
	if err != nil {
		return err
	}
	// --> IF CheckMessage was ok, we can now handle applying the message
	// apply directories directly
	if msg.Object.Directory {
		// no merge because it should never happen for directories
		return c.tin.model.ApplyUpdateMessage(msg)
	}
	op := msg.Operation
	// create and modify must first fetch the file
	if op == shared.OpCreate || op == shared.OpModify {
		rm := shared.CreateRequestMessage(shared.OtObject, msg.Object.Identification)
		var wg sync.WaitGroup
		wg.Add(1)
		c.requestFile(address, rm, func(address, path string) {
			// force calling function to wait until this has been handled
			defer func() { wg.Done() }()
			// rename to correct name for model
			err := os.Rename(path, c.temppath+"/"+rm.Identification)
			if err != nil {
				c.log("Failed to move file to temp: " + err.Error())
				return
			}
			// TODO decrypt file!
			// log.Println("DEBUG: TODO: decrypt file here!")
			// apply
			err = c.mergeUpdate(*msg)
			if err != nil {
				c.log("File application error: " + err.Error())
			}
			// done
		})
		// wait for file to be received before returning
		wg.Wait()
		// errors may turn up but only when the file has been received, so done here
		return nil
	} else if op == shared.OpRemove {
		// remove is without file transfer, so directly apply
		return c.mergeUpdate(*msg)
	}
	c.warn("Unknown operation received, ignoring update message!")
	return shared.ErrIllegalParameters
}

/*
encSendPush sends a PushMessage for the given parameters, setting the ObjType
according to the path.
*/
func (c *chaninterface) encSendPush(address, path, identification string) {
	peerDir := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.PEERSDIR
	authPath := shared.TINZENITEDIR + "/" + shared.ORGDIR + "/" + shared.AUTHJSON
	// default object type is OtObject
	objectType := shared.OtObject
	// if peer --> update objectType
	if strings.HasPrefix(path, peerDir) {
		objectType = shared.OtPeer
	}
	// if auth file --> update objectType
	if path == authPath {
		objectType = shared.OtAuth
	}
	pm := shared.CreatePushMessage(identification, objectType)
	c.tin.channel.Send(address, pm.JSON())
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"

	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/core/coreunix"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"

	u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName       = "quiet"
	silentOptionName      = "silent"
	progressOptionName    = "progress"
	trickleOptionName     = "trickle"
	wrapOptionName        = "wrap-with-directory"
	hiddenOptionName      = "hidden"
	onlyHashOptionName    = "only-hash"
	chunkerOptionName     = "chunker"
	pinOptionName         = "pin"
	rawLeavesOptionName   = "raw-leaves"
	noCopyOptionName      = "nocopy"
	fstoreCacheOptionName = "fscache"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file or directory to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
		cmds.BoolOption(silentOptionName, "Write no output."),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. (experimental)"),
		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
	},
	PreRun: func(req cmds.Request) error {
		quiet, _, _ := req.Option(quietOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()

		if quiet || silent {
			return nil
		}

		// ipfs cli progress bar defaults to true unless quiet or silent is used
		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		cfg, err := n.Repo.Config()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()
		rawblks, rbset, _ := req.Option(rawLeavesOptionName).Bool()
		nocopy, _, _ := req.Option(noCopyOptionName).Bool()
		fscache, _, _ := req.Option(fstoreCacheOptionName).Bool()

		if nocopy && !cfg.Experimental.FilestoreEnabled {
			res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"),
				cmds.ErrClient)
			return
		}

		if nocopy && !rbset {
			rawblks = true
		}

		if nocopy && !rawblks {
			res.SetError(fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well"), cmds.ErrNormal)
			return
		}

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		addblockstore := n.Blockstore
		if !(fscache || nocopy) {
			addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker)
		}

		exch := n.Exchange
		local, _, _ := req.Option("local").Bool()
		if local {
			exch = offline.Exchange(addblockstore)
		}

		bserv := blockservice.New(addblockstore, exch)
		dserv := dag.NewDAGService(bserv)

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent
		fileAdder.RawLeaves = rawblks
		fileAdder.NoCopy = nocopy

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := req.Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		progress, _, err := req.Option(progressOptionName).Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		var bar *pb.ProgressBar
		if progress {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					if progress {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !progress {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if progress {
					bar.Update()
				}
			case size := <-sizeChan:
				if progress {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<commit_msg>feat: add quieter option to add<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"

	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	blockservice "github.com/ipfs/go-ipfs/blockservice"
	cmds "github.com/ipfs/go-ipfs/commands"
	files "github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/core/coreunix"
	offline "github.com/ipfs/go-ipfs/exchange/offline"
	dag "github.com/ipfs/go-ipfs/merkledag"
	dagtest "github.com/ipfs/go-ipfs/merkledag/test"
	mfs "github.com/ipfs/go-ipfs/mfs"
	ft "github.com/ipfs/go-ipfs/unixfs"

	u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
	"gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

const (
	quietOptionName       = "quiet"
	quieterOptionName     = "quieter"
	silentOptionName      = "silent"
	progressOptionName    = "progress"
	trickleOptionName     = "trickle"
	wrapOptionName        = "wrap-with-directory"
	hiddenOptionName      = "hidden"
	onlyHashOptionName    = "only-hash"
	chunkerOptionName     = "chunker"
	pinOptionName         = "pin"
	rawLeavesOptionName   = "raw-leaves"
	noCopyOptionName      = "nocopy"
	fstoreCacheOptionName = "fscache"
)

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add a file or directory to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories (recursively).
`,
		LongDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG.

The wrap option, '-w', wraps the file (or files, if using the
recursive option) in a directory. This directory contains only
the files which have been added, and means that the file retains
its filename. For example:

  > ipfs add example.jpg
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  > ipfs add example.jpg -w
  added QmbFMke1KXqnYyBBWxB74N4c5SBnJMVAiMNRcGu6x1AwQH example.jpg
  added QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx

You can now refer to the added file in a gateway, like so:

  /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to ipfs.").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption(quietOptionName, "q", "Write minimal output."),
		cmds.BoolOption(quieterOptionName, "Q", "Write only final hash."),
		cmds.BoolOption(silentOptionName, "Write no output."),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data."),
		cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
		cmds.BoolOption(hiddenOptionName, "H", "Include files that are hidden. Only takes effect on recursive add."),
		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm to use."),
		cmds.BoolOption(pinOptionName, "Pin this object when adding.").Default(true),
		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes. (experimental)"),
		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. (experimental)"),
		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
	},
	PreRun: func(req cmds.Request) error {
		quiet, _, _ := req.Option(quietOptionName).Bool()
		quieter, _, _ := req.Option(quieterOptionName).Bool()
		quiet = quiet || quieter

		silent, _, _ := req.Option(silentOptionName).Bool()

		if quiet || silent {
			return nil
		}

		// ipfs cli progress bar defaults to true unless quiet or silent is used
		_, found, _ := req.Option(progressOptionName).Bool()
		if !found {
			req.SetOption(progressOptionName, true)
		}

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			log.Warning("cannnot determine size of input file")
			return nil
		}

		sizeCh := make(chan int64, 1)
		req.Values()["size"] = sizeCh

		go func() {
			size, err := sizeFile.Size()
			if err != nil {
				log.Warningf("error getting files size: %s", err)
				// see comment above
				return
			}

			log.Debugf("Total size of file being added: %v\n", size)
			sizeCh <- size
		}()

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		cfg, err := n.Repo.Config()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		// check if repo will exceed storage limit if added
		// TODO: this doesn't handle the case if the hashed file is already in blocks (deduplicated)
		// TODO: conditional GC is disabled due to it is somehow not possible to pass the size to the daemon
		//if err := corerepo.ConditionalGC(req.Context(), n, uint64(size)); err != nil {
		//	res.SetError(err, cmds.ErrNormal)
		//	return
		//}

		progress, _, _ := req.Option(progressOptionName).Bool()
		trickle, _, _ := req.Option(trickleOptionName).Bool()
		wrap, _, _ := req.Option(wrapOptionName).Bool()
		hash, _, _ := req.Option(onlyHashOptionName).Bool()
		hidden, _, _ := req.Option(hiddenOptionName).Bool()
		silent, _, _ := req.Option(silentOptionName).Bool()
		chunker, _, _ := req.Option(chunkerOptionName).String()
		dopin, _, _ := req.Option(pinOptionName).Bool()
		rawblks, rbset, _ := req.Option(rawLeavesOptionName).Bool()
		nocopy, _, _ := req.Option(noCopyOptionName).Bool()
		fscache, _, _ := req.Option(fstoreCacheOptionName).Bool()

		if nocopy && !cfg.Experimental.FilestoreEnabled {
			res.SetError(errors.New("filestore is not enabled, see https://git.io/vy4XN"),
				cmds.ErrClient)
			return
		}

		if nocopy && !rbset {
			rawblks = true
		}

		if nocopy && !rawblks {
			res.SetError(fmt.Errorf("nocopy option requires '--raw-leaves' to be enabled as well"), cmds.ErrNormal)
			return
		}

		if hash {
			nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
				//TODO: need this to be true or all files
				// hashed will be stored in memory!
				NilRepo: true,
			})
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			n = nilnode
		}

		addblockstore := n.Blockstore
		if !(fscache || nocopy) {
			addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker)
		}

		exch := n.Exchange
		local, _, _ := req.Option("local").Bool()
		if local {
			exch = offline.Exchange(addblockstore)
		}

		bserv := blockservice.New(addblockstore, exch)
		dserv := dag.NewDAGService(bserv)

		outChan := make(chan interface{}, 8)
		res.SetOutput((<-chan interface{})(outChan))

		fileAdder, err := coreunix.NewAdder(req.Context(), n.Pinning, n.Blockstore, dserv)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		fileAdder.Out = outChan
		fileAdder.Chunker = chunker
		fileAdder.Progress = progress
		fileAdder.Hidden = hidden
		fileAdder.Trickle = trickle
		fileAdder.Wrap = wrap
		fileAdder.Pin = dopin
		fileAdder.Silent = silent
		fileAdder.RawLeaves = rawblks
		fileAdder.NoCopy = nocopy

		if hash {
			md := dagtest.Mock()
			mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

			fileAdder.SetMfsRoot(mr)
		}

		addAllAndPin := func(f files.File) error {
			// Iterate over each top-level file and add individually. Otherwise the
			// single files.File f is treated as a directory, affecting hidden file
			// semantics.
			for {
				file, err := f.NextFile()
				if err == io.EOF {
					// Finished the list of files.
					break
				} else if err != nil {
					return err
				}
				if err := fileAdder.AddFile(file); err != nil {
					return err
				}
			}

			// copy intermediary nodes from editor to our actual dagservice
			_, err := fileAdder.Finalize()
			if err != nil {
				return err
			}

			if hash {
				return nil
			}

			return fileAdder.PinRoot()
		}

		go func() {
			defer close(outChan)
			if err := addAllAndPin(req.Files()); err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}

		}()
	},
	PostRun: func(req cmds.Request, res cmds.Response) {
		if res.Error() != nil {
			return
		}
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, _ := req.Option(quietOptionName).Bool()
		quieter, _, _ := req.Option(quieterOptionName).Bool()
		quiet = quiet || quieter

		progress, _, _ := req.Option(progressOptionName).Bool()

		var bar *pb.ProgressBar
		if progress {
			bar = pb.New64(0).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.ShowTimeLeft = false
			bar.ShowPercent = false
			bar.Output = res.Stderr()
			bar.Start()
		}

		var sizeChan chan int64
		s, found := req.Values()["size"]
		if found {
			sizeChan = s.(chan int64)
		}

		lastFile := ""
		lastHash := ""
		var totalProgress, prevFiles, lastBytes int64

	LOOP:
		for {
			select {
			case out, ok := <-outChan:
				if !ok {
					if quieter {
						fmt.Fprintln(res.Stdout(), lastHash)
					}
					break LOOP
				}
				output := out.(*coreunix.AddedObject)
				if len(output.Hash) > 0 {
					lastHash = output.Hash
					if quieter {
						continue
					}

					if progress {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(res.Stderr(), "\033[2K\r")
					}
					if quiet {
						fmt.Fprintf(res.Stdout(), "%s\n", output.Hash)
					} else {
						fmt.Fprintf(res.Stdout(), "added %s %s\n", output.Hash, output.Name)
					}
				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !progress {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if progress {
					bar.Update()
				}
			case size := <-sizeChan:
				if progress {
					bar.Total = size
					bar.ShowPercent = true
					bar.ShowBar = true
					bar.ShowTimeLeft = true
				}
			case <-req.Context().Done():
				res.SetError(req.Context().Err(), cmds.ErrNormal)
				return
			}
		}
	},
	Type: coreunix.AddedObject{},
}
<|endoftext|>