<commit_before>package checklog

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	"github.com/mackerelio/golib/pluginutil"
	"github.com/mattn/go-encoding"
	"github.com/mattn/go-zglob"
	enc "golang.org/x/text/encoding"
)

type logOpts struct {
	LogFile             string   `short:"f" long:"file" value-name:"FILE" description:"Path to log file"`
	Pattern             []string `short:"p" long:"pattern" required:"true" value-name:"PAT" description:"Pattern to search for. If specified multiple, they will be treated together with the AND operator"`
	Exclude             string   `short:"E" long:"exclude" value-name:"PAT" description:"Pattern to exclude from matching"`
	WarnOver            int64    `short:"w" long:"warning-over" description:"Trigger a warning if matched lines is over a number"`
	CritOver            int64    `short:"c" long:"critical-over" description:"Trigger a critical if matched lines is over a number"`
	WarnLevel           float64  `long:"warning-level" value-name:"N" description:"Warning level if pattern has a group"`
	CritLevel           float64  `long:"critical-level" value-name:"N" description:"Critical level if pattern has a group"`
	ReturnContent       bool     `short:"r" long:"return" description:"Return matched line"`
	ReturnFileNames     bool     `long:"return-filenames" description:"Return matched file names"`
	FilePattern         string   `short:"F" long:"file-pattern" value-name:"FILE" description:"Check a pattern of files, instead of one file"`
	CaseInsensitive     bool     `short:"i" long:"icase" description:"Run a case insensitive match"`
	StateDir            string   `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
	NoState             bool     `long:"no-state" description:"Don't use state file and read whole logs"`
	Encoding            string   `long:"encoding" description:"Encoding of log file"`
	Missing             string   `long:"missing" default:"UNKNOWN" value-name:"(CRITICAL|WARNING|OK|UNKNOWN)" description:"Exit status when log files missing"`
	CheckFirst          bool     `long:"check-first" description:"Check the log on the first run"`
	patternReg          []*regexp.Regexp
	excludeReg          *regexp.Regexp
	fileListFromGlob    []string
	fileListFromPattern []string
	origArgs            []string
	decoder             *enc.Decoder
}

func (opts *logOpts) prepare() error {
	if opts.LogFile == "" && opts.FilePattern == "" {
		return fmt.Errorf("No log file specified")
	}

	var err error
	var reg *regexp.Regexp
	for _, ptn := range opts.Pattern {
		if reg, err = regCompileWithCase(ptn, opts.CaseInsensitive); err != nil {
			return fmt.Errorf("pattern is invalid")
		}
		opts.patternReg = append(opts.patternReg, reg)
	}

	if len(opts.patternReg) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
		return fmt.Errorf("When multiple patterns specified, --warning-level --critical-level can not be used")
	}

	if opts.Exclude != "" {
		opts.excludeReg, err = regCompileWithCase(opts.Exclude, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("exclude pattern is invalid")
		}
	}

	if opts.LogFile != "" {
		opts.fileListFromGlob, err = zglob.Glob(opts.LogFile)
		// unless --missing specified, we should ignore file not found error
		if err != nil && err != os.ErrNotExist {
			return fmt.Errorf("invalid glob for --file")
		}
	}

	if opts.FilePattern != "" {
		dirStr := filepath.Dir(opts.FilePattern)
		filePat := filepath.Base(opts.FilePattern)
		reg, err := regCompileWithCase(filePat, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("file-pattern is invalid")
		}

		fileInfos, err := ioutil.ReadDir(dirStr)
		if err != nil {
			return fmt.Errorf("cannot read the directory:" + err.Error())
		}

		for _, fileInfo := range fileInfos {
			if fileInfo.IsDir() {
				continue
			}
			fname := fileInfo.Name()
			if reg.MatchString(fname) {
				opts.fileListFromPattern = append(opts.fileListFromPattern, dirStr+string(filepath.Separator)+fileInfo.Name())
			}
		}
	}
	if !validateMissing(opts.Missing) {
		return fmt.Errorf("missing option is invalid")
	}
	return nil
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "LOG"
	ckr.Exit()
}

func regCompileWithCase(ptn string, caseInsensitive bool) (*regexp.Regexp, error) {
	if caseInsensitive {
		ptn = "(?i)" + ptn
	}
	return regexp.Compile(ptn)
}

func validateMissing(missing string) bool {
	switch missing {
	case "CRITICAL", "WARNING", "OK", "UNKNOWN", "":
		return true
	default:
		return false
	}
}

func parseArgs(args []string) (*logOpts, error) {
	origArgs := make([]string, len(args))
	copy(origArgs, args)
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	opts.origArgs = origArgs
	if opts.StateDir == "" {
		workdir := pluginutil.PluginWorkDir()
		opts.StateDir = filepath.Join(workdir, "check-log")
	}
	return opts, err
}

func run(args []string) *checkers.Checker {
	opts, err := parseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	err = opts.prepare()
	if err != nil {
		return checkers.Unknown(err.Error())
	}

	warnNum := int64(0)
	critNum := int64(0)
	var missingFiles []string
	errorOverall := ""

	if opts.LogFile != "" && len(opts.fileListFromGlob) == 0 {
		missingFiles = append(missingFiles, opts.LogFile)
	}

	for _, f := range append(opts.fileListFromGlob, opts.fileListFromPattern...) {
		_, err := os.Stat(f)
		if err != nil {
			missingFiles = append(missingFiles, f)
			continue
		}
		w, c, errLines, err := opts.searchLog(f)
		if err != nil {
			return checkers.Unknown(err.Error())
		}
		warnNum += w
		critNum += c
		if opts.ReturnFileNames && errLines != "" {
			errorOverall += "[" + f + "]\n"
		}
		if opts.ReturnContent {
			errorOverall += errLines
		}
	}

	var patterns []string
	for _, ptn := range opts.Pattern {
		patterns = append(patterns, fmt.Sprintf("/%s/", ptn))
	}
	msg := fmt.Sprintf("%d warnings, %d criticals for pattern %s.", warnNum, critNum, strings.Join(patterns, " and "))
	if errorOverall != "" {
		msg += "\n" + errorOverall
	}
	checkSt := checkers.OK
	if len(missingFiles) > 0 {
		switch opts.Missing {
		case "OK":
		case "WARNING":
			checkSt = checkers.WARNING
		case "CRITICAL":
			checkSt = checkers.CRITICAL
		default:
			checkSt = checkers.UNKNOWN
		}
		msg += "\n" + fmt.Sprintf("The following %d files are missing.", len(missingFiles))
		for _, f := range missingFiles {
			msg += "\n" + f
		}
	}
	if warnNum > opts.WarnOver {
		checkSt = checkers.WARNING
	}
	if critNum > opts.CritOver {
		checkSt = checkers.CRITICAL
	}
	return checkers.NewChecker(checkSt, msg)
}

func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
	skipBytes := int64(0)
	if !opts.NoState {
		s, err := getBytesToSkip(stateFile)
		if err != nil {
			return 0, 0, "", err
		}
		skipBytes = s
	}

	f, err := os.Open(logFile)
	if err != nil {
		return 0, 0, "", err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return 0, 0, "", err
	}

	if !opts.NoState && !opts.CheckFirst && skipBytes == 0 {
		skipBytes = stat.Size()
	}

	rotated := false
	if stat.Size() < skipBytes {
		rotated = true
	} else if skipBytes > 0 {
		f.Seek(skipBytes, 0)
	}

	var r io.Reader = f
	if opts.Encoding != "" {
		e := encoding.GetEncoding(opts.Encoding)
		if e == nil {
			return 0, 0, "", fmt.Errorf("unknown encoding:" + opts.Encoding)
		}
		opts.decoder = e.NewDecoder()
	}

	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)
	if err != nil {
		return warnNum, critNum, errLines, err
	}

	if rotated {
		skipBytes = readBytes
	} else {
		skipBytes += readBytes
	}

	if !opts.NoState {
		err = writeBytesToSkip(stateFile, skipBytes)
		if err != nil {
			log.Printf("writeByteToSkip failed: %s\n", err.Error())
		}
	}
	return warnNum, critNum, errLines, nil
}

func (opts *logOpts) searchReader(rdr io.Reader) (warnNum, critNum, readBytes int64, errLines string, err error) {
	r := bufio.NewReader(rdr)
	for {
		lineBytes, rErr := r.ReadBytes('\n')
		if rErr != nil {
			if rErr != io.EOF {
				err = rErr
			}
			break
		}
		readBytes += int64(len(lineBytes))

		if opts.decoder != nil {
			lineBytes, err = opts.decoder.Bytes(lineBytes)
			if err != nil {
				break
			}
		}
		line := strings.Trim(string(lineBytes), "\r\n")
		if matched, matches := opts.match(line); matched {
			if len(matches) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
				level, err := strconv.ParseFloat(matches[1], 64)
				if err != nil {
					warnNum++
					critNum++
					errLines += line + "\n"
				} else {
					levelOver := false
					if level > opts.WarnLevel {
						levelOver = true
						warnNum++
					}
					if level > opts.CritLevel {
						levelOver = true
						critNum++
					}
					if levelOver {
						errLines += line + "\n"
					}
				}
			} else {
				warnNum++
				critNum++
				errLines += line + "\n"
			}
		}
	}
	return
}

func (opts *logOpts) match(line string) (bool, []string) {
	var matches []string
	for _, pReg := range opts.patternReg {
		eReg := opts.excludeReg

		matches = pReg.FindStringSubmatch(line)
		if len(matches) == 0 || (eReg != nil && eReg.MatchString(line)) {
			return false, nil
		}
	}
	return true, matches
}

var stateRe = regexp.MustCompile(`^([a-zA-Z]):[/\\]`)

func getStateFile(stateDir, f string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			stateRe.ReplaceAllString(f, `$1`+string(filepath.Separator)),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func getBytesToSkip(f string) (int64, error) {
	_, err := os.Stat(f)
	if err != nil {
		return 0, nil
	}
	b, err := ioutil.ReadFile(f)
	if err != nil {
		return 0, err
	}
	i, err := strconv.ParseInt(strings.Trim(string(b), " \r\n"), 10, 64)
	if err != nil {
		log.Printf("failed to getBytesToSkip (ignoring): %s", err)
	}
	return i, nil
}

func writeBytesToSkip(f string, num int64) error {
	err := os.MkdirAll(filepath.Dir(f), 0755)
	if err != nil {
		return err
	}
	return writeFileAtomically(f, []byte(fmt.Sprintf("%d", num)))
}

func writeFileAtomically(f string, contents []byte) error {
	// MUST be located on same disk partition
	tmpf, err := ioutil.TempFile(filepath.Dir(f), "tmp")
	if err != nil {
		return err
	}
	// os.Remove here works successfully when tmpf.Write fails or os.Rename fails.
	// In successful case, os.Remove fails because the temporary file is already renamed.
	defer os.Remove(tmpf.Name())
	_, err = tmpf.Write(contents)
	tmpf.Close() // should be called before rename
	if err != nil {
		return err
	}
	return os.Rename(tmpf.Name(), f)
}
<commit_msg>always output filename if we use `--return` option<commit_after>package checklog

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	"github.com/mackerelio/golib/pluginutil"
	"github.com/mattn/go-encoding"
	"github.com/mattn/go-zglob"
	enc "golang.org/x/text/encoding"
)

type logOpts struct {
	LogFile             string   `short:"f" long:"file" value-name:"FILE" description:"Path to log file"`
	Pattern             []string `short:"p" long:"pattern" required:"true" value-name:"PAT" description:"Pattern to search for. If specified multiple, they will be treated together with the AND operator"`
	Exclude             string   `short:"E" long:"exclude" value-name:"PAT" description:"Pattern to exclude from matching"`
	WarnOver            int64    `short:"w" long:"warning-over" description:"Trigger a warning if matched lines is over a number"`
	CritOver            int64    `short:"c" long:"critical-over" description:"Trigger a critical if matched lines is over a number"`
	WarnLevel           float64  `long:"warning-level" value-name:"N" description:"Warning level if pattern has a group"`
	CritLevel           float64  `long:"critical-level" value-name:"N" description:"Critical level if pattern has a group"`
	ReturnContent       bool     `short:"r" long:"return" description:"Return matched line"`
	FilePattern         string   `short:"F" long:"file-pattern" value-name:"FILE" description:"Check a pattern of files, instead of one file"`
	CaseInsensitive     bool     `short:"i" long:"icase" description:"Run a case insensitive match"`
	StateDir            string   `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
	NoState             bool     `long:"no-state" description:"Don't use state file and read whole logs"`
	Encoding            string   `long:"encoding" description:"Encoding of log file"`
	Missing             string   `long:"missing" default:"UNKNOWN" value-name:"(CRITICAL|WARNING|OK|UNKNOWN)" description:"Exit status when log files missing"`
	CheckFirst          bool     `long:"check-first" description:"Check the log on the first run"`
	patternReg          []*regexp.Regexp
	excludeReg          *regexp.Regexp
	fileListFromGlob    []string
	fileListFromPattern []string
	origArgs            []string
	decoder             *enc.Decoder
}

func (opts *logOpts) prepare() error {
	if opts.LogFile == "" && opts.FilePattern == "" {
		return fmt.Errorf("No log file specified")
	}

	var err error
	var reg *regexp.Regexp
	for _, ptn := range opts.Pattern {
		if reg, err = regCompileWithCase(ptn, opts.CaseInsensitive); err != nil {
			return fmt.Errorf("pattern is invalid")
		}
		opts.patternReg = append(opts.patternReg, reg)
	}

	if len(opts.patternReg) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
		return fmt.Errorf("When multiple patterns specified, --warning-level --critical-level can not be used")
	}

	if opts.Exclude != "" {
		opts.excludeReg, err = regCompileWithCase(opts.Exclude, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("exclude pattern is invalid")
		}
	}

	if opts.LogFile != "" {
		opts.fileListFromGlob, err = zglob.Glob(opts.LogFile)
		// unless --missing specified, we should ignore file not found error
		if err != nil && err != os.ErrNotExist {
			return fmt.Errorf("invalid glob for --file")
		}
	}

	if opts.FilePattern != "" {
		dirStr := filepath.Dir(opts.FilePattern)
		filePat := filepath.Base(opts.FilePattern)
		reg, err := regCompileWithCase(filePat, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("file-pattern is invalid")
		}

		fileInfos, err := ioutil.ReadDir(dirStr)
		if err != nil {
			return fmt.Errorf("cannot read the directory:" + err.Error())
		}

		for _, fileInfo := range fileInfos {
			if fileInfo.IsDir() {
				continue
			}
			fname := fileInfo.Name()
			if reg.MatchString(fname) {
				opts.fileListFromPattern = append(opts.fileListFromPattern, dirStr+string(filepath.Separator)+fileInfo.Name())
			}
		}
	}
	if !validateMissing(opts.Missing) {
		return fmt.Errorf("missing option is invalid")
	}
	return nil
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "LOG"
	ckr.Exit()
}

func regCompileWithCase(ptn string, caseInsensitive bool) (*regexp.Regexp, error) {
	if caseInsensitive {
		ptn = "(?i)" + ptn
	}
	return regexp.Compile(ptn)
}

func validateMissing(missing string) bool {
	switch missing {
	case "CRITICAL", "WARNING", "OK", "UNKNOWN", "":
		return true
	default:
		return false
	}
}

func parseArgs(args []string) (*logOpts, error) {
	origArgs := make([]string, len(args))
	copy(origArgs, args)
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	opts.origArgs = origArgs
	if opts.StateDir == "" {
		workdir := pluginutil.PluginWorkDir()
		opts.StateDir = filepath.Join(workdir, "check-log")
	}
	return opts, err
}

func run(args []string) *checkers.Checker {
	opts, err := parseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	err = opts.prepare()
	if err != nil {
		return checkers.Unknown(err.Error())
	}

	warnNum := int64(0)
	critNum := int64(0)
	var missingFiles []string
	errorOverall := ""

	if opts.LogFile != "" && len(opts.fileListFromGlob) == 0 {
		missingFiles = append(missingFiles, opts.LogFile)
	}

	for _, f := range append(opts.fileListFromGlob, opts.fileListFromPattern...) {
		_, err := os.Stat(f)
		if err != nil {
			missingFiles = append(missingFiles, f)
			continue
		}
		w, c, errLines, err := opts.searchLog(f)
		if err != nil {
			return checkers.Unknown(err.Error())
		}
		warnNum += w
		critNum += c
		if opts.ReturnContent && errLines != "" {
			errorOverall += "[" + f + "]\n" + errLines
		}
	}

	var patterns []string
	for _, ptn := range opts.Pattern {
		patterns = append(patterns, fmt.Sprintf("/%s/", ptn))
	}
	msg := fmt.Sprintf("%d warnings, %d criticals for pattern %s.", warnNum, critNum, strings.Join(patterns, " and "))
	if errorOverall != "" {
		msg += "\n" + errorOverall
	}
	checkSt := checkers.OK
	if len(missingFiles) > 0 {
		switch opts.Missing {
		case "OK":
		case "WARNING":
			checkSt = checkers.WARNING
		case "CRITICAL":
			checkSt = checkers.CRITICAL
		default:
			checkSt = checkers.UNKNOWN
		}
		msg += "\n" + fmt.Sprintf("The following %d files are missing.", len(missingFiles))
		for _, f := range missingFiles {
			msg += "\n" + f
		}
	}
	if warnNum > opts.WarnOver {
		checkSt = checkers.WARNING
	}
	if critNum > opts.CritOver {
		checkSt = checkers.CRITICAL
	}
	return checkers.NewChecker(checkSt, msg)
}

func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
	skipBytes := int64(0)
	if !opts.NoState {
		s, err := getBytesToSkip(stateFile)
		if err != nil {
			return 0, 0, "", err
		}
		skipBytes = s
	}

	f, err := os.Open(logFile)
	if err != nil {
		return 0, 0, "", err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return 0, 0, "", err
	}

	if !opts.NoState && !opts.CheckFirst && skipBytes == 0 {
		skipBytes = stat.Size()
	}

	rotated := false
	if stat.Size() < skipBytes {
		rotated = true
	} else if skipBytes > 0 {
		f.Seek(skipBytes, 0)
	}

	var r io.Reader = f
	if opts.Encoding != "" {
		e := encoding.GetEncoding(opts.Encoding)
		if e == nil {
			return 0, 0, "", fmt.Errorf("unknown encoding:" + opts.Encoding)
		}
		opts.decoder = e.NewDecoder()
	}

	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)
	if err != nil {
		return warnNum, critNum, errLines, err
	}

	if rotated {
		skipBytes = readBytes
	} else {
		skipBytes += readBytes
	}

	if !opts.NoState {
		err = writeBytesToSkip(stateFile, skipBytes)
		if err != nil {
			log.Printf("writeByteToSkip failed: %s\n", err.Error())
		}
	}
	return warnNum, critNum, errLines, nil
}

func (opts *logOpts) searchReader(rdr io.Reader) (warnNum, critNum, readBytes int64, errLines string, err error) {
	r := bufio.NewReader(rdr)
	for {
		lineBytes, rErr := r.ReadBytes('\n')
		if rErr != nil {
			if rErr != io.EOF {
				err = rErr
			}
			break
		}
		readBytes += int64(len(lineBytes))

		if opts.decoder != nil {
			lineBytes, err = opts.decoder.Bytes(lineBytes)
			if err != nil {
				break
			}
		}
		line := strings.Trim(string(lineBytes), "\r\n")
		if matched, matches := opts.match(line); matched {
			if len(matches) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
				level, err := strconv.ParseFloat(matches[1], 64)
				if err != nil {
					warnNum++
					critNum++
					errLines += line + "\n"
				} else {
					levelOver := false
					if level > opts.WarnLevel {
						levelOver = true
						warnNum++
					}
					if level > opts.CritLevel {
						levelOver = true
						critNum++
					}
					if levelOver {
						errLines += line + "\n"
					}
				}
			} else {
				warnNum++
				critNum++
				errLines += line + "\n"
			}
		}
	}
	return
}

func (opts *logOpts) match(line string) (bool, []string) {
	var matches []string
	for _, pReg := range opts.patternReg {
		eReg := opts.excludeReg

		matches = pReg.FindStringSubmatch(line)
		if len(matches) == 0 || (eReg != nil && eReg.MatchString(line)) {
			return false, nil
		}
	}
	return true, matches
}

var stateRe = regexp.MustCompile(`^([a-zA-Z]):[/\\]`)

func getStateFile(stateDir, f string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			stateRe.ReplaceAllString(f, `$1`+string(filepath.Separator)),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func getBytesToSkip(f string) (int64, error) {
	_, err := os.Stat(f)
	if err != nil {
		return 0, nil
	}
	b, err := ioutil.ReadFile(f)
	if err != nil {
		return 0, err
	}
	i, err := strconv.ParseInt(strings.Trim(string(b), " \r\n"), 10, 64)
	if err != nil {
		log.Printf("failed to getBytesToSkip (ignoring): %s", err)
	}
	return i, nil
}

func writeBytesToSkip(f string, num int64) error {
	err := os.MkdirAll(filepath.Dir(f), 0755)
	if err != nil {
		return err
	}
	return writeFileAtomically(f, []byte(fmt.Sprintf("%d", num)))
}

func writeFileAtomically(f string, contents []byte) error {
	// MUST be located on same disk partition
	tmpf, err := ioutil.TempFile(filepath.Dir(f), "tmp")
	if err != nil {
		return err
	}
	// os.Remove here works successfully when tmpf.Write fails or os.Rename fails.
	// In successful case, os.Remove fails because the temporary file is already renamed.
	defer os.Remove(tmpf.Name())
	_, err = tmpf.Write(contents)
	tmpf.Close() // should be called before rename
	if err != nil {
		return err
	}
	return os.Rename(tmpf.Name(), f)
}
<|endoftext|>


<commit_before>package xd

import (
	"bufio"
	"io"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"time"
	"xd/lib/bittorrent/swarm"
	"xd/lib/config"
	"xd/lib/log"
	"xd/lib/rpc"
	"xd/lib/sync"
	t "xd/lib/translate"
	"xd/lib/util"
	"xd/lib/version"
)

type httpRPC struct {
	w http.ResponseWriter
	r *http.Request
}

func printHelp(cmd string) {
	log.Infof("usage: %s [config.ini] | --genconf config.ini\n", cmd)
}

func NewContext() *Context {
	return &Context{
		sigchnl: make(chan os.Signal),
		stdin:   os.Stdin,
	}
}

type Context struct {
	closers    sync.Map
	numClosers int
	quit       bool
	swarms     []*swarm.Swarm
	sigchnl    chan os.Signal
	stdin      io.ReadCloser
	netlost    bool
}

func (c *Context) Run() {
	r := bufio.NewReader(c.stdin)
	for {
		line, err := r.ReadString(10)
		if err == nil {
			if strings.ToLower(line) == "f\n" {
				if c.netlost {
					log.Debug("respec paid")
				}
			}
		} else {
			return
		}
	}
}

func (c *Context) Running() bool {
	return !c.quit
}

func (c *Context) RunSignals() {
	c.AddCloser(c.stdin)
	signal.Notify(c.sigchnl, os.Interrupt)
	for {
		sig := <-c.sigchnl
		if sig == os.Interrupt {
			log.Info("Interrupted")
			c.Close()
			return
		} else {
			log.Warnf("got wierd signal wtf: %s", sig)
			continue
		}
	}
}

func (c *Context) AddCloser(cl io.Closer) int {
	c.numClosers++
	c.closers.Store(c.numClosers, cl)
	return c.numClosers
}

func (c *Context) ReplaceCloser(id int, cl io.Closer) {
	c.closers.Store(id, cl)
}

func (c *Context) AddSwarm(sw *swarm.Swarm) {
	c.swarms = append(c.swarms, sw)
}

func (c *Context) Close() error {
	c.quit = true
	// close swarms first
	for _, sw := range c.swarms {
		sw.Close()
	}
	c.closers.Range(func(k, v interface{}) bool {
		cl := v.(io.Closer)
		cl.Close()
		return true
	})
	return nil
}

// Run runs XD main function
func Run() {

	ctx := NewContext()

	v := version.Version()
	conf := new(config.Config)
	fname := "torrents.ini"
	if len(os.Args) > 1 {
		fname = os.Args[1]
	}
	if fname == "-h" || fname == "--help" {
		printHelp(os.Args[0])
		return
	}
	var err error
	if fname == "--genconf" {
		if len(os.Args) == 3 {
			conf.Load("")
			err = conf.Save(os.Args[2])
			if err != nil {
				log.Errorf("failed to save config: %s", err)
			}
		} else {
			printHelp(os.Args[0])
		}
		return
	}

	log.Info(t.T("starting %s", v))
	if !util.CheckFile(fname) {
		conf.Load(fname)
		err = conf.Save(fname)
		if err != nil {
			log.Errorf("failed to save initial config: %s", err)
			return
		}
		log.Info(t.T("auto-generated new config at %s", fname))
	}
	err = conf.Load(fname)
	if err != nil {
		log.Errorf("failed to config %s", err)
		return
	}
	log.Info(t.T("loaded config %s", fname))
	log.SetLevel(conf.Log.Level)

	if conf.Log.Pprof {
		go func() {
			pprofaddr := "127.0.0.1:6060"
			log.Infof("spawning pprof at %s", pprofaddr)
			log.Warnf("pprof exited: %s", http.ListenAndServe(pprofaddr, nil))
		}()
	}

	st := conf.Storage.CreateStorage()
	err = st.Init()
	if err != nil {
		log.Errorf("error initializing storage: %s", err)
		return
	}
	// start io thread
	go st.Run()
	count := 0
	for count < conf.Bittorrent.Swarms {
		gnutella := conf.Gnutella.CreateSwarm()
		sw := conf.Bittorrent.CreateSwarm(st, gnutella)
		if gnutella != nil {
			ctx.AddCloser(gnutella)
		}
		ctx.AddSwarm(sw)
		count++
	}

	ts, err := st.OpenAllTorrents()
	if err != nil {
		log.Errorf("error opening all torrents: %s", err)
		return
	}
	for _, t := range ts {
		for _, sw := range ctx.swarms {
			err = sw.AddTorrent(t)
			if err != nil {
				log.Errorf("error adding torrent: %s", err)
			}
		}
	}

	// torrent auto adder
	go func() {
		for ctx.Running() {
			nt := st.PollNewTorrents()
			for _, t := range nt {
				e := t.VerifyAll()
				if e != nil {
					log.Errorf("failed to add %s: %s", t.Name(), e.Error())
					continue
				}
				for _, sw := range ctx.swarms {
					sw.AddTorrent(t)
				}
			}
			time.Sleep(time.Second)
		}
	}()

	// start rpc server
	if conf.RPC.Enabled {
		log.Infof("RPC enabled")
		var host string
		var l net.Listener
		var e error
		var cleanSock func()
		if strings.HasPrefix(conf.RPC.Bind, "unix:") {
			sock := conf.RPC.Bind[5:]
			cleanSock = func() {
				os.Remove(sock)
			}
			l, e = net.Listen("unix", sock)
			if e == nil {
				e = os.Chmod(sock, 0640)
			}
		} else {
			l, e = net.Listen("tcp", conf.RPC.Bind)
			cleanSock = func() {
			}
			host = conf.RPC.ExpectedHost
		}
		if e == nil {
			ctx.AddCloser(l)
			s := &http.Server{
				Handler: rpc.NewServer(ctx.swarms, host),
			}
			go func(serv *http.Server) {
				log.Errorf("rpc died: %s", serv.Serve(l))
				cleanSock()
			}(s)
		} else {
			log.Errorf("failed to bind rpc: %s", e)
		}
	}

	runFunc := func(netConf config.I2PConfig, sw *swarm.Swarm) {
		n := netConf.CreateSession()
		id := ctx.AddCloser(n)
		for sw.Running() {
			log.Info("opening i2p session")
			err := n.Open()
			if err == nil {
				log.Infof("i2p session made, we are %s", n.B32Addr())
				sw.ObtainedNetwork(n)
				ctx.netlost = false
				err = sw.Run()
				if err != nil {
					ctx.netlost = true
					log.Errorf("lost i2p session: %s", err)
					sw.LostNetwork()
					n = netConf.CreateSession()
					ctx.ReplaceCloser(id, n)
				}
			} else {
				ctx.netlost = true
				n = netConf.CreateSession()
				ctx.ReplaceCloser(id, n)
				log.Errorf("failed to create i2p session: %s", err)
				time.Sleep(time.Second)
			}
		}
	}

	for idx := range ctx.swarms {
		go runFunc(conf.I2P, ctx.swarms[idx])
	}
	ctx.AddCloser(st)
	go ctx.RunSignals()
	ctx.Run()
}
<commit_msg>properly handle stdin<commit_after>package xd

import (
	"bufio"
	"io"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"time"
	"xd/lib/bittorrent/swarm"
	"xd/lib/config"
	"xd/lib/log"
	"xd/lib/rpc"
	"xd/lib/sync"
	t "xd/lib/translate"
	"xd/lib/util"
	"xd/lib/version"
)

type httpRPC struct {
	w http.ResponseWriter
	r *http.Request
}

func printHelp(cmd string) {
	log.Infof("usage: %s [config.ini] | --genconf config.ini\n", cmd)
}

func NewContext() *Context {
	pr, pw := io.Pipe()
	return &Context{
		pr:      pr,
		pw:      pw,
		sigchnl: make(chan os.Signal),
	}
}

type Context struct {
	pr         io.ReadCloser
	pw         io.WriteCloser
	closers    sync.Map
	numClosers int
	quit       bool
	swarms     []*swarm.Swarm
	sigchnl    chan os.Signal
	netlost    bool
}

func (c *Context) Run() {
	go func() {
		io.Copy(c.pw, os.Stdin)
	}()
	r := bufio.NewReader(c.pr)
	for {
		line, err := r.ReadString(10)
		if err == nil {
			if strings.ToLower(line) == "f\n" {
				if c.netlost {
					log.Debug("respec paid")
				}
			} else if line == "\n" && c.quit {
				break
			} else if line == "" {
				break
			}
		} else {
			break
		}
	}
	c.pr.Close()
}

func (c *Context) Running() bool {
	return !c.quit
}

func (c *Context) RunSignals() {
	signal.Notify(c.sigchnl, os.Interrupt)
	for {
		sig := <-c.sigchnl
		if sig == os.Interrupt {
			log.Info("Interrupted")
			c.Close()
			return
		} else {
			log.Warnf("got wierd signal wtf: %s", sig)
			continue
		}
	}
}

func (c *Context) AddCloser(cl io.Closer) int {
	c.numClosers++
	c.closers.Store(c.numClosers, cl)
	return c.numClosers
}

func (c *Context) ReplaceCloser(id int, cl io.Closer) {
	c.closers.Store(id, cl)
}

func (c *Context) AddSwarm(sw *swarm.Swarm) {
	c.swarms = append(c.swarms, sw)
}

func (c *Context) Close() error {
	c.quit = true
	c.pw.Close()
	// close swarms first
	for _, sw := range c.swarms {
		sw.Close()
	}
	c.closers.Range(func(k, v interface{}) bool {
		cl := v.(io.Closer)
		cl.Close()
		return true
	})
	return nil
}

// Run runs XD main function
func Run() {

	ctx := NewContext()

	v := version.Version()
	conf := new(config.Config)
	fname := "torrents.ini"
	if len(os.Args) > 1 {
		fname = os.Args[1]
	}
	if fname == "-h" || fname == "--help" {
		printHelp(os.Args[0])
		return
	}
	var err error
	if fname == "--genconf" {
		if len(os.Args) == 3 {
			conf.Load("")
			err = conf.Save(os.Args[2])
			if err != nil {
				log.Errorf("failed to save config: %s", err)
			}
		} else {
			printHelp(os.Args[0])
		}
		return
	}

	log.Info(t.T("starting %s", v))
	if !util.CheckFile(fname) {
		conf.Load(fname)
		err = conf.Save(fname)
		if err != nil {
			log.Errorf("failed to save initial config: %s", err)
			return
		}
		log.Info(t.T("auto-generated new config at %s", fname))
	}
	err = conf.Load(fname)
	if err != nil {
		log.Errorf("failed to config %s", err)
		return
	}
	log.Info(t.T("loaded config %s", fname))
	log.SetLevel(conf.Log.Level)

	if conf.Log.Pprof {
		go func() {
			pprofaddr := "127.0.0.1:6060"
			l, err := net.Listen("tcp", pprofaddr)
			if err == nil {
				ctx.AddCloser(l)
				log.Infof("spawning pprof at %s", pprofaddr)
				log.Warnf("pprof exited: %s", http.Serve(l, nil))
			}
		}()
	}

	st := conf.Storage.CreateStorage()
	err = st.Init()
	if err != nil {
		log.Errorf("error initializing storage: %s", err)
		return
	}
	// start io thread
	go st.Run()
	count := 0
	for count < conf.Bittorrent.Swarms {
		gnutella := conf.Gnutella.CreateSwarm()
		sw := conf.Bittorrent.CreateSwarm(st, gnutella)
		if gnutella != nil {
			ctx.AddCloser(gnutella)
		}
		ctx.AddSwarm(sw)
		count++
	}

	ts, err := st.OpenAllTorrents()
	if err != nil {
		log.Errorf("error opening all torrents: %s", err)
		return
	}
	for _, t := range ts {
		for _, sw := range ctx.swarms {
			err = sw.AddTorrent(t)
			if err != nil {
				log.Errorf("error adding torrent: %s", err)
			}
		}
	}

	// torrent auto adder
	go func() {
		for ctx.Running() {
			nt := st.PollNewTorrents()
			for _, t := range nt {
				e := t.VerifyAll()
				if e != nil {
					log.Errorf("failed to add %s: %s", t.Name(), e.Error())
					continue
				}
				for _, sw := range ctx.swarms {
					sw.AddTorrent(t)
				}
			}
			time.Sleep(time.Second)
		}
	}()

	// start rpc server
	if conf.RPC.Enabled {
		log.Infof("RPC enabled")
		var host string
		var l net.Listener
		var e error
		var cleanSock func()
		if strings.HasPrefix(conf.RPC.Bind, "unix:") {
			sock := conf.RPC.Bind[5:]
			cleanSock = func() {
				os.Remove(sock)
			}
			l, e = net.Listen("unix", sock)
			if e == nil {
				e = os.Chmod(sock, 0640)
			}
		} else {
			l, e = net.Listen("tcp", conf.RPC.Bind)
			cleanSock = func() {
			}
			host = conf.RPC.ExpectedHost
		}
		if e == nil {
			ctx.AddCloser(l)
			s := &http.Server{
				Handler: rpc.NewServer(ctx.swarms, host),
			}
			go func(serv *http.Server) {
				log.Errorf("rpc died: %s", serv.Serve(l))
				cleanSock()
			}(s)
		} else {
			log.Errorf("failed to bind rpc: %s", e)
		}
	}

	runFunc := func(netConf config.I2PConfig, sw *swarm.Swarm) {
		n := netConf.CreateSession()
		id := ctx.AddCloser(n)
		for sw.Running() {
			log.Info("opening i2p session")
			err := n.Open()
			if err == nil {
				log.Infof("i2p session made, we are %s", n.B32Addr())
				sw.ObtainedNetwork(n)
				ctx.netlost = false
				err = sw.Run()
				if err != nil {
					ctx.netlost = true
					log.Errorf("lost i2p session: %s", err)
					sw.LostNetwork()
					n = netConf.CreateSession()
					ctx.ReplaceCloser(id, n)
				}
			} else {
				ctx.netlost = true
				n = netConf.CreateSession()
				ctx.ReplaceCloser(id, n)
				log.Errorf("failed to create i2p session: %s", err)
				time.Sleep(time.Second)
			}
		}
	}

	for idx := range ctx.swarms {
		go runFunc(conf.I2P, ctx.swarms[idx])
	}
	ctx.AddCloser(st)
	go ctx.RunSignals()
	ctx.Run()
}
<|endoftext|>


<commit_before>package f5

import (
	"encoding/json"
	"fmt"
)

// Backup REST paths
const (
	PathBackup = "/mgmt/tm/shared/sys/backup"
)

// BackupResponse holds attributes returned by requests on the backup API.
type BackupResponse struct {
	// Unique ID to identify uniquely the backup action.
	ID string `json:"id"`

	// Name of the file in which the backup is saved to or restord from.
	File string `json:"file"`

	// Type of action performed. Possible values are:
	//    - BACKUP
	//    - RESTORE
	//    - RESTORE_WITH_NO_LICENSE
	//    - BACKUP_WITH_NO_PRIVATE_KEYS
	//    - BACKUP_WITH_ENCRYPTION
	//    - BACKUP_WITH_NO_PRIVATE_KEYS_WITH_ENCRYPTION
	//    - RESTORE_WITH_ENCRYPTION
	//    - RESTORE_WITH_NO_LICENSE_WITH_ENCRYPTION
	//    - CLEANUP
	Action string `json:"action"`

	// Status of the backup. Possible values are:
	//    - CREATED
	//    - STARTED
	//    - CANCEL_REQUESTED
	//    - CANCELED
	//    - FAILED
	//    - FINISHED
	Status string `json:"status"`
}

// Backup creates a backup remotely saved into a file named according to the
// provided filename.
func (c *Client) Backup(filename string) (*BackupResponse, error) {
	return c.backupRequest("BACKUP", filename)
}

// CheckBackup fetches the status of a backup process.
func (c *Client) CheckBackup(id string) (*BackupResponse, error) {
	var backupResp BackupResponse
	if err := c.ReadQuery(PathBackup+"/"+id, &backupResp); err != nil {
		return nil, fmt.Errorf("cannot read backup status: %v", err)
	}
	return &backupResp, nil
}

// RestoreBackup restores a backup from a file having the provided filename and
// located into /var/local/ucs directory.
func (c *Client) RestoreBackup(filename string) (*BackupResponse, error) {
	return c.backupRequest("RESTORE", filename)
}

func (c *Client) backupRequest(action, filename string) (*BackupResponse, error) {
	data := map[string]interface{}{
		"file":   filename,
		"action": action,
	}

	resp, err := c.SendRequest("POST", PathBackup, data)
	if err != nil {
		return nil, fmt.Errorf("error while requesting ucs backup: %v", err)
	}
	defer resp.Body.Close()

	if err := c.ReadError(resp); err != nil {
		return nil, fmt.Errorf("backup api returned an error: %v", err)
	}

	var backupResp BackupResponse
	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(&backupResp); err != nil {
		return nil, fmt.Errorf("backup api returned a malformed json response: %v", err)
	}

	return &backupResp, nil
}
<commit_msg>f5: add getter to BackupResponse type<commit_after>package f5

import (
	"encoding/json"
	"fmt"
)

// Backup REST paths
const (
	PathBackup = "/mgmt/tm/shared/sys/backup"
)

// BackupResponse holds attributes returned by requests on the backup API.
type BackupResponse struct {
	// Unique ID to identify uniquely the backup action.
	ID string `json:"id"`

	// Name of the file in which the backup is saved to or restord from.
	File string `json:"file"`

	// Type of action performed. Possible values are:
	//    - BACKUP
	//    - RESTORE
	//    - RESTORE_WITH_NO_LICENSE
	//    - BACKUP_WITH_NO_PRIVATE_KEYS
	//    - BACKUP_WITH_ENCRYPTION
	//    - BACKUP_WITH_NO_PRIVATE_KEYS_WITH_ENCRYPTION
	//    - RESTORE_WITH_ENCRYPTION
	//    - RESTORE_WITH_NO_LICENSE_WITH_ENCRYPTION
	//    - CLEANUP
	Action string `json:"action"`

	// Status of the backup. Possible values are:
	//    - CREATED
	//    - STARTED
	//    - CANCEL_REQUESTED
	//    - CANCELED
	//    - FAILED
	//    - FINISHED
	Status string `json:"status"`
}

// IsDone reports whether the status indicates that the action is terminated,
// even if it is an error or that the task has been canceled.
func (resp BackupResponse) IsDone() bool {
	switch resp.Status {
	case "STARTED", "CANCEL_REQUESTED":
		return false
	}
	return true
}

// IsFailure reports whether the status is FAILED.
func (resp BackupResponse) IsFailure() bool {
	return resp.Status == "FAILED"
}

// IsCanceled reports whether the status is CANCELED.
func (resp BackupResponse) IsCanceled() bool {
	return resp.Status == "CANCELED"
}

// IsSuccess reports whether the status is FINISHED.
func (resp BackupResponse) IsSuccess() bool {
	return resp.Status == "FINISHED"
}

// Backup creates a backup remotely saved into a file named according to the
// provided filename.
func (c *Client) Backup(filename string) (*BackupResponse, error) {
	return c.backupRequest("BACKUP", filename)
}

// CheckBackup fetches the status of a backup process.
func (c *Client) CheckBackup(id string) (*BackupResponse, error) {
	var backupResp BackupResponse
	if err := c.ReadQuery(PathBackup+"/"+id, &backupResp); err != nil {
		return nil, fmt.Errorf("cannot read backup status: %v", err)
	}
	return &backupResp, nil
}

// RestoreBackup restores a backup from a file having the provided filename and
// located into /var/local/ucs directory.
func (c *Client) RestoreBackup(filename string) (*BackupResponse, error) {
	return c.backupRequest("RESTORE", filename)
}

func (c *Client) backupRequest(action, filename string) (*BackupResponse, error) {
	data := map[string]interface{}{
		"file":   filename,
		"action": action,
	}

	resp, err := c.SendRequest("POST", PathBackup, data)
	if err != nil {
		return nil, fmt.Errorf("error while requesting ucs backup: %v", err)
	}
	defer resp.Body.Close()

	if err := c.ReadError(resp); err != nil {
		return nil, fmt.Errorf("backup api returned an error: %v", err)
	}

	var backupResp BackupResponse
	dec := json.NewDecoder(resp.Body)
	if err := dec.Decode(&backupResp); err != nil {
		return nil, fmt.Errorf("backup api returned a malformed json response: %v", err)
	}

	return &backupResp, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"testing"

	"github.com/anz-bank/sysl/src/proto"
	"github.com/golang/protobuf/proto"
	"github.com/stretchr/testify/assert"
)

func readSyslModule(filename string) (*sysl.Module, error) {
	var buf bytes.Buffer

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	io.Copy(&buf, f)
	f.Close()

	module := &sysl.Module{}
	if err := proto.UnmarshalText(buf.String(), module); err != nil {
		return nil, err
	}
	return module, nil
}

var pySysl = func() string {
	if pySysl, ok := os.LookupEnv("SYSL_PYTHON_BIN"); ok {
		return pySysl
	}
	return "sysl"
}()

func pyParse(filename, root, output string) (*sysl.Module, error) {
	args := []string{"textpb", "-o", output, filename}
	if len(root) > 0 {
		rootArg := []string{"--root", root}
		output = args[2]
		args = append(rootArg, args...)
	}

	cmd := exec.Command(pySysl, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("Error running %#v %#v: %s", pySysl, args, err)
	}

	return readSyslModule(output)
}

func parseComparable(filename, root string) (*sysl.Module, error) {
	fs := &osFileSystem{root}
	module, err := FSParse(filename, fs)
	if err != nil {
		return nil, err
	}

	// remove stuff that does not match legacy.
	for _, app := range module.Apps {
		app.SourceContext = nil
		for _, ep := range app.Endpoints {
			ep.SourceContext = nil
		}
		for _, t := range app.Types {
			t.SourceContext = nil
		}
	}

	return module, nil
}

func parseAndCompare(filename, root, golden string, goldenModule *sysl.Module) (bool, error) {
	module, err := parseComparable(filename, root)
	if err != nil {
		return false, err
	}

	if proto.Equal(goldenModule, module) {
		return true, nil
	}

	if err = TextPB(goldenModule, golden); err != nil {
		return false, err
	}

	generated, err := ioutil.TempFile("", "sysl-test-*.textpb")
	if err != nil {
		return false, err
	}
	generatedClosed := false
	defer func() {
		if !generatedClosed {
			generated.Close()
		}
	}()

	if err = FTextPB(generated, module); err != nil {
		fmt.Printf("%#v retained for checking", generated.Name())
		return false, err
	}
	if err := generated.Close(); err != nil {
		return false, err
	}
	generatedClosed = true

	cmd := exec.Command("diff", "-y", golden, generated.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		fmt.Printf("%#v retained for checking", generated.Name())
		return false, err
	}

	os.Remove(generated.Name())
	return false, nil
}

func parseAndCompareWithPython(filename, root string) (bool, error) {
	fmt.Printf("%35s <=> $(%s %[1]s)\n", filename, pySysl)

	golden, err := ioutil.TempFile("", "sysl-test-golden-*.textpb")
	if err != nil {
		return false, err
	}
	defer golden.Close()

	pyModule, err := pyParse(filename, root, golden.Name())
	if err != nil {
		fmt.Printf("%#v retained for checking", golden.Name())
		return false, err
	}

	equal, err := parseAndCompare(filename, root, golden.Name(), pyModule)
	if err != nil {
		fmt.Printf("%#v retained for checking", golden.Name())
	}
	os.Remove(golden.Name())
	return equal, err
}

func parseAndCompareWithGolden(filename, root string) (bool, error) {
	golden := filename + ".golden.textpb"
	fmt.Printf("%35s <=> %s\n", filename, golden)

	goldenModule, err := readSyslModule(golden)
	if err != nil {
		return false, err
	}
	return parseAndCompare(filename, root, golden, goldenModule)
}

func parseAndPrint(t *testing.T, filename, root string) error {
	goModule, err := parseComparable(filename, root)
	if err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "Outputting %#v:\n", filename)
	FTextPB(os.Stderr, goModule)
	fmt.Fprintf(os.Stderr, "------------------------\n")
	return nil
}

func testParseAgainstPython(t *testing.T, filename, root string) {
	equal, err := parseAndCompareWithPython(filename, root)
	assert.NoError(t, err)
	assert.True(t, equal, "Mismatch")
}

func testParseAgainstGolden(t *testing.T, filename, root string) {
	equal, err := parseAndCompareWithGolden(filename, root)
	assert.NoError(t, err)
	assert.True(t, equal, "Mismatch")
}

func TestParseMissingFile(t *testing.T) {
	_, err := parseAndCompareWithGolden("tests/doesn't.exist", "")
	assert.Error(t, err)
}

func TestParseBadFile(t *testing.T) {
	_, err := parseAndCompareWithGolden("sysl.go", "")
	assert.Error(t, err)
}

func TestSimpleEP(t *testing.T) {
	testParseAgainstPython(t, "tests/test1.sysl", "")
}

func TestAttribs(t *testing.T) {
	testParseAgainstPython(t, "tests/attribs.sysl", "")
}

func TestIfElse(t *testing.T) {
	testParseAgainstPython(t, "tests/if_else.sysl", "")
}

func TestArgs(t *testing.T) {
	testParseAgainstPython(t, "tests/args.sysl", "")
}

func TestSimpleEPWithSpaces(t *testing.T) {
	testParseAgainstPython(t, "tests/with_spaces.sysl", "")
}

func TestSimpleEP2(t *testing.T) {
	testParseAgainstPython(t, "tests/test4.sysl", "")
}

func TestSimpleEndpointParams(t *testing.T) {
	testParseAgainstPython(t, "tests/ep_params.sysl", "")
}

func TestOneOfStatements(t *testing.T) {
	testParseAgainstPython(t, "tests/oneof.sysl", "")
}

func TestDuplicateEndpoints(t *testing.T) {
	testParseAgainstPython(t, "tests/duplicate.sysl", "")
}

func TestEventing(t *testing.T) {
	testParseAgainstPython(t, "tests/eventing.sysl", "")
}

func TestCollector(t *testing.T) {
	testParseAgainstPython(t, "tests/collector.sysl", "")
}

func TestPubSubCollector(t *testing.T) {
	testParseAgainstPython(t, "tests/pubsub_collector.sysl", "")
}

func TestDocstrings(t *testing.T) {
	testParseAgainstPython(t, "tests/docstrings.sysl", "")
}

func TestMixins(t *testing.T) {
	testParseAgainstPython(t, "tests/mixin.sysl", "")
}
func TestForLoops(t *testing.T) {
	testParseAgainstPython(t, "tests/for_loop.sysl", "")
}

func TestGroupStmt(t *testing.T) {
	testParseAgainstPython(t, "tests/group_stmt.sysl", "")
}

func TestUntilLoop(t *testing.T) {
	testParseAgainstPython(t, "tests/until_loop.sysl", "")
}

func TestTuple(t *testing.T) {
	testParseAgainstPython(t, "tests/test2.sysl", "")
}

func TestInplaceTuple(t *testing.T) {
	testParseAgainstPython(t, "tests/inplace_tuple.sysl", "")
}

func TestRelational(t *testing.T) {
	testParseAgainstPython(t, "tests/school.sysl", "")
}

func TestImports(t *testing.T) {
	testParseAgainstPython(t, "tests/library.sysl", "")
}

func TestRootArg(t *testing.T) {
	testParseAgainstPython(t, "school.sysl", "tests")
}

func TestRestApi(t *testing.T) {
	testParseAgainstPython(t, "tests/test_rest_api.sysl", "")
}

func TestRestApiQueryParams(t *testing.T) {
	testParseAgainstPython(t, "tests/rest_api_query_params.sysl", "")
}

func TestSimpleProject(t *testing.T) {
	testParseAgainstPython(t, "tests/project.sysl", "")
}

func TestUrlParamOrder(t *testing.T) {
	filename := "tests/rest_url_params.sysl"
	parseAndCompareWithPython(filename, "")
	fmt.Printf("Output for %#v won't match legacy. Visually inspect the above diff.\n", filename)
}

func TestUrlParamOrderAgainstGolden(t *testing.T) {
	testParseAgainstGolden(t, "tests/rest_url_params.sysl", "")
}

func TestRestApi_WrongOrder(t *testing.T) {
	testParseAgainstPython(t, "tests/bad_order.sysl", "")
}

func TestTransform(t *testing.T) {
	testParseAgainstPython(t, "tests/transform.sysl", "")
}

func TestImpliedDot(t *testing.T) {
	testParseAgainstPython(t, "tests/implied.sysl", "")
}

func TestStmts(t *testing.T) {
	testParseAgainstPython(t, "tests/stmts.sysl", "")
}

func TestMath(t *testing.T) {
	testParseAgainstPython(t, "tests/math.sysl", "")
}

func TestTableof(t *testing.T) {
	testParseAgainstPython(t, "tests/tableof.sysl", "")
}

func TestRank(t *testing.T) {
	testParseAgainstPython(t, "tests/rank.sysl", "")
}

func TestMatching(t *testing.T) {
	testParseAgainstPython(t, "tests/matching.sysl", "")
}

func TestNavigate(t *testing.T) {
	testParseAgainstPython(t, "tests/navigate.sysl", "")
}

func TestFuncs(t *testing.T) {
	testParseAgainstPython(t, "tests/funcs.sysl", "")
}

func TestPetshop(t *testing.T) {
	testParseAgainstPython(t, "petshop.sysl", "../../demo/petshop")
}

func TestCrash(t *testing.T) {
	testParseAgainstPython(t, "tests/crash.sysl", "")
}
<commit_msg>:art: Remove redundant code<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"testing"

	"github.com/anz-bank/sysl/src/proto"
	"github.com/golang/protobuf/proto"
	"github.com/stretchr/testify/assert"
)

func readSyslModule(filename string) (*sysl.Module, error) {
	var buf bytes.Buffer

	f, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	io.Copy(&buf, f)
	f.Close()

	module := &sysl.Module{}
	if err := proto.UnmarshalText(buf.String(), module); err != nil {
		return nil, err
	}
	return module, nil
}

var pySysl = func() string {
	if pySysl, ok := os.LookupEnv("SYSL_PYTHON_BIN"); ok {
		return pySysl
	}
	return "sysl"
}()

func pyParse(filename, root, output string) (*sysl.Module, error) {
	args := []string{"textpb", "-o", output, filename}
	if len(root) > 0 {
		rootArg := []string{"--root", root}
		args = append(rootArg, args...)
	}

	cmd := exec.Command(pySysl, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return nil, fmt.Errorf("Error running %#v %#v: %s", pySysl, args, err)
	}

	return readSyslModule(output)
}

func parseComparable(filename, root string) (*sysl.Module, error) {
	fs := &osFileSystem{root}
	module, err := FSParse(filename, fs)
	if err != nil {
		return nil, err
	}

	// remove stuff that does not match legacy.
	for _, app := range module.Apps {
		app.SourceContext = nil
		for _, ep := range app.Endpoints {
			ep.SourceContext = nil
		}
		for _, t := range app.Types {
			t.SourceContext = nil
		}
	}

	return module, nil
}

func parseAndCompare(filename, root, golden string, goldenModule *sysl.Module) (bool, error) {
	module, err := parseComparable(filename, root)
	if err != nil {
		return false, err
	}

	if proto.Equal(goldenModule, module) {
		return true, nil
	}

	if err = TextPB(goldenModule, golden); err != nil {
		return false, err
	}

	generated, err := ioutil.TempFile("", "sysl-test-*.textpb")
	if err != nil {
		return false, err
	}
	generatedClosed := false
	defer func() {
		if !generatedClosed {
			generated.Close()
		}
	}()

	if err = FTextPB(generated, module); err != nil {
		fmt.Printf("%#v retained for checking", generated.Name())
		return false, err
	}
	if err := generated.Close(); err != nil {
		return false, err
	}
	generatedClosed = true

	cmd := exec.Command("diff", "-y", golden, generated.Name())
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		fmt.Printf("%#v retained for checking", generated.Name())
		return false, err
	}

	os.Remove(generated.Name())
	return false, nil
}

func parseAndCompareWithPython(filename, root string) (bool, error) {
	fmt.Printf("%35s <=> $(%s %[1]s)\n", filename, pySysl)

	golden, err := ioutil.TempFile("", "sysl-test-golden-*.textpb")
	if err != nil {
		return false, err
	}
	defer golden.Close()

	pyModule, err := pyParse(filename, root, golden.Name())
	if err != nil {
		fmt.Printf("%#v retained for checking", golden.Name())
		return false, err
	}

	equal, err := parseAndCompare(filename, root, golden.Name(), pyModule)
	if err != nil {
		fmt.Printf("%#v retained for checking", golden.Name())
	}
	os.Remove(golden.Name())
	return equal, err
}

func parseAndCompareWithGolden(filename, root string) (bool, error) {
	golden := filename + ".golden.textpb"
	fmt.Printf("%35s <=> %s\n", filename, golden)

	goldenModule, err := readSyslModule(golden)
	if err != nil {
		return false, err
	}
	return parseAndCompare(filename, root, golden, goldenModule)
}

func parseAndPrint(t *testing.T, filename, root string) error {
	goModule, err := parseComparable(filename, root)
	if err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "Outputting %#v:\n", filename)
	FTextPB(os.Stderr, goModule)
	fmt.Fprintf(os.Stderr, "------------------------\n")
	return nil
}

func testParseAgainstPython(t *testing.T, filename, root string) {
	equal, err := parseAndCompareWithPython(filename, root)
	assert.NoError(t, err)
	assert.True(t, equal, "Mismatch")
}

func testParseAgainstGolden(t *testing.T, filename, root string) {
	equal, err := parseAndCompareWithGolden(filename, root)
	assert.NoError(t, err)
	assert.True(t, equal, "Mismatch")
}

func TestParseMissingFile(t *testing.T) {
	_, err := parseAndCompareWithGolden("tests/doesn't.exist", "")
	assert.Error(t, err)
}

func TestParseBadFile(t *testing.T) {
	_, err := parseAndCompareWithGolden("sysl.go", "")
	assert.Error(t, err)
}

func TestSimpleEP(t *testing.T) {
	testParseAgainstPython(t, "tests/test1.sysl", "")
}

func TestAttribs(t *testing.T) {
	testParseAgainstPython(t, "tests/attribs.sysl", "")
}

func TestIfElse(t *testing.T) {
	testParseAgainstPython(t, "tests/if_else.sysl", "")
}

func TestArgs(t *testing.T) {
	testParseAgainstPython(t, "tests/args.sysl", "")
}

func TestSimpleEPWithSpaces(t *testing.T) {
	testParseAgainstPython(t, "tests/with_spaces.sysl", "")
}

func TestSimpleEP2(t *testing.T) {
	testParseAgainstPython(t, "tests/test4.sysl", "")
}

func TestSimpleEndpointParams(t *testing.T) {
	testParseAgainstPython(t, "tests/ep_params.sysl", "")
}

func TestOneOfStatements(t *testing.T) {
	testParseAgainstPython(t, "tests/oneof.sysl", "")
}

func TestDuplicateEndpoints(t *testing.T) {
	testParseAgainstPython(t, "tests/duplicate.sysl", "")
}

func TestEventing(t *testing.T) {
	testParseAgainstPython(t, "tests/eventing.sysl", "")
}

func TestCollector(t *testing.T) {
	testParseAgainstPython(t, "tests/collector.sysl", "")
}

func TestPubSubCollector(t *testing.T) {
	testParseAgainstPython(t, "tests/pubsub_collector.sysl", "")
}

func TestDocstrings(t *testing.T) {
	testParseAgainstPython(t, "tests/docstrings.sysl", "")
}

func TestMixins(t *testing.T) {
	testParseAgainstPython(t, "tests/mixin.sysl", "")
}
func TestForLoops(t *testing.T) {
	testParseAgainstPython(t, "tests/for_loop.sysl", "")
}

func TestGroupStmt(t *testing.T) {
	testParseAgainstPython(t, "tests/group_stmt.sysl", "")
}

func TestUntilLoop(t *testing.T) {
	testParseAgainstPython(t, "tests/until_loop.sysl", "")
}

func TestTuple(t *testing.T) {
	testParseAgainstPython(t, "tests/test2.sysl", "")
}

func TestInplaceTuple(t *testing.T) {
	testParseAgainstPython(t, "tests/inplace_tuple.sysl", "")
}

func TestRelational(t *testing.T) {
	testParseAgainstPython(t, "tests/school.sysl", "")
}

func TestImports(t *testing.T) {
	testParseAgainstPython(t, "tests/library.sysl", "")
}

func TestRootArg(t *testing.T) {
	testParseAgainstPython(t, "school.sysl", "tests")
}

func TestRestApi(t *testing.T) {
	testParseAgainstPython(t, "tests/test_rest_api.sysl", "")
}

func TestRestApiQueryParams(t *testing.T) {
	testParseAgainstPython(t, "tests/rest_api_query_params.sysl", "")
}

func TestSimpleProject(t *testing.T) {
	testParseAgainstPython(t, "tests/project.sysl", "")
}

func TestUrlParamOrder(t *testing.T) {
	filename := "tests/rest_url_params.sysl"
	parseAndCompareWithPython(filename, "")
	fmt.Printf("Output for %#v won't match legacy. Visually inspect the above diff.\n", filename)
}

func TestUrlParamOrderAgainstGolden(t *testing.T) {
	testParseAgainstGolden(t, "tests/rest_url_params.sysl", "")
}

func TestRestApi_WrongOrder(t *testing.T) {
	testParseAgainstPython(t, "tests/bad_order.sysl", "")
}

func TestTransform(t *testing.T) {
	testParseAgainstPython(t, "tests/transform.sysl", "")
}

func TestImpliedDot(t *testing.T) {
	testParseAgainstPython(t, "tests/implied.sysl", "")
}

func TestStmts(t *testing.T) {
	testParseAgainstPython(t, "tests/stmts.sysl", "")
}

func TestMath(t *testing.T) {
	testParseAgainstPython(t, "tests/math.sysl", "")
}

func TestTableof(t *testing.T) {
	testParseAgainstPython(t, "tests/tableof.sysl", "")
}

func TestRank(t *testing.T) {
	testParseAgainstPython(t, "tests/rank.sysl", "")
}

func TestMatching(t *testing.T) {
	testParseAgainstPython(t, "tests/matching.sysl", "")
}

func TestNavigate(t *testing.T) {
	testParseAgainstPython(t, "tests/navigate.sysl", "")
}

func TestFuncs(t *testing.T) {
	testParseAgainstPython(t, "tests/funcs.sysl", "")
}

func TestPetshop(t *testing.T) {
	testParseAgainstPython(t, "petshop.sysl", "../../demo/petshop")
}

func TestCrash(t *testing.T) {
	testParseAgainstPython(t, "tests/crash.sysl", "")
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package har

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"net/url"
	"sort"
	"strings"
)

func Convert(h HAR, includeCodeCheck bool, batchTime uint, nobatch bool, only, skip []string) (string, error) {
	var b bytes.Buffer
	w := bufio.NewWriter(&b)
	if includeCodeCheck {
		fmt.Fprint(w, "import { group, check, sleep } from 'k6';\n")
	} else {
		fmt.Fprint(w, "import { group, sleep } from 'k6';\n")
	}
	fmt.Fprint(w, "import http from 'k6/http';\n\n")

	fmt.Fprintf(w, "// Version: %v\n", h.Log.Version)
	fmt.Fprintf(w, "// Creator: %v\n", h.Log.Creator.Name)
	if h.Log.Browser != nil {
		fmt.Fprintf(w, "// Browser: %v\n", h.Log.Browser.Name)
	}
	if h.Log.Comment != "" {
		fmt.Fprintf(w, "// %v\n", h.Log.Comment)
	}

	fmt.Fprint(w, "\n")
	fmt.Fprint(w, "export default function() {\n\n")

	pages := h.Log.Pages
	sort.Sort(PageByStarted(pages))

	// Grouping by page and URL filtering
	pageEntries := make(map[string][]*Entry)
	for _, e := range h.Log.Entries {

		// URL filtering
		u, err := url.Parse(e.Request.URL)
		if err != nil {
			return "", err
		}
		if !IsAllowedURL(u.Host, only, skip) {
			continue
		}

		// Avoid multipart/form-data requests until k6 scripts can support binary data
		if e.Request.PostData != nil && strings.HasPrefix(e.Request.PostData.MimeType, "multipart/form-data") {
			continue
		}

		// Create new group o adding page to a existing one
		if _, ok := pageEntries[e.Pageref]; !ok {
			pageEntries[e.Pageref] = append([]*Entry{}, e)
		} else {
			pageEntries[e.Pageref] = append(pageEntries[e.Pageref], e)
		}
	}

	for i, page := range pages {

		entries := pageEntries[page.ID]
		fmt.Fprintf(w, "\tgroup(\"%s - %s\", function() {\n", page.ID, page.Title)

		sort.Sort(EntryByStarted(entries))

		if nobatch {
			fmt.Fprint(w, "\t\tlet res;\n")

			for _, e := range entries {

				var params []string
				var cookies []string
				var body string

				if e.Request.PostData != nil {
					body = e.Request.PostData.Text
				}

				for _, c := range e.Request.Cookies {
					cookies = append(cookies, fmt.Sprintf(`%q: %q`, c.Name, c.Value))
				}
				if len(cookies) > 0 {
					params = append(params, fmt.Sprintf("\"cookies\": {\n\t\t\t\t%s\n\t\t\t}", strings.Join(cookies, ",\n\t\t\t\t\t")))
				}

				if headers := buildK6Headers(e.Request.Headers); len(headers) > 0 {
					params = append(params, fmt.Sprintf("\"headers\": {\n\t\t\t\t\t%s\n\t\t\t\t}", strings.Join(headers, ",\n\t\t\t\t\t")))
				}

				fmt.Fprintf(w, "\t\tres = http.%s(%q,\n\t\t\t%q",
					strings.ToLower(e.Request.Method), e.Request.URL, body)

				if len(params) > 0 {
					fmt.Fprintf(w, ",\n\t\t\t{\n\t\t\t\t%s\n\t\t\t}", strings.Join(params, ",\n\t\t\t"))
				}

				fmt.Fprintf(w, "\n\t\t)\n")

				if includeCodeCheck {
					if e.Response.Status > 0 {
						fmt.Fprintf(w, "\t\tcheck(res, {\n\t\t\t\"status is %v\": (r) => r.status === %v\n\t\t});\n", e.Response.Status, e.Response.Status)
					}
				}
			}
		} else {
			batches := SplitEntriesInBatches(entries, batchTime)

			fmt.Fprint(w, "\t\tlet req, res;\n")

			for j, batchEntries := range batches {

				fmt.Fprint(w, "\t\treq = [")
				for k, e := range batchEntries {
					r, err := buildK6RequestObject(e.Request)
					if err != nil {
						return "", err
					}
					fmt.Fprintf(w, "%v", r)
					if k != len(batchEntries)-1 {
						fmt.Fprint(w, ",")
					}
				}
				fmt.Fprint(w, "];\n")
				fmt.Fprint(w, "\t\tres = http.batch(req);\n")

				if includeCodeCheck {
					for k, e := range batchEntries {
						if e.Response.Status > 0 {
							fmt.Fprintf(w, "\t\tcheck(res[%v], {\n\t\t\"status is %v\": (r) => r.status === %v,\n\t});\n", k, e.Response.Status, e.Response.Status)
						}
					}
				}

				if j != len(batches)-1 {
					lastBatchEntry := batchEntries[len(batchEntries)-1]
					firstBatchEntry := batches[j+1][0]
					t := firstBatchEntry.StartedDateTime.Sub(lastBatchEntry.StartedDateTime).Seconds()
					fmt.Fprintf(w, "\t\tsleep(%.2f);\n", t)
				}
			}

			if i == len(pages)-1 {
				// Last page; add random sleep time at the group completion
				fmt.Fprint(w, "\t\t// Random sleep between 2s and 4s\n")
				fmt.Fprint(w, "\t\tsleep(Math.floor(Math.random()*3+2));\n")
			} else {
				// Add sleep time at the end of the group
				nextPage := pages[i+1]
				lastEntry := entries[len(entries)-1]
				t := nextPage.StartedDateTime.Sub(lastEntry.StartedDateTime).Seconds()
				if t < 0.01 {
					t = 0.5
				}
				fmt.Fprintf(w, "\t\tsleep(%.2f);\n", t)
			}
		}

		fmt.Fprint(w, "\t});\n")
	}

	fmt.Fprint(w, "\n}\n")
	if err := w.Flush(); err != nil {
		return "", err
	}
	return b.String(), nil
}

func buildK6RequestObject(req *Request) (string, error) {
	var b bytes.Buffer
	w := bufio.NewWriter(&b)

	fmt.Fprint(w, "{\n")

	method := strings.ToLower(req.Method)
	if method == "delete" {
		method = "del"
	}
	fmt.Fprintf(w, `"method": %q, "url": %q`, method, req.URL)

	if req.PostData != nil && method != "get" {
		postParams, plainText, err := buildK6Body(req)
		if err != nil {
			return "", err
		} else if len(postParams) > 0 {
			fmt.Fprintf(w, `, "body": { %s }`, strings.Join(postParams, ", "))
		} else if plainText != "" {
			fmt.Fprintf(w, `, "body": %q`, plainText)
		}
	}

	var params []string
	var cookies []string
	for _, c := range req.Cookies {
		cookies = append(cookies, fmt.Sprintf(`%q: %q`, c.Name, c.Value))
	}
	if len(cookies) > 0 {
		params = append(params, fmt.Sprintf(`"cookies": { %s }`, strings.Join(cookies, ", ")))
	}

	if headers := buildK6Headers(req.Headers); len(headers) > 0 {
		params = append(params, fmt.Sprintf(`"headers": { %s }`, strings.Join(headers, ", ")))
	}

	if len(params) > 0 {
		fmt.Fprintf(w, `, "params": { %s }`, strings.Join(params, ", "))
	}

	fmt.Fprint(w, "}")
	if err := w.Flush(); err != nil {
		return "", err
	}

	var buffer bytes.Buffer
	err := json.Indent(&buffer, b.Bytes(), "\t\t", "\t")
	if err != nil {
		return "", err
	}
	return buffer.String(), nil
}

func buildK6Headers(headers []Header) []string {
	var h []string
	if len(headers) > 0 {
		m := make(map[string]Header)
		for _, header := range headers {
			name := strings.ToLower(header.Name)
			_, exists := m[name]
			// Avoid SPDY's, duplicated or cookie headers
			if !exists && name[0] != ':' && name != "cookie" {
				m[strings.ToLower(header.Name)] = header
				h = append(h, fmt.Sprintf("%q: %q", header.Name, header.Value))
			}
		}
	}
	return h
}

func buildK6Body(req *Request) ([]string, string, error) {
	var postParams []string
	if req.PostData.MimeType == "application/x-www-form-urlencoded" && len(req.PostData.Params) > 0 {
		for _, p := range req.PostData.Params {
			n, err := url.QueryUnescape(p.Name)
			if err != nil {
				return postParams, "", err
			}
			v, err := url.QueryUnescape(p.Value)
			if err != nil {
				return postParams, "", err
			}
			postParams = append(postParams, fmt.Sprintf(`%q: %q`, n, v))
		}
		return postParams, "", nil
	}
	return postParams, req.PostData.Text, nil
}
<commit_msg>Break early on unexpected return codes.<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package har

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"net/url"
	"sort"
	"strings"
)

func Convert(h HAR, includeCodeCheck bool, batchTime uint, nobatch bool, only, skip []string) (string, error) {
	var b bytes.Buffer
	w := bufio.NewWriter(&b)
	if includeCodeCheck {
		fmt.Fprint(w, "import { group, check, sleep } from 'k6';\n")
	} else {
		fmt.Fprint(w, "import { group, sleep } from 'k6';\n")
	}
	fmt.Fprint(w, "import http from 'k6/http';\n\n")

	fmt.Fprintf(w, "// Version: %v\n", h.Log.Version)
	fmt.Fprintf(w, "// Creator: %v\n", h.Log.Creator.Name)
	if h.Log.Browser != nil {
		fmt.Fprintf(w, "// Browser: %v\n", h.Log.Browser.Name)
	}
	if h.Log.Comment != "" {
		fmt.Fprintf(w, "// %v\n", h.Log.Comment)
	}

	fmt.Fprint(w, "\n")
	fmt.Fprint(w, "export default function() {\n\n")

	pages := h.Log.Pages
	sort.Sort(PageByStarted(pages))

	// Grouping by page and URL filtering
	pageEntries := make(map[string][]*Entry)
	for _, e := range h.Log.Entries {

		// URL filtering
		u, err := url.Parse(e.Request.URL)
		if err != nil {
			return "", err
		}
		if !IsAllowedURL(u.Host, only, skip) {
			continue
		}

		// Avoid multipart/form-data requests until k6 scripts can support binary data
		if e.Request.PostData != nil && strings.HasPrefix(e.Request.PostData.MimeType, "multipart/form-data") {
			continue
		}

		// Create new group o adding page to a existing one
		if _, ok := pageEntries[e.Pageref]; !ok {
			pageEntries[e.Pageref] = append([]*Entry{}, e)
		} else {
			pageEntries[e.Pageref] = append(pageEntries[e.Pageref], e)
		}
	}

	for i, page := range pages {

		entries := pageEntries[page.ID]
		fmt.Fprintf(w, "\tgroup(\"%s - %s\", function() {\n", page.ID, page.Title)

		sort.Sort(EntryByStarted(entries))

		if nobatch {
			fmt.Fprint(w, "\t\tlet res;\n")

			for _, e := range entries {

				var params []string
				var cookies []string
				var body string

				if e.Request.PostData != nil {
					body = e.Request.PostData.Text
				}

				for _, c := range e.Request.Cookies {
					cookies = append(cookies, fmt.Sprintf(`%q: %q`, c.Name, c.Value))
				}
				if len(cookies) > 0 {
					params = append(params, fmt.Sprintf("\"cookies\": {\n\t\t\t\t%s\n\t\t\t}", strings.Join(cookies, ",\n\t\t\t\t\t")))
				}

				if headers := buildK6Headers(e.Request.Headers); len(headers) > 0 {
					params = append(params, fmt.Sprintf("\"headers\": {\n\t\t\t\t\t%s\n\t\t\t\t}", strings.Join(headers, ",\n\t\t\t\t\t")))
				}

				fmt.Fprintf(w, "\t\tres = http.%s(%q,\n\t\t\t%q",
					strings.ToLower(e.Request.Method), e.Request.URL, body)

				if len(params) > 0 {
					fmt.Fprintf(w, ",\n\t\t\t{\n\t\t\t\t%s\n\t\t\t}", strings.Join(params, ",\n\t\t\t"))
				}

				fmt.Fprintf(w, "\n\t\t)\n")

				if includeCodeCheck {
					if e.Response.Status > 0 {
						fmt.Fprintf(w, "\t\tif (!check(res, {\"status is %v\": (r) => r.status === %v })) { return };\n", e.Response.Status, e.Response.Status)
					}
				}
			}
		} else {
			batches := SplitEntriesInBatches(entries, batchTime)

			fmt.Fprint(w, "\t\tlet req, res;\n")

			for j, batchEntries := range batches {

				fmt.Fprint(w, "\t\treq = [")
				for k, e := range batchEntries {
					r, err := buildK6RequestObject(e.Request)
					if err != nil {
						return "", err
					}
					fmt.Fprintf(w, "%v", r)
					if k != len(batchEntries)-1 {
						fmt.Fprint(w, ",")
					}
				}
				fmt.Fprint(w, "];\n")
				fmt.Fprint(w, "\t\tres = http.batch(req);\n")

				if includeCodeCheck {
					for k, e := range batchEntries {
						if e.Response.Status > 0 {
							fmt.Fprintf(w, "\t\tcheck(res[%v], {\n\t\t\"status is %v\": (r) => r.status === %v,\n\t});\n", k, e.Response.Status, e.Response.Status)
						}
					}
				}

				if j != len(batches)-1 {
					lastBatchEntry := batchEntries[len(batchEntries)-1]
					firstBatchEntry := batches[j+1][0]
					t := firstBatchEntry.StartedDateTime.Sub(lastBatchEntry.StartedDateTime).Seconds()
					fmt.Fprintf(w, "\t\tsleep(%.2f);\n", t)
				}
			}

			if i == len(pages)-1 {
				// Last page; add random sleep time at the group completion
				fmt.Fprint(w, "\t\t// Random sleep between 2s and 4s\n")
				fmt.Fprint(w, "\t\tsleep(Math.floor(Math.random()*3+2));\n")
			} else {
				// Add sleep time at the end of the group
				nextPage := pages[i+1]
				lastEntry := entries[len(entries)-1]
				t := nextPage.StartedDateTime.Sub(lastEntry.StartedDateTime).Seconds()
				if t < 0.01 {
					t = 0.5
				}
				fmt.Fprintf(w, "\t\tsleep(%.2f);\n", t)
			}
		}

		fmt.Fprint(w, "\t});\n")
	}

	fmt.Fprint(w, "\n}\n")
	if err := w.Flush(); err != nil {
		return "", err
	}
	return b.String(), nil
}

func buildK6RequestObject(req *Request) (string, error) {
	var b bytes.Buffer
	w := bufio.NewWriter(&b)

	fmt.Fprint(w, "{\n")

	method := strings.ToLower(req.Method)
	if method == "delete" {
		method = "del"
	}
	fmt.Fprintf(w, `"method": %q, "url": %q`, method, req.URL)

	if req.PostData != nil && method != "get" {
		postParams, plainText, err := buildK6Body(req)
		if err != nil {
			return "", err
		} else if len(postParams) > 0 {
			fmt.Fprintf(w, `, "body": { %s }`, strings.Join(postParams, ", "))
		} else if plainText != "" {
			fmt.Fprintf(w, `, "body": %q`, plainText)
		}
	}

	var params []string
	var cookies []string
	for _, c := range req.Cookies {
		cookies = append(cookies, fmt.Sprintf(`%q: %q`, c.Name, c.Value))
	}
	if len(cookies) > 0 {
		params = append(params, fmt.Sprintf(`"cookies": { %s }`, strings.Join(cookies, ", ")))
	}

	if headers := buildK6Headers(req.Headers); len(headers) > 0 {
		params = append(params, fmt.Sprintf(`"headers": { %s }`, strings.Join(headers, ", ")))
	}

	if len(params) > 0 {
		fmt.Fprintf(w, `, "params": { %s }`, strings.Join(params, ", "))
	}

	fmt.Fprint(w, "}")
	if err := w.Flush(); err != nil {
		return "", err
	}

	var buffer bytes.Buffer
	err := json.Indent(&buffer, b.Bytes(), "\t\t", "\t")
	if err != nil {
		return "", err
	}
	return buffer.String(), nil
}

func buildK6Headers(headers []Header) []string {
	var h []string
	if len(headers) > 0 {
		m := make(map[string]Header)
		for _, header := range headers {
			name := strings.ToLower(header.Name)
			_, exists := m[name]
			// Avoid SPDY's, duplicated or cookie headers
			if !exists && name[0] != ':' && name != "cookie" {
				m[strings.ToLower(header.Name)] = header
				h = append(h, fmt.Sprintf("%q: %q", header.Name, header.Value))
			}
		}
	}
	return h
}

func buildK6Body(req *Request) ([]string, string, error) {
	var postParams []string
	if req.PostData.MimeType == "application/x-www-form-urlencoded" && len(req.PostData.Params) > 0 {
		for _, p := range req.PostData.Params {
			n, err := url.QueryUnescape(p.Name)
			if err != nil {
				return postParams, "", err
			}
			v, err := url.QueryUnescape(p.Value)
			if err != nil {
				return postParams, "", err
			}
			postParams = append(postParams, fmt.Sprintf(`%q: %q`, n, v))
		}
		return postParams, "", nil
	}
	return postParams, req.PostData.Text, nil
}
<|endoftext|>


<commit_before>package slack

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"golang.org/x/net/websocket"
)

const (
	API_BASE = "https://slack.com/api/"
)

type Credentials struct {
	HookToken   string
	Bot         Bot
	SlackbotUrl string
	Commands    map[string]string
}

type Message struct {
	Type    string     `json:"type"`
	SubType string     `json:"subtype"`
	Channel string     `json:"channel"`
	User    string     `json:"user"`
	Text    string     `json:"text"`
	Ts      string     `json:"ts"`
	File    FileObject `json:"file"`
}

type FileObject struct {
	Mimetype   string `json:"mimetype"`
	Filetype   string `json:"filetype"`
	PrettyType string `json:"pretty_type"`
}

type Attachment struct {
	Fallback   string   `json:"fallback"`
	Color      string   `json:"color"`
	Pretext    string   `json:"pretext"`
	AuthorName string   `json:"author_name"`
	Title      string   `json:"title"`
	TitleLink  string   `json:"title_link"`
	Text       string   `json:"text"`
	Fields     []Field  `json:"fields"`
	MrkdwnIn   []string `json:"mrkdwn_in"`
}

type Field struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}

type Bot struct {
	Token  string `json:"token"`
	UserId string `json:"user_id"`
	User   string `json:"user"`
	Client *http.Client
}

func (b Bot) WithClient(c *http.Client) Bot {
	b.Client = c
	return b
}

func (b Bot) PostForm(endpoint string, data url.Values) (respJson map[string]interface{}, err error) {
	data.Add("token", b.Token)
	if _, ok := data["as_user"]; !ok {
		data.Add("as_user", "true")
	}

	if b.Client == nil {
		b.Client = http.DefaultClient
	}
	resp, err := b.Client.PostForm(API_BASE+endpoint, data)
	if err != nil {
		return
	}
	respJson, err = asJson(resp)
	if err != nil {
		return
	}
	if !respJson["ok"].(bool) {
		err = errors.New(respJson["error"].(string))
	}
	return
}

func asJson(resp *http.Response) (map[string]interface{}, error) {
	defer resp.Body.Close()
	m := make(map[string]interface{})
	dec := json.NewDecoder(resp.Body)
	for {
		if err := dec.Decode(&m); err == io.EOF {
			return m, nil
		} else if err != nil {
			return nil, err
		}
	}
}

func (b Bot) ChatPostMessage(data url.Values) (err error) {
	_, err = b.PostForm("chat.postMessage", data)
	return
}

func (b Bot) UsersGetPresence(user string) (presence string, err error) {
	resp, err := b.PostForm("users.getPresence", url.Values{"user": {user}})
	if err != nil {
		return
	} else {
		presence = resp["presence"].(string)
		return
	}
}

// for presence now
func (b Bot) UsersList(presence string) (present []string, err error) {
	resp, err := b.PostForm("users.list", url.Values{"presence": {presence}})
	if err != nil {
		return
	}
	members := resp["members"].([]interface{})
	present = make([]string, 0)
	for i := range members {
		m := members[i].(map[string]interface{})
		if p, ok := m["presence"]; ok {
			if p == "active" {
				present = append(present, m["id"].(string))
			}
		}
	}
	return
}

// Calls rtm.start API, return websocket url and bot id
func RtmStart(token string) (wsurl string, err error) {
	resp, err := http.PostForm(API_BASE+"rtm.start", url.Values{"token": {token}})
	if err != nil {
		return
	}
	respRtmStart, err := asJson(resp)
	if err != nil {
		return
	}
	wsurl = respRtmStart["url"].(string)
	return
}

func RtmReceive(ws *websocket.Conn) (m Message, err error) {
	err = websocket.JSON.Receive(ws, &m)
	return
}

func RtmSend(ws *websocket.Conn, m Message) error {
	return websocket.JSON.Send(ws, m)
}

func LoadCredentials(filename string) (credentials Credentials, err error) {
	f, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	err = json.Unmarshal(f, &credentials)
	return
}

func ValidateCommand(handler http.Handler, commands map[string]string) http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		command := r.PostFormValue("command")
		token := r.PostFormValue("token")
		if t, ok := commands[command]; ok && t == token {
			handler.ServeHTTP(w, r)
		} else {
			w.WriteHeader(400)
		}
	})
}
<commit_msg>add NewBot<commit_after>package slack

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"golang.org/x/net/websocket"
)

const (
	API_BASE = "https://slack.com/api/"
)

type Credentials struct {
	HookToken   string
	Bot         Bot
	SlackbotUrl string
	Commands    map[string]string
}

type Message struct {
	Type    string     `json:"type"`
	SubType string     `json:"subtype"`
	Channel string     `json:"channel"`
	User    string     `json:"user"`
	Text    string     `json:"text"`
	Ts      string     `json:"ts"`
	File    FileObject `json:"file"`
}

type FileObject struct {
	Mimetype   string `json:"mimetype"`
	Filetype   string `json:"filetype"`
	PrettyType string `json:"pretty_type"`
}

type Attachment struct {
	Fallback   string   `json:"fallback"`
	Color      string   `json:"color"`
	Pretext    string   `json:"pretext"`
	AuthorName string   `json:"author_name"`
	Title      string   `json:"title"`
	TitleLink  string   `json:"title_link"`
	Text       string   `json:"text"`
	Fields     []Field  `json:"fields"`
	MrkdwnIn   []string `json:"mrkdwn_in"`
}

type Field struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short"`
}

type Bot struct {
	Token  string `json:"token"`
	UserId string `json:"user_id"`
	User   string `json:"user"`
	Client *http.Client
}

func NewBot(token string) *Bot {
	return &Bot{Token: token}
}

func (b Bot) WithClient(c *http.Client) Bot {
	b.Client = c
	return b
}

func (b Bot) PostForm(endpoint string, data url.Values) (respJson map[string]interface{}, err error) {
	data.Add("token", b.Token)
	if _, ok := data["as_user"]; !ok {
		data.Add("as_user", "true")
	}

	if b.Client == nil {
		b.Client = http.DefaultClient
	}
	resp, err := b.Client.PostForm(API_BASE+endpoint, data)
	if err != nil {
		return
	}
	respJson, err = asJson(resp)
	if err != nil {
		return
	}
	if !respJson["ok"].(bool) {
		err = errors.New(respJson["error"].(string))
	}
	return
}

func asJson(resp *http.Response) (map[string]interface{}, error) {
	defer resp.Body.Close()
	m := make(map[string]interface{})
	dec := json.NewDecoder(resp.Body)
	for {
		if err := dec.Decode(&m); err == io.EOF {
			return m, nil
		} else if err != nil {
			return nil, err
		}
	}
}

func (b Bot) ChatPostMessage(data url.Values) (err error) {
	_, err = b.PostForm("chat.postMessage", data)
	return
}

func (b Bot) UsersGetPresence(user string) (presence string, err error) {
	resp, err := b.PostForm("users.getPresence", url.Values{"user": {user}})
	if err != nil {
		return
	} else {
		presence = resp["presence"].(string)
		return
	}
}

// for presence now
func (b Bot) UsersList(presence string) (present []string, err error) {
	resp, err := b.PostForm("users.list", url.Values{"presence": {presence}})
	if err != nil {
		return
	}
	members := resp["members"].([]interface{})
	present = make([]string, 0)
	for i := range members {
		m := members[i].(map[string]interface{})
		if p, ok := m["presence"]; ok {
			if p == "active" {
				present = append(present, m["id"].(string))
			}
		}
	}
	return
}

// Calls rtm.start API, return websocket url and bot id
func RtmStart(token string) (wsurl string, err error) {
	resp, err := http.PostForm(API_BASE+"rtm.start", url.Values{"token": {token}})
	if err != nil {
		return
	}
	respRtmStart, err := asJson(resp)
	if err != nil {
		return
	}
	wsurl = respRtmStart["url"].(string)
	return
}

func RtmReceive(ws *websocket.Conn) (m Message, err error) {
	err = websocket.JSON.Receive(ws, &m)
	return
}

func RtmSend(ws *websocket.Conn, m Message) error {
	return websocket.JSON.Send(ws, m)
}

func LoadCredentials(filename string) (credentials Credentials, err error) {
	f, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}
	err = json.Unmarshal(f, &credentials)
	return
}

func ValidateCommand(handler http.Handler, commands map[string]string) http.HandlerFunc {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		command := r.PostFormValue("command")
		token := r.PostFormValue("token")
		if t, ok := commands[command]; ok && t == token {
			handler.ServeHTTP(w, r)
		} else {
			w.WriteHeader(400)
		}
	})
}
<|endoftext|>


<commit_before>package oauth2

import (
	"encoding/json"
	"fmt"
)

const (
	ErrorAccessDenied            = "access_denied"
	ErrorInvalidClient           = "invalid_client"
	ErrorInvalidGrant            = "invalid_grant"
	ErrorInvalidRequest          = "invalid_request"
	ErrorServerError             = "server_error"
	ErrorUnsupportedGrantType    = "unsupported_grant_type"
	ErrorUnsupportedResponseType = "unsupported_response_type"
)

type Error struct {
	Type  string `json:"error"`
	State string `json:"state,omitempty"`
}

func (e *Error) Error() string {
	return e.Type
}

func NewError(typ string) *Error {
	return &Error{Type: typ}
}

func unmarshalError(b []byte) error {
	var oerr Error
	err := json.Unmarshal(b, &oerr)
	if err != nil {
		return fmt.Errorf("unrecognized error: %s", string(b))
	}
	return &oerr
}
<commit_msg>oauth2: add unauthorized_client error type.<commit_after>package oauth2

import (
	"encoding/json"
	"fmt"
)

const (
	ErrorAccessDenied            = "access_denied"
	ErrorInvalidClient           = "invalid_client"
	ErrorInvalidGrant            = "invalid_grant"
	ErrorInvalidRequest          = "invalid_request"
	ErrorServerError             = "server_error"
	ErrorUnauthorizedClient      = "unauthorized_client"
	ErrorUnsupportedGrantType    = "unsupported_grant_type"
	ErrorUnsupportedResponseType = "unsupported_response_type"
)

type Error struct {
	Type  string `json:"error"`
	State string `json:"state,omitempty"`
}

func (e *Error) Error() string {
	return e.Type
}

func NewError(typ string) *Error {
	return &Error{Type: typ}
}

func unmarshalError(b []byte) error {
	var oerr Error
	err := json.Unmarshal(b, &oerr)
	if err != nil {
		return fmt.Errorf("unrecognized error: %s", string(b))
	}
	return &oerr
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/chrislusf/weed-fs/go/glog"
	"github.com/chrislusf/weed-fs/go/util"
	"github.com/chrislusf/weed-fs/go/weed/weed_server"
)

var (
	f FilerOptions
)

type FilerOptions struct {
	master                  *string
	port                    *int
	collection              *string
	defaultReplicaPlacement *string
	dir                     *string
	redirectOnRead          *bool
	cassandra_server        *string
	cassandra_keyspace      *string
	redis_server            *string
	redis_database          *int
}

func init() {
	cmdFiler.Run = runFiler // break init cycle
	f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location")
	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
	f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
	f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
	f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
	f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the cassandra server, e.g., 127.0.0.1:6379")
	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
}

var cmdFiler = &Command{
	UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>",
	Short:     "start a file server that points to a master server",
	Long: `start a file server which accepts REST operation for any files.

	//create or overwrite the file, the directories /path/to will be automatically created
	POST /path/to/file
	//get the file content
	GET /path/to/file
	//create or overwrite the file, the filename in the multipart request will be used
	POST /path/to/
	//return a json format subdirectory and files listing
	GET /path/to/

  Current <fullpath~fileid> mapping metadata store is local embedded leveldb.
  It should be highly scalable to hundreds of millions of files on a modest machine.

  Future we will ensure it can avoid of being SPOF.

  `,
}

func runFiler(cmd *Command, args []string) bool {

	if err := util.TestFolderWritable(*f.dir); err != nil {
		glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
	}

	r := http.NewServeMux()
	_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection,
		*f.defaultReplicaPlacement, *f.redirectOnRead,
		*f.cassandra_server, *f.cassandra_keyspace,
		*f.redis_server, *f.redis_database,
	)
	if nfs_err != nil {
		glog.Fatalf(nfs_err.Error())
	}
	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
	filerListener, e := util.NewListener(
		":"+strconv.Itoa(*f.port),
		time.Duration(10)*time.Second,
	)
	if e != nil {
		glog.Fatalf(e.Error())
	}
	if e := http.Serve(filerListener, r); e != nil {
		glog.Fatalf("Filer Fail to serve:%s", e.Error())
	}

	return true
}
<commit_msg>fixed description error in redis.database<commit_after>package main

import (
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/chrislusf/weed-fs/go/glog"
	"github.com/chrislusf/weed-fs/go/util"
	"github.com/chrislusf/weed-fs/go/weed/weed_server"
)

var (
	f FilerOptions
)

type FilerOptions struct {
	master                  *string
	port                    *int
	collection              *string
	defaultReplicaPlacement *string
	dir                     *string
	redirectOnRead          *bool
	cassandra_server        *string
	cassandra_keyspace      *string
	redis_server            *string
	redis_database          *int
}

func init() {
	cmdFiler.Run = runFiler // break init cycle
	f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location")
	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
	f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
	f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
	f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server")
	f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server")
	f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379")
	f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server")
}

var cmdFiler = &Command{
	UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>",
	Short:     "start a file server that points to a master server",
	Long: `start a file server which accepts REST operation for any files.

	//create or overwrite the file, the directories /path/to will be automatically created
	POST /path/to/file
	//get the file content
	GET /path/to/file
	//create or overwrite the file, the filename in the multipart request will be used
	POST /path/to/
	//return a json format subdirectory and files listing
	GET /path/to/

  Current <fullpath~fileid> mapping metadata store is local embedded leveldb.
  It should be highly scalable to hundreds of millions of files on a modest machine.

  Future we will ensure it can avoid of being SPOF.

  `,
}

func runFiler(cmd *Command, args []string) bool {

	if err := util.TestFolderWritable(*f.dir); err != nil {
		glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err)
	}

	r := http.NewServeMux()
	_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection,
		*f.defaultReplicaPlacement, *f.redirectOnRead,
		*f.cassandra_server, *f.cassandra_keyspace,
		*f.redis_server, *f.redis_database,
	)
	if nfs_err != nil {
		glog.Fatalf(nfs_err.Error())
	}
	glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port))
	filerListener, e := util.NewListener(
		":"+strconv.Itoa(*f.port),
		time.Duration(10)*time.Second,
	)
	if e != nil {
		glog.Fatalf(e.Error())
	}
	if e := http.Serve(filerListener, r); e != nil {
		glog.Fatalf("Filer Fail to serve:%s", e.Error())
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2009 by Maurice Gilden. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package godata

import (
	"strings"
	"os"
	"go/ast"
	"go/parser"
	"./logger"
	"container/vector"
)

var DefaultOutputFileName string

// Returns the greater of both numbers.
func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// ================================
// ============ GoFile ============
// ================================

type GoFile struct {
	Filename           string         // file name with relative path
	Pack               *GoPackage     // the package this file belongs to
	HasMain            bool           // main function found (only true for main package)
	IsCGOFile          bool           // imports "C"
	IsTestFile         bool           // files with "_test.go" suffix
	TestFunctions      *vector.Vector // vector of all test functions (name only)
	BenchmarkFunctions *vector.Vector // vector of all benchmark functions (name only)
}

/*
 Parses the content of a .go file and searches for package name, imports and
 main function.
*/
func (this *GoFile) ParseFile(packs *GoPackageContainer) (err os.Error) {
	var packName string
	var fileast *ast.File

	if fileast, err = parser.ParseFile(this.Filename, nil, 0); err != nil {
		logger.Error("%s\n", err)
		os.Exit(1)
	}

	packName = fileast.Name.String()

	if err != nil {
		logger.Warn("Parsing file %s returned with errors: %s\n", this.Filename, err)
	}

	// check if package is in the correct path
	if packName != "main" {
		switch strings.Count(this.Filename, "/") {
		case 0: // no sub-directory
			logger.Warn("File %s from package %s is not in the correct path. Should be %s.\n",
				this.Filename, fileast.Name,
				strings.Join([]string{packName, this.Filename}, "/"))
		case 1: // one sub-directory
			if this.Filename[0:strings.Index(this.Filename, "/")] != packName {
				logger.Warn("File %s from package %s is not in the correct directory. Should be %s.\n",
					this.Filename, packName,
					strings.Join([]string{packName,
						this.Filename[strings.Index(this.Filename, "/")+1 : len(this.Filename)],
					},
						"/"))
			}
		default: // >1 sub-directories
			if this.Filename[max(strings.LastIndex(this.Filename, "/")-len(packName), 0):strings.LastIndex(this.Filename, "/")] != packName {

				// NOTE: this case will result in a link-error (exit with error here?)
				logger.Warn("File %s from package %s is not in the expected directory.\n",
					this.Filename, packName)
			}
			packName = strings.Join([]string{
				this.Filename[0:strings.LastIndex(this.Filename[0:strings.LastIndex(this.Filename, "/")], "/")],
				packName,
			},
				"/")
		}
	}

	// create empty temporary package, will be merged later
	this.Pack = NewGoPackage(packName)

	// find the local imports in this file
	visitor := astVisitor{this, packs}
	ast.Walk(visitor, fileast)

	packs.AddFile(this, packName)

	return
}

// ================================
// ========== astVisitor ==========
// ================================

// this visitor looks for imports and the main function in an AST
type astVisitor struct {
	file  *GoFile
	packs *GoPackageContainer
}

/*
 Implementation of the visitor interface for ast walker.
 Returning nil stops the walker, anything else continues into the subtree
*/
func (v astVisitor) Visit(node interface{}) (w ast.Visitor) {
	switch n := node.(type) {
	case *ast.ImportSpec:
		var packName string
		var packType int
		for _, bl := range n.Path {
			if (len(bl.Value) > 4) &&
				(bl.Value[1] == '.') &&
				(bl.Value[2] == '/') {

				// local package found
				packName = string(bl.Value[3 : len(bl.Value)-1])
				packType = LOCAL_PACKAGE

			} else {
				packName = string(bl.Value[1 : len(bl.Value)-1])
				packType = UNKNOWN_PACKAGE
			}

			dep, exists := v.packs.Get(packName)
			if !exists {
				dep = v.packs.AddNewPackage(packName)
			} else if dep.Type == LOCAL_PACKAGE {
				packType = LOCAL_PACKAGE
			}

			dep.Type = packType
			v.file.Pack.Depends.Push(dep)

			if string(bl.Value) == "\"C\"" {
				v.file.IsCGOFile = true // not used yet
			}

		}
		return nil
	case *ast.FuncDecl:
		if n.Recv == nil && n.Name.String() == "main" && v.file.Pack.Name == "main" {
			v.file.HasMain = true
		} else if n.Recv == nil && v.file.IsTestFile &&
			strings.HasPrefix(n.Name.String(), "Test") &&
			n.Body != nil {
			v.file.TestFunctions.Push(n.Name.String())
		} else if n.Recv == nil && v.file.IsTestFile &&
			strings.HasPrefix(n.Name.String(), "Benchmark") &&
			n.Body != nil {
			v.file.BenchmarkFunctions.Push(n.Name.String())
		}
		return nil
	case *ast.Package, *ast.File, *ast.BadDecl,
		*ast.GenDecl, *ast.Ident, []ast.Decl:
		return v

	default:
		return nil // makes parsing faster
			   // this may cause some files to not parse correctly
	}

	return nil // unreachable
}
<commit_msg>quick fix for new parameter for parser.ParseFile<commit_after>// Copyright 2009 by Maurice Gilden. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package godata

import (
	"strings"
	"os"
	"go/ast"
	"go/parser"
	"./logger"
	"container/vector"
)

var DefaultOutputFileName string

// Returns the greater of both numbers.
func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

// ================================
// ============ GoFile ============
// ================================

type GoFile struct {
	Filename           string         // file name with relative path
	Pack               *GoPackage     // the package this file belongs to
	HasMain            bool           // main function found (only true for main package)
	IsCGOFile          bool           // imports "C"
	IsTestFile         bool           // files with "_test.go" suffix
	TestFunctions      *vector.Vector // vector of all test functions (name only)
	BenchmarkFunctions *vector.Vector // vector of all benchmark functions (name only)
}

/*
 Parses the content of a .go file and searches for package name, imports and
 main function.
*/
func (this *GoFile) ParseFile(packs *GoPackageContainer) (err os.Error) {
	var packName string
	var fileast *ast.File

	if fileast, err = parser.ParseFile(this.Filename, nil, nil, 0); err != nil {
		logger.Error("%s\n", err)
		os.Exit(1)
	}

	packName = fileast.Name.String()

	if err != nil {
		logger.Warn("Parsing file %s returned with errors: %s\n", this.Filename, err)
	}

	// check if package is in the correct path
	if packName != "main" {
		switch strings.Count(this.Filename, "/") {
		case 0: // no sub-directory
			logger.Warn("File %s from package %s is not in the correct path. Should be %s.\n",
				this.Filename, fileast.Name,
				strings.Join([]string{packName, this.Filename}, "/"))
		case 1: // one sub-directory
			if this.Filename[0:strings.Index(this.Filename, "/")] != packName {
				logger.Warn("File %s from package %s is not in the correct directory. Should be %s.\n",
					this.Filename, packName,
					strings.Join([]string{packName,
						this.Filename[strings.Index(this.Filename, "/")+1 : len(this.Filename)],
					},
						"/"))
			}
		default: // >1 sub-directories
			if this.Filename[max(strings.LastIndex(this.Filename, "/")-len(packName), 0):strings.LastIndex(this.Filename, "/")] != packName {

				// NOTE: this case will result in a link-error (exit with error here?)
				logger.Warn("File %s from package %s is not in the expected directory.\n",
					this.Filename, packName)
			}
			packName = strings.Join([]string{
				this.Filename[0:strings.LastIndex(this.Filename[0:strings.LastIndex(this.Filename, "/")], "/")],
				packName,
			},
				"/")
		}
	}

	// create empty temporary package, will be merged later
	this.Pack = NewGoPackage(packName)

	// find the local imports in this file
	visitor := astVisitor{this, packs}
	ast.Walk(visitor, fileast)

	packs.AddFile(this, packName)

	return
}

// ================================
// ========== astVisitor ==========
// ================================

// this visitor looks for imports and the main function in an AST
type astVisitor struct {
	file  *GoFile
	packs *GoPackageContainer
}

/*
 Implementation of the visitor interface for ast walker.
 Returning nil stops the walker, anything else continues into the subtree
*/
func (v astVisitor) Visit(node interface{}) (w ast.Visitor) {
	switch n := node.(type) {
	case *ast.ImportSpec:
		var packName string
		var packType int
		for _, bl := range n.Path {
			if (len(bl.Value) > 4) &&
				(bl.Value[1] == '.') &&
				(bl.Value[2] == '/') {

				// local package found
				packName = string(bl.Value[3 : len(bl.Value)-1])
				packType = LOCAL_PACKAGE

			} else {
				packName = string(bl.Value[1 : len(bl.Value)-1])
				packType = UNKNOWN_PACKAGE
			}

			dep, exists := v.packs.Get(packName)
			if !exists {
				dep = v.packs.AddNewPackage(packName)
			} else if dep.Type == LOCAL_PACKAGE {
				packType = LOCAL_PACKAGE
			}

			dep.Type = packType
			v.file.Pack.Depends.Push(dep)

			if string(bl.Value) == "\"C\"" {
				v.file.IsCGOFile = true // not used yet
			}

		}
		return nil
	case *ast.FuncDecl:
		if n.Recv == nil && n.Name.String() == "main" && v.file.Pack.Name == "main" {
			v.file.HasMain = true
		} else if n.Recv == nil && v.file.IsTestFile &&
			strings.HasPrefix(n.Name.String(), "Test") &&
			n.Body != nil {
			v.file.TestFunctions.Push(n.Name.String())
		} else if n.Recv == nil && v.file.IsTestFile &&
			strings.HasPrefix(n.Name.String(), "Benchmark") &&
			n.Body != nil {
			v.file.BenchmarkFunctions.Push(n.Name.String())
		}
		return nil
	case *ast.Package, *ast.File, *ast.BadDecl,
		*ast.GenDecl, *ast.Ident, []ast.Decl:
		return v

	default:
		return nil // makes parsing faster
			   // this may cause some files to not parse correctly
	}

	return nil // unreachable
}
<|endoftext|>