<commit_before>package checklog

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	"github.com/mackerelio/golib/pluginutil"
	"github.com/mattn/go-encoding"
	"github.com/mattn/go-zglob"
	enc "golang.org/x/text/encoding"
)

type logOpts struct {
	LogFile             string   `short:"f" long:"file" value-name:"FILE" description:"Path to log file"`
	Pattern             []string `short:"p" long:"pattern" required:"true" value-name:"PAT" description:"Pattern to search for. If specified multiple, they will be treated together with the AND operator"`
	Exclude             string   `short:"E" long:"exclude" value-name:"PAT" description:"Pattern to exclude from matching"`
	WarnOver            int64    `short:"w" long:"warning-over" description:"Trigger a warning if matched lines is over a number"`
	CritOver            int64    `short:"c" long:"critical-over" description:"Trigger a critical if matched lines is over a number"`
	WarnLevel           float64  `long:"warning-level" value-name:"N" description:"Warning level if pattern has a group"`
	CritLevel           float64  `long:"critical-level" value-name:"N" description:"Critical level if pattern has a group"`
	ReturnContent       bool     `short:"r" long:"return" description:"Return matched line"`
	FilePattern         string   `short:"F" long:"file-pattern" value-name:"FILE" description:"Check a pattern of files, instead of one file"`
	CaseInsensitive     bool     `short:"i" long:"icase" description:"Run a case insensitive match"`
	StateDir            string   `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
	NoState             bool     `long:"no-state" description:"Don't use state file and read whole logs"`
	Encoding            string   `long:"encoding" description:"Encoding of log file"`
	Missing             string   `long:"missing" default:"UNKNOWN" value-name:"(CRITICAL|WARNING|OK|UNKNOWN)" description:"Exit status when log files missing"`
	CheckFirst          bool     `long:"check-first" description:"Check the log on the first run"`
	patternReg          []*regexp.Regexp
	excludeReg          *regexp.Regexp
	fileListFromGlob    []string
	fileListFromPattern []string
	origArgs            []string
	decoder             *enc.Decoder
}

func (opts *logOpts) prepare() error {
	if opts.LogFile == "" && opts.FilePattern == "" {
		return fmt.Errorf("No log file specified")
	}

	var err error
	var reg *regexp.Regexp
	for _, ptn := range opts.Pattern {
		if reg, err = regCompileWithCase(ptn, opts.CaseInsensitive); err != nil {
			return fmt.Errorf("pattern is invalid")
		}
		opts.patternReg = append(opts.patternReg, reg)
	}

	if len(opts.patternReg) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
		return fmt.Errorf("When multiple patterns specified, --warning-level --critical-level can not be used")
	}

	if opts.Exclude != "" {
		opts.excludeReg, err = regCompileWithCase(opts.Exclude, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("exclude pattern is invalid")
		}
	}

	if opts.LogFile != "" {
		opts.fileListFromGlob, err = zglob.Glob(opts.LogFile)
		// unless --missing specified, we should ignore file not found error
		if err != nil && err != os.ErrNotExist {
			return fmt.Errorf("invalid glob for --file")
		}
	}

	if opts.FilePattern != "" {
		dirStr := filepath.Dir(opts.FilePattern)
		filePat := filepath.Base(opts.FilePattern)
		reg, err := regCompileWithCase(filePat, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("file-pattern is invalid")
		}

		fileInfos, err := ioutil.ReadDir(dirStr)
		if err != nil {
			return fmt.Errorf("cannot read the directory:" + err.Error())
		}

		for _, fileInfo := range fileInfos {
			if fileInfo.IsDir() {
				continue
			}
			fname := fileInfo.Name()
			if reg.MatchString(fname) {
				opts.fileListFromPattern = append(opts.fileListFromPattern, dirStr+string(filepath.Separator)+fileInfo.Name())
			}
		}
	}
	if !validateMissing(opts.Missing) {
		return fmt.Errorf("missing option is invalid")
	}
	return nil
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "LOG"
	ckr.Exit()
}

func regCompileWithCase(ptn string, caseInsensitive bool) (*regexp.Regexp, error) {
	if caseInsensitive {
		ptn = "(?i)" + ptn
	}
	return regexp.Compile(ptn)
}

func validateMissing(missing string) bool {
	switch missing {
	case "CRITICAL", "WARNING", "OK", "UNKNOWN", "":
		return true
	default:
		return false
	}
}

func parseArgs(args []string) (*logOpts, error) {
	origArgs := make([]string, len(args))
	copy(origArgs, args)
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	opts.origArgs = origArgs
	if opts.StateDir == "" {
		workdir := pluginutil.PluginWorkDir()
		opts.StateDir = filepath.Join(workdir, "check-log")
	}
	return opts, err
}

func run(args []string) *checkers.Checker {
	opts, err := parseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	err = opts.prepare()
	if err != nil {
		return checkers.Unknown(err.Error())
	}

	warnNum := int64(0)
	critNum := int64(0)
	var missingFiles []string
	errorOverall := ""

	if opts.LogFile != "" && len(opts.fileListFromGlob) == 0 {
		missingFiles = append(missingFiles, opts.LogFile)
	}

	for _, f := range append(opts.fileListFromGlob, opts.fileListFromPattern...) {
		_, err := os.Stat(f)
		if err != nil {
			missingFiles = append(missingFiles, f)
			continue
		}
		w, c, errLines, err := opts.searchLog(f)
		if err != nil {
			return checkers.Unknown(err.Error())
		}
		warnNum += w
		critNum += c
		if opts.ReturnContent && errLines != "" {
			errorOverall += "[" + f + "]\n" + errLines
		}
	}

	var patterns []string
	for _, ptn := range opts.Pattern {
		patterns = append(patterns, fmt.Sprintf("/%s/", ptn))
	}
	msg := fmt.Sprintf("%d warnings, %d criticals for pattern %s.", warnNum, critNum, strings.Join(patterns, " and "))
	if errorOverall != "" {
		msg += "\n" + errorOverall
	}
	checkSt := checkers.OK
	if len(missingFiles) > 0 {
		switch opts.Missing {
		case "OK":
		case "WARNING":
			checkSt = checkers.WARNING
		case "CRITICAL":
			checkSt = checkers.CRITICAL
		default:
			checkSt = checkers.UNKNOWN
		}
		msg += "\n" + fmt.Sprintf("The following %d files are missing.", len(missingFiles))
		for _, f := range missingFiles {
			msg += "\n" + f
		}
	}
	if warnNum > opts.WarnOver {
		checkSt = checkers.WARNING
	}
	if critNum > opts.CritOver {
		checkSt = checkers.CRITICAL
	}
	return checkers.NewChecker(checkSt, msg)
}

func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
	skipBytes := int64(0)
	if !opts.NoState {
		s, err := getBytesToSkip(stateFile)
		if err != nil {
			return 0, 0, "", err
		}
		skipBytes = s
	}

	f, err := os.Open(logFile)
	if err != nil {
		return 0, 0, "", err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return 0, 0, "", err
	}

	_, err = os.Stat(stateFile)
	if !opts.NoState && !opts.CheckFirst && os.IsNotExist(err) {
		skipBytes = stat.Size()
	}

	rotated := false
	if stat.Size() < skipBytes {
		rotated = true
	} else if skipBytes > 0 {
		f.Seek(skipBytes, 0)
	}

	var r io.Reader = f
	if opts.Encoding != "" {
		e := encoding.GetEncoding(opts.Encoding)
		if e == nil {
			return 0, 0, "", fmt.Errorf("unknown encoding:" + opts.Encoding)
		}
		opts.decoder = e.NewDecoder()
	}

	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)
	if err != nil {
		return warnNum, critNum, errLines, err
	}

	if rotated {
		skipBytes = readBytes
	} else {
		skipBytes += readBytes
	}

	if !opts.NoState {
		err = writeBytesToSkip(stateFile, skipBytes)
		if err != nil {
			log.Printf("writeByteToSkip failed: %s\n", err.Error())
		}
	}
	return warnNum, critNum, errLines, nil
}

func (opts *logOpts) searchReader(rdr io.Reader) (warnNum, critNum, readBytes int64, errLines string, err error) {
	r := bufio.NewReader(rdr)
	for {
		lineBytes, rErr := r.ReadBytes('\n')
		if rErr != nil {
			if rErr != io.EOF {
				err = rErr
			}
			break
		}
		readBytes += int64(len(lineBytes))

		if opts.decoder != nil {
			lineBytes, err = opts.decoder.Bytes(lineBytes)
			if err != nil {
				break
			}
		}
		line := strings.Trim(string(lineBytes), "\r\n")
		if matched, matches := opts.match(line); matched {
			if len(matches) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
				level, err := strconv.ParseFloat(matches[1], 64)
				if err != nil {
					warnNum++
					critNum++
					errLines += line + "\n"
				} else {
					levelOver := false
					if level > opts.WarnLevel {
						levelOver = true
						warnNum++
					}
					if level > opts.CritLevel {
						levelOver = true
						critNum++
					}
					if levelOver {
						errLines += line + "\n"
					}
				}
			} else {
				warnNum++
				critNum++
				errLines += line + "\n"
			}
		}
	}
	return
}

func (opts *logOpts) match(line string) (bool, []string) {
	var matches []string
	for _, pReg := range opts.patternReg {
		eReg := opts.excludeReg

		matches = pReg.FindStringSubmatch(line)
		if len(matches) == 0 || (eReg != nil && eReg.MatchString(line)) {
			return false, nil
		}
	}
	return true, matches
}

var stateRe = regexp.MustCompile(`^([a-zA-Z]):[/\\]`)

func getStateFile(stateDir, f string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			stateRe.ReplaceAllString(f, `$1`+string(filepath.Separator)),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func getBytesToSkip(f string) (int64, error) {
	_, err := os.Stat(f)
	if err != nil {
		return 0, nil
	}
	b, err := ioutil.ReadFile(f)
	if err != nil {
		return 0, err
	}
	i, err := strconv.ParseInt(strings.Trim(string(b), " \r\n"), 10, 64)
	if err != nil {
		log.Printf("failed to getBytesToSkip (ignoring): %s", err)
	}
	return i, nil
}

func writeBytesToSkip(f string, num int64) error {
	err := os.MkdirAll(filepath.Dir(f), 0755)
	if err != nil {
		return err
	}
	return writeFileAtomically(f, []byte(fmt.Sprintf("%d", num)))
}

func writeFileAtomically(f string, contents []byte) error {
	// MUST be located on same disk partition
	tmpf, err := ioutil.TempFile(filepath.Dir(f), "tmp")
	if err != nil {
		return err
	}
	// os.Remove here works successfully when tmpf.Write fails or os.Rename fails.
	// In successful case, os.Remove fails because the temporary file is already renamed.
	defer os.Remove(tmpf.Name())
	_, err = tmpf.Write(contents)
	tmpf.Close() // should be called before rename
	if err != nil {
		return err
	}
	return os.Rename(tmpf.Name(), f)
}
<commit_msg>improve for reducing unnecessary syscall<commit_after>package checklog

import (
	"bufio"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
	"github.com/mackerelio/golib/pluginutil"
	"github.com/mattn/go-encoding"
	"github.com/mattn/go-zglob"
	enc "golang.org/x/text/encoding"
)

type logOpts struct {
	LogFile             string   `short:"f" long:"file" value-name:"FILE" description:"Path to log file"`
	Pattern             []string `short:"p" long:"pattern" required:"true" value-name:"PAT" description:"Pattern to search for. If specified multiple, they will be treated together with the AND operator"`
	Exclude             string   `short:"E" long:"exclude" value-name:"PAT" description:"Pattern to exclude from matching"`
	WarnOver            int64    `short:"w" long:"warning-over" description:"Trigger a warning if matched lines is over a number"`
	CritOver            int64    `short:"c" long:"critical-over" description:"Trigger a critical if matched lines is over a number"`
	WarnLevel           float64  `long:"warning-level" value-name:"N" description:"Warning level if pattern has a group"`
	CritLevel           float64  `long:"critical-level" value-name:"N" description:"Critical level if pattern has a group"`
	ReturnContent       bool     `short:"r" long:"return" description:"Return matched line"`
	FilePattern         string   `short:"F" long:"file-pattern" value-name:"FILE" description:"Check a pattern of files, instead of one file"`
	CaseInsensitive     bool     `short:"i" long:"icase" description:"Run a case insensitive match"`
	StateDir            string   `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
	NoState             bool     `long:"no-state" description:"Don't use state file and read whole logs"`
	Encoding            string   `long:"encoding" description:"Encoding of log file"`
	Missing             string   `long:"missing" default:"UNKNOWN" value-name:"(CRITICAL|WARNING|OK|UNKNOWN)" description:"Exit status when log files missing"`
	CheckFirst          bool     `long:"check-first" description:"Check the log on the first run"`
	patternReg          []*regexp.Regexp
	excludeReg          *regexp.Regexp
	fileListFromGlob    []string
	fileListFromPattern []string
	origArgs            []string
	decoder             *enc.Decoder
}

func (opts *logOpts) prepare() error {
	if opts.LogFile == "" && opts.FilePattern == "" {
		return fmt.Errorf("No log file specified")
	}

	var err error
	var reg *regexp.Regexp
	for _, ptn := range opts.Pattern {
		if reg, err = regCompileWithCase(ptn, opts.CaseInsensitive); err != nil {
			return fmt.Errorf("pattern is invalid")
		}
		opts.patternReg = append(opts.patternReg, reg)
	}

	if len(opts.patternReg) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
		return fmt.Errorf("When multiple patterns specified, --warning-level --critical-level can not be used")
	}

	if opts.Exclude != "" {
		opts.excludeReg, err = regCompileWithCase(opts.Exclude, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("exclude pattern is invalid")
		}
	}

	if opts.LogFile != "" {
		opts.fileListFromGlob, err = zglob.Glob(opts.LogFile)
		// unless --missing specified, we should ignore file not found error
		if err != nil && err != os.ErrNotExist {
			return fmt.Errorf("invalid glob for --file")
		}
	}

	if opts.FilePattern != "" {
		dirStr := filepath.Dir(opts.FilePattern)
		filePat := filepath.Base(opts.FilePattern)
		reg, err := regCompileWithCase(filePat, opts.CaseInsensitive)
		if err != nil {
			return fmt.Errorf("file-pattern is invalid")
		}

		fileInfos, err := ioutil.ReadDir(dirStr)
		if err != nil {
			return fmt.Errorf("cannot read the directory:" + err.Error())
		}

		for _, fileInfo := range fileInfos {
			if fileInfo.IsDir() {
				continue
			}
			fname := fileInfo.Name()
			if reg.MatchString(fname) {
				opts.fileListFromPattern = append(opts.fileListFromPattern, dirStr+string(filepath.Separator)+fileInfo.Name())
			}
		}
	}
	if !validateMissing(opts.Missing) {
		return fmt.Errorf("missing option is invalid")
	}
	return nil
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "LOG"
	ckr.Exit()
}

func regCompileWithCase(ptn string, caseInsensitive bool) (*regexp.Regexp, error) {
	if caseInsensitive {
		ptn = "(?i)" + ptn
	}
	return regexp.Compile(ptn)
}

func validateMissing(missing string) bool {
	switch missing {
	case "CRITICAL", "WARNING", "OK", "UNKNOWN", "":
		return true
	default:
		return false
	}
}

func parseArgs(args []string) (*logOpts, error) {
	origArgs := make([]string, len(args))
	copy(origArgs, args)
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	opts.origArgs = origArgs
	if opts.StateDir == "" {
		workdir := pluginutil.PluginWorkDir()
		opts.StateDir = filepath.Join(workdir, "check-log")
	}
	return opts, err
}

func run(args []string) *checkers.Checker {
	opts, err := parseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	err = opts.prepare()
	if err != nil {
		return checkers.Unknown(err.Error())
	}

	warnNum := int64(0)
	critNum := int64(0)
	var missingFiles []string
	errorOverall := ""

	if opts.LogFile != "" && len(opts.fileListFromGlob) == 0 {
		missingFiles = append(missingFiles, opts.LogFile)
	}

	for _, f := range append(opts.fileListFromGlob, opts.fileListFromPattern...) {
		_, err := os.Stat(f)
		if err != nil {
			missingFiles = append(missingFiles, f)
			continue
		}
		w, c, errLines, err := opts.searchLog(f)
		if err != nil {
			return checkers.Unknown(err.Error())
		}
		warnNum += w
		critNum += c
		if opts.ReturnContent && errLines != "" {
			errorOverall += "[" + f + "]\n" + errLines
		}
	}

	var patterns []string
	for _, ptn := range opts.Pattern {
		patterns = append(patterns, fmt.Sprintf("/%s/", ptn))
	}
	msg := fmt.Sprintf("%d warnings, %d criticals for pattern %s.", warnNum, critNum, strings.Join(patterns, " and "))
	if errorOverall != "" {
		msg += "\n" + errorOverall
	}
	checkSt := checkers.OK
	if len(missingFiles) > 0 {
		switch opts.Missing {
		case "OK":
		case "WARNING":
			checkSt = checkers.WARNING
		case "CRITICAL":
			checkSt = checkers.CRITICAL
		default:
			checkSt = checkers.UNKNOWN
		}
		msg += "\n" + fmt.Sprintf("The following %d files are missing.", len(missingFiles))
		for _, f := range missingFiles {
			msg += "\n" + f
		}
	}
	if warnNum > opts.WarnOver {
		checkSt = checkers.WARNING
	}
	if critNum > opts.CritOver {
		checkSt = checkers.CRITICAL
	}
	return checkers.NewChecker(checkSt, msg)
}

func (opts *logOpts) searchLog(logFile string) (int64, int64, string, error) {
	stateFile := getStateFile(opts.StateDir, logFile, opts.origArgs)
	skipBytes := int64(0)
	if !opts.NoState {
		s, err := getBytesToSkip(stateFile)
		if err != nil {
			return 0, 0, "", err
		}
		skipBytes = s
	}

	f, err := os.Open(logFile)
	if err != nil {
		return 0, 0, "", err
	}
	defer f.Close()

	stat, err := f.Stat()
	if err != nil {
		return 0, 0, "", err
	}

	if !opts.NoState && !opts.CheckFirst {
		if _, err = os.Stat(stateFile); os.IsNotExist(err) {
			skipBytes = stat.Size()
		}
	}

	rotated := false
	if stat.Size() < skipBytes {
		rotated = true
	} else if skipBytes > 0 {
		f.Seek(skipBytes, 0)
	}

	var r io.Reader = f
	if opts.Encoding != "" {
		e := encoding.GetEncoding(opts.Encoding)
		if e == nil {
			return 0, 0, "", fmt.Errorf("unknown encoding:" + opts.Encoding)
		}
		opts.decoder = e.NewDecoder()
	}

	warnNum, critNum, readBytes, errLines, err := opts.searchReader(r)
	if err != nil {
		return warnNum, critNum, errLines, err
	}

	if rotated {
		skipBytes = readBytes
	} else {
		skipBytes += readBytes
	}

	if !opts.NoState {
		err = writeBytesToSkip(stateFile, skipBytes)
		if err != nil {
			log.Printf("writeByteToSkip failed: %s\n", err.Error())
		}
	}
	return warnNum, critNum, errLines, nil
}

func (opts *logOpts) searchReader(rdr io.Reader) (warnNum, critNum, readBytes int64, errLines string, err error) {
	r := bufio.NewReader(rdr)
	for {
		lineBytes, rErr := r.ReadBytes('\n')
		if rErr != nil {
			if rErr != io.EOF {
				err = rErr
			}
			break
		}
		readBytes += int64(len(lineBytes))

		if opts.decoder != nil {
			lineBytes, err = opts.decoder.Bytes(lineBytes)
			if err != nil {
				break
			}
		}
		line := strings.Trim(string(lineBytes), "\r\n")
		if matched, matches := opts.match(line); matched {
			if len(matches) > 1 && (opts.WarnLevel > 0 || opts.CritLevel > 0) {
				level, err := strconv.ParseFloat(matches[1], 64)
				if err != nil {
					warnNum++
					critNum++
					errLines += line + "\n"
				} else {
					levelOver := false
					if level > opts.WarnLevel {
						levelOver = true
						warnNum++
					}
					if level > opts.CritLevel {
						levelOver = true
						critNum++
					}
					if levelOver {
						errLines += line + "\n"
					}
				}
			} else {
				warnNum++
				critNum++
				errLines += line + "\n"
			}
		}
	}
	return
}

func (opts *logOpts) match(line string) (bool, []string) {
	var matches []string
	for _, pReg := range opts.patternReg {
		eReg := opts.excludeReg

		matches = pReg.FindStringSubmatch(line)
		if len(matches) == 0 || (eReg != nil && eReg.MatchString(line)) {
			return false, nil
		}
	}
	return true, matches
}

var stateRe = regexp.MustCompile(`^([a-zA-Z]):[/\\]`)

func getStateFile(stateDir, f string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			stateRe.ReplaceAllString(f, `$1`+string(filepath.Separator)),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func getBytesToSkip(f string) (int64, error) {
	_, err := os.Stat(f)
	if err != nil {
		return 0, nil
	}
	b, err := ioutil.ReadFile(f)
	if err != nil {
		return 0, err
	}
	i, err := strconv.ParseInt(strings.Trim(string(b), " \r\n"), 10, 64)
	if err != nil {
		log.Printf("failed to getBytesToSkip (ignoring): %s", err)
	}
	return i, nil
}

func writeBytesToSkip(f string, num int64) error {
	err := os.MkdirAll(filepath.Dir(f), 0755)
	if err != nil {
		return err
	}
	return writeFileAtomically(f, []byte(fmt.Sprintf("%d", num)))
}

func writeFileAtomically(f string, contents []byte) error {
	// MUST be located on same disk partition
	tmpf, err := ioutil.TempFile(filepath.Dir(f), "tmp")
	if err != nil {
		return err
	}
	// os.Remove here works successfully when tmpf.Write fails or os.Rename fails.
	// In successful case, os.Remove fails because the temporary file is already renamed.
	defer os.Remove(tmpf.Name())
	_, err = tmpf.Write(contents)
	tmpf.Close() // should be called before rename
	if err != nil {
		return err
	}
	return os.Rename(tmpf.Name(), f)
}
<|endoftext|>


<commit_before>package etl

import "testing"

type given map[int][]string
type expectation map[string]int

var transformTests = []struct {
	input  given
	output expectation
}{
	{
		given{1: {"A"}},
		expectation{"a": 1},
	},
	{
		given{1: {"A", "E", "I", "O", "U"}},
		expectation{"a": 1, "e": 1, "i": 1, "o": 1, "u": 1},
	},
	{
		given{
			1: {"A", "E"},
			2: {"D", "G"},
		},
		expectation{
			"a": 1,
			"e": 1,
			"d": 2,
			"g": 2,
		},
	},
	{
		given{
			1:  {"A", "E", "I", "O", "U", "L", "N", "R", "S", "T"},
			2:  {"D", "G"},
			3:  {"B", "C", "M", "P"},
			4:  {"F", "H", "V", "W", "Y"},
			5:  {"K"},
			8:  {"J", "X"},
			10: {"Q", "Z"},
		},
		expectation{
			"a": 1, "e": 1, "i": 1, "o": 1, "u": 1, "l": 1, "n": 1, "r": 1, "s": 1, "t": 1,
			"d": 2, "g": 2,
			"b": 3, "c": 3, "m": 3, "p": 3,
			"f": 4, "h": 4, "v": 4, "w": 4, "y": 4,
			"k": 5,
			"j": 8, "x": 8,
			"q": 10, "z": 10,
		},
	},
}

func equal(actual map[string]int, expectation map[string]int) bool {
	if len(actual) != len(expectation) {
		return false
	}

	for k, actualVal := range actual {
		expectationVal, present := expectation[k]

		if !present || actualVal != expectationVal {
			return false
		}
	}

	return true
}

func TestTranform(t *testing.T) {
	for _, tt := range transformTests {
		actual := Transform(tt.input)
		if !equal(actual, tt.output) {
			t.Fatalf("Transform(%v). Expected [%v], Actual [%v]", tt.input, tt.output, actual)
		}
	}
}
<commit_msg>Added benchmark in go/etl<commit_after>package etl

import "testing"

type given map[int][]string
type expectation map[string]int

var transformTests = []struct {
	input  given
	output expectation
}{
	{
		given{1: {"A"}},
		expectation{"a": 1},
	},
	{
		given{1: {"A", "E", "I", "O", "U"}},
		expectation{"a": 1, "e": 1, "i": 1, "o": 1, "u": 1},
	},
	{
		given{
			1: {"A", "E"},
			2: {"D", "G"},
		},
		expectation{
			"a": 1,
			"e": 1,
			"d": 2,
			"g": 2,
		},
	},
	{
		given{
			1:  {"A", "E", "I", "O", "U", "L", "N", "R", "S", "T"},
			2:  {"D", "G"},
			3:  {"B", "C", "M", "P"},
			4:  {"F", "H", "V", "W", "Y"},
			5:  {"K"},
			8:  {"J", "X"},
			10: {"Q", "Z"},
		},
		expectation{
			"a": 1, "e": 1, "i": 1, "o": 1, "u": 1, "l": 1, "n": 1, "r": 1, "s": 1, "t": 1,
			"d": 2, "g": 2,
			"b": 3, "c": 3, "m": 3, "p": 3,
			"f": 4, "h": 4, "v": 4, "w": 4, "y": 4,
			"k": 5,
			"j": 8, "x": 8,
			"q": 10, "z": 10,
		},
	},
}

func equal(actual map[string]int, expectation map[string]int) bool {
	if len(actual) != len(expectation) {
		return false
	}

	for k, actualVal := range actual {
		expectationVal, present := expectation[k]

		if !present || actualVal != expectationVal {
			return false
		}
	}

	return true
}

func TestTranform(t *testing.T) {
	for _, tt := range transformTests {
		actual := Transform(tt.input)
		if !equal(actual, tt.output) {
			t.Fatalf("Transform(%v). Expected [%v], Actual [%v]", tt.input, tt.output, actual)
		}
	}
}

func BenchmarkTranform(b *testing.B) {
	b.StopTimer()
	for _, tt := range transformTests {
		b.StartTimer()

		for i := 0; i < b.N; i++ {
			Transform(tt.input)
		}

		b.StopTimer()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io"
	"os"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		info("Usage: errignorer file.go")
		os.Exit(1)
	}
	in := os.Args[1]
	info("Processing", in)

	err := errorIgnoreFilter(in, os.Stdout)
	if err != nil {
		panic(err)
	}
}

func errorIgnoreFilter(in string, out io.Writer) error {
	fset := token.NewFileSet()
	file, err := parser.ParseFile(fset, in, nil, parser.AllErrors)
	if err != nil {
		return err
	}

	killer := &errignorer{fset}
	ast.Walk(killer, file)

	// Print altered program
	format.Node(out, fset, file)

	return nil
}

type errignorer struct {
	fset *token.FileSet
}

func (ei *errignorer) Visit(node ast.Node) (w ast.Visitor) {
	if node != nil {
		switch parent := node.(type) {
		case *ast.BlockStmt:
			kept := make([]ast.Stmt, 0, len(parent.List))
			for _, stmt := range parent.List {
				keep := true
				//infof("stmt = %T %v", stmt, stmt)
				switch child := stmt.(type) {
				case *ast.IfStmt:
					be, ok := child.Cond.(*ast.BinaryExpr)
					if !ok {
						break
					}
					identL, ok := be.X.(*ast.Ident)
					if !ok {
						break
					}
					// TODO: checking name only is a dirty approximation.
					// Try to check type instead.
					isErr := strings.HasPrefix(identL.Obj.Name, "err")
					isDiff := be.Op == token.NEQ
					identR, ok := be.Y.(*ast.Ident)
					if !ok {
						break
					}
					isNil := identR.Name == "nil"
					isTestErrDiffNil := isErr && isDiff && isNil
					if isTestErrDiffNil {
						fmt.Fprintln(os.Stderr, "Found err check", be)
					}
				}
				if keep {
					kept = append(kept, stmt)
				} else {
					fmt.Fprintf(os.Stderr, "DESTROYING %T \n", stmt)
				}
			}
			parent.List = kept
		}
	}
	// For recursion
	return ei
}
<commit_msg>errignorer: remove err check if blocks, keep else block.<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io"
	"os"
	"strings"
)

func main() {
	if len(os.Args) < 2 {
		info("Usage: errignorer file.go")
		os.Exit(1)
	}
	in := os.Args[1]
	info("Processing", in)

	err := errorIgnoreFilter(in, os.Stdout)
	if err != nil {
		panic(err)
	}
}

func errorIgnoreFilter(in string, out io.Writer) error {
	fset := token.NewFileSet()
	file, err := parser.ParseFile(fset, in, nil, parser.AllErrors)
	if err != nil {
		return err
	}

	killer := &errignorer{fset}
	ast.Walk(killer, file)

	// Print altered program
	format.Node(out, fset, file)

	return nil
}

type errignorer struct {
	fset *token.FileSet
}

func (ei *errignorer) Visit(node ast.Node) (w ast.Visitor) {
	if node != nil {
		switch parent := node.(type) {
		case *ast.BlockStmt:
			kept := make([]ast.Stmt, 0, len(parent.List))
			for _, stmt := range parent.List {
				keep := true
				//infof("stmt = %T %v", stmt, stmt)
				switch child := stmt.(type) {
				case *ast.IfStmt:
					be, ok := child.Cond.(*ast.BinaryExpr)
					if !ok {
						break
					}
					identL, ok := be.X.(*ast.Ident)
					if !ok {
						break
					}
					// TODO: checking name only is a dirty approximation.
					// Try to check type instead.
					isErr := strings.HasPrefix(identL.Obj.Name, "err")
					isDiff := be.Op == token.NEQ
					identR, ok := be.Y.(*ast.Ident)
					if !ok {
						break
					}
					isNil := identR.Name == "nil"
					isTestErrDiffNil := isErr && isDiff && isNil
					if isTestErrDiffNil {
						fmt.Fprintln(os.Stderr, "Found err check", be)
						keep = false
						// But we want to keep the ELSE block!!
						// If exists, it is the actual main code.
						if child.Else != nil {
							kept = append(kept, child.Else)
						}
					}
				}
				if keep {
					kept = append(kept, stmt)
				} else {
					fmt.Fprintf(os.Stderr, "DESTROYING %T \n", stmt)
				}
			}
			parent.List = kept
		}
	}
	// For recursion
	return ei
}
<|endoftext|>


<commit_before>// Copyright 2010 Hannes Baldursson. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is part of go-idn

// This package implements a mechanism called  IDNA for handling
// International Domain Names (IDN) in applications in a standard fashion
// as described RFC 3490
package idna

import (
	"code.google.com/p/go-idn/idna/punycode"
	"code.google.com/p/go-idn/idna/stringprep"
	"os"
	"strings"
	//"fmt"
)

// IDNA section 5
const (
	ACE_PREFIX = "xn--"
)

//
// Converts a Unicode string to ASCII using the procedure in RFC 3490
// section 4.1. Unassigned characters are not allowed and STD3 ASCII
// rules are enforced. The input string may be a domain name
// containing dots.
//
func ToASCII(label string) (string, os.Error) {

	label = strings.ToLower(label)
	o := ""
	h := ""

	for _, cp := range label {

		if cp == 0x2E /* dot */ || cp == 0x3002 || cp == 0xff0e || cp == 0xff61 {
			uh, err := toASCIIRaw(h)
			if err != nil {
				return label, err
			}
			o += uh
			o += string(cp)
			h = ""
		} else {
			h += string(cp)
		}
	}
	uh, err := toASCIIRaw(h)
	if err != nil {
		return label, err
	}
	o += uh
	return o, nil
}

func toASCIIRaw(label string) (string, os.Error) {
	original := label

	// Step 1: If the sequence contains any code points outside the ASCII range
	// (0..7F) then proceed to step 2, otherwise skip to step 3.
	for i := 0; i < len(label); i++ {
		if label[i] > 127 {
			// Step 2: Perform the smake teps specified in [NAMEPREP] and fail if there is an error.
			// The AllowUnassigned flag is used in [NAMEPREP].
			label = stringprep.Nameprep(label)
			break
		}
	}

	// Step 3: - Verify the absence of non-LDH ASCII code points
	for _, c := range label {
		if (c <= 0x2c) || (c >= 0x2e && c <= 0x2f) || (c >= 0x3a && c <= 0x40) || (c >= 0x5b && c <= 0x60) || (c >= 0x7b && c <= 0x7f) {
			return original, os.NewError("Contains non-LDH ASCII codepoints")
		}

	}
	if strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
		return original, os.NewError("Contains hyphen at either end of the string")
	}

	// Step 4: If the sequence contains any code points outside the ASCII range
	// (0..7F) then proceed to step 5, otherwise skip to step 8.

	isASCII := true
	for i := 0; i < len(label); i++ {
		if label[i] > 127 {
			isASCII = false
			break
		}

	}

	if !isASCII {

		// Step 5 Verify that the sequence does NOT begin with the ACE prefix.
		if strings.HasPrefix(label, ACE_PREFIX) {
			return label, os.NewError("Label starts with ACE prefix")
		}

		var err os.Error

		// Step 6: Encode with punycode
		label, err = punycode.ToASCII(label)
		if err != nil {
			return "", err // delegate err
		}
		// Step 7: Prepend ACE prefix
		label = ACE_PREFIX + label
	}

	// 8. Verify that the number of code points is in the range 1 to 63 inclusive.
	if 0 < len(label) && len(label) < 64 {
		return label, nil
	}

	return original, os.NewError("label empty or too long")
}

//
// Converts a Punycode string to Unicode using the procedure in RFC 3490
// section 4.2. Unassigned characters are not allowed and STD3 ASCII
// rules are enforced. The input string may be a domain name
// containing dots.
//
// ToUnicode never fails.  If any step fails, then the original input
// sequence is returned immediately in that step.
func ToUnicode(label string) (string, os.Error) {

	label = strings.ToLower(label)
	o := ""
	h := ""

	for _, cp := range label {

		if cp == 0x2E /* dot */ || cp == 0x3002 || cp == 0xff0e || cp == 0xff61 {
			uh, err := toUnicodeRaw(h)
			if err != nil {
				return label, err
			}
			o += uh
			o += string(cp)
			h = ""
		} else {
			h += string(cp)
		}
	}
	uh, err := toUnicodeRaw(h)
	if err != nil {
		return label, err
	}
	o += uh
	return o, nil
}

func toUnicodeRaw(label string) (string, os.Error) {

	original := label

	// Step 1: If all code points in the sequence are in the ASCII range (0..7F) then skip to step 3.
	for i := 0; i < len(label); i++ {
		if label[i] > 127 {
			// Step 2: Perform the steps specified in [NAMEPREP] and fail if there is an error.
			label = stringprep.Nameprep(label)
			break
		}
	}

	// Step 3: Verify that the sequence begins with the ACE prefix, and save a copy of the sequence.
	if !strings.HasPrefix(label, ACE_PREFIX) {
		return label, os.NewError("Label doesn't begin with the ACE prefix")
	} // else

	// 4. Remove the ACE prefix.
	label = strings.SplitN(label, ACE_PREFIX, -1)[1]

	// 5. Decode the sequence using the decoding algorithm in [PUNYCODE] and fail if there is an error.
	//fmt.Printf(label+"\n")
	results, err := punycode.ToUnicode(label)

	if err != nil {
		return original, os.NewError("Failed punycode decoding: " + err.String())
	}

	// 6. Apply ToASCII.
	verification, err := ToASCII(label)

	if err != nil {
		return original, os.NewError("Failed ToASCII on the decoded sequence: " + err.String())
	}

	// 7. Verify that the result of step 6 matches the saved copy from step 3,
	// 	  using a case-insensitive ASCII comparison.
	if strings.ToLower(verification) == strings.ToLower(original) {
		return results, nil
	}

	return original, os.NewError("Failed verification step")
}
<commit_msg>idna: write isSeparator function<commit_after>// Copyright 2010 Hannes Baldursson. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file is part of go-idn

// This package implements a mechanism called  IDNA for handling
// International Domain Names (IDN) in applications in a standard fashion
// as described RFC 3490
package idna

import (
	"code.google.com/p/go-idn/idna/punycode"
	"code.google.com/p/go-idn/idna/stringprep"
	"os"
	"strings"
	//"fmt"
)

// IDNA section 5
const (
	ACE_PREFIX = "xn--"
)

//
// Converts a Unicode string to ASCII using the procedure in RFC 3490
// section 4.1. Unassigned characters are not allowed and STD3 ASCII
// rules are enforced. The input string may be a domain name
// containing dots.
//
func ToASCII(label string) (string, os.Error) {

	label = strings.ToLower(label)
	o := ""
	h := ""

	for _, cp := range label {

		if cp == 0x2E /* dot */ || cp == 0x3002 || cp == 0xff0e || cp == 0xff61 {
			uh, err := toASCIIRaw(h)
			if err != nil {
				return label, err
			}
			o += uh
			o += string(cp)
			h = ""
		} else {
			h += string(cp)
		}
	}
	uh, err := toASCIIRaw(h)
	if err != nil {
		return label, err
	}
	o += uh
	return o, nil
}

func toASCIIRaw(label string) (string, os.Error) {
	original := label

	// Step 1: If the sequence contains any code points outside the ASCII range
	// (0..7F) then proceed to step 2, otherwise skip to step 3.
	for i := 0; i < len(label); i++ {
		if label[i] > 127 {
			// Step 2: Perform the smake teps specified in [NAMEPREP] and fail if there is an error.
			// The AllowUnassigned flag is used in [NAMEPREP].
			label = stringprep.Nameprep(label)
			break
		}
	}

	// Step 3: - Verify the absence of non-LDH ASCII code points
	for _, c := range label {
		if (c <= 0x2c) || (c >= 0x2e && c <= 0x2f) || (c >= 0x3a && c <= 0x40) || (c >= 0x5b && c <= 0x60) || (c >= 0x7b && c <= 0x7f) {
			return original, os.NewError("Contains non-LDH ASCII codepoints")
		}

	}
	if strings.HasPrefix(label, "-") || strings.HasSuffix(label, "-") {
		return original, os.NewError("Contains hyphen at either end of the string")
	}

	// Step 4: If the sequence contains any code points outside the ASCII range
	// (0..7F) then proceed to step 5, otherwise skip to step 8.

	isASCII := true
	for i := 0; i < len(label); i++ {
		if label[i] > 127 {
			isASCII = false
			break
		}

	}

	if !isASCII {

		// Step 5 Verify that the sequence does NOT begin with the ACE prefix.
		if strings.HasPrefix(label, ACE_PREFIX) {
			return label, os.NewError("Label starts with ACE prefix")
		}

		var err os.Error

		// Step 6: Encode with punycode
		label, err = punycode.ToASCII(label)
		if err != nil {
			return "", err // delegate err
		}
		// Step 7: Prepend ACE prefix
		label = ACE_PREFIX + label
	}

	// 8. Verify that the number of code points is in the range 1 to 63 inclusive.
	if 0 < len(label) && len(label) < 64 {
		return label, nil
	}

	return original, os.NewError("label empty or too long")
}

//
// Converts a Punycode string to Unicode using the procedure in RFC 3490
// section 4.2. Unassigned characters are not allowed and STD3 ASCII
// rules are enforced. The input string may be a domain name
// containing dots.
//
// ToUnicode never fails.  If any step fails, then the original input
// sequence is returned immediately in that step.
func ToUnicode(label string) (string, os.Error) {

	label = strings.ToLower(label)
	o := ""
	h := ""

	for _, cp := range label {

		if cp == 0x2E /* dot */ || cp == 0x3002 || cp == 0xff0e || cp == 0xff61 {
			uh, err := toUnicodeRaw(h)
			if err != nil {
				return label, err
			}
			o += uh
			o += string(cp)
			h = ""
		} else {
			h += string(cp)
		}
	}
	uh, err := toUnicodeRaw(h)
	if err != nil {
		return label, err
	}
	o += uh
	return o, nil
}

func toUnicodeRaw(label string) (string, os.Error) {

	original := label

	// Step 1: If all code points in the sequence are in the ASCII range (0..7F) then skip to step 3.
	for i := 0; i < len(label); i++ {
		if label[i] > 127 {
			// Step 2: Perform the steps specified in [NAMEPREP] and fail if there is an error.
			label = stringprep.Nameprep(label)
			break
		}
	}

	// Step 3: Verify that the sequence begins with the ACE prefix, and save a copy of the sequence.
	if !strings.HasPrefix(label, ACE_PREFIX) {
		return label, os.NewError("Label doesn't begin with the ACE prefix")
	} // else

	// 4. Remove the ACE prefix.
	label = strings.SplitN(label, ACE_PREFIX, -1)[1]

	// 5. Decode the sequence using the decoding algorithm in [PUNYCODE] and fail if there is an error.
	//fmt.Printf(label+"\n")
	results, err := punycode.ToUnicode(label)

	if err != nil {
		return original, os.NewError("Failed punycode decoding: " + err.String())
	}

	// 6. Apply ToASCII.
	verification, err := ToASCII(label)

	if err != nil {
		return original, os.NewError("Failed ToASCII on the decoded sequence: " + err.String())
	}

	// 7. Verify that the result of step 6 matches the saved copy from step 3,
	// 	  using a case-insensitive ASCII comparison.
	if strings.ToLower(verification) == strings.ToLower(original) {
		return results, nil
	}

	return original, os.NewError("Failed verification step")
}

// returns true if c is a label separator
func isSeparator(c rune) bool {
	if c == 0x02E || c == 0x3002 || c == 0xFF0E || c == 0xFF61 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package commands

// Version
const version = "v0.70-beta"
<commit_msg>version bump<commit_after>package commands

// Version
const version = "v0.71-beta"
<|endoftext|>


<commit_before>package commands

import (
	"github.com/hexbotio/hex/models"
)

func Version(message *models.Message, config models.Config) {
	response := "Version: Non Standard Build"
	if config.Version != "" {
		response = "Version: " + config.Version
	}
	message.Outputs = append(message.Outputs, models.Output{
		Rule:     "version",
		Response: response,
	})
}
<commit_msg>adding latest version check<commit_after>package commands

import (
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/hexbotio/hex/models"
)

func Version(message *models.Message, config models.Config) {
	response := "Version: Non Standard Build"
	if config.Version != "" {
		response = "Version: " + config.Version
	}
	var u url.URL
	u.Scheme = "https"
	u.Host = "hexbot.io"
	u.Path = "/downloads/latest"
	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		config.Logger.Error("Version New Request - " + err.Error())
	} else {
		req.Header.Add("User-Agent", "HexBot")
		client := &http.Client{}
		client.Timeout = time.Duration(3) * time.Second
		resp, err := client.Do(req)
		if err != nil {
			config.Logger.Error("Version Response - " + err.Error())
		} else {
			if resp.StatusCode != 200 {
				config.Logger.Error("Version Response Error")
			} else {
				rawbody, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					config.Logger.Error("Version Body Read - " + err.Error())
				}
				body := strings.TrimSpace(string(rawbody))
				response = response + " (latest is " + body + ")"
			}
		}
	}
	message.Outputs = append(message.Outputs, models.Output{
		Rule:     "version",
		Response: response,
	})
}
<|endoftext|>


<commit_before>package commands

// Version
const version = "v0.53-beta"
<commit_msg>version bump<commit_after>package commands

// Version
const version = "v0.60-beta"
<|endoftext|>


<commit_before>package rel

import (
	"testing"
)

func TestAttributeNotEqSql(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" != 10"
	if sql != expected {
		t.Logf("TestAttributeNotEqSql sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeNotEqAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEqAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" != 1 OR \"users\".\"id\" != 2)"
	if sql != expected {
		t.Logf("TestAttributeNotEqAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeNotEqNil(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEq(nil))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" IS NOT NULL"
	if sql != expected {
		t.Logf("TestAttributeNotEqNil sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeNotEqAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEqAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" != 1 AND \"users\".\"id\" != 2)"
	if sql != expected {
		t.Logf("TestAttributeNotEqAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGt(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").Gt(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" > 10"
	if sql != expected {
		t.Logf("TestAttributeGt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtEq(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtEq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" >= 10"
	if sql != expected {
		t.Logf("TestAttributeGtEq sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtEqAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtEqAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" >= 1 OR \"users\".\"id\" >= 2)"
	if sql != expected {
		t.Logf("TestAttributeGtEqAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtEqAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtEqAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" >= 1 AND \"users\".\"id\" >= 2)"
	if sql != expected {
		t.Logf("TestAttributeGtEqAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" > 1 AND \"users\".\"id\" > 2)"
	if sql != expected {
		t.Logf("TestAttributeGtAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" > 1 OR \"users\".\"id\" > 2)"
	if sql != expected {
		t.Logf("TestAttributeGtAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLt(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").Lt(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" < 10"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtEq(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtEq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" <= 10"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtEqAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtEqAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" <= 1 OR \"users\".\"id\" <= 2)"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtEqAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtEqAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" <= 1 AND \"users\".\"id\" <= 2)"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" < 1 OR \"users\".\"id\" < 2)"
	if sql != expected {
		t.Logf("TestAttributeLtAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" < 1 AND \"users\".\"id\" < 2)"
	if sql != expected {
		t.Logf("TestAttributeLtAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeCount(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id").Count())
	sql := mgr.ToSql()
	expected := "SELECT COUNT(\"users\".\"id\") FROM \"users\""
	if sql != expected {
		t.Logf("TestAttributeLtAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}
<commit_msg>Created test for AttributeNode#Eq<commit_after>package rel

import (
	"testing"
)

func TestAttributeNotEqSql(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" != 10"
	if sql != expected {
		t.Logf("TestAttributeNotEqSql sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeNotEqAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEqAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" != 1 OR \"users\".\"id\" != 2)"
	if sql != expected {
		t.Logf("TestAttributeNotEqAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeNotEqNil(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEq(nil))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" IS NOT NULL"
	if sql != expected {
		t.Logf("TestAttributeNotEqNil sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeNotEqAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").NotEqAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" != 1 AND \"users\".\"id\" != 2)"
	if sql != expected {
		t.Logf("TestAttributeNotEqAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGt(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").Gt(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" > 10"
	if sql != expected {
		t.Logf("TestAttributeGt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtEq(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtEq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" >= 10"
	if sql != expected {
		t.Logf("TestAttributeGtEq sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtEqAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtEqAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" >= 1 OR \"users\".\"id\" >= 2)"
	if sql != expected {
		t.Logf("TestAttributeGtEqAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtEqAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtEqAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" >= 1 AND \"users\".\"id\" >= 2)"
	if sql != expected {
		t.Logf("TestAttributeGtEqAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" > 1 AND \"users\".\"id\" > 2)"
	if sql != expected {
		t.Logf("TestAttributeGtAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeGtAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").GtAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" > 1 OR \"users\".\"id\" > 2)"
	if sql != expected {
		t.Logf("TestAttributeGtAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLt(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").Lt(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" < 10"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtEq(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtEq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" <= 10"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtEqAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtEqAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" <= 1 OR \"users\".\"id\" <= 2)"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtEqAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtEqAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" <= 1 AND \"users\".\"id\" <= 2)"
	if sql != expected {
		t.Logf("TestAttributeLt sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtAny(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtAny(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" < 1 OR \"users\".\"id\" < 2)"
	if sql != expected {
		t.Logf("TestAttributeLtAny sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeLtAll(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").LtAll(Sql(1), Sql(2)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE (\"users\".\"id\" < 1 AND \"users\".\"id\" < 2)"
	if sql != expected {
		t.Logf("TestAttributeLtAll sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeCount(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id").Count())
	sql := mgr.ToSql()
	expected := "SELECT COUNT(\"users\".\"id\") FROM \"users\""
	if sql != expected {
		t.Logf("TestAttributeCount sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}

func TestAttributeEq(t *testing.T) {
	users := NewTable("users")
	mgr := users.Select(users.Attr("id"))
	mgr.Where(users.Attr("id").Eq(Sql(10)))
	sql := mgr.ToSql()
	expected := "SELECT \"users\".\"id\" FROM \"users\" WHERE \"users\".\"id\" = 10"
	if sql != expected {
		t.Logf("TestAttributeEq sql: \n%s != \n%s", sql, expected)
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package es

import (
	"log"
	"os"
	"time"
)

var timer = time.NewTimer(1 * time.Second)

type IndexerStats struct {
	Runs        int64
	IndexedDocs int64
	TotalTime   time.Duration
}

func (stats *IndexerStats) Add(count int, dur time.Duration) {
	stats.Runs++
	stats.IndexedDocs += int64(count)
	stats.TotalTime += dur
}

type Indexer struct {
	Index *Index

	IndexEvery time.Duration // triggers a new index run after that duration, will be reset when BatchSize reached
	BatchSize  int           // triggers a new index run when the batch reaches that size

	docsBatch   []*Doc
	docsChannel chan *Doc
	timer       *time.Timer
	Stats       IndexerStats
}

func (indexer *Indexer) Finish() error {
	return indexer.indexBatch()
}

func (indexer *Indexer) resetBatch() {
	indexer.docsBatch = make([]*Doc, 0, indexer.BatchSize)
}

func (indexer *Indexer) resetTimer() bool {
	return indexer.timer.Reset(indexer.IndexEvery)
}

var logger = log.New(os.Stderr, "", 0)

func (indexer *Indexer) Start() chan *Doc {
	if indexer.BatchSize == 0 {
		indexer.BatchSize = 100
	}
	if indexer.IndexEvery == 0 {
		indexer.IndexEvery = 1 * time.Hour
	}
	indexer.timer = time.NewTimer(indexer.IndexEvery)
	indexer.docsChannel = make(chan *Doc, indexer.BatchSize)
	indexer.resetBatch()
	go func(indexer *Indexer) {
		for {
			select {
			case <-indexer.timer.C:
				e := indexer.indexBatch()
				if e != nil {
					logger.Printf("ERROR=%q", e)
				}
				indexer.resetTimer()
			case doc, ok := <-indexer.docsChannel:
				if !ok {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR=%q", e)
					}
					return
				}
				if doc.Index == "" {
					doc.Index = indexer.Index.Index
				}
				if doc.Type == "" {
					doc.Type = indexer.Index.Type
				}
				indexer.docsBatch = append(indexer.docsBatch, doc)
				if len(indexer.docsBatch) >= indexer.BatchSize {
					indexer.timer.Stop()
					indexer.indexBatch()
					indexer.resetTimer()
				}
			}
		}
	}(indexer)
	return indexer.docsChannel
}

func (indexer *Indexer) Close() error {
	close(indexer.docsChannel)
	return indexer.indexBatch()
}

func (indexer *Indexer) indexBatch() error {
	if len(indexer.docsBatch) < 1 {
		return nil
	}
	started := time.Now()
	e := indexer.Index.IndexDocs(indexer.docsBatch)
	if e != nil {
		return e
	}
	indexer.Stats.Add(len(indexer.docsBatch), time.Now().Sub(started))
	indexer.resetBatch()
	return nil
}
<commit_msg>reminder to refactor indexer<commit_after>package es

import (
	"log"
	"os"
	"time"
)

var timer = time.NewTimer(1 * time.Second)

type IndexerStats struct {
	Runs        int64
	IndexedDocs int64
	TotalTime   time.Duration
}

func (stats *IndexerStats) Add(count int, dur time.Duration) {
	stats.Runs++
	stats.IndexedDocs += int64(count)
	stats.TotalTime += dur
}

type Indexer struct {
	Index *Index

	IndexEvery time.Duration // triggers a new index run after that duration, will be reset when BatchSize reached
	BatchSize  int           // triggers a new index run when the batch reaches that size

	docsBatch   []*Doc
	docsChannel chan *Doc
	timer       *time.Timer
	Stats       IndexerStats
}

func (indexer *Indexer) Finish() error {
	return indexer.indexBatch()
}

func (indexer *Indexer) resetBatch() {
	indexer.docsBatch = make([]*Doc, 0, indexer.BatchSize)
}

func (indexer *Indexer) resetTimer() bool {
	return indexer.timer.Reset(indexer.IndexEvery)
}

var logger = log.New(os.Stderr, "", 0)

// TODO change handling to e.g. Add command with implizit opening of channel
// to not expose the channel to the public
func (indexer *Indexer) Start() chan *Doc {
	if indexer.BatchSize == 0 {
		indexer.BatchSize = 100
	}
	if indexer.IndexEvery == 0 {
		indexer.IndexEvery = 1 * time.Hour
	}
	indexer.timer = time.NewTimer(indexer.IndexEvery)
	indexer.docsChannel = make(chan *Doc, indexer.BatchSize)
	indexer.resetBatch()
	go func(indexer *Indexer) {
		for {
			select {
			case <-indexer.timer.C:
				e := indexer.indexBatch()
				if e != nil {
					logger.Printf("ERROR=%q", e)
				}
				indexer.resetTimer()
			case doc, ok := <-indexer.docsChannel:
				if !ok {
					indexer.timer.Stop()
					e := indexer.indexBatch()
					if e != nil {
						logger.Printf("ERROR=%q", e)
					}
					return
				}
				if doc.Index == "" {
					doc.Index = indexer.Index.Index
				}
				if doc.Type == "" {
					doc.Type = indexer.Index.Type
				}
				indexer.docsBatch = append(indexer.docsBatch, doc)
				if len(indexer.docsBatch) >= indexer.BatchSize {
					indexer.timer.Stop()
					indexer.indexBatch()
					indexer.resetTimer()
				}
			}
		}
	}(indexer)
	return indexer.docsChannel
}

func (indexer *Indexer) Close() error {
	close(indexer.docsChannel)
	return indexer.indexBatch()
}

func (indexer *Indexer) indexBatch() error {
	if len(indexer.docsBatch) < 1 {
		return nil
	}
	started := time.Now()
	e := indexer.Index.IndexDocs(indexer.docsBatch)
	if e != nil {
		return e
	}
	indexer.Stats.Add(len(indexer.docsBatch), time.Now().Sub(started))
	indexer.resetBatch()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"runtime"
	d "runtime/debug"
	"strconv"
	"strings"
	"time"

	bimg "gopkg.in/h2non/bimg.v1"

	. "github.com/tj/go-debug"
)

var debug = Debug("imaginary")

var (
	aAddr              = flag.String("a", "", "bind address")
	aPort              = flag.Int("p", 8088, "port to listen")
	aVers              = flag.Bool("v", false, "Show version")
	aVersl             = flag.Bool("version", false, "Show version")
	aHelp              = flag.Bool("h", false, "Show help")
	aHelpl             = flag.Bool("help", false, "Show help")
	aPathPrefix        = flag.String("path-prefix", "/", "Url path prefix to listen to")
	aCors              = flag.Bool("cors", false, "Enable CORS support")
	aGzip              = flag.Bool("gzip", false, "Enable gzip compression")
	aAuthForwarding    = flag.Bool("enable-auth-forwarding", false, "Forwards X-Forward-Authorization or Authorization header to the image source server. -enable-url-source flag must be defined. Tip: secure your server from public access to prevent attack vectors")
	aEnableURLSource   = flag.Bool("enable-url-source", false, "Enable remote HTTP URL image source processing")
	aEnablePlaceholder = flag.Bool("enable-placeholder", false, "Enable image response placeholder to be used in case of error")
	aAlloweOrigins     = flag.String("allowed-origins", "", "Restrict remote image source processing to certain origins (separated by commas)")
	aKey               = flag.String("key", "", "Define API key for authorization")
	aMount             = flag.String("mount", "", "Mount server local directory")
	aCertFile          = flag.String("certfile", "", "TLS certificate file path")
	aKeyFile           = flag.String("keyfile", "", "TLS private key file path")
	aAuthorization     = flag.String("authorization", "", "Defines a constant Authorization header value passed to all the image source servers. -enable-url-source flag must be defined. This overwrites authorization headers forwarding behavior via X-Forward-Authorization")
	aPlaceholder       = flag.String("placeholder", "", "Image path to image custom placeholder to be used in case of error. Recommended minimum image size is: 1200x1200")
	aHttpCacheTtl      = flag.Int("http-cache-ttl", -1, "The TTL in seconds")
	aReadTimeout       = flag.Int("http-read-timeout", 60, "HTTP read timeout in seconds")
	aWriteTimeout      = flag.Int("http-write-timeout", 60, "HTTP write timeout in seconds")
	aConcurrency       = flag.Int("concurrency", 0, "Throttle concurrency limit per second")
	aBurst             = flag.Int("burst", 100, "Throttle burst max cache size")
	aMRelease          = flag.Int("mrelease", 30, "OS memory release interval in seconds")
	aCpus              = flag.Int("cpus", runtime.GOMAXPROCS(-1), "Number of cpu cores to use")
)

const usage = `imaginary %s

Usage:
  imaginary -p 80
  imaginary -cors -gzip
  imaginary -concurrency 10
  imaginary -path-prefix /api/v1
  imaginary -enable-url-source
  imaginary -enable-url-source -allowed-origins http://localhost,http://server.com
  imaginary -enable-url-source -enable-auth-forwarding
  imaginary -enable-url-source -authorization "Basic AwDJdL2DbwrD=="
	imaginary -enable-placeholder
	imaginery -enable-url-source -placeholder ./placeholder.jpg
	imaginary -h | -help
  imaginary -v | -version

Options:
  -a <addr>                 bind address [default: *]
  -p <port>                 bind port [default: 8088]
  -h, -help                 output help
  -v, -version              output version
  -path-prefix <value>      Url path prefix to listen to [default: "/"]
  -cors                     Enable CORS support [default: false]
  -gzip                     Enable gzip compression [default: false]
  -key <key>                Define API key for authorization
  -mount <path>             Mount server local directory
  -http-cache-ttl <num>     The TTL in seconds. Adds caching headers to locally served files.
  -http-read-timeout <num>  HTTP read timeout in seconds [default: 30]
  -http-write-timeout <num> HTTP write timeout in seconds [default: 30]
  -enable-url-source        Restrict remote image source processing to certain origins (separated by commas)
	-enable-placeholder       Enable image response placeholder to be used in case of error [default: false]
  -enable-auth-forwarding   Forwards X-Forward-Authorization or Authorization header to the image source server. -enable-url-source flag must be defined. Tip: secure your server from public access to prevent attack vectors
  -allowed-origins <urls>   TLS certificate file path
  -certfile <path>          TLS certificate file path
  -keyfile <path>           TLS private key file path
  -authorization <value>    Defines a constant Authorization header value passed to all the image source servers. -enable-url-source flag must be defined. This overwrites authorization headers forwarding behavior via X-Forward-Authorization
  -placeholder <path>       Image path to image custom placeholder to be used in case of error. Recommended minimum image size is: 1200x1200
	-concurreny <num>         Throttle concurrency limit per second [default: disabled]
  -burst <num>              Throttle burst max cache size [default: 100]
  -mrelease <num>           OS memory release interval in seconds [default: 30]
  -cpus <num>               Number of used cpu cores.
                            (default for current machine is %d cores)
`

func main() {
	flag.Usage = func() {
		fmt.Fprint(os.Stderr, fmt.Sprintf(usage, Version, runtime.NumCPU()))
	}
	flag.Parse()

	if *aHelp || *aHelpl {
		showUsage()
	}
	if *aVers || *aVersl {
		showVersion()
	}

	// Only required in Go < 1.5
	runtime.GOMAXPROCS(*aCpus)

	port := getPort(*aPort)
	opts := ServerOptions{
		Port:              port,
		Address:           *aAddr,
		Gzip:              *aGzip,
		CORS:              *aCors,
		AuthForwarding:    *aAuthForwarding,
		EnableURLSource:   *aEnableURLSource,
		EnablePlaceholder: *aEnablePlaceholder,
		PathPrefix:        *aPathPrefix,
		ApiKey:            *aKey,
		Concurrency:       *aConcurrency,
		Burst:             *aBurst,
		Mount:             *aMount,
		CertFile:          *aCertFile,
		KeyFile:           *aKeyFile,
		Placeholder:       *aPlaceholder,
		HttpCacheTtl:      *aHttpCacheTtl,
		HttpReadTimeout:   *aReadTimeout,
		HttpWriteTimeout:  *aWriteTimeout,
		Authorization:     *aAuthorization,
		AlloweOrigins:     parseOrigins(*aAlloweOrigins),
	}

	// Create a memory release goroutine
	if *aMRelease > 0 {
		memoryRelease(*aMRelease)
	}

	// Check if the mount directory exists, if present
	if *aMount != "" {
		checkMountDirectory(*aMount)
	}

	// Validate HTTP cache param, if present
	if *aHttpCacheTtl != -1 {
		checkHttpCacheTtl(*aHttpCacheTtl)
	}

	// Read placeholder image, if required
	if *aPlaceholder != "" {
		buf, err := ioutil.ReadFile(*aPlaceholder)
		if err != nil {
			exitWithError("cannot start the server: %s", err)
		}

		imageType := bimg.DetermineImageType(buf)
		if !bimg.IsImageTypeSupportedByVips(imageType) {
			exitWithError("Placeholder image type is not supported. Only JPEG, PNG or WEBP are supported")
		}

		opts.PlaceholderImage = buf
	} else if *aEnablePlaceholder {
		// Expose default placeholder
		opts.PlaceholderImage = placeholder
	}

	debug("imaginary server listening on port :%d/%s", opts.Port, strings.TrimPrefix(opts.PathPrefix, "/"))

	// Load image source providers
	LoadSources(opts)

	// Start the server
	err := Server(opts)
	if err != nil {
		exitWithError("cannot start the server: %s", err)
	}
}

func getPort(port int) int {
	if portEnv := os.Getenv("PORT"); portEnv != "" {
		newPort, _ := strconv.Atoi(portEnv)
		if newPort > 0 {
			port = newPort
		}
	}
	return port
}

func showUsage() {
	flag.Usage()
	os.Exit(1)
}

func showVersion() {
	fmt.Println(Version)
	os.Exit(1)
}

func checkMountDirectory(path string) {
	src, err := os.Stat(path)
	if err != nil {
		exitWithError("error while mounting directory: %s", err)
	}
	if src.IsDir() == false {
		exitWithError("mount path is not a directory: %s", path)
	}
	if path == "/" {
		exitWithError("cannot mount root directory for security reasons")
	}
}

func checkHttpCacheTtl(ttl int) {
	if ttl < -1 || ttl > 31556926 {
		exitWithError("The -http-cache-ttl flag only accepts a value from 0 to 31556926")
	}

	if ttl == 0 {
		debug("Adding HTTP cache control headers set to prevent caching.")
	}
}

func parseOrigins(origins string) []*url.URL {
	urls := []*url.URL{}
	if origins == "" {
		return urls
	}
	for _, origin := range strings.Split(origins, ",") {
		u, err := url.Parse(origin)
		if err != nil {
			continue
		}
		urls = append(urls, u)
	}
	return urls
}

func memoryRelease(interval int) {
	ticker := time.NewTicker(time.Duration(interval) * time.Second)
	go func() {
		for _ = range ticker.C {
			debug("FreeOSMemory()")
			d.FreeOSMemory()
		}
	}()
}

func exitWithError(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args)
	os.Exit(1)
}
<commit_msg>fix(usage): correct help message of 'allowed-origins'<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"runtime"
	d "runtime/debug"
	"strconv"
	"strings"
	"time"

	bimg "gopkg.in/h2non/bimg.v1"

	. "github.com/tj/go-debug"
)

var debug = Debug("imaginary")

var (
	aAddr              = flag.String("a", "", "bind address")
	aPort              = flag.Int("p", 8088, "port to listen")
	aVers              = flag.Bool("v", false, "Show version")
	aVersl             = flag.Bool("version", false, "Show version")
	aHelp              = flag.Bool("h", false, "Show help")
	aHelpl             = flag.Bool("help", false, "Show help")
	aPathPrefix        = flag.String("path-prefix", "/", "Url path prefix to listen to")
	aCors              = flag.Bool("cors", false, "Enable CORS support")
	aGzip              = flag.Bool("gzip", false, "Enable gzip compression")
	aAuthForwarding    = flag.Bool("enable-auth-forwarding", false, "Forwards X-Forward-Authorization or Authorization header to the image source server. -enable-url-source flag must be defined. Tip: secure your server from public access to prevent attack vectors")
	aEnableURLSource   = flag.Bool("enable-url-source", false, "Enable remote HTTP URL image source processing")
	aEnablePlaceholder = flag.Bool("enable-placeholder", false, "Enable image response placeholder to be used in case of error")
	aAlloweOrigins     = flag.String("allowed-origins", "", "Restrict remote image source processing to certain origins (separated by commas)")
	aKey               = flag.String("key", "", "Define API key for authorization")
	aMount             = flag.String("mount", "", "Mount server local directory")
	aCertFile          = flag.String("certfile", "", "TLS certificate file path")
	aKeyFile           = flag.String("keyfile", "", "TLS private key file path")
	aAuthorization     = flag.String("authorization", "", "Defines a constant Authorization header value passed to all the image source servers. -enable-url-source flag must be defined. This overwrites authorization headers forwarding behavior via X-Forward-Authorization")
	aPlaceholder       = flag.String("placeholder", "", "Image path to image custom placeholder to be used in case of error. Recommended minimum image size is: 1200x1200")
	aHttpCacheTtl      = flag.Int("http-cache-ttl", -1, "The TTL in seconds")
	aReadTimeout       = flag.Int("http-read-timeout", 60, "HTTP read timeout in seconds")
	aWriteTimeout      = flag.Int("http-write-timeout", 60, "HTTP write timeout in seconds")
	aConcurrency       = flag.Int("concurrency", 0, "Throttle concurrency limit per second")
	aBurst             = flag.Int("burst", 100, "Throttle burst max cache size")
	aMRelease          = flag.Int("mrelease", 30, "OS memory release interval in seconds")
	aCpus              = flag.Int("cpus", runtime.GOMAXPROCS(-1), "Number of cpu cores to use")
)

const usage = `imaginary %s

Usage:
  imaginary -p 80
  imaginary -cors -gzip
  imaginary -concurrency 10
  imaginary -path-prefix /api/v1
  imaginary -enable-url-source
  imaginary -enable-url-source -allowed-origins http://localhost,http://server.com
  imaginary -enable-url-source -enable-auth-forwarding
  imaginary -enable-url-source -authorization "Basic AwDJdL2DbwrD=="
	imaginary -enable-placeholder
	imaginery -enable-url-source -placeholder ./placeholder.jpg
	imaginary -h | -help
  imaginary -v | -version

Options:
  -a <addr>                 bind address [default: *]
  -p <port>                 bind port [default: 8088]
  -h, -help                 output help
  -v, -version              output version
  -path-prefix <value>      Url path prefix to listen to [default: "/"]
  -cors                     Enable CORS support [default: false]
  -gzip                     Enable gzip compression [default: false]
  -key <key>                Define API key for authorization
  -mount <path>             Mount server local directory
  -http-cache-ttl <num>     The TTL in seconds. Adds caching headers to locally served files.
  -http-read-timeout <num>  HTTP read timeout in seconds [default: 30]
  -http-write-timeout <num> HTTP write timeout in seconds [default: 30]
  -enable-url-source        Restrict remote image source processing to certain origins (separated by commas)
	-enable-placeholder       Enable image response placeholder to be used in case of error [default: false]
  -enable-auth-forwarding   Forwards X-Forward-Authorization or Authorization header to the image source server. -enable-url-source flag must be defined. Tip: secure your server from public access to prevent attack vectors
  -allowed-origins <urls>   Restrict remote image source processing to certain origins (separated by commas)
  -certfile <path>          TLS certificate file path
  -keyfile <path>           TLS private key file path
  -authorization <value>    Defines a constant Authorization header value passed to all the image source servers. -enable-url-source flag must be defined. This overwrites authorization headers forwarding behavior via X-Forward-Authorization
  -placeholder <path>       Image path to image custom placeholder to be used in case of error. Recommended minimum image size is: 1200x1200
	-concurreny <num>         Throttle concurrency limit per second [default: disabled]
  -burst <num>              Throttle burst max cache size [default: 100]
  -mrelease <num>           OS memory release interval in seconds [default: 30]
  -cpus <num>               Number of used cpu cores.
                            (default for current machine is %d cores)
`

func main() {
	flag.Usage = func() {
		fmt.Fprint(os.Stderr, fmt.Sprintf(usage, Version, runtime.NumCPU()))
	}
	flag.Parse()

	if *aHelp || *aHelpl {
		showUsage()
	}
	if *aVers || *aVersl {
		showVersion()
	}

	// Only required in Go < 1.5
	runtime.GOMAXPROCS(*aCpus)

	port := getPort(*aPort)
	opts := ServerOptions{
		Port:              port,
		Address:           *aAddr,
		Gzip:              *aGzip,
		CORS:              *aCors,
		AuthForwarding:    *aAuthForwarding,
		EnableURLSource:   *aEnableURLSource,
		EnablePlaceholder: *aEnablePlaceholder,
		PathPrefix:        *aPathPrefix,
		ApiKey:            *aKey,
		Concurrency:       *aConcurrency,
		Burst:             *aBurst,
		Mount:             *aMount,
		CertFile:          *aCertFile,
		KeyFile:           *aKeyFile,
		Placeholder:       *aPlaceholder,
		HttpCacheTtl:      *aHttpCacheTtl,
		HttpReadTimeout:   *aReadTimeout,
		HttpWriteTimeout:  *aWriteTimeout,
		Authorization:     *aAuthorization,
		AlloweOrigins:     parseOrigins(*aAlloweOrigins),
	}

	// Create a memory release goroutine
	if *aMRelease > 0 {
		memoryRelease(*aMRelease)
	}

	// Check if the mount directory exists, if present
	if *aMount != "" {
		checkMountDirectory(*aMount)
	}

	// Validate HTTP cache param, if present
	if *aHttpCacheTtl != -1 {
		checkHttpCacheTtl(*aHttpCacheTtl)
	}

	// Read placeholder image, if required
	if *aPlaceholder != "" {
		buf, err := ioutil.ReadFile(*aPlaceholder)
		if err != nil {
			exitWithError("cannot start the server: %s", err)
		}

		imageType := bimg.DetermineImageType(buf)
		if !bimg.IsImageTypeSupportedByVips(imageType) {
			exitWithError("Placeholder image type is not supported. Only JPEG, PNG or WEBP are supported")
		}

		opts.PlaceholderImage = buf
	} else if *aEnablePlaceholder {
		// Expose default placeholder
		opts.PlaceholderImage = placeholder
	}

	debug("imaginary server listening on port :%d/%s", opts.Port, strings.TrimPrefix(opts.PathPrefix, "/"))

	// Load image source providers
	LoadSources(opts)

	// Start the server
	err := Server(opts)
	if err != nil {
		exitWithError("cannot start the server: %s", err)
	}
}

func getPort(port int) int {
	if portEnv := os.Getenv("PORT"); portEnv != "" {
		newPort, _ := strconv.Atoi(portEnv)
		if newPort > 0 {
			port = newPort
		}
	}
	return port
}

func showUsage() {
	flag.Usage()
	os.Exit(1)
}

func showVersion() {
	fmt.Println(Version)
	os.Exit(1)
}

func checkMountDirectory(path string) {
	src, err := os.Stat(path)
	if err != nil {
		exitWithError("error while mounting directory: %s", err)
	}
	if src.IsDir() == false {
		exitWithError("mount path is not a directory: %s", path)
	}
	if path == "/" {
		exitWithError("cannot mount root directory for security reasons")
	}
}

func checkHttpCacheTtl(ttl int) {
	if ttl < -1 || ttl > 31556926 {
		exitWithError("The -http-cache-ttl flag only accepts a value from 0 to 31556926")
	}

	if ttl == 0 {
		debug("Adding HTTP cache control headers set to prevent caching.")
	}
}

func parseOrigins(origins string) []*url.URL {
	urls := []*url.URL{}
	if origins == "" {
		return urls
	}
	for _, origin := range strings.Split(origins, ",") {
		u, err := url.Parse(origin)
		if err != nil {
			continue
		}
		urls = append(urls, u)
	}
	return urls
}

func memoryRelease(interval int) {
	ticker := time.NewTicker(time.Duration(interval) * time.Second)
	go func() {
		for _ = range ticker.C {
			debug("FreeOSMemory()")
			d.FreeOSMemory()
		}
	}()
}

func exitWithError(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", args)
	os.Exit(1)
}
<|endoftext|>


<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"image"
	"image/color"
	"runtime"
	"sync"

	"github.com/hajimehoshi/ebiten/internal/opengl"
)

func glContext() *opengl.Context {
	// This is called from finalizers even when the context or the program is not set.
	g, ok := theGraphicsContext.Load().(*graphicsContext)
	if !ok {
		return nil
	}
	if g == nil {
		return nil
	}
	return g.GLContext()
}

type images struct {
	images      map[*imageImpl]struct{}
	m           sync.Mutex
	lastChecked *imageImpl
}

var theImagesForRestoring = images{
	images: map[*imageImpl]struct{}{},
}

func (i *images) add(img *imageImpl) (*Image, error) {
	i.m.Lock()
	defer i.m.Unlock()
	i.images[img] = struct{}{}
	eimg := &Image{img}
	runtime.SetFinalizer(eimg, theImagesForRestoring.remove)
	return eimg, nil
}

func (i *images) remove(img *Image) {
	if err := img.Dispose(); err != nil {
		panic(err)
	}
	i.m.Lock()
	defer i.m.Unlock()
	delete(i.images, img.impl)
	runtime.SetFinalizer(img, nil)
}

func (i *images) resolveStalePixels(context *opengl.Context) error {
	i.m.Lock()
	defer i.m.Unlock()
	i.lastChecked = nil
	for img := range i.images {
		if err := img.resolveStalePixels(context); err != nil {
			return err
		}
	}
	return nil
}

func (i *images) resetPixelsIfDependingOn(target *Image, context *opengl.Context) {
	i.m.Lock()
	defer i.m.Unlock()
	if i.lastChecked == target.impl {
		return
	}
	i.lastChecked = target.impl
	if target.impl.isDisposed() {
		return
	}
	for img := range i.images {
		img.resetPixelsIfDependingOn(target.impl, context)
	}
}

func (i *images) restore(context *opengl.Context) error {
	i.m.Lock()
	defer i.m.Unlock()
	// Framebuffers/textures cannot be disposed since framebuffers/textures that
	// don't belong to the current context.
	imagesWithoutDependency := []*imageImpl{}
	imagesWithDependency := []*imageImpl{}
	for img := range i.images {
		if img.hasDependency() {
			imagesWithDependency = append(imagesWithDependency, img)
		} else {
			imagesWithoutDependency = append(imagesWithoutDependency, img)
		}
	}
	// Images depending on other images should be processed first.
	for _, img := range imagesWithoutDependency {
		if err := img.restore(context); err != nil {
			return err
		}
	}
	for _, img := range imagesWithDependency {
		if err := img.restore(context); err != nil {
			return err
		}
	}
	return nil
}

func (i *images) clearVolatileImages() error {
	i.m.Lock()
	defer i.m.Unlock()
	for img := range i.images {
		img.clearIfVolatile()
	}
	return nil
}

// Image represents an image.
// The pixel format is alpha-premultiplied.
// Image implements image.Image.
type Image struct {
	impl *imageImpl
}

// Size returns the size of the image.
//
// This function is concurrent-safe.
func (i *Image) Size() (width, height int) {
	return i.impl.restorable.Size()
}

// Clear resets the pixels of the image into 0.
//
// When the image is disposed, this function does nothing.
//
// This function always returns nil as of 1.5.0-alpha.
//
// This function is concurrent-safe.
func (i *Image) Clear() error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	i.impl.Fill(color.Transparent)
	return nil
}

// Fill fills the image with a solid color.
//
// When the image is disposed, this function does nothing.
//
// This function always returns nil as of 1.5.0-alpha.
//
// This function is concurrent-safe.
func (i *Image) Fill(clr color.Color) error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	i.impl.Fill(clr)
	return nil
}

// DrawImage draws the given image on the receiver image.
//
// This method accepts the options.
// The parts of the given image at the parts of the destination.
// After determining parts to draw, this applies the geometry matrix and the color matrix.
//
// Here are the default values:
//     ImageParts:    (0, 0) - (source width, source height) to (0, 0) - (source width, source height)
//                    (i.e. the whole source image)
//     GeoM:          Identity matrix
//     ColorM:        Identity matrix (that changes no colors)
//     CompositeMode: CompositeModeSourceOver (regular alpha blending)
//
// For drawing, the pixels of the argument image at the time of this call is adopted.
// Even if the argument image is mutated after this call,
// the drawing result is never affected.
//
// This function is concurrent-safe.
func (i *Image) DrawImage(image *Image, options *DrawImageOptions) error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	return i.impl.DrawImage(image, options)
}

// Bounds returns the bounds of the image.
//
// This function is concurrent-safe.
func (i *Image) Bounds() image.Rectangle {
	w, h := i.impl.restorable.Size()
	return image.Rect(0, 0, w, h)
}

// ColorModel returns the color model of the image.
//
// This function is concurrent-safe.
func (i *Image) ColorModel() color.Model {
	return color.RGBAModel
}

// At returns the color of the image at (x, y).
//
// This method loads pixels from VRAM to system memory if necessary.
//
// This method can't be called before the main loop (ebiten.Run) starts (as of version 1.4.0-alpha).
//
// This function is concurrent-safe.
func (i *Image) At(x, y int) color.Color {
	return i.impl.At(x, y, glContext())
}

// Dispose disposes the image data. After disposing, the image becomes invalid.
// This is useful to save memory.
//
// The behavior of any functions for a disposed image is undefined.
//
// This function is concurrent-safe.
func (i *Image) Dispose() error {
	if i.impl.isDisposed() {
		return nil
	}
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	return i.impl.Dispose()
}

// ReplacePixels replaces the pixels of the image with p.
//
// The given p must represent RGBA pre-multiplied alpha values. len(p) must equal to 4 * (image width) * (image height).
//
// This function may be slow (as for implementation, this calls glTexSubImage2D).
//
// This function is concurrent-safe.
func (i *Image) ReplacePixels(p []uint8) error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	return i.impl.ReplacePixels(p)
}

// A DrawImageOptions represents options to render an image on an image.
type DrawImageOptions struct {
	ImageParts    ImageParts
	GeoM          GeoM
	ColorM        ColorM
	CompositeMode CompositeMode

	// Deprecated (as of 1.1.0-alpha): Use ImageParts instead.
	Parts []ImagePart
}

// NewImage returns an empty image.
//
// NewImage generates a new texture and a new framebuffer.
//
// This function is concurrent-safe.
func NewImage(width, height int, filter Filter) (*Image, error) {
	img, err := newImageImpl(width, height, filter, false)
	if err != nil {
		return nil, err
	}
	img.Fill(color.Transparent)
	eimg, err := theImagesForRestoring.add(img)
	if err != nil {
		return nil, err
	}
	return eimg, nil
}

// newVolatileImage returns an empty 'volatile' image.
// A volatile image is always cleared at the start of a frame.
//
// This is suitable for offscreen images that pixels are changed often.
//
// Pixels in regular non-volatile images are saved at each end of a frame if the image
// is changed, and restored automatically from the saved pixels on GL context lost.
// On the other hand, pixels in volatile images are not saved.
// Saving pixels is an expensive operation, and it is desirable to avoid it if possible.
//
// This function is concurrent-safe.
func newVolatileImage(width, height int, filter Filter) (*Image, error) {
	img, err := newImageImpl(width, height, filter, true)
	if err != nil {
		return nil, err
	}
	img.Fill(color.Transparent)
	eimg, err := theImagesForRestoring.add(img)
	if err != nil {
		return nil, err
	}
	return eimg, nil
}

// NewImageFromImage creates a new image with the given image (source).
//
// NewImageFromImage generates a new texture and a new framebuffer.
//
// This function is concurrent-safe.
func NewImageFromImage(source image.Image, filter Filter) (*Image, error) {
	img, err := newImageImplFromImage(source, filter)
	if err != nil {
		return nil, err
	}
	eimg, err := theImagesForRestoring.add(img)
	if err != nil {
		return nil, err
	}
	return eimg, nil
}

func newImageWithScreenFramebuffer(width, height int) (*Image, error) {
	img, err := newScreenImageImpl(width, height)
	if err != nil {
		return nil, err
	}
	eimg, err := theImagesForRestoring.add(img)
	if err != nil {
		return nil, err
	}
	return eimg, nil
}
<commit_msg>Simplify internal API not to return errors<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"image"
	"image/color"
	"runtime"
	"sync"

	"github.com/hajimehoshi/ebiten/internal/opengl"
)

func glContext() *opengl.Context {
	// This is called from finalizers even when the context or the program is not set.
	g, ok := theGraphicsContext.Load().(*graphicsContext)
	if !ok {
		return nil
	}
	if g == nil {
		return nil
	}
	return g.GLContext()
}

type images struct {
	images      map[*imageImpl]struct{}
	m           sync.Mutex
	lastChecked *imageImpl
}

var theImagesForRestoring = images{
	images: map[*imageImpl]struct{}{},
}

func (i *images) add(img *imageImpl) *Image {
	i.m.Lock()
	defer i.m.Unlock()
	i.images[img] = struct{}{}
	eimg := &Image{img}
	runtime.SetFinalizer(eimg, theImagesForRestoring.remove)
	return eimg
}

func (i *images) remove(img *Image) {
	if err := img.Dispose(); err != nil {
		panic(err)
	}
	i.m.Lock()
	defer i.m.Unlock()
	delete(i.images, img.impl)
	runtime.SetFinalizer(img, nil)
}

func (i *images) resolveStalePixels(context *opengl.Context) error {
	i.m.Lock()
	defer i.m.Unlock()
	i.lastChecked = nil
	for img := range i.images {
		if err := img.resolveStalePixels(context); err != nil {
			return err
		}
	}
	return nil
}

func (i *images) resetPixelsIfDependingOn(target *Image, context *opengl.Context) {
	i.m.Lock()
	defer i.m.Unlock()
	if i.lastChecked == target.impl {
		return
	}
	i.lastChecked = target.impl
	if target.impl.isDisposed() {
		return
	}
	for img := range i.images {
		img.resetPixelsIfDependingOn(target.impl, context)
	}
}

func (i *images) restore(context *opengl.Context) error {
	i.m.Lock()
	defer i.m.Unlock()
	// Framebuffers/textures cannot be disposed since framebuffers/textures that
	// don't belong to the current context.
	imagesWithoutDependency := []*imageImpl{}
	imagesWithDependency := []*imageImpl{}
	for img := range i.images {
		if img.hasDependency() {
			imagesWithDependency = append(imagesWithDependency, img)
		} else {
			imagesWithoutDependency = append(imagesWithoutDependency, img)
		}
	}
	// Images depending on other images should be processed first.
	for _, img := range imagesWithoutDependency {
		if err := img.restore(context); err != nil {
			return err
		}
	}
	for _, img := range imagesWithDependency {
		if err := img.restore(context); err != nil {
			return err
		}
	}
	return nil
}

func (i *images) clearVolatileImages() error {
	i.m.Lock()
	defer i.m.Unlock()
	for img := range i.images {
		img.clearIfVolatile()
	}
	return nil
}

// Image represents an image.
// The pixel format is alpha-premultiplied.
// Image implements image.Image.
type Image struct {
	impl *imageImpl
}

// Size returns the size of the image.
//
// This function is concurrent-safe.
func (i *Image) Size() (width, height int) {
	return i.impl.restorable.Size()
}

// Clear resets the pixels of the image into 0.
//
// When the image is disposed, this function does nothing.
//
// This function always returns nil as of 1.5.0-alpha.
//
// This function is concurrent-safe.
func (i *Image) Clear() error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	i.impl.Fill(color.Transparent)
	return nil
}

// Fill fills the image with a solid color.
//
// When the image is disposed, this function does nothing.
//
// This function always returns nil as of 1.5.0-alpha.
//
// This function is concurrent-safe.
func (i *Image) Fill(clr color.Color) error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	i.impl.Fill(clr)
	return nil
}

// DrawImage draws the given image on the receiver image.
//
// This method accepts the options.
// The parts of the given image at the parts of the destination.
// After determining parts to draw, this applies the geometry matrix and the color matrix.
//
// Here are the default values:
//     ImageParts:    (0, 0) - (source width, source height) to (0, 0) - (source width, source height)
//                    (i.e. the whole source image)
//     GeoM:          Identity matrix
//     ColorM:        Identity matrix (that changes no colors)
//     CompositeMode: CompositeModeSourceOver (regular alpha blending)
//
// For drawing, the pixels of the argument image at the time of this call is adopted.
// Even if the argument image is mutated after this call,
// the drawing result is never affected.
//
// This function is concurrent-safe.
func (i *Image) DrawImage(image *Image, options *DrawImageOptions) error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	return i.impl.DrawImage(image, options)
}

// Bounds returns the bounds of the image.
//
// This function is concurrent-safe.
func (i *Image) Bounds() image.Rectangle {
	w, h := i.impl.restorable.Size()
	return image.Rect(0, 0, w, h)
}

// ColorModel returns the color model of the image.
//
// This function is concurrent-safe.
func (i *Image) ColorModel() color.Model {
	return color.RGBAModel
}

// At returns the color of the image at (x, y).
//
// This method loads pixels from VRAM to system memory if necessary.
//
// This method can't be called before the main loop (ebiten.Run) starts (as of version 1.4.0-alpha).
//
// This function is concurrent-safe.
func (i *Image) At(x, y int) color.Color {
	return i.impl.At(x, y, glContext())
}

// Dispose disposes the image data. After disposing, the image becomes invalid.
// This is useful to save memory.
//
// The behavior of any functions for a disposed image is undefined.
//
// This function is concurrent-safe.
func (i *Image) Dispose() error {
	if i.impl.isDisposed() {
		return nil
	}
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	return i.impl.Dispose()
}

// ReplacePixels replaces the pixels of the image with p.
//
// The given p must represent RGBA pre-multiplied alpha values. len(p) must equal to 4 * (image width) * (image height).
//
// This function may be slow (as for implementation, this calls glTexSubImage2D).
//
// This function is concurrent-safe.
func (i *Image) ReplacePixels(p []uint8) error {
	theImagesForRestoring.resetPixelsIfDependingOn(i, glContext())
	return i.impl.ReplacePixels(p)
}

// A DrawImageOptions represents options to render an image on an image.
type DrawImageOptions struct {
	ImageParts    ImageParts
	GeoM          GeoM
	ColorM        ColorM
	CompositeMode CompositeMode

	// Deprecated (as of 1.1.0-alpha): Use ImageParts instead.
	Parts []ImagePart
}

// NewImage returns an empty image.
//
// NewImage generates a new texture and a new framebuffer.
//
// This function is concurrent-safe.
func NewImage(width, height int, filter Filter) (*Image, error) {
	img, err := newImageImpl(width, height, filter, false)
	if err != nil {
		return nil, err
	}
	img.Fill(color.Transparent)
	return theImagesForRestoring.add(img), nil
}

// newVolatileImage returns an empty 'volatile' image.
// A volatile image is always cleared at the start of a frame.
//
// This is suitable for offscreen images that pixels are changed often.
//
// Pixels in regular non-volatile images are saved at each end of a frame if the image
// is changed, and restored automatically from the saved pixels on GL context lost.
// On the other hand, pixels in volatile images are not saved.
// Saving pixels is an expensive operation, and it is desirable to avoid it if possible.
//
// This function is concurrent-safe.
func newVolatileImage(width, height int, filter Filter) (*Image, error) {
	img, err := newImageImpl(width, height, filter, true)
	if err != nil {
		return nil, err
	}
	img.Fill(color.Transparent)
	return theImagesForRestoring.add(img), nil
}

// NewImageFromImage creates a new image with the given image (source).
//
// NewImageFromImage generates a new texture and a new framebuffer.
//
// This function is concurrent-safe.
func NewImageFromImage(source image.Image, filter Filter) (*Image, error) {
	img, err := newImageImplFromImage(source, filter)
	if err != nil {
		return nil, err
	}
	return theImagesForRestoring.add(img), nil
}

func newImageWithScreenFramebuffer(width, height int) (*Image, error) {
	img, err := newScreenImageImpl(width, height)
	if err != nil {
		return nil, err
	}
	return theImagesForRestoring.add(img), nil
}
<|endoftext|>