<commit_before>package flags

import (
	"bytes"
	"fmt"
	"os"
	"sort"
	"strings"
	"unicode/utf8"
)

type parseState struct {
	arg        string
	args       []string
	retargs    []string
	positional []*Arg
	err        error

	command *Command
	lookup  lookup
}

func (p *parseState) eof() bool {
	return len(p.args) == 0
}

func (p *parseState) pop() string {
	if p.eof() {
		return ""
	}

	p.arg = p.args[0]
	p.args = p.args[1:]

	return p.arg
}

func (p *parseState) peek() string {
	if p.eof() {
		return ""
	}

	return p.args[0]
}

func (p *parseState) checkRequired(parser *Parser) error {
	c := parser.Command

	var required []*Option

	for c != nil {
		c.eachGroup(func(g *Group) {
			for _, option := range g.options {
				if !option.isSet && option.Required {
					required = append(required, option)
				}
			}
		})

		c = c.Active
	}

	if len(required) == 0 {
		if len(p.positional) > 0 && p.command.ArgsRequired {
			var reqnames []string

			for _, arg := range p.positional {
				if arg.isRemaining() {
					break
				}

				reqnames = append(reqnames, "`"+arg.Name+"`")
			}

			if len(reqnames) == 0 {
				return nil
			}

			var msg string

			if len(reqnames) == 1 {
				msg = fmt.Sprintf("the required argument %s was not provided", reqnames[0])
			} else {
				msg = fmt.Sprintf("the required arguments %s and %s were not provided",
					strings.Join(reqnames[:len(reqnames)-1], ", "), reqnames[len(reqnames)-1])
			}

			p.err = newError(ErrRequired, msg)
			return p.err
		}

		return nil
	}

	names := make([]string, 0, len(required))

	for _, k := range required {
		names = append(names, "`"+k.String()+"'")
	}

	sort.Strings(names)

	var msg string

	if len(names) == 1 {
		msg = fmt.Sprintf("the required flag %s was not specified", names[0])
	} else {
		msg = fmt.Sprintf("the required flags %s and %s were not specified",
			strings.Join(names[:len(names)-1], ", "), names[len(names)-1])
	}

	p.err = newError(ErrRequired, msg)
	return p.err
}

func (p *parseState) estimateCommand() error {
	commands := p.command.sortedCommands()
	cmdnames := make([]string, len(commands))

	for i, v := range commands {
		cmdnames[i] = v.Name
	}

	var msg string
	var errtype ErrorType

	if len(p.retargs) != 0 {
		c, l := closestChoice(p.retargs[0], cmdnames)
		msg = fmt.Sprintf("Unknown command `%s'", p.retargs[0])
		errtype = ErrUnknownCommand

		if float32(l)/float32(len(c)) < 0.5 {
			msg = fmt.Sprintf("%s, did you mean `%s'?", msg, c)
		} else if len(cmdnames) == 1 {
			msg = fmt.Sprintf("%s. You should use the %s command",
				msg,
				cmdnames[0])
		} else {
			msg = fmt.Sprintf("%s. Please specify one command of: %s or %s",
				msg,
				strings.Join(cmdnames[:len(cmdnames)-1], ", "),
				cmdnames[len(cmdnames)-1])
		}
	} else {
		errtype = ErrCommandRequired

		if len(cmdnames) == 1 {
			msg = fmt.Sprintf("Please specify the %s command", cmdnames[0])
		} else {
			msg = fmt.Sprintf("Please specify one command of: %s or %s",
				strings.Join(cmdnames[:len(cmdnames)-1], ", "),
				cmdnames[len(cmdnames)-1])
		}
	}

	return newError(errtype, msg)
}

func (p *Parser) parseOption(s *parseState, name string, option *Option, canarg bool, argument *string) (err error) {
	if !option.canArgument() {
		if argument != nil {
			return newErrorf(ErrNoArgumentForBool, "bool flag `%s' cannot have an argument", option)
		}

		err = option.set(nil)
	} else if argument != nil || (canarg && !s.eof()) {
		var arg string

		if argument != nil {
			arg = *argument
		} else {
			arg = s.pop()
			// Accept any single character arguments including '-'.
			// '-' is the special file name for the standard input or the standard output in many cases.
			if len(arg) > 1 && argumentIsOption(arg) {
				return newErrorf(ErrExpectedArgument, "expected argument for flag `%s', but got option `%s'", option, arg)
			}
		}

		if option.tag.Get("unquote") != "false" {
			arg, err = unquoteIfPossible(arg)
		}

		if err == nil {
			err = option.set(&arg)
		}
	} else if option.OptionalArgument {
		option.empty()

		for _, v := range option.OptionalValue {
			err = option.set(&v)

			if err != nil {
				break
			}
		}
	} else {
		err = newErrorf(ErrExpectedArgument, "expected argument for flag `%s'", option)
	}

	if err != nil {
		if _, ok := err.(*Error); !ok {
			err = newErrorf(ErrMarshal, "invalid argument for flag `%s' (expected %s): %s",
				option,
				option.value.Type(),
				err.Error())
		}
	}

	return err
}

func (p *Parser) parseLong(s *parseState, name string, argument *string) error {
	if option := s.lookup.longNames[name]; option != nil {
		// Only long options that are required can consume an argument
		// from the argument list
		canarg := !option.OptionalArgument

		return p.parseOption(s, name, option, canarg, argument)
	}

	return newErrorf(ErrUnknownFlag, "unknown flag `%s'", name)
}

func (p *Parser) splitShortConcatArg(s *parseState, optname string) (string, *string) {
	c, n := utf8.DecodeRuneInString(optname)

	if n == len(optname) {
		return optname, nil
	}

	first := string(c)

	if option := s.lookup.shortNames[first]; option != nil && option.canArgument() {
		arg := optname[n:]
		return first, &arg
	}

	return optname, nil
}

func (p *Parser) parseShort(s *parseState, optname string, argument *string) error {
	if argument == nil {
		optname, argument = p.splitShortConcatArg(s, optname)
	}

	for i, c := range optname {
		shortname := string(c)

		if option := s.lookup.shortNames[shortname]; option != nil {
			// Only the last short argument can consume an argument from
			// the arguments list, and only if it's non optional
			canarg := (i+utf8.RuneLen(c) == len(optname)) && !option.OptionalArgument

			if err := p.parseOption(s, shortname, option, canarg, argument); err != nil {
				return err
			}
		} else {
			return newErrorf(ErrUnknownFlag, "unknown flag `%s'", shortname)
		}

		// Only the first option can have a concatted argument, so just
		// clear argument here
		argument = nil
	}

	return nil
}

func (p *parseState) addArgs(args ...string) error {
	for len(p.positional) > 0 && len(args) > 0 {
		arg := p.positional[0]

		if err := convert(args[0], arg.value, arg.tag); err != nil {
			return err
		}

		if !arg.isRemaining() {
			p.positional = p.positional[1:]
		}

		args = args[1:]
	}

	p.retargs = append(p.retargs, args...)
	return nil
}

func (p *Parser) parseNonOption(s *parseState) error {
	if len(s.positional) > 0 {
		return s.addArgs(s.arg)
	}

	if cmd := s.lookup.commands[s.arg]; cmd != nil {
		s.command.Active = cmd
		cmd.fillParseState(s)
	} else if (p.Options & PassAfterNonOption) != None {
		// If PassAfterNonOption is set then all remaining arguments
		// are considered positional
		if err := s.addArgs(s.arg); err != nil {
			return err
		}

		if err := s.addArgs(s.args...); err != nil {
			return err
		}

		s.args = []string{}
	} else {
		return s.addArgs(s.arg)
	}

	return nil
}

func (p *Parser) showBuiltinHelp() error {
	var b bytes.Buffer

	p.WriteHelp(&b)
	return newError(ErrHelp, b.String())
}

func (p *Parser) printError(err error) error {
	if err != nil && (p.Options&PrintErrors) != None {
		fmt.Fprintln(os.Stderr, err)
	}

	return err
}

func (p *Parser) clearIsSet() {
	p.eachCommand(func(c *Command) {
		c.eachGroup(func(g *Group) {
			for _, option := range g.options {
				option.isSet = false
			}
		})
	}, true)
}
<commit_msg>Added empty line for improved readability<commit_after>package flags

import (
	"bytes"
	"fmt"
	"os"
	"sort"
	"strings"
	"unicode/utf8"
)

type parseState struct {
	arg        string
	args       []string
	retargs    []string
	positional []*Arg
	err        error

	command *Command
	lookup  lookup
}

func (p *parseState) eof() bool {
	return len(p.args) == 0
}

func (p *parseState) pop() string {
	if p.eof() {
		return ""
	}

	p.arg = p.args[0]
	p.args = p.args[1:]

	return p.arg
}

func (p *parseState) peek() string {
	if p.eof() {
		return ""
	}

	return p.args[0]
}

func (p *parseState) checkRequired(parser *Parser) error {
	c := parser.Command

	var required []*Option

	for c != nil {
		c.eachGroup(func(g *Group) {
			for _, option := range g.options {
				if !option.isSet && option.Required {
					required = append(required, option)
				}
			}
		})

		c = c.Active
	}

	if len(required) == 0 {
		if len(p.positional) > 0 && p.command.ArgsRequired {
			var reqnames []string

			for _, arg := range p.positional {
				if arg.isRemaining() {
					break
				}

				reqnames = append(reqnames, "`"+arg.Name+"`")
			}

			if len(reqnames) == 0 {
				return nil
			}

			var msg string

			if len(reqnames) == 1 {
				msg = fmt.Sprintf("the required argument %s was not provided", reqnames[0])
			} else {
				msg = fmt.Sprintf("the required arguments %s and %s were not provided",
					strings.Join(reqnames[:len(reqnames)-1], ", "), reqnames[len(reqnames)-1])
			}

			p.err = newError(ErrRequired, msg)
			return p.err
		}

		return nil
	}

	names := make([]string, 0, len(required))

	for _, k := range required {
		names = append(names, "`"+k.String()+"'")
	}

	sort.Strings(names)

	var msg string

	if len(names) == 1 {
		msg = fmt.Sprintf("the required flag %s was not specified", names[0])
	} else {
		msg = fmt.Sprintf("the required flags %s and %s were not specified",
			strings.Join(names[:len(names)-1], ", "), names[len(names)-1])
	}

	p.err = newError(ErrRequired, msg)
	return p.err
}

func (p *parseState) estimateCommand() error {
	commands := p.command.sortedCommands()
	cmdnames := make([]string, len(commands))

	for i, v := range commands {
		cmdnames[i] = v.Name
	}

	var msg string
	var errtype ErrorType

	if len(p.retargs) != 0 {
		c, l := closestChoice(p.retargs[0], cmdnames)
		msg = fmt.Sprintf("Unknown command `%s'", p.retargs[0])
		errtype = ErrUnknownCommand

		if float32(l)/float32(len(c)) < 0.5 {
			msg = fmt.Sprintf("%s, did you mean `%s'?", msg, c)
		} else if len(cmdnames) == 1 {
			msg = fmt.Sprintf("%s. You should use the %s command",
				msg,
				cmdnames[0])
		} else {
			msg = fmt.Sprintf("%s. Please specify one command of: %s or %s",
				msg,
				strings.Join(cmdnames[:len(cmdnames)-1], ", "),
				cmdnames[len(cmdnames)-1])
		}
	} else {
		errtype = ErrCommandRequired

		if len(cmdnames) == 1 {
			msg = fmt.Sprintf("Please specify the %s command", cmdnames[0])
		} else {
			msg = fmt.Sprintf("Please specify one command of: %s or %s",
				strings.Join(cmdnames[:len(cmdnames)-1], ", "),
				cmdnames[len(cmdnames)-1])
		}
	}

	return newError(errtype, msg)
}

func (p *Parser) parseOption(s *parseState, name string, option *Option, canarg bool, argument *string) (err error) {
	if !option.canArgument() {
		if argument != nil {
			return newErrorf(ErrNoArgumentForBool, "bool flag `%s' cannot have an argument", option)
		}

		err = option.set(nil)
	} else if argument != nil || (canarg && !s.eof()) {
		var arg string

		if argument != nil {
			arg = *argument
		} else {
			arg = s.pop()

			// Accept any single character arguments including '-'.
			// '-' is the special file name for the standard input or the standard output in many cases.
			if len(arg) > 1 && argumentIsOption(arg) {
				return newErrorf(ErrExpectedArgument, "expected argument for flag `%s', but got option `%s'", option, arg)
			}
		}

		if option.tag.Get("unquote") != "false" {
			arg, err = unquoteIfPossible(arg)
		}

		if err == nil {
			err = option.set(&arg)
		}
	} else if option.OptionalArgument {
		option.empty()

		for _, v := range option.OptionalValue {
			err = option.set(&v)

			if err != nil {
				break
			}
		}
	} else {
		err = newErrorf(ErrExpectedArgument, "expected argument for flag `%s'", option)
	}

	if err != nil {
		if _, ok := err.(*Error); !ok {
			err = newErrorf(ErrMarshal, "invalid argument for flag `%s' (expected %s): %s",
				option,
				option.value.Type(),
				err.Error())
		}
	}

	return err
}

func (p *Parser) parseLong(s *parseState, name string, argument *string) error {
	if option := s.lookup.longNames[name]; option != nil {
		// Only long options that are required can consume an argument
		// from the argument list
		canarg := !option.OptionalArgument

		return p.parseOption(s, name, option, canarg, argument)
	}

	return newErrorf(ErrUnknownFlag, "unknown flag `%s'", name)
}

func (p *Parser) splitShortConcatArg(s *parseState, optname string) (string, *string) {
	c, n := utf8.DecodeRuneInString(optname)

	if n == len(optname) {
		return optname, nil
	}

	first := string(c)

	if option := s.lookup.shortNames[first]; option != nil && option.canArgument() {
		arg := optname[n:]
		return first, &arg
	}

	return optname, nil
}

func (p *Parser) parseShort(s *parseState, optname string, argument *string) error {
	if argument == nil {
		optname, argument = p.splitShortConcatArg(s, optname)
	}

	for i, c := range optname {
		shortname := string(c)

		if option := s.lookup.shortNames[shortname]; option != nil {
			// Only the last short argument can consume an argument from
			// the arguments list, and only if it's non optional
			canarg := (i+utf8.RuneLen(c) == len(optname)) && !option.OptionalArgument

			if err := p.parseOption(s, shortname, option, canarg, argument); err != nil {
				return err
			}
		} else {
			return newErrorf(ErrUnknownFlag, "unknown flag `%s'", shortname)
		}

		// Only the first option can have a concatted argument, so just
		// clear argument here
		argument = nil
	}

	return nil
}

func (p *parseState) addArgs(args ...string) error {
	for len(p.positional) > 0 && len(args) > 0 {
		arg := p.positional[0]

		if err := convert(args[0], arg.value, arg.tag); err != nil {
			return err
		}

		if !arg.isRemaining() {
			p.positional = p.positional[1:]
		}

		args = args[1:]
	}

	p.retargs = append(p.retargs, args...)
	return nil
}

func (p *Parser) parseNonOption(s *parseState) error {
	if len(s.positional) > 0 {
		return s.addArgs(s.arg)
	}

	if cmd := s.lookup.commands[s.arg]; cmd != nil {
		s.command.Active = cmd
		cmd.fillParseState(s)
	} else if (p.Options & PassAfterNonOption) != None {
		// If PassAfterNonOption is set then all remaining arguments
		// are considered positional
		if err := s.addArgs(s.arg); err != nil {
			return err
		}

		if err := s.addArgs(s.args...); err != nil {
			return err
		}

		s.args = []string{}
	} else {
		return s.addArgs(s.arg)
	}

	return nil
}

func (p *Parser) showBuiltinHelp() error {
	var b bytes.Buffer

	p.WriteHelp(&b)
	return newError(ErrHelp, b.String())
}

func (p *Parser) printError(err error) error {
	if err != nil && (p.Options&PrintErrors) != None {
		fmt.Fprintln(os.Stderr, err)
	}

	return err
}

func (p *Parser) clearIsSet() {
	p.eachCommand(func(c *Command) {
		c.eachGroup(func(g *Group) {
			for _, option := range g.options {
				option.isSet = false
			}
		})
	}, true)
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// syz-ci is a continuous fuzzing system for syzkaller.
// It runs several syz-manager's, polls and rebuilds images for managers
// and polls and rebuilds syzkaller binaries.
// For usage instructions see: docs/ci.md
package main

// Implementation details:
//
// 2 main components:
//  - SyzUpdater: handles syzkaller updates
//  - Manager: handles kernel build and syz-manager process (one per manager)
// Both operate in a similar way and keep 2 builds:
//  - latest: latest known good build (i.e. we tested it)
//    preserved across restarts/reboots, i.e. we can start fuzzing even when
//    current syzkaller/kernel git head is broken, or git is down, or anything else
//  - current: currently used build (a copy of one of the latest builds)
// Other important points:
//  - syz-ci is always built on the same revision as the rest of syzkaller binaries,
//    this allows us to handle e.g. changes in manager config format.
//  - consequently, syzkaller binaries are never updated on-the-fly,
//    instead we re-exec and then update
//  - we understand when the latest build is fresh even after reboot,
//    i.e. we store enough information to identify it (git hash, compiler identity, etc),
//    so we don't rebuild unnecessary (kernel builds take time)
//  - we generally avoid crashing the process and handle all errors gracefully
//    (this is a continuous system), except for some severe/user errors during start
//    (e.g. bad config file, or can't create necessary dirs)
//
// Directory/file structure:
// syz-ci			: current executable
// syzkaller/
//	latest/			: latest good syzkaller build
//	current/		: syzkaller build currently in use
// managers/
//	manager1/		: one dir per manager
//		kernel/		: kernel checkout
//		workdir/	: manager workdir (never deleted)
//		latest/		: latest good kernel image build
//		current/	: kernel image currently in use
// jobs/
//	linux/			: one dir per target OS
//		kernel/		: kernel checkout
//		image/		: currently used image
//		workdir/	: some temp files
//
// Current executable, syzkaller and kernel builds are marked with tag files.
// Tag files uniquely identify the build (git hash, compiler identity, kernel config, etc).
// For tag files both contents and modification time are important,
// modification time allows us to understand if we need to rebuild after a restart.

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"sync"

	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
)

var (
	flagConfig     = flag.String("config", "", "config file")
	flagAutoUpdate = flag.Bool("autoupdate", true, "auto-update the binary (for testing)")
	flagManagers   = flag.Bool("managers", true, "start managers (for testing)")
)

type Config struct {
	Name string `json:"name"`
	HTTP string `json:"http"`
	// If manager http address is not specified, give it an address starting from this port. Optional.
	ManagerPort     int    `json:"manager_port_start"`
	DashboardAddr   string `json:"dashboard_addr"`   // Optional.
	DashboardClient string `json:"dashboard_client"` // Optional.
	DashboardKey    string `json:"dashboard_key"`    // Optional.
	HubAddr         string `json:"hub_addr"`         // Optional.
	HubKey          string `json:"hub_key"`          // Optional.
	Goroot          string `json:"goroot"`           // Go 1.8+ toolchain dir.
	SyzkallerRepo   string `json:"syzkaller_repo"`
	SyzkallerBranch string `json:"syzkaller_branch"` // Defaults to "master".
	// Dir with additional syscall descriptions (.txt and .const files).
	SyzkallerDescriptions string `json:"syzkaller_descriptions"`
	// GCS path to upload coverage reports from managers (optional).
	CoverUploadPath string `json:"cover_upload_path"`
	// Enable patch testing jobs.
	EnableJobs   bool             `json:"enable_jobs"`
	BisectBinDir string           `json:"bisect_bin_dir"`
	Managers     []*ManagerConfig `json:"managers"`
}

type ManagerConfig struct {
	Name            string `json:"name"`
	DashboardClient string `json:"dashboard_client"`
	DashboardKey    string `json:"dashboard_key"`
	Repo            string `json:"repo"`
	// Short name of the repo (e.g. "linux-next"), used only for reporting.
	RepoAlias    string `json:"repo_alias"`
	Branch       string `json:"branch"` // Defaults to "master".
	Compiler     string `json:"compiler"`
	Userspace    string `json:"userspace"`
	KernelConfig string `json:"kernel_config"`
	// File with kernel cmdline values (optional).
	KernelCmdline string `json:"kernel_cmdline"`
	// File with sysctl values (e.g. output of sysctl -a, optional).
	KernelSysctl string `json:"kernel_sysctl"`
	PollCommits  bool   `json:"poll_commits"`
	Bisect       bool   `json:"bisect"`

	ManagerConfig json.RawMessage `json:"manager_config"`
	managercfg    *mgrconfig.Config
}

func main() {
	flag.Parse()
	log.EnableLogCaching(1000, 1<<20)
	cfg, err := loadConfig(*flagConfig)
	if err != nil {
		log.Fatalf("failed to load config: %v", err)
	}

	shutdownPending := make(chan struct{})
	osutil.HandleInterrupts(shutdownPending)

	serveHTTP(cfg)

	os.Unsetenv("GOPATH")
	if cfg.Goroot != "" {
		os.Setenv("GOROOT", cfg.Goroot)
		os.Setenv("PATH", filepath.Join(cfg.Goroot, "bin")+
			string(filepath.ListSeparator)+os.Getenv("PATH"))
	}

	updatePending := make(chan struct{})
	updater := NewSyzUpdater(cfg)
	updater.UpdateOnStart(*flagAutoUpdate, shutdownPending)
	if *flagAutoUpdate {
		go func() {
			updater.WaitForUpdate()
			close(updatePending)
		}()
	}

	var wg sync.WaitGroup
	wg.Add(1)
	stop := make(chan struct{})
	go func() {
		select {
		case <-shutdownPending:
		case <-updatePending:
		}
		kernelBuildSem <- struct{}{} // wait for all current builds
		close(stop)
		wg.Done()
	}()

	var managers []*Manager
	for _, mgrcfg := range cfg.Managers {
		mgr, err := createManager(cfg, mgrcfg, stop)
		if err != nil {
			log.Logf(0, "failed to create manager %v: %v", mgrcfg.Name, err)
			continue
		}
		managers = append(managers, mgr)
	}
	if len(managers) == 0 {
		log.Fatalf("failed to create all managers")
	}
	if *flagManagers {
		for _, mgr := range managers {
			mgr := mgr
			wg.Add(1)
			go func() {
				defer wg.Done()
				mgr.loop()
			}()
		}
	}

	jp := newJobProcessor(cfg, managers, stop, shutdownPending)
	wg.Add(1)
	go func() {
		defer wg.Done()
		jp.loop()
	}()

	// For testing. Racy. Use with care.
	http.HandleFunc("/upload_cover", func(w http.ResponseWriter, r *http.Request) {
		for _, mgr := range managers {
			if err := mgr.uploadCoverReport(); err != nil {
				w.Write([]byte(fmt.Sprintf("failed for %v: %v
\n", mgr.name, err)))
				return
			}
			w.Write([]byte(fmt.Sprintf("upload cover for %v
\n", mgr.name)))
		}
	})

	wg.Wait()

	select {
	case <-shutdownPending:
	case <-updatePending:
		updater.UpdateAndRestart()
	}
}

func serveHTTP(cfg *Config) {
	ln, err := net.Listen("tcp4", cfg.HTTP)
	if err != nil {
		log.Fatalf("failed to listen on %v: %v", cfg.HTTP, err)
	}
	log.Logf(0, "serving http on http://%v", ln.Addr())
	go func() {
		err := http.Serve(ln, nil)
		log.Fatalf("failed to serve http: %v", err)
	}()
}

func loadConfig(filename string) (*Config, error) {
	cfg := &Config{
		SyzkallerRepo:   "https://github.com/google/syzkaller.git",
		SyzkallerBranch: "master",
		ManagerPort:     10000,
		Goroot:          os.Getenv("GOROOT"),
	}
	if err := config.LoadFile(filename, cfg); err != nil {
		return nil, err
	}
	if cfg.Name == "" {
		return nil, fmt.Errorf("param 'name' is empty")
	}
	if cfg.HTTP == "" {
		return nil, fmt.Errorf("param 'http' is empty")
	}
	if len(cfg.Managers) == 0 {
		return nil, fmt.Errorf("no managers specified")
	}
	if cfg.EnableJobs && (cfg.DashboardAddr == "" || cfg.DashboardClient == "") {
		return nil, fmt.Errorf("enabled_jobs is set but no dashboard info")
	}
	if cfg.EnableJobs && cfg.BisectBinDir == "" {
		return nil, fmt.Errorf("enabled_jobs is set but no bisect_bin_dir")
	}
	for i, mgr := range cfg.Managers {
		if mgr.Name == "" {
			return nil, fmt.Errorf("param 'managers[%v].name' is empty", i)
		}
		if mgr.Branch == "" {
			mgr.Branch = "master"
		}
		managercfg, err := mgrconfig.LoadPartialData(mgr.ManagerConfig)
		if err != nil {
			return nil, fmt.Errorf("manager %v: %v", mgr.Name, err)
		}
		if mgr.PollCommits && (cfg.DashboardAddr == "" || mgr.DashboardClient == "") {
			return nil, fmt.Errorf("manager %v: commit_poll is set but no dashboard info", mgr.Name)
		}
		mgr.managercfg = managercfg
		managercfg.Name = cfg.Name + "-" + mgr.Name
		managercfg.Syzkaller = filepath.FromSlash("syzkaller/current")
		if managercfg.HTTP == "" {
			managercfg.HTTP = fmt.Sprintf(":%v", cfg.ManagerPort)
			cfg.ManagerPort++
		}
	}
	return cfg, nil
}
<commit_msg>syz-ci: stricter verification of instance names<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// syz-ci is a continuous fuzzing system for syzkaller.
// It runs several syz-manager's, polls and rebuilds images for managers
// and polls and rebuilds syzkaller binaries.
// For usage instructions see: docs/ci.md
package main

// Implementation details:
//
// 2 main components:
//  - SyzUpdater: handles syzkaller updates
//  - Manager: handles kernel build and syz-manager process (one per manager)
// Both operate in a similar way and keep 2 builds:
//  - latest: latest known good build (i.e. we tested it)
//    preserved across restarts/reboots, i.e. we can start fuzzing even when
//    current syzkaller/kernel git head is broken, or git is down, or anything else
//  - current: currently used build (a copy of one of the latest builds)
// Other important points:
//  - syz-ci is always built on the same revision as the rest of syzkaller binaries,
//    this allows us to handle e.g. changes in manager config format.
//  - consequently, syzkaller binaries are never updated on-the-fly,
//    instead we re-exec and then update
//  - we understand when the latest build is fresh even after reboot,
//    i.e. we store enough information to identify it (git hash, compiler identity, etc),
//    so we don't rebuild unnecessary (kernel builds take time)
//  - we generally avoid crashing the process and handle all errors gracefully
//    (this is a continuous system), except for some severe/user errors during start
//    (e.g. bad config file, or can't create necessary dirs)
//
// Directory/file structure:
// syz-ci			: current executable
// syzkaller/
//	latest/			: latest good syzkaller build
//	current/		: syzkaller build currently in use
// managers/
//	manager1/		: one dir per manager
//		kernel/		: kernel checkout
//		workdir/	: manager workdir (never deleted)
//		latest/		: latest good kernel image build
//		current/	: kernel image currently in use
// jobs/
//	linux/			: one dir per target OS
//		kernel/		: kernel checkout
//		image/		: currently used image
//		workdir/	: some temp files
//
// Current executable, syzkaller and kernel builds are marked with tag files.
// Tag files uniquely identify the build (git hash, compiler identity, kernel config, etc).
// For tag files both contents and modification time are important,
// modification time allows us to understand if we need to rebuild after a restart.

import (
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path/filepath"
	"regexp"
	"sync"

	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
)

var (
	flagConfig     = flag.String("config", "", "config file")
	flagAutoUpdate = flag.Bool("autoupdate", true, "auto-update the binary (for testing)")
	flagManagers   = flag.Bool("managers", true, "start managers (for testing)")
)

type Config struct {
	Name string `json:"name"`
	HTTP string `json:"http"`
	// If manager http address is not specified, give it an address starting from this port. Optional.
	ManagerPort     int    `json:"manager_port_start"`
	DashboardAddr   string `json:"dashboard_addr"`   // Optional.
	DashboardClient string `json:"dashboard_client"` // Optional.
	DashboardKey    string `json:"dashboard_key"`    // Optional.
	HubAddr         string `json:"hub_addr"`         // Optional.
	HubKey          string `json:"hub_key"`          // Optional.
	Goroot          string `json:"goroot"`           // Go 1.8+ toolchain dir.
	SyzkallerRepo   string `json:"syzkaller_repo"`
	SyzkallerBranch string `json:"syzkaller_branch"` // Defaults to "master".
	// Dir with additional syscall descriptions (.txt and .const files).
	SyzkallerDescriptions string `json:"syzkaller_descriptions"`
	// GCS path to upload coverage reports from managers (optional).
	CoverUploadPath string `json:"cover_upload_path"`
	// Enable patch testing jobs.
	EnableJobs   bool             `json:"enable_jobs"`
	BisectBinDir string           `json:"bisect_bin_dir"`
	Managers     []*ManagerConfig `json:"managers"`
}

type ManagerConfig struct {
	Name            string `json:"name"`
	DashboardClient string `json:"dashboard_client"`
	DashboardKey    string `json:"dashboard_key"`
	Repo            string `json:"repo"`
	// Short name of the repo (e.g. "linux-next"), used only for reporting.
	RepoAlias    string `json:"repo_alias"`
	Branch       string `json:"branch"` // Defaults to "master".
	Compiler     string `json:"compiler"`
	Userspace    string `json:"userspace"`
	KernelConfig string `json:"kernel_config"`
	// File with kernel cmdline values (optional).
	KernelCmdline string `json:"kernel_cmdline"`
	// File with sysctl values (e.g. output of sysctl -a, optional).
	KernelSysctl string `json:"kernel_sysctl"`
	PollCommits  bool   `json:"poll_commits"`
	Bisect       bool   `json:"bisect"`

	ManagerConfig json.RawMessage `json:"manager_config"`
	managercfg    *mgrconfig.Config
}

func main() {
	flag.Parse()
	log.EnableLogCaching(1000, 1<<20)
	cfg, err := loadConfig(*flagConfig)
	if err != nil {
		log.Fatalf("failed to load config: %v", err)
	}

	shutdownPending := make(chan struct{})
	osutil.HandleInterrupts(shutdownPending)

	serveHTTP(cfg)

	os.Unsetenv("GOPATH")
	if cfg.Goroot != "" {
		os.Setenv("GOROOT", cfg.Goroot)
		os.Setenv("PATH", filepath.Join(cfg.Goroot, "bin")+
			string(filepath.ListSeparator)+os.Getenv("PATH"))
	}

	updatePending := make(chan struct{})
	updater := NewSyzUpdater(cfg)
	updater.UpdateOnStart(*flagAutoUpdate, shutdownPending)
	if *flagAutoUpdate {
		go func() {
			updater.WaitForUpdate()
			close(updatePending)
		}()
	}

	var wg sync.WaitGroup
	wg.Add(1)
	stop := make(chan struct{})
	go func() {
		select {
		case <-shutdownPending:
		case <-updatePending:
		}
		kernelBuildSem <- struct{}{} // wait for all current builds
		close(stop)
		wg.Done()
	}()

	var managers []*Manager
	for _, mgrcfg := range cfg.Managers {
		mgr, err := createManager(cfg, mgrcfg, stop)
		if err != nil {
			log.Logf(0, "failed to create manager %v: %v", mgrcfg.Name, err)
			continue
		}
		managers = append(managers, mgr)
	}
	if len(managers) == 0 {
		log.Fatalf("failed to create all managers")
	}
	if *flagManagers {
		for _, mgr := range managers {
			mgr := mgr
			wg.Add(1)
			go func() {
				defer wg.Done()
				mgr.loop()
			}()
		}
	}

	jp := newJobProcessor(cfg, managers, stop, shutdownPending)
	wg.Add(1)
	go func() {
		defer wg.Done()
		jp.loop()
	}()

	// For testing. Racy. Use with care.
	http.HandleFunc("/upload_cover", func(w http.ResponseWriter, r *http.Request) {
		for _, mgr := range managers {
			if err := mgr.uploadCoverReport(); err != nil {
				w.Write([]byte(fmt.Sprintf("failed for %v: %v
\n", mgr.name, err)))
				return
			}
			w.Write([]byte(fmt.Sprintf("upload cover for %v
\n", mgr.name)))
		}
	})

	wg.Wait()

	select {
	case <-shutdownPending:
	case <-updatePending:
		updater.UpdateAndRestart()
	}
}

func serveHTTP(cfg *Config) {
	ln, err := net.Listen("tcp4", cfg.HTTP)
	if err != nil {
		log.Fatalf("failed to listen on %v: %v", cfg.HTTP, err)
	}
	log.Logf(0, "serving http on http://%v", ln.Addr())
	go func() {
		err := http.Serve(ln, nil)
		log.Fatalf("failed to serve http: %v", err)
	}()
}

func loadConfig(filename string) (*Config, error) {
	cfg := &Config{
		SyzkallerRepo:   "https://github.com/google/syzkaller.git",
		SyzkallerBranch: "master",
		ManagerPort:     10000,
		Goroot:          os.Getenv("GOROOT"),
	}
	if err := config.LoadFile(filename, cfg); err != nil {
		return nil, err
	}
	if cfg.Name == "" {
		return nil, fmt.Errorf("param 'name' is empty")
	}
	if cfg.HTTP == "" {
		return nil, fmt.Errorf("param 'http' is empty")
	}
	if len(cfg.Managers) == 0 {
		return nil, fmt.Errorf("no managers specified")
	}
	if cfg.EnableJobs && (cfg.DashboardAddr == "" || cfg.DashboardClient == "") {
		return nil, fmt.Errorf("enabled_jobs is set but no dashboard info")
	}
	if cfg.EnableJobs && cfg.BisectBinDir == "" {
		return nil, fmt.Errorf("enabled_jobs is set but no bisect_bin_dir")
	}
	// Manager name must not contain dots because it is used as GCE image name prefix.
	managerNameRe := regexp.MustCompile("^[a-zA-Z0-9-_]{4,64}$")
	for i, mgr := range cfg.Managers {
		if !managerNameRe.MatchString(mgr.Name) {
			return nil, fmt.Errorf("param 'managers[%v].name' has bad value: %q", i, mgr.Name)
		}
		if mgr.Branch == "" {
			mgr.Branch = "master"
		}
		managercfg, err := mgrconfig.LoadPartialData(mgr.ManagerConfig)
		if err != nil {
			return nil, fmt.Errorf("manager %v: %v", mgr.Name, err)
		}
		if mgr.PollCommits && (cfg.DashboardAddr == "" || mgr.DashboardClient == "") {
			return nil, fmt.Errorf("manager %v: commit_poll is set but no dashboard info", mgr.Name)
		}
		mgr.managercfg = managercfg
		managercfg.Name = cfg.Name + "-" + mgr.Name
		managercfg.Syzkaller = filepath.FromSlash("syzkaller/current")
		if managercfg.HTTP == "" {
			managercfg.HTTP = fmt.Sprintf(":%v", cfg.ManagerPort)
			cfg.ManagerPort++
		}
	}
	return cfg, nil
}
<|endoftext|>


<commit_before>// Exception handler like "parent.child.sub-child...sub-child".
//
// "A.B" is the parent exception is "A.B.C", "A.B.D" and "A.B.C.D", not "A.C".
//
package exception

import (
	"fmt"
	"strings"
)

var (
	SEPARATOR = "."
)

type Exception struct {
	_type string
	msg   string
}

func New(_type string, msg string) Exception {
	return Exception{_type: fixType(_type), msg: msg}
}

func fixType(_type string) string {
	_type = strings.TrimSpace(_type)
	return strings.Trim(_type, SEPARATOR)
}

func (e Exception) Error() string {
	return e.String()
}

func (e Exception) String() string {
	return fmt.Sprintf("Exception<%s>: %s", e._type, e.msg)
}

func (e Exception) GetType() string {
	return e._type
}

func (e Exception) GetMsg() string {
	return e.msg
}

func (e Exception) IsChildByString(parent string) bool {
	return IsChildByString(e._type, parent)
}

func (e Exception) IsChild(parent Exception) bool {
	return IsChildByString(e._type, parent._type)
}

func (e Exception) IsSame(other Exception) bool {
	return IsSame(e._type, other._type)
}

func (e Exception) GetTypeList() (list []string) {
	return getListBySep(e._type, SEPARATOR)
}

func getListBySep(s string, sep string) (list []string) {
	orig_list := strings.Split(s, sep)
	for _, t := range orig_list {
		if t = strings.TrimSpace(t); t != "" {
			list = append(list, t)
		}
	}
	return
}

func IsChildByString(child, parent string) bool {
	type1 := fixType(child)
	type2 := fixType(parent)

	// "aa.bb.cc" is not the child of "aa.bb.cc" or "aa.bb"
	if len(type1) <= len(type2) {
		return false
	}

	// "aa.bb.cc" is the child of "aa.bb" and "aa"
	return strings.Contains(type1, type2)
}

func IsSame(type1, type2 string) bool {
	type1 = fixType(type1)
	type2 = fixType(type2)

	if type1 == type2 {
		return true
	}
	return false
}

func IsChildByExc(child, parent Exception) bool {
	return IsChildByString(child._type, parent._type)
}
<commit_msg>Add the comment of the package exception<commit_after>// Package exception is a exception handler like "parent.child.sub-child...sub-child".
//
// "A.B" is the parent exception is "A.B.C", "A.B.D" and "A.B.C.D", not "A.C".
//
package exception

import (
	"fmt"
	"strings"
)

var (
	// SEPARATOR is the separator between the parent and the child exceptions.
	SEPARATOR = "."
)

// Exception is the exception handler.
type Exception struct {
	_type string
	msg   string
}

// New returns a new Exception.
func New(_type string, msg string) Exception {
	return Exception{_type: fixType(_type), msg: msg}
}

func fixType(_type string) string {
	_type = strings.TrimSpace(_type)
	return strings.Trim(_type, SEPARATOR)
}

// Error implements the error interface.
func (e Exception) Error() string {
	return e.String()
}

// String implements the interface fmt.Stringer.
func (e Exception) String() string {
	return fmt.Sprintf("Exception<%s>: %s", e._type, e.msg)
}

// GetType returns the exception type.
func (e Exception) GetType() string {
	return e._type
}

// GetMsg returns the exception message.
func (e Exception) GetMsg() string {
	return e.msg
}

// IsChildByString is equal to IsChildByString(e.GetType(), parent).
func (e Exception) IsChildByString(parent string) bool {
	return IsChildByString(e._type, parent)
}

// IsChild is equal to IsChildByString(e.GetType(), parent.GetType()).
func (e Exception) IsChild(parent Exception) bool {
	return IsChildByString(e._type, parent._type)
}

// IsSame is equal to IsSame(e.GetType(), other.GetType()).
func (e Exception) IsSame(other Exception) bool {
	return IsSame(e._type, other._type)
}

// GetTypeList returns a list of the exception types.
//
// For example, GetTypeList() on the exception type "e1.e2.e3" returns ["e1", "e2", "e3"].
func (e Exception) GetTypeList() (list []string) {
	return getListBySep(e._type, SEPARATOR)
}

func getListBySep(s string, sep string) (list []string) {
	origList := strings.Split(s, sep)
	for _, t := range origList {
		if t = strings.TrimSpace(t); t != "" {
			list = append(list, t)
		}
	}
	return
}

// IsChildByString check whether child is sub-exception of parent or not.
//
// Return true if yes, or false.
func IsChildByString(child, parent string) bool {
	type1 := fixType(child)
	type2 := fixType(parent)

	// "aa.bb.cc" is not the child of "aa.bb.cc" or "aa.bb"
	if len(type1) <= len(type2) {
		return false
	}

	// "aa.bb.cc" is the child of "aa.bb" and "aa"
	return strings.Contains(type1, type2)
}

// IsSame check whether two exception types is the same.
//
// Return ture if yes, or false.
func IsSame(type1, type2 string) bool {
	type1 = fixType(type1)
	type2 = fixType(type2)

	if type1 == type2 {
		return true
	}
	return false
}

// IsChildByExc is same as IsChildByString, but the type is Exception, not string.
func IsChildByExc(child, parent Exception) bool {
	return IsChildByString(child._type, parent._type)
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/hashicorp/hil/ast"
)

// An InterpolatedVariable is a variable reference within an interpolation.
//
// Implementations of this interface represents various sources where
// variables can come from: user variables, resources, etc.
type InterpolatedVariable interface {
	FullKey() string
}

// CountVariable is a variable for referencing information about
// the count.
type CountVariable struct {
	Type CountValueType
	key  string
}

// CountValueType is the type of the count variable that is referenced.
type CountValueType byte

const (
	CountValueInvalid CountValueType = iota
	CountValueIndex
)

// A ModuleVariable is a variable that is referencing the output
// of a module, such as "${module.foo.bar}"
type ModuleVariable struct {
	Name  string
	Field string
	key   string
}

// A PathVariable is a variable that references path information about the
// module.
type PathVariable struct {
	Type PathValueType
	key  string
}

type PathValueType byte

const (
	PathValueInvalid PathValueType = iota
	PathValueCwd
	PathValueModule
	PathValueRoot
)

// A ResourceVariable is a variable that is referencing the field
// of a resource, such as "${aws_instance.foo.ami}"
type ResourceVariable struct {
	Mode  ResourceMode
	Type  string // Resource type, i.e. "aws_instance"
	Name  string // Resource name
	Field string // Resource field

	Multi bool // True if multi-variable: aws_instance.foo.*.id
	Index int  // Index for multi-variable: aws_instance.foo.1.id == 1

	key string
}

// SelfVariable is a variable that is referencing the same resource
// it is running on: "${self.address}"
type SelfVariable struct {
	Field string

	key string
}

// SimpleVariable is an unprefixed variable, which can show up when users have
// strings they are passing down to resources that use interpolation
// internally. The template_file resource is an example of this.
type SimpleVariable struct {
	Key string
}

// TerraformVariable is a "terraform."-prefixed variable used to access
// metadata about the Terraform run.
type TerraformVariable struct {
	Field string
	key   string
}

// A UserVariable is a variable that is referencing a user variable
// that is inputted from outside the configuration. This looks like
// "${var.foo}"
type UserVariable struct {
	Name string
	Elem string

	key string
}

// A LocalVariable is a variable that references a local value defined within
// the current module, via a "locals" block. This looks like "${local.foo}".
type LocalVariable struct {
	Name string
}

func NewInterpolatedVariable(v string) (InterpolatedVariable, error) {
	if strings.HasPrefix(v, "count.") {
		return NewCountVariable(v)
	} else if strings.HasPrefix(v, "path.") {
		return NewPathVariable(v)
	} else if strings.HasPrefix(v, "self.") {
		return NewSelfVariable(v)
	} else if strings.HasPrefix(v, "terraform.") {
		return NewTerraformVariable(v)
	} else if strings.HasPrefix(v, "var.") {
		return NewUserVariable(v)
	} else if strings.HasPrefix(v, "local.") {
		return NewLocalVariable(v)
	} else if strings.HasPrefix(v, "module.") {
		return NewModuleVariable(v)
	} else if !strings.ContainsRune(v, '.') {
		return NewSimpleVariable(v)
	} else {
		return NewResourceVariable(v)
	}
}

func NewCountVariable(key string) (*CountVariable, error) {
	var fieldType CountValueType
	parts := strings.SplitN(key, ".", 2)
	switch parts[1] {
	case "index":
		fieldType = CountValueIndex
	}

	return &CountVariable{
		Type: fieldType,
		key:  key,
	}, nil
}

func (c *CountVariable) FullKey() string {
	return c.key
}

func NewModuleVariable(key string) (*ModuleVariable, error) {
	parts := strings.SplitN(key, ".", 3)
	if len(parts) < 3 {
		return nil, fmt.Errorf(
			"%s: module variables must be three parts: module.name.attr",
			key)
	}

	return &ModuleVariable{
		Name:  parts[1],
		Field: parts[2],
		key:   key,
	}, nil
}

func (v *ModuleVariable) FullKey() string {
	return v.key
}

func (v *ModuleVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewPathVariable(key string) (*PathVariable, error) {
	var fieldType PathValueType
	parts := strings.SplitN(key, ".", 2)
	switch parts[1] {
	case "cwd":
		fieldType = PathValueCwd
	case "module":
		fieldType = PathValueModule
	case "root":
		fieldType = PathValueRoot
	}

	return &PathVariable{
		Type: fieldType,
		key:  key,
	}, nil
}

func (v *PathVariable) FullKey() string {
	return v.key
}

func NewResourceVariable(key string) (*ResourceVariable, error) {
	var mode ResourceMode
	var parts []string
	if strings.HasPrefix(key, "data.") {
		mode = DataResourceMode
		parts = strings.SplitN(key, ".", 4)
		if len(parts) < 4 {
			return nil, fmt.Errorf(
				"%s: data variables must be four parts: data.TYPE.NAME.ATTR",
				key)
		}

		// Don't actually need the "data." prefix for parsing, since it's
		// always constant.
		parts = parts[1:]
	} else {
		mode = ManagedResourceMode
		parts = strings.SplitN(key, ".", 3)
		if len(parts) < 3 {
			return nil, fmt.Errorf(
				"%s: resource variables must be three parts: TYPE.NAME.ATTR",
				key)
		}
	}

	field := parts[2]
	multi := false
	var index int

	if idx := strings.Index(field, "."); idx != -1 {
		indexStr := field[:idx]
		multi = indexStr == "*"
		index = -1

		if !multi {
			indexInt, err := strconv.ParseInt(indexStr, 0, 0)
			if err == nil {
				multi = true
				index = int(indexInt)
			}
		}

		if multi {
			field = field[idx+1:]
		}
	}

	return &ResourceVariable{
		Mode:  mode,
		Type:  parts[0],
		Name:  parts[1],
		Field: field,
		Multi: multi,
		Index: index,
		key:   key,
	}, nil
}

func (v *ResourceVariable) ResourceId() string {
	switch v.Mode {
	case ManagedResourceMode:
		return fmt.Sprintf("%s.%s", v.Type, v.Name)
	case DataResourceMode:
		return fmt.Sprintf("data.%s.%s", v.Type, v.Name)
	default:
		panic(fmt.Errorf("unknown resource mode %s", v.Mode))
	}
}

func (v *ResourceVariable) FullKey() string {
	return v.key
}

func NewSelfVariable(key string) (*SelfVariable, error) {
	field := key[len("self."):]

	return &SelfVariable{
		Field: field,

		key: key,
	}, nil
}

func (v *SelfVariable) FullKey() string {
	return v.key
}

func (v *SelfVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewSimpleVariable(key string) (*SimpleVariable, error) {
	return &SimpleVariable{key}, nil
}

func (v *SimpleVariable) FullKey() string {
	return v.Key
}

func (v *SimpleVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewTerraformVariable(key string) (*TerraformVariable, error) {
	field := key[len("terraform."):]
	return &TerraformVariable{
		Field: field,
		key:   key,
	}, nil
}

func (v *TerraformVariable) FullKey() string {
	return v.key
}

func (v *TerraformVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewUserVariable(key string) (*UserVariable, error) {
	name := key[len("var."):]
	elem := ""
	if idx := strings.Index(name, "."); idx > -1 {
		elem = name[idx+1:]
		name = name[:idx]
	}

	if len(elem) > 0 {
		return nil, fmt.Errorf("Invalid dot index found: 'var.%s.%s'. Values in maps and lists can be referenced using square bracket indexing, like: 'var.mymap[\"key\"]' or 'var.mylist[1]'.", name, elem)
	}

	return &UserVariable{
		key: key,

		Name: name,
		Elem: elem,
	}, nil
}

func (v *UserVariable) FullKey() string {
	return v.key
}

func (v *UserVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewLocalVariable(key string) (*LocalVariable, error) {
	name := key[len("local."):]
	if idx := strings.Index(name, "."); idx > -1 {
		return nil, fmt.Errorf("Can't use dot (.) attribute access in local.%s; use square bracket indexing", name)
	}

	return &LocalVariable{
		Name: name,
	}, nil
}

func (v *LocalVariable) FullKey() string {
	return fmt.Sprintf("local.%s", v.Name)
}

func (v *LocalVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

// DetectVariables takes an AST root and returns all the interpolated
// variables that are detected in the AST tree.
func DetectVariables(root ast.Node) ([]InterpolatedVariable, error) {
	var result []InterpolatedVariable
	var resultErr error

	// Visitor callback
	fn := func(n ast.Node) ast.Node {
		if resultErr != nil {
			return n
		}

		switch vn := n.(type) {
		case *ast.VariableAccess:
			v, err := NewInterpolatedVariable(vn.Name)
			if err != nil {
				resultErr = err
				return n
			}
			result = append(result, v)
		case *ast.Index:
			if va, ok := vn.Target.(*ast.VariableAccess); ok {
				v, err := NewInterpolatedVariable(va.Name)
				if err != nil {
					resultErr = err
					return n
				}
				result = append(result, v)
			}
			if va, ok := vn.Key.(*ast.VariableAccess); ok {
				v, err := NewInterpolatedVariable(va.Name)
				if err != nil {
					resultErr = err
					return n
				}
				result = append(result, v)
			}
		default:
			return n
		}

		return n
	}

	// Visitor pattern
	root.Accept(fn)

	if resultErr != nil {
		return nil, resultErr
	}

	return result, nil
}
<commit_msg>config: source code ranges for InterpolatedVariable<commit_after>package config

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/tfdiags"

	"github.com/hashicorp/hil/ast"
)

// An InterpolatedVariable is a variable reference within an interpolation.
//
// Implementations of this interface represents various sources where
// variables can come from: user variables, resources, etc.
type InterpolatedVariable interface {
	FullKey() string
	SourceRange() tfdiags.SourceRange
}

// varRange can be embedded into an InterpolatedVariable implementation to
// implement the SourceRange method.
type varRange struct {
	rng tfdiags.SourceRange
}

func (r varRange) SourceRange() tfdiags.SourceRange {
	return r.rng
}

func makeVarRange(rng tfdiags.SourceRange) varRange {
	return varRange{rng}
}

// CountVariable is a variable for referencing information about
// the count.
type CountVariable struct {
	Type CountValueType
	key  string
	varRange
}

// CountValueType is the type of the count variable that is referenced.
type CountValueType byte

const (
	CountValueInvalid CountValueType = iota
	CountValueIndex
)

// A ModuleVariable is a variable that is referencing the output
// of a module, such as "${module.foo.bar}"
type ModuleVariable struct {
	Name  string
	Field string
	key   string
	varRange
}

// A PathVariable is a variable that references path information about the
// module.
type PathVariable struct {
	Type PathValueType
	key  string
	varRange
}

type PathValueType byte

const (
	PathValueInvalid PathValueType = iota
	PathValueCwd
	PathValueModule
	PathValueRoot
)

// A ResourceVariable is a variable that is referencing the field
// of a resource, such as "${aws_instance.foo.ami}"
type ResourceVariable struct {
	Mode  ResourceMode
	Type  string // Resource type, i.e. "aws_instance"
	Name  string // Resource name
	Field string // Resource field

	Multi bool // True if multi-variable: aws_instance.foo.*.id
	Index int  // Index for multi-variable: aws_instance.foo.1.id == 1

	key string
	varRange
}

// SelfVariable is a variable that is referencing the same resource
// it is running on: "${self.address}"
type SelfVariable struct {
	Field string

	key string
	varRange
}

// SimpleVariable is an unprefixed variable, which can show up when users have
// strings they are passing down to resources that use interpolation
// internally. The template_file resource is an example of this.
type SimpleVariable struct {
	Key string
	varRange
}

// TerraformVariable is a "terraform."-prefixed variable used to access
// metadata about the Terraform run.
type TerraformVariable struct {
	Field string
	key   string
	varRange
}

// A UserVariable is a variable that is referencing a user variable
// that is inputted from outside the configuration. This looks like
// "${var.foo}"
type UserVariable struct {
	Name string
	Elem string

	key string
	varRange
}

// A LocalVariable is a variable that references a local value defined within
// the current module, via a "locals" block. This looks like "${local.foo}".
type LocalVariable struct {
	Name string
	varRange
}

func NewInterpolatedVariable(v string) (InterpolatedVariable, error) {
	if strings.HasPrefix(v, "count.") {
		return NewCountVariable(v)
	} else if strings.HasPrefix(v, "path.") {
		return NewPathVariable(v)
	} else if strings.HasPrefix(v, "self.") {
		return NewSelfVariable(v)
	} else if strings.HasPrefix(v, "terraform.") {
		return NewTerraformVariable(v)
	} else if strings.HasPrefix(v, "var.") {
		return NewUserVariable(v)
	} else if strings.HasPrefix(v, "local.") {
		return NewLocalVariable(v)
	} else if strings.HasPrefix(v, "module.") {
		return NewModuleVariable(v)
	} else if !strings.ContainsRune(v, '.') {
		return NewSimpleVariable(v)
	} else {
		return NewResourceVariable(v)
	}
}

func NewCountVariable(key string) (*CountVariable, error) {
	var fieldType CountValueType
	parts := strings.SplitN(key, ".", 2)
	switch parts[1] {
	case "index":
		fieldType = CountValueIndex
	}

	return &CountVariable{
		Type: fieldType,
		key:  key,
	}, nil
}

func (c *CountVariable) FullKey() string {
	return c.key
}

func NewModuleVariable(key string) (*ModuleVariable, error) {
	parts := strings.SplitN(key, ".", 3)
	if len(parts) < 3 {
		return nil, fmt.Errorf(
			"%s: module variables must be three parts: module.name.attr",
			key)
	}

	return &ModuleVariable{
		Name:  parts[1],
		Field: parts[2],
		key:   key,
	}, nil
}

func (v *ModuleVariable) FullKey() string {
	return v.key
}

func (v *ModuleVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewPathVariable(key string) (*PathVariable, error) {
	var fieldType PathValueType
	parts := strings.SplitN(key, ".", 2)
	switch parts[1] {
	case "cwd":
		fieldType = PathValueCwd
	case "module":
		fieldType = PathValueModule
	case "root":
		fieldType = PathValueRoot
	}

	return &PathVariable{
		Type: fieldType,
		key:  key,
	}, nil
}

func (v *PathVariable) FullKey() string {
	return v.key
}

func NewResourceVariable(key string) (*ResourceVariable, error) {
	var mode ResourceMode
	var parts []string
	if strings.HasPrefix(key, "data.") {
		mode = DataResourceMode
		parts = strings.SplitN(key, ".", 4)
		if len(parts) < 4 {
			return nil, fmt.Errorf(
				"%s: data variables must be four parts: data.TYPE.NAME.ATTR",
				key)
		}

		// Don't actually need the "data." prefix for parsing, since it's
		// always constant.
		parts = parts[1:]
	} else {
		mode = ManagedResourceMode
		parts = strings.SplitN(key, ".", 3)
		if len(parts) < 3 {
			return nil, fmt.Errorf(
				"%s: resource variables must be three parts: TYPE.NAME.ATTR",
				key)
		}
	}

	field := parts[2]
	multi := false
	var index int

	if idx := strings.Index(field, "."); idx != -1 {
		indexStr := field[:idx]
		multi = indexStr == "*"
		index = -1

		if !multi {
			indexInt, err := strconv.ParseInt(indexStr, 0, 0)
			if err == nil {
				multi = true
				index = int(indexInt)
			}
		}

		if multi {
			field = field[idx+1:]
		}
	}

	return &ResourceVariable{
		Mode:  mode,
		Type:  parts[0],
		Name:  parts[1],
		Field: field,
		Multi: multi,
		Index: index,
		key:   key,
	}, nil
}

func (v *ResourceVariable) ResourceId() string {
	switch v.Mode {
	case ManagedResourceMode:
		return fmt.Sprintf("%s.%s", v.Type, v.Name)
	case DataResourceMode:
		return fmt.Sprintf("data.%s.%s", v.Type, v.Name)
	default:
		panic(fmt.Errorf("unknown resource mode %s", v.Mode))
	}
}

func (v *ResourceVariable) FullKey() string {
	return v.key
}

func NewSelfVariable(key string) (*SelfVariable, error) {
	field := key[len("self."):]

	return &SelfVariable{
		Field: field,

		key: key,
	}, nil
}

func (v *SelfVariable) FullKey() string {
	return v.key
}

func (v *SelfVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewSimpleVariable(key string) (*SimpleVariable, error) {
	return &SimpleVariable{Key: key}, nil
}

func (v *SimpleVariable) FullKey() string {
	return v.Key
}

func (v *SimpleVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewTerraformVariable(key string) (*TerraformVariable, error) {
	field := key[len("terraform."):]
	return &TerraformVariable{
		Field: field,
		key:   key,
	}, nil
}

func (v *TerraformVariable) FullKey() string {
	return v.key
}

func (v *TerraformVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewUserVariable(key string) (*UserVariable, error) {
	name := key[len("var."):]
	elem := ""
	if idx := strings.Index(name, "."); idx > -1 {
		elem = name[idx+1:]
		name = name[:idx]
	}

	if len(elem) > 0 {
		return nil, fmt.Errorf("Invalid dot index found: 'var.%s.%s'. Values in maps and lists can be referenced using square bracket indexing, like: 'var.mymap[\"key\"]' or 'var.mylist[1]'.", name, elem)
	}

	return &UserVariable{
		key: key,

		Name: name,
		Elem: elem,
	}, nil
}

func (v *UserVariable) FullKey() string {
	return v.key
}

func (v *UserVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

func NewLocalVariable(key string) (*LocalVariable, error) {
	name := key[len("local."):]
	if idx := strings.Index(name, "."); idx > -1 {
		return nil, fmt.Errorf("Can't use dot (.) attribute access in local.%s; use square bracket indexing", name)
	}

	return &LocalVariable{
		Name: name,
	}, nil
}

func (v *LocalVariable) FullKey() string {
	return fmt.Sprintf("local.%s", v.Name)
}

func (v *LocalVariable) GoString() string {
	return fmt.Sprintf("*%#v", *v)
}

// DetectVariables takes an AST root and returns all the interpolated
// variables that are detected in the AST tree.
func DetectVariables(root ast.Node) ([]InterpolatedVariable, error) {
	var result []InterpolatedVariable
	var resultErr error

	// Visitor callback
	fn := func(n ast.Node) ast.Node {
		if resultErr != nil {
			return n
		}

		switch vn := n.(type) {
		case *ast.VariableAccess:
			v, err := NewInterpolatedVariable(vn.Name)
			if err != nil {
				resultErr = err
				return n
			}
			result = append(result, v)
		case *ast.Index:
			if va, ok := vn.Target.(*ast.VariableAccess); ok {
				v, err := NewInterpolatedVariable(va.Name)
				if err != nil {
					resultErr = err
					return n
				}
				result = append(result, v)
			}
			if va, ok := vn.Key.(*ast.VariableAccess); ok {
				v, err := NewInterpolatedVariable(va.Name)
				if err != nil {
					resultErr = err
					return n
				}
				result = append(result, v)
			}
		default:
			return n
		}

		return n
	}

	// Visitor pattern
	root.Accept(fn)

	if resultErr != nil {
		return nil, resultErr
	}

	return result, nil
}
<|endoftext|>


<commit_before>package tagger

import (
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io/ioutil"
	"runtime"
	"sort"
	"time"
	"unsafe"

	"github.com/uber-go/zap"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/helper/RowBinary"
	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
)

func unsafeString(b []byte) string {
	return *(*string)(unsafe.Pointer(&b))
}

func countMetrics(body []byte) (int, error) {
	var namelen uint64
	bodyLen := len(body)
	var count, offset, readBytes int
	var err error

	for {
		if offset >= bodyLen {
			if offset == bodyLen {
				return count, nil
			}
			return 0, clickhouse.ErrClickHouseResponse
		}

		namelen, readBytes, err = clickhouse.ReadUvarint(body[offset:])
		if err != nil {
			return 0, err
		}
		offset += readBytes + int(namelen)
		count++
	}

	return 0, nil
}

func pathLevel(path []byte) int {
	if path == nil || len(path) == 0 {
		return 0
	}

	if path[len(path)-1] == '.' {
		return bytes.Count(path, []byte{'.'})
	}

	return bytes.Count(path, []byte{'.'}) + 1
}

func Make(cfg *config.Config, logger zap.Logger) error {
	var start time.Time
	var block string
	begin := func(b string) {
		block = b
		start = time.Now()
		logger.Info(block)
	}

	end := func() {
		var m runtime.MemStats
		runtime.ReadMemStats(&m)
		d := time.Since(start)
		logger.Info(block,
			zap.String("time", d.String()),
			zap.Duration("time_ns", d),
			zap.Uint64("mem_rss_mb", (m.Sys-m.HeapReleased)/1048576),
		)
	}

	version := uint32(time.Now().Unix())

	// Parse rules
	begin("parse rules")
	rules, err := ParseFile(cfg.Tags.Rules)
	if err != nil {
		return err
	}

	date, err := time.ParseInLocation("2006-01-02", cfg.Tags.Date, time.Local)
	if err != nil {
		return err
	}
	end()

	// Read clickhouse
	begin("read and parse tree")
	var body []byte

	if cfg.Tags.InputFile != "" {
		body, err = ioutil.ReadFile(cfg.Tags.InputFile)
		if err != nil {
			return err
		}
	} else {
		body, err = clickhouse.Query(
			context.WithValue(context.Background(), "logger", logger),
			cfg.ClickHouse.Url,
			fmt.Sprintf("SELECT Path FROM %s GROUP BY Path FORMAT RowBinary", cfg.ClickHouse.TreeTable),
			cfg.ClickHouse.TreeTimeout.Value(),
		)
	}

	if err != nil {
		return err
	}

	count, err := countMetrics(body)
	if err != nil {
		return err
	}

	metricList := make([]Metric, count)

	var namelen uint64
	bodyLen := len(body)
	var offset, readBytes int
	var maxLevel int

	for index := 0; ; index++ {
		if offset >= bodyLen {
			if offset == bodyLen {
				break
			}
			return clickhouse.ErrClickHouseResponse
		}

		namelen, readBytes, err = clickhouse.ReadUvarint(body[offset:])
		if err != nil {
			return err
		}

		metricList[index].Path = body[offset+readBytes : offset+readBytes+int(namelen)]
		metricList[index].Level = pathLevel(metricList[index].Path)

		if metricList[index].Level > maxLevel {
			maxLevel = metricList[index].Level
		}

		offset += readBytes + int(namelen)
	}
	end()

	begin("sort")
	start = time.Now()
	sort.Sort(ByPath(metricList))
	end()

	begin("make map")
	levelMap := make([]int, maxLevel+1)
	for index := 0; index < len(metricList); index++ {
		m := &metricList[index]
		levelMap[m.Level] = index

		if m.Level > 0 {
			parentIndex := levelMap[m.Level-1]
			if bytes.Equal(m.ParentPath(), metricList[parentIndex].Path) {
				m.ParentIndex = parentIndex
			} else {
				m.ParentIndex = -1
			}
		}
	}
	end()

	begin("match")
	for index := 0; index < count; index++ {
		m := &metricList[index]

		if m.ParentIndex < 0 {
			m.Tags = EmptySet
		} else {
			m.Tags = metricList[m.ParentIndex].Tags
		}

		rules.Match(m)
	}
	end()

	// copy from childs to parents
	begin("copy tags from childs to parents")
	for index := 0; index < count; index++ {
		m := &metricList[index]

		for p := m.ParentIndex; p >= 0; p = metricList[p].ParentIndex {
			metricList[p].Tags = metricList[p].Tags.Merge(m.Tags)
		}
	}
	end()

	begin("marshal RowBinary + gzip")
	// INSERT INTO graphite_tag (Date,Version,Level,Path,IsLeaf,Tags,Tag1) FORMAT RowBinary
	// with Content-Encoding: gzip

	outBuf := new(bytes.Buffer)
	writer := gzip.NewWriter(outBuf)

	encoder := RowBinary.NewEncoder(writer)

	days := RowBinary.DateToUint16(date)
	metricBuffer := new(bytes.Buffer)
	metricEncoder := RowBinary.NewEncoder(metricBuffer)

	for i := 0; i < len(metricList); i++ {
		m := &metricList[i]

		if m.Tags == nil || m.Tags.Len() == 0 {
			continue
		}

		metricBuffer.Reset()

		// Date
		err := metricEncoder.Uint16(days)
		if err != nil {
			return err
		}
		// Version
		err = metricEncoder.Uint32(version)
		if err != nil {
			return err
		}
		// Level
		err = metricEncoder.Uint32(uint32(m.Level))
		if err != nil {
			return err
		}
		// Path
		err = metricEncoder.Bytes(m.Path)
		if err != nil {
			return err
		}
		// IsLeaf
		err = metricEncoder.Uint8(m.IsLeaf())
		if err != nil {
			return err
		}
		// Tags
		err = metricEncoder.StringList(m.Tags.List())
		if err != nil {
			return err
		}

		for _, tag := range m.Tags.List() {
			_, err = writer.Write(metricBuffer.Bytes())
			if err != nil {
				return err
			}

			// Tag1
			err = encoder.String(tag)
			if err != nil {
				return err
			}
		}
	}

	writer.Close()
	end()

	if cfg.Tags.OutputFile != "" {
		begin(fmt.Sprintf("write to %#v", cfg.Tags.OutputFile))
		ioutil.WriteFile(cfg.Tags.OutputFile, outBuf.Bytes(), 0644)
		end()
	} else {
		begin("upload to clickhouse")
		_, err = clickhouse.PostGzip(
			context.WithValue(context.Background(), "logger", logger),
			cfg.ClickHouse.Url,
			fmt.Sprintf("INSERT INTO %s (Date,Version,Level,Path,IsLeaf,Tags,Tag1) FORMAT RowBinary", cfg.ClickHouse.TagTable),
			outBuf,
			cfg.ClickHouse.TreeTimeout.Value(),
		)
		if err != nil {
			return err
		}
		end()
	}

	return nil
}
<commit_msg>empty record with last version<commit_after>package tagger

import (
	"bytes"
	"compress/gzip"
	"context"
	"fmt"
	"io/ioutil"
	"runtime"
	"sort"
	"time"
	"unsafe"

	"github.com/uber-go/zap"

	"github.com/lomik/graphite-clickhouse/config"
	"github.com/lomik/graphite-clickhouse/helper/RowBinary"
	"github.com/lomik/graphite-clickhouse/helper/clickhouse"
)

func unsafeString(b []byte) string {
	return *(*string)(unsafe.Pointer(&b))
}

func countMetrics(body []byte) (int, error) {
	var namelen uint64
	bodyLen := len(body)
	var count, offset, readBytes int
	var err error

	for {
		if offset >= bodyLen {
			if offset == bodyLen {
				return count, nil
			}
			return 0, clickhouse.ErrClickHouseResponse
		}

		namelen, readBytes, err = clickhouse.ReadUvarint(body[offset:])
		if err != nil {
			return 0, err
		}
		offset += readBytes + int(namelen)
		count++
	}

	return 0, nil
}

func pathLevel(path []byte) int {
	if path == nil || len(path) == 0 {
		return 0
	}

	if path[len(path)-1] == '.' {
		return bytes.Count(path, []byte{'.'})
	}

	return bytes.Count(path, []byte{'.'}) + 1
}

func Make(cfg *config.Config, logger zap.Logger) error {
	var start time.Time
	var block string
	begin := func(b string) {
		block = b
		start = time.Now()
		logger.Info(block)
	}

	end := func() {
		var m runtime.MemStats
		runtime.ReadMemStats(&m)
		d := time.Since(start)
		logger.Info(block,
			zap.String("time", d.String()),
			zap.Duration("time_ns", d),
			zap.Uint64("mem_rss_mb", (m.Sys-m.HeapReleased)/1048576),
		)
	}

	version := uint32(time.Now().Unix())

	// Parse rules
	begin("parse rules")
	rules, err := ParseFile(cfg.Tags.Rules)
	if err != nil {
		return err
	}

	date, err := time.ParseInLocation("2006-01-02", cfg.Tags.Date, time.Local)
	if err != nil {
		return err
	}
	end()

	// Read clickhouse
	begin("read and parse tree")
	var body []byte

	if cfg.Tags.InputFile != "" {
		body, err = ioutil.ReadFile(cfg.Tags.InputFile)
		if err != nil {
			return err
		}
	} else {
		body, err = clickhouse.Query(
			context.WithValue(context.Background(), "logger", logger),
			cfg.ClickHouse.Url,
			fmt.Sprintf("SELECT Path FROM %s GROUP BY Path FORMAT RowBinary", cfg.ClickHouse.TreeTable),
			cfg.ClickHouse.TreeTimeout.Value(),
		)
	}

	if err != nil {
		return err
	}

	count, err := countMetrics(body)
	if err != nil {
		return err
	}

	metricList := make([]Metric, count)

	var namelen uint64
	bodyLen := len(body)
	var offset, readBytes int
	var maxLevel int

	for index := 0; ; index++ {
		if offset >= bodyLen {
			if offset == bodyLen {
				break
			}
			return clickhouse.ErrClickHouseResponse
		}

		namelen, readBytes, err = clickhouse.ReadUvarint(body[offset:])
		if err != nil {
			return err
		}

		metricList[index].Path = body[offset+readBytes : offset+readBytes+int(namelen)]
		metricList[index].Level = pathLevel(metricList[index].Path)

		if metricList[index].Level > maxLevel {
			maxLevel = metricList[index].Level
		}

		offset += readBytes + int(namelen)
	}
	end()

	begin("sort")
	start = time.Now()
	sort.Sort(ByPath(metricList))
	end()

	begin("make map")
	levelMap := make([]int, maxLevel+1)
	for index := 0; index < len(metricList); index++ {
		m := &metricList[index]
		levelMap[m.Level] = index

		if m.Level > 0 {
			parentIndex := levelMap[m.Level-1]
			if bytes.Equal(m.ParentPath(), metricList[parentIndex].Path) {
				m.ParentIndex = parentIndex
			} else {
				m.ParentIndex = -1
			}
		}
	}
	end()

	begin("match")
	for index := 0; index < count; index++ {
		m := &metricList[index]

		if m.ParentIndex < 0 {
			m.Tags = EmptySet
		} else {
			m.Tags = metricList[m.ParentIndex].Tags
		}

		rules.Match(m)
	}
	end()

	// copy from childs to parents
	begin("copy tags from childs to parents")
	for index := 0; index < count; index++ {
		m := &metricList[index]

		for p := m.ParentIndex; p >= 0; p = metricList[p].ParentIndex {
			metricList[p].Tags = metricList[p].Tags.Merge(m.Tags)
		}
	}
	end()

	begin("marshal RowBinary + gzip")
	// INSERT INTO graphite_tag (Date,Version,Level,Path,IsLeaf,Tags,Tag1) FORMAT RowBinary
	// with Content-Encoding: gzip

	outBuf := new(bytes.Buffer)
	writer := gzip.NewWriter(outBuf)

	encoder := RowBinary.NewEncoder(writer)

	days := RowBinary.DateToUint16(date)
	metricBuffer := new(bytes.Buffer)
	metricEncoder := RowBinary.NewEncoder(metricBuffer)

	for i := 0; i < len(metricList); i++ {
		m := &metricList[i]

		if m.Tags == nil || m.Tags.Len() == 0 {
			continue
		}

		metricBuffer.Reset()

		// Date
		err := metricEncoder.Uint16(days)
		if err != nil {
			return err
		}
		// Version
		err = metricEncoder.Uint32(version)
		if err != nil {
			return err
		}
		// Level
		err = metricEncoder.Uint32(uint32(m.Level))
		if err != nil {
			return err
		}
		// Path
		err = metricEncoder.Bytes(m.Path)
		if err != nil {
			return err
		}
		// IsLeaf
		err = metricEncoder.Uint8(m.IsLeaf())
		if err != nil {
			return err
		}
		// Tags
		err = metricEncoder.StringList(m.Tags.List())
		if err != nil {
			return err
		}

		for _, tag := range m.Tags.List() {
			_, err = writer.Write(metricBuffer.Bytes())
			if err != nil {
				return err
			}

			// Tag1
			err = encoder.String(tag)
			if err != nil {
				return err
			}
		}
	}

	// AND Empty record With Level=0, Path=0 and Without Tags

	// Date
	err = encoder.Uint16(days)
	if err != nil {
		return err
	}
	// Version
	err = encoder.Uint32(version)
	if err != nil {
		return err
	}
	// Level=0
	err = encoder.Uint32(0)
	if err != nil {
		return err
	}
	// Path=""
	err = encoder.Bytes([]byte{})
	if err != nil {
		return err
	}
	// IsLeaf=0
	err = encoder.Uint8(0)
	if err != nil {
		return err
	}
	// Tags=[]
	err = encoder.StringList([]string{})
	if err != nil {
		return err
	}
	// Tag1=""
	err = encoder.String("")
	if err != nil {
		return err
	}

	writer.Close()
	end()

	if cfg.Tags.OutputFile != "" {
		begin(fmt.Sprintf("write to %#v", cfg.Tags.OutputFile))
		ioutil.WriteFile(cfg.Tags.OutputFile, outBuf.Bytes(), 0644)
		end()
	} else {
		begin("upload to clickhouse")
		_, err = clickhouse.PostGzip(
			context.WithValue(context.Background(), "logger", logger),
			cfg.ClickHouse.Url,
			fmt.Sprintf("INSERT INTO %s (Date,Version,Level,Path,IsLeaf,Tags,Tag1) FORMAT RowBinary", cfg.ClickHouse.TagTable),
			outBuf,
			cfg.ClickHouse.TreeTimeout.Value(),
		)
		if err != nil {
			return err
		}
		end()
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/cenkalti/backoff"
	logrus "github.com/sirupsen/logrus"
)

type WorkerConfig struct {
	Workers  int `json:"workers,omitempty"`
	MaxTries int `json:"max_tries,omitempty"`
}

func (c *WorkerConfig) setup() *ConfigError {
	if c.Workers < 1 {
		c.Workers = 1
	}
	if c.MaxTries < 1 {
		c.MaxTries = 5
	}
	return nil
}

type Target struct {
	Bucket    string
	Object    string
	LocalPath string
	Error     error
}

func (t *Target) URL() string {
	return fmt.Sprintf("gs://%s/%s", t.Bucket, t.Object)
}

type Targets []*Target

func (targets Targets) error() error {
	messages := []string{}
	for _, t := range targets {
		if t.Error != nil {
			messages = append(messages, t.Error.Error())
		}
	}
	if len(messages) == 0 {
		return nil
	}
	return fmt.Errorf(strings.Join(messages, "\n"))
}

type TargetWorker struct {
	name     string
	targets  chan *Target
	impl     func(bucket, object, srcPath string) error
	done     bool
	maxTries int
}

func (w *TargetWorker) run() {
	for {
		flds := logrus.Fields{}
		log.Debugln("Getting a target")
		var t *Target
		select {
		case t = <-w.targets:
		default: // Do nothing to break
		}
		if t == nil {
			log.Debugln("No target found any more")
			w.done = true
			break
		}
		if t.Error != nil {
			continue
		}

		flds["target"] = t
		log.WithFields(flds).Debugf("Worker Start to %v\n", w.name)

		f := func() error {
			return w.impl(t.Bucket, t.Object, t.LocalPath)
		}

		eb := backoff.NewExponentialBackOff()
		eb.InitialInterval = 30 * time.Second
		b := backoff.WithMaxTries(eb, uint64(w.maxTries))
		err := backoff.Retry(f, b)
		flds["error"] = err
		if err != nil {
			log.WithFields(flds).Errorf("Worker Failed to %v %v\n", w.name, t.URL())
			t.Error = err
			continue
		}
		log.WithFields(flds).Debugf("Worker Finished to %v\n", w.name)
	}
}

type TargetWorkers []*TargetWorker

func (ws TargetWorkers) process(targets Targets) error {
	c := make(chan *Target, len(targets))
	for _, t := range targets {
		c <- t
	}

	for _, w := range ws {
		w.targets = c
		go w.run()
	}

	for {
		time.Sleep(100 * time.Millisecond)
		if ws.done() {
			break
		}
	}

	return targets.error()
}

func (ws TargetWorkers) done() bool {
	for _, w := range ws {
		if !w.done {
			return false
		}
	}
	return true
}
<commit_msg>:+1: Reset done flags before workers run<commit_after>package main

import (
	"fmt"
	"strings"
	"time"

	"github.com/cenkalti/backoff"
	logrus "github.com/sirupsen/logrus"
)

type WorkerConfig struct {
	Workers  int `json:"workers,omitempty"`
	MaxTries int `json:"max_tries,omitempty"`
}

func (c *WorkerConfig) setup() *ConfigError {
	if c.Workers < 1 {
		c.Workers = 1
	}
	if c.MaxTries < 1 {
		c.MaxTries = 5
	}
	return nil
}

type Target struct {
	Bucket    string
	Object    string
	LocalPath string
	Error     error
}

func (t *Target) URL() string {
	return fmt.Sprintf("gs://%s/%s", t.Bucket, t.Object)
}

type Targets []*Target

func (targets Targets) error() error {
	messages := []string{}
	for _, t := range targets {
		if t.Error != nil {
			messages = append(messages, t.Error.Error())
		}
	}
	if len(messages) == 0 {
		return nil
	}
	return fmt.Errorf(strings.Join(messages, "\n"))
}

type TargetWorker struct {
	name     string
	targets  chan *Target
	impl     func(bucket, object, srcPath string) error
	done     bool
	maxTries int
}

func (w *TargetWorker) run() {
	for {
		flds := logrus.Fields{}
		log.Debugln("Getting a target")
		var t *Target
		select {
		case t = <-w.targets:
		default: // Do nothing to break
		}
		if t == nil {
			log.Debugln("No target found any more")
			w.done = true
			break
		}
		if t.Error != nil {
			continue
		}

		flds["target"] = t
		log.WithFields(flds).Debugf("Worker Start to %v\n", w.name)

		f := func() error {
			return w.impl(t.Bucket, t.Object, t.LocalPath)
		}

		eb := backoff.NewExponentialBackOff()
		eb.InitialInterval = 30 * time.Second
		b := backoff.WithMaxTries(eb, uint64(w.maxTries))
		err := backoff.Retry(f, b)
		flds["error"] = err
		if err != nil {
			log.WithFields(flds).Errorf("Worker Failed to %v %v\n", w.name, t.URL())
			t.Error = err
			continue
		}
		log.WithFields(flds).Debugf("Worker Finished to %v\n", w.name)
	}
}

type TargetWorkers []*TargetWorker

func (ws TargetWorkers) process(targets Targets) error {
	c := make(chan *Target, len(targets))
	for _, t := range targets {
		c <- t
	}

	for _, w := range ws {
		w.done = false
		w.targets = c
		go w.run()
	}

	for {
		time.Sleep(100 * time.Millisecond)
		if ws.done() {
			break
		}
	}

	return targets.error()
}

func (ws TargetWorkers) done() bool {
	for _, w := range ws {
		if !w.done {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either Version 3 of the License, or
// (at your option) any later Version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package version

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
)

// CurrentGaugeVersion represents the current version of Gauge
var CurrentGaugeVersion = &Version{1, 0, 4}

// BuildMetadata represents build information of current release (e.g, nightly build information)
var BuildMetadata = ""
var CommitHash = ""

type Version struct {
	Major int
	Minor int
	Patch int
}

type VersionSupport struct {
	Minimum string
	Maximum string
}

func ParseVersion(versionText string) (*Version, error) {
	splits := strings.Split(versionText, ".")
	if len(splits) != 3 {
		return nil, fmt.Errorf("Incorrect Version format. Version should be in the form 1.5.7")
	}
	Major, err := strconv.Atoi(splits[0])
	if err != nil {
		return nil, VersionError("major", splits[0], err)
	}
	Minor, err := strconv.Atoi(splits[1])
	if err != nil {
		return nil, VersionError("minor", splits[1], err)
	}
	Patch, err := strconv.Atoi(splits[2])
	if err != nil {
		return nil, VersionError("patch", splits[2], err)
	}

	return &Version{Major, Minor, Patch}, nil
}

func VersionError(level, text string, err error) error {
	return fmt.Errorf("Error parsing %s Version %s to integer. %s", level, text, err.Error())
}

func (Version *Version) IsBetween(lower *Version, greater *Version) bool {
	return Version.IsGreaterThanEqualTo(lower) && Version.IsLesserThanEqualTo(greater)
}

func (Version *Version) IsLesserThan(version1 *Version) bool {
	return CompareVersions(Version, version1, LesserThanFunc)
}

func (Version *Version) IsGreaterThan(version1 *Version) bool {
	return CompareVersions(Version, version1, GreaterThanFunc)
}

func (Version *Version) IsLesserThanEqualTo(version1 *Version) bool {
	return Version.IsLesserThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsGreaterThanEqualTo(version1 *Version) bool {
	return Version.IsGreaterThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsEqualTo(version1 *Version) bool {
	return IsEqual(Version.Major, version1.Major) && IsEqual(Version.Minor, version1.Minor) && IsEqual(Version.Patch, version1.Patch)
}

func CompareVersions(first *Version, second *Version, compareFunc func(int, int) bool) bool {
	if compareFunc(first.Major, second.Major) {
		return true
	} else if IsEqual(first.Major, second.Major) {
		if compareFunc(first.Minor, second.Minor) {
			return true
		} else if IsEqual(first.Minor, second.Minor) {
			if compareFunc(first.Patch, second.Patch) {
				return true
			}
			return false
		}
	}
	return false
}

func LesserThanFunc(first, second int) bool {
	return first < second
}

func GreaterThanFunc(first, second int) bool {
	return first > second
}

func IsEqual(first, second int) bool {
	return first == second
}

func (Version *Version) String() string {
	return fmt.Sprintf("%d.%d.%d", Version.Major, Version.Minor, Version.Patch)
}

// FullVersion returns the CurrentGaugeVersion including build metadata.
func FullVersion() string {
	var metadata string
	if BuildMetadata != "" {
		metadata = fmt.Sprintf(".%s", BuildMetadata)
	}
	return fmt.Sprintf("%s%s", CurrentGaugeVersion.String(), metadata)
}

func GetCommitHash() string {
	var commitHash string
	if CommitHash != "" {
		commitHash = fmt.Sprintf("%s", CommitHash)
	}
	return fmt.Sprintf("%s", commitHash)
}

type byDecreasingVersion []*Version

func (a byDecreasingVersion) Len() int      { return len(a) }
func (a byDecreasingVersion) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byDecreasingVersion) Less(i, j int) bool {
	return a[i].IsGreaterThan(a[j])
}

func GetLatestVersion(versions []*Version) *Version {
	sort.Sort(byDecreasingVersion(versions))
	return versions[0]
}

func CheckCompatibility(currentVersion *Version, versionSupport *VersionSupport) error {
	minSupportVersion, err := ParseVersion(versionSupport.Minimum)
	if err != nil {
		return fmt.Errorf("Invalid minimum support version %s. : %s. ", versionSupport.Minimum, err.Error())
	}
	if versionSupport.Maximum != "" {
		maxSupportVersion, err := ParseVersion(versionSupport.Maximum)
		if err != nil {
			return fmt.Errorf("Invalid maximum support version %s. : %s. ", versionSupport.Maximum, err.Error())
		}
		if currentVersion.IsBetween(minSupportVersion, maxSupportVersion) {
			return nil
		}
		return fmt.Errorf("Version %s is not between %s and %s", currentVersion, minSupportVersion, maxSupportVersion)
	}

	if minSupportVersion.IsLesserThanEqualTo(currentVersion) {
		return nil
	}
	return fmt.Errorf("Incompatible version. Minimum support version %s is higher than current version %s", minSupportVersion, currentVersion)
}
<commit_msg>bumping gauge version to 1.0.5, #1298<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either Version 3 of the License, or
// (at your option) any later Version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package version

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
)

// CurrentGaugeVersion represents the current version of Gauge
var CurrentGaugeVersion = &Version{1, 0, 5}

// BuildMetadata represents build information of current release (e.g, nightly build information)
var BuildMetadata = ""
var CommitHash = ""

type Version struct {
	Major int
	Minor int
	Patch int
}

type VersionSupport struct {
	Minimum string
	Maximum string
}

func ParseVersion(versionText string) (*Version, error) {
	splits := strings.Split(versionText, ".")
	if len(splits) != 3 {
		return nil, fmt.Errorf("Incorrect Version format. Version should be in the form 1.5.7")
	}
	Major, err := strconv.Atoi(splits[0])
	if err != nil {
		return nil, VersionError("major", splits[0], err)
	}
	Minor, err := strconv.Atoi(splits[1])
	if err != nil {
		return nil, VersionError("minor", splits[1], err)
	}
	Patch, err := strconv.Atoi(splits[2])
	if err != nil {
		return nil, VersionError("patch", splits[2], err)
	}

	return &Version{Major, Minor, Patch}, nil
}

func VersionError(level, text string, err error) error {
	return fmt.Errorf("Error parsing %s Version %s to integer. %s", level, text, err.Error())
}

func (Version *Version) IsBetween(lower *Version, greater *Version) bool {
	return Version.IsGreaterThanEqualTo(lower) && Version.IsLesserThanEqualTo(greater)
}

func (Version *Version) IsLesserThan(version1 *Version) bool {
	return CompareVersions(Version, version1, LesserThanFunc)
}

func (Version *Version) IsGreaterThan(version1 *Version) bool {
	return CompareVersions(Version, version1, GreaterThanFunc)
}

func (Version *Version) IsLesserThanEqualTo(version1 *Version) bool {
	return Version.IsLesserThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsGreaterThanEqualTo(version1 *Version) bool {
	return Version.IsGreaterThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsEqualTo(version1 *Version) bool {
	return IsEqual(Version.Major, version1.Major) && IsEqual(Version.Minor, version1.Minor) && IsEqual(Version.Patch, version1.Patch)
}

func CompareVersions(first *Version, second *Version, compareFunc func(int, int) bool) bool {
	if compareFunc(first.Major, second.Major) {
		return true
	} else if IsEqual(first.Major, second.Major) {
		if compareFunc(first.Minor, second.Minor) {
			return true
		} else if IsEqual(first.Minor, second.Minor) {
			if compareFunc(first.Patch, second.Patch) {
				return true
			}
			return false
		}
	}
	return false
}

func LesserThanFunc(first, second int) bool {
	return first < second
}

func GreaterThanFunc(first, second int) bool {
	return first > second
}

func IsEqual(first, second int) bool {
	return first == second
}

func (Version *Version) String() string {
	return fmt.Sprintf("%d.%d.%d", Version.Major, Version.Minor, Version.Patch)
}

// FullVersion returns the CurrentGaugeVersion including build metadata.
func FullVersion() string {
	var metadata string
	if BuildMetadata != "" {
		metadata = fmt.Sprintf(".%s", BuildMetadata)
	}
	return fmt.Sprintf("%s%s", CurrentGaugeVersion.String(), metadata)
}

func GetCommitHash() string {
	var commitHash string
	if CommitHash != "" {
		commitHash = fmt.Sprintf("%s", CommitHash)
	}
	return fmt.Sprintf("%s", commitHash)
}

type byDecreasingVersion []*Version

func (a byDecreasingVersion) Len() int      { return len(a) }
func (a byDecreasingVersion) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byDecreasingVersion) Less(i, j int) bool {
	return a[i].IsGreaterThan(a[j])
}

func GetLatestVersion(versions []*Version) *Version {
	sort.Sort(byDecreasingVersion(versions))
	return versions[0]
}

func CheckCompatibility(currentVersion *Version, versionSupport *VersionSupport) error {
	minSupportVersion, err := ParseVersion(versionSupport.Minimum)
	if err != nil {
		return fmt.Errorf("Invalid minimum support version %s. : %s. ", versionSupport.Minimum, err.Error())
	}
	if versionSupport.Maximum != "" {
		maxSupportVersion, err := ParseVersion(versionSupport.Maximum)
		if err != nil {
			return fmt.Errorf("Invalid maximum support version %s. : %s. ", versionSupport.Maximum, err.Error())
		}
		if currentVersion.IsBetween(minSupportVersion, maxSupportVersion) {
			return nil
		}
		return fmt.Errorf("Version %s is not between %s and %s", currentVersion, minSupportVersion, maxSupportVersion)
	}

	if minSupportVersion.IsLesserThanEqualTo(currentVersion) {
		return nil
	}
	return fmt.Errorf("Incompatible version. Minimum support version %s is higher than current version %s", minSupportVersion, currentVersion)
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	Version = "0.2.1"
	GitSHA  = "Not provided (use ./build instead of go build)"
)
<commit_msg>*: bump version to v0.2.1+git<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

var (
	Version = "0.2.1+git"
	GitSHA  = "Not provided (use ./build instead of go build)"
)
<|endoftext|>


<commit_before>package version

const (
	APP_VERSION = "1.1.1"
)

var ( // from Makefile
	KERNEL_VERSION string
	GIT_COMMIT     string
)
<commit_msg>Start v1.1.2-dev<commit_after>package version

const (
	APP_VERSION = "1.1.2-dev"
)

var ( // from Makefile
	KERNEL_VERSION string
	GIT_COMMIT     string
)
<|endoftext|>


<commit_before>package version

// Version is the version of the build.
const Version = "0.1.27-dev"
<commit_msg>version: bump to v0.1.27<commit_after>package version

// Version is the version of the build.
const Version = "0.1.27"
<|endoftext|>


<commit_before>package version

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 5
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 13
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 1

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = ""
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<commit_msg>Move to v5.13.2-dev<commit_after>package version

import "fmt"

const (
	// VersionMajor is for an API incompatible changes
	VersionMajor = 5
	// VersionMinor is for functionality in a backwards-compatible manner
	VersionMinor = 13
	// VersionPatch is for backwards-compatible bug fixes
	VersionPatch = 2

	// VersionDev indicates development branch. Releases will be empty string.
	VersionDev = "-dev"
)

// Version is the specification version that the package types support.
var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
<|endoftext|>


<commit_before>package version

import (
	"fmt"
	"strings"
)

var (
	Version = "dev"

	// GitCommit will be overwritten automatically by the build system
	GitCommit = "HEAD"
)

// FullVersion formats the version to be printed
func FullVersion() string {
	return fmt.Sprintf("%s, build %s", Version, GitCommit)
}

// RC checks if the Machine version is a release candidate or not
func RC() bool {
	return strings.Contains(Version, "rc")
}
<commit_msg>Bump version to 0.12.2<commit_after>package version

import (
	"fmt"
	"strings"
)

var (
	Version = "0.12.2"

	// GitCommit will be overwritten automatically by the build system
	GitCommit = "HEAD"
)

// FullVersion formats the version to be printed
func FullVersion() string {
	return fmt.Sprintf("%s, build %s", Version, GitCommit)
}

// RC checks if the Machine version is a release candidate or not
func RC() bool {
	return strings.Contains(Version, "rc")
}
<|endoftext|>


<commit_before>package version

import (
	"bytes"
	"fmt"
)

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string

// The main version number that is being run at the moment.
const Version = "1.0.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = ""

func FormattedVersion() string {
	var versionString bytes.Buffer
	fmt.Fprintf(&versionString, "%s", Version)
	if VersionPrerelease != "" {
		fmt.Fprintf(&versionString, "-%s", VersionPrerelease)

		if GitCommit != "" {
			fmt.Fprintf(&versionString, " (%s)", GitCommit)
		}
	}

	return versionString.String()
}
<commit_msg>next version is 1.0.2<commit_after>package version

import (
	"bytes"
	"fmt"
)

// The git commit that was compiled. This will be filled in by the compiler.
var GitCommit string

// The main version number that is being run at the moment.
const Version = "1.0.2"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"

func FormattedVersion() string {
	var versionString bytes.Buffer
	fmt.Fprintf(&versionString, "%s", Version)
	if VersionPrerelease != "" {
		fmt.Fprintf(&versionString, "-%s", VersionPrerelease)

		if GitCommit != "" {
			fmt.Fprintf(&versionString, " (%s)", GitCommit)
		}
	}

	return versionString.String()
}
<|endoftext|>


<commit_before>package version

// VERSION ...
const VERSION = "1.1.36"
<commit_msg>v1.1.37<commit_after>package version

// VERSION ...
const VERSION = "1.1.37"
<|endoftext|>


<commit_before>package version

// Version is the version of the build.
const Version = "1.8.0-dev"
<commit_msg>version: bump v1.8.0<commit_after>package version

// Version is the version of the build.
const Version = "1.8.0"
<|endoftext|>


<commit_before>package isdomain

// ExtendedTLDs is a set of additional "TLDs", allowing decentralized name
// systems, like TOR and Namecoin.
var ExtendedTLDs = map[string]bool{
	"BIT":   true,
	"ONION": true,
	"ETH":   true,
	"BBS":   true,
	"CHAN":  true,
	"CYB":   true,
	"DYN":   true,
	"EPIC":  true,
	"GEEK":  true,
	"GOPHER":true,
	"INDY":  true,
	"LIBRE": true,
	"NEO":   true,
	"NULL":  true,
	"O":     true,
	"OSS":   true,
	"OZ":    true,
	"PARODY":true,
	"PIRATE":true,
	"CRYPTO": true, // unstoppabledomains.com
	"ZIL":    true, // unstoppabledomains.com
}
<commit_msg>Fix conflict.<commit_after>package isdomain

// ExtendedTLDs is a set of additional "TLDs", allowing decentralized name
// systems, like TOR and Namecoin.
var ExtendedTLDs = map[string]bool{
	"BIT":    true, // namecoin.org
	"ONION":  true, // torproject.org
	"ETH":    true, // ens.domains
	"CRYPTO": true, // unstoppabledomains.com
	"ZIL":    true, // unstoppabledomains.com
	"BBS":    true, // opennic.org
	"CHAN":   true, // opennic.org
	"CYB":    true, // opennic.org
	"DYN":    true, // opennic.org
	"EPIC":   true, // opennic.org
	"GEEK":   true, // opennic.org
	"GOPHER": true, // opennic.org
	"INDY":   true, // opennic.org
	"LIBRE":  true, // opennic.org
	"NEO":    true, // opennic.org
	"NULL":   true, // opennic.org
	"O":      true, // opennic.org
	"OSS":    true, // opennic.org
	"OZ":     true, // opennic.org
	"PARODY": true, // opennic.org
	"PIRATE": true, // opennic.org
}
<|endoftext|>


<commit_before>package filesystem

import (
	"errors"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
	"syscall"
)

var stashDirectory string = ".stash"

func (objSrv *ObjectServer) commitObject(hashVal hash.Hash) error {
	hashName := objectcache.HashToFilename(hashVal)
	filename := path.Join(objSrv.baseDir, hashName)
	stashFilename := path.Join(objSrv.baseDir, stashDirectory, hashName)
	fi, err := os.Lstat(stashFilename)
	if err != nil {
		return err
	}
	if !fi.Mode().IsRegular() {
		fsutil.ForceRemove(stashFilename)
		return errors.New("Existing non-file: " + stashFilename)
	}
	if err = os.MkdirAll(path.Dir(filename), syscall.S_IRWXU); err != nil {
		return err
	}
	objSrv.rwLock.Lock()
	defer objSrv.rwLock.Unlock()
	if _, ok := objSrv.sizesMap[hashVal]; ok {
		fsutil.ForceRemove(stashFilename)
		return nil
	} else {
		objSrv.sizesMap[hashVal] = uint64(fi.Size())
		return os.Rename(stashFilename, filename)
	}
}

func (objSrv *ObjectServer) deleteStashedObject(hashVal hash.Hash) error {
	filename := path.Join(objSrv.baseDir, stashDirectory,
		objectcache.HashToFilename(hashVal))
	return os.Remove(filename)
}

func (objSrv *ObjectServer) stashOrVerifyObject(reader io.Reader,
	length uint64, expectedHash *hash.Hash) (hash.Hash, []byte, error) {
	hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
	if err != nil {
		return hashVal, nil, err
	}
	length = uint64(len(data))
	hashName := objectcache.HashToFilename(hashVal)
	filename := path.Join(objSrv.baseDir, hashName)
	// Check for existing object and collision.
	if isNew, err := addOrCompare(hashVal, data, filename); err != nil {
		return hashVal, nil, err
	} else if !isNew {
		return hashVal, nil, nil
	}
	// Check for existing stashed object and collision.
	stashFilename := path.Join(objSrv.baseDir, stashDirectory, hashName)
	if _, err := addOrCompare(hashVal, data, stashFilename); err != nil {
		return hashVal, nil, err
	} else {
		return hashVal, data, nil
	}
}
<commit_msg>Fix bug in ObjectServer.StashOrVerifyObject(): was auto-committing.<commit_after>package filesystem

import (
	"errors"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
	"syscall"
)

var stashDirectory string = ".stash"

func (objSrv *ObjectServer) commitObject(hashVal hash.Hash) error {
	hashName := objectcache.HashToFilename(hashVal)
	filename := path.Join(objSrv.baseDir, hashName)
	stashFilename := path.Join(objSrv.baseDir, stashDirectory, hashName)
	fi, err := os.Lstat(stashFilename)
	if err != nil {
		return err
	}
	if !fi.Mode().IsRegular() {
		fsutil.ForceRemove(stashFilename)
		return errors.New("Existing non-file: " + stashFilename)
	}
	if err = os.MkdirAll(path.Dir(filename), syscall.S_IRWXU); err != nil {
		return err
	}
	objSrv.rwLock.Lock()
	defer objSrv.rwLock.Unlock()
	if _, ok := objSrv.sizesMap[hashVal]; ok {
		fsutil.ForceRemove(stashFilename)
		return nil
	} else {
		objSrv.sizesMap[hashVal] = uint64(fi.Size())
		return os.Rename(stashFilename, filename)
	}
}

func (objSrv *ObjectServer) deleteStashedObject(hashVal hash.Hash) error {
	filename := path.Join(objSrv.baseDir, stashDirectory,
		objectcache.HashToFilename(hashVal))
	return os.Remove(filename)
}

func (objSrv *ObjectServer) stashOrVerifyObject(reader io.Reader,
	length uint64, expectedHash *hash.Hash) (hash.Hash, []byte, error) {
	hashVal, data, err := objectcache.ReadObject(reader, length, expectedHash)
	if err != nil {
		return hashVal, nil, err
	}
	length = uint64(len(data))
	hashName := objectcache.HashToFilename(hashVal)
	filename := path.Join(objSrv.baseDir, hashName)
	// Check for existing object and collision.
	if length, err := objSrv.checkObject(hashVal); err != nil {
		return hashVal, nil, err
	} else if length > 0 {
		if err := collisionCheck(data, filename, int64(length)); err != nil {
			return hashVal, nil, err
		}
		return hashVal, nil, nil
	}
	// Check for existing stashed object and collision.
	stashFilename := path.Join(objSrv.baseDir, stashDirectory, hashName)
	if _, err := addOrCompare(hashVal, data, stashFilename); err != nil {
		return hashVal, nil, err
	} else {
		return hashVal, data, nil
	}
}
<|endoftext|>