<commit_before>package mp4

import (
	"fmt"
	"io"
)

// FtypBox - File Type Box (ftyp - mandatory)
//
// Status: decoded
type FtypBox struct {
	MajorBrand       string
	MinorVersion     []byte
	CompatibleBrands []string
}

// Decode decodes the ftyp box
func DecodeFtyp(r io.Reader, size uint64) (Box, error) {
	data, err := read(r, size)
	if err != nil {
		return nil, err
	}
	b := &FtypBox{
		MajorBrand:   string(data[0:4]),
		MinorVersion: data[4:8],
	}
	if len(data) > 8 {
		b.CompatibleBrands = make([]string, len(data)-8)
		for i := 8; i < len(data); i += 4 {
			b.CompatibleBrands[(i-8)/4] = string(data[i : i+4])
		}
	}
	return b, nil
}

func (b *FtypBox) Type() string {
	return "ftyp"
}

func (b *FtypBox) Size() uint64 {
	return uint64(8 + 4*len(b.CompatibleBrands))
}

func (b *FtypBox) Dump() {
	fmt.Printf("File Type: %s\n", b.MajorBrand)
}

func (b *FtypBox) Encode(w io.Writer) error {
	err := EncodeHeader(b, w)
	if err != nil {
		return err
	}
	buf := makebuf(b)
	strtobuf(buf, b.MajorBrand, 4)
	copy(buf[4:], b.MinorVersion)
	for i, c := range b.CompatibleBrands {
		strtobuf(buf[8+i*4:], c, 4)
	}
	_, err = w.Write(buf)
	return err
}
<commit_msg>fix: wrongly calculating compatible brands in ftyp atom<commit_after>package mp4

import (
	"fmt"
	"io"
)

// FtypBox - File Type Box (ftyp - mandatory)
//
// Status: decoded
type FtypBox struct {
	MajorBrand       string
	MinorVersion     []byte
	CompatibleBrands []string
}

// Decode decodes the ftyp box
func DecodeFtyp(r io.Reader, size uint64) (Box, error) {
	data, err := read(r, size)
	if err != nil {
		return nil, err
	}
	b := &FtypBox{
		MajorBrand:   string(data[0:4]),
		MinorVersion: data[4:8],
	}
	if len(data) > 8 {
		b.CompatibleBrands = make([]string, (len(data)-8)/4)
		for i := 8; i < len(data); i += 4 {
			b.CompatibleBrands[(i-8)/4] = string(data[i : i+4])
		}
	}
	return b, nil
}

func (b *FtypBox) Type() string {
	return "ftyp"
}

func (b *FtypBox) Size() uint64 {
	return uint64(8 + 4*len(b.CompatibleBrands))
}

func (b *FtypBox) Dump() {
	fmt.Printf("File Type: %s\n", b.MajorBrand)
}

func (b *FtypBox) Encode(w io.Writer) error {
	err := EncodeHeader(b, w)
	if err != nil {
		return err
	}
	buf := makebuf(b)
	strtobuf(buf, b.MajorBrand, 4)
	copy(buf[4:], b.MinorVersion)
	for i, c := range b.CompatibleBrands {
		strtobuf(buf[8+i*4:], c, 4)
	}
	_, err = w.Write(buf)
	return err
}
<|endoftext|>


<commit_before>// +build gofuzz

package amqp

import (
	"bytes"
	"context"
	"time"

	"pack.ag/amqp/testconn"
)

func FuzzConn(data []byte) int {
	client, err := New(testconn.New(data),
		ConnSASLPlain("listen", "3aCXZYFcuZA89xe6lZkfYJvOPnTGipA3ap7NvPruBhI="),
		ConnIdleTimeout(10*time.Millisecond),
	)
	if err != nil {
		return 0
	}
	defer client.Close()

	s, err := client.NewSession()
	if err != nil {
		return 0
	}

	r, err := s.NewReceiver(LinkSource("source"), LinkCredit(2))
	if err != nil {
		return 0
	}

	msg, err := r.Receive(context.Background())
	if err != nil {
		return 0
	}

	msg.Accept()

	// r.Close() // disabled until link close timeout implemented

	s.Close()

	return 1
}

func FuzzUnmarshal(data []byte) int {
	types := []interface{}{
		new(performAttach),
		new(*performAttach),
		new(performBegin),
		new(*performBegin),
		new(performClose),
		new(*performClose),
		new(performDetach),
		new(*performDetach),
		new(performDisposition),
		new(*performDisposition),
		new(performEnd),
		new(*performEnd),
		new(performFlow),
		new(*performFlow),
		new(performOpen),
		new(*performOpen),
		new(performTransfer),
		new(*performTransfer),
		new(source),
		new(*source),
		new(target),
		new(*target),
		new(Error),
		new(*Error),
		new(saslCode),
		new(*saslCode),
		new(saslMechanisms),
		new(*saslMechanisms),
		new(saslOutcome),
		new(*saslOutcome),
		new(Message),
		new(*Message),
		new(MessageHeader),
		new(*MessageHeader),
		new(MessageProperties),
		new(*MessageProperties),
		new(stateReceived),
		new(*stateReceived),
		new(stateAccepted),
		new(*stateAccepted),
		new(stateRejected),
		new(*stateRejected),
		new(stateReleased),
		new(*stateReleased),
		new(stateModified),
		new(*stateModified),
		new(mapAnyAny),
		new(*mapAnyAny),
		new(mapStringAny),
		new(*mapStringAny),
		new(mapSymbolAny),
		new(*mapSymbolAny),
		new(unsettled),
		new(*unsettled),
		new(milliseconds),
		new(*milliseconds),
		new(bool),
		new(*bool),
		new(int8),
		new(*int8),
		new(int16),
		new(*int16),
		new(int32),
		new(*int32),
		new(int64),
		new(*int64),
		new(uint8),
		new(*uint8),
		new(uint16),
		new(*uint16),
		new(uint32),
		new(*uint32),
		new(uint64),
		new(*uint64),
		new(time.Time),
		new(*time.Time),
		new(time.Duration),
		new(*time.Duration),
		new(Symbol),
		new(*Symbol),
		new([]byte),
		new(*[]byte),
		new([]string),
		new(*[]string),
		new([]Symbol),
		new(*[]Symbol),
		new(map[interface{}]interface{}),
		new(*map[interface{}]interface{}),
		new(map[string]interface{}),
		new(*map[string]interface{}),
		new(map[Symbol]interface{}),
		new(*map[Symbol]interface{}),
		new(interface{}),
		new(*interface{}),
		new(ErrorCondition),
		new(*ErrorCondition),
		new(role),
		new(*role),
	}

	for _, t := range types {
		unmarshal(bytes.NewBuffer(data), t)
	}
	return 0
}
<commit_msg>Rename Symbol -> symbol in fuzz.go<commit_after>// +build gofuzz

package amqp

import (
	"bytes"
	"context"
	"time"

	"pack.ag/amqp/testconn"
)

func FuzzConn(data []byte) int {
	client, err := New(testconn.New(data),
		ConnSASLPlain("listen", "3aCXZYFcuZA89xe6lZkfYJvOPnTGipA3ap7NvPruBhI="),
		ConnIdleTimeout(10*time.Millisecond),
	)
	if err != nil {
		return 0
	}
	defer client.Close()

	s, err := client.NewSession()
	if err != nil {
		return 0
	}

	r, err := s.NewReceiver(LinkSource("source"), LinkCredit(2))
	if err != nil {
		return 0
	}

	msg, err := r.Receive(context.Background())
	if err != nil {
		return 0
	}

	msg.Accept()

	// r.Close() // disabled until link close timeout implemented

	s.Close()

	return 1
}

func FuzzUnmarshal(data []byte) int {
	types := []interface{}{
		new(performAttach),
		new(*performAttach),
		new(performBegin),
		new(*performBegin),
		new(performClose),
		new(*performClose),
		new(performDetach),
		new(*performDetach),
		new(performDisposition),
		new(*performDisposition),
		new(performEnd),
		new(*performEnd),
		new(performFlow),
		new(*performFlow),
		new(performOpen),
		new(*performOpen),
		new(performTransfer),
		new(*performTransfer),
		new(source),
		new(*source),
		new(target),
		new(*target),
		new(Error),
		new(*Error),
		new(saslCode),
		new(*saslCode),
		new(saslMechanisms),
		new(*saslMechanisms),
		new(saslOutcome),
		new(*saslOutcome),
		new(Message),
		new(*Message),
		new(MessageHeader),
		new(*MessageHeader),
		new(MessageProperties),
		new(*MessageProperties),
		new(stateReceived),
		new(*stateReceived),
		new(stateAccepted),
		new(*stateAccepted),
		new(stateRejected),
		new(*stateRejected),
		new(stateReleased),
		new(*stateReleased),
		new(stateModified),
		new(*stateModified),
		new(mapAnyAny),
		new(*mapAnyAny),
		new(mapStringAny),
		new(*mapStringAny),
		new(mapSymbolAny),
		new(*mapSymbolAny),
		new(unsettled),
		new(*unsettled),
		new(milliseconds),
		new(*milliseconds),
		new(bool),
		new(*bool),
		new(int8),
		new(*int8),
		new(int16),
		new(*int16),
		new(int32),
		new(*int32),
		new(int64),
		new(*int64),
		new(uint8),
		new(*uint8),
		new(uint16),
		new(*uint16),
		new(uint32),
		new(*uint32),
		new(uint64),
		new(*uint64),
		new(time.Time),
		new(*time.Time),
		new(time.Duration),
		new(*time.Duration),
		new(symbol),
		new(*symbol),
		new([]byte),
		new(*[]byte),
		new([]string),
		new(*[]string),
		new([]symbol),
		new(*[]symbol),
		new(map[interface{}]interface{}),
		new(*map[interface{}]interface{}),
		new(map[string]interface{}),
		new(*map[string]interface{}),
		new(map[symbol]interface{}),
		new(*map[symbol]interface{}),
		new(interface{}),
		new(*interface{}),
		new(ErrorCondition),
		new(*ErrorCondition),
		new(role),
		new(*role),
	}

	for _, t := range types {
		unmarshal(bytes.NewBuffer(data), t)
	}
	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	validate                     bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	environmentParam             string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	needSyncDirs                 []string
	needSyncEnvs                 map[string]struct{}
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJSONParseTime           float64
	metadataJSONParseTime        float64
	gmetadataJSONParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
	desiredContent               []string
	unchangedModuleDirs          []string
	mapModulesRefsToPuppetEnv    map[string]string
)

// LatestForgeModules contains a map of unique Forge modules
// that should be the latest versions of them
type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int            `yaml:"timeout"`
	IgnoreUnreachableModules    bool           `yaml:"ignore_unreachable_modules"`
	Maxworker                   int            `yaml:"maxworker"`
	MaxExtractworker            int            `yaml:"maxextractworker"`
	UseCacheFallback            bool           `yaml:"use_cache_fallback"`
	RetryGitCommands            bool           `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool           `yaml:"git_object_syntax_not_supported"`
	PostRunCommand              []string       `yaml:"postrun"`
	Deploy                      DeploySettings `yaml:"deploy"`
	PurgeLevels                 []string       `yaml:"purge_levels"`
	PurgeWhitelist              []string       `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist    []string       `yaml:"deployment_purge_whitelist"`
	WriteLock                   string         `yaml:"write_lock"`
	GenerateTypes               bool           `yaml:"generate_types"`
	PuppetPath                  string         `yaml:"puppet_path"`
	PurgeBlacklist              []string       `yaml:"purge_blacklist"`
	CloneGitModules             bool           `yaml:"clone_git_modules"`
}

// DeploySettings is a struct for settings for controlling how g10k deploys behave.
// Trying to emulate r10k https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy
type DeploySettings struct {
	PurgeLevels              []string `yaml:"purge_levels"`
	PurgeWhitelist           []string `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist []string `yaml:"deployment_purge_whitelist"`
	WriteLock                string   `yaml:"write_lock"`
	GenerateTypes            bool     `yaml:"generate_types"`
	PuppetPath               string   `yaml:"puppet_path"`
	PurgeBlacklist           []string `yaml:"purge_blacklist"`
}

// Forge is a simple struct that contains the base URL of
// the Forge that g10k should use. Defaults to: https://forgeapi.puppetlabs.com
type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

// Git is a simple struct that contains the optional SSH private key to
// use for authentication
type Git struct {
	privateKey string `yaml:"private_key"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	forgeBaseURL      string
	forgeCacheTTL     time.Duration
	forgeModules      map[string]ForgeModule
	gitModules        map[string]GitModule
	privateKey        string
	source            string
	sourceBranch      string
	workDir           string
	moduleDirs        []string
	controlRepoBranch string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version      string
	name         string
	author       string
	md5sum       string
	fileSize     int64
	baseURL      string
	cacheTTL     time.Duration
	sha256sum    string
	moduleDir    string
	sourceBranch string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	tree              string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
	moduleDir         string
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

// DeployResult contains information about the Puppet environment which was deployed by g10k and tries to emulate the .r10k-deploy.json
type DeployResult struct {
	Name               string    `json:"name"`
	Signature          string    `json:"signature"`
	StartedAt          time.Time `json:"started_at"`
	FinishedAt         time.Time `json:"finished_at"`
	DeploySuccess      bool      `json:"deploy_success"`
	PuppetfileChecksum string    `json:"puppetfile_checksum"`
}

func init() {
	// initialize global maps
	needSyncEnvs = make(map[string]struct{})
	uniqueForgeModules = make(map[string]ForgeModule)
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update. Just the branch name, e.g. master, qa, dev")
	flag.StringVar(&environmentParam, "environment", "", "which Puppet environment to update. Source name inside the config + '_' + branch name, e.g. foo_master, foo_qa, foo_dev")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&validate, "validate", false, "only validate given configuration and exit")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.8.8 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		checkDirAndCreate(config.CacheDir, "cachedir configured value")
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			branchParam = ""
			resolvePuppetEnvironment(tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "./"}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			// default purge_levels
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			config.PurgeLevels = []string{"puppetfile"}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", "cmdlineparam", false, false)
			puppetfile.workDir = ""
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJSONParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJSONParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolve and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}

	checkForAndExecutePostrunCommand()
}
<commit_msg>bump version to v0.8.9<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	validate                     bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	environmentParam             string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	needSyncDirs                 []string
	needSyncEnvs                 map[string]struct{}
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJSONParseTime           float64
	metadataJSONParseTime        float64
	gmetadataJSONParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
	desiredContent               []string
	unchangedModuleDirs          []string
	mapModulesRefsToPuppetEnv    map[string]string
)

// LatestForgeModules contains a map of unique Forge modules
// that should be the latest versions of them
type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int            `yaml:"timeout"`
	IgnoreUnreachableModules    bool           `yaml:"ignore_unreachable_modules"`
	Maxworker                   int            `yaml:"maxworker"`
	MaxExtractworker            int            `yaml:"maxextractworker"`
	UseCacheFallback            bool           `yaml:"use_cache_fallback"`
	RetryGitCommands            bool           `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool           `yaml:"git_object_syntax_not_supported"`
	PostRunCommand              []string       `yaml:"postrun"`
	Deploy                      DeploySettings `yaml:"deploy"`
	PurgeLevels                 []string       `yaml:"purge_levels"`
	PurgeWhitelist              []string       `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist    []string       `yaml:"deployment_purge_whitelist"`
	WriteLock                   string         `yaml:"write_lock"`
	GenerateTypes               bool           `yaml:"generate_types"`
	PuppetPath                  string         `yaml:"puppet_path"`
	PurgeBlacklist              []string       `yaml:"purge_blacklist"`
	CloneGitModules             bool           `yaml:"clone_git_modules"`
}

// DeploySettings is a struct for settings for controlling how g10k deploys behave.
// Trying to emulate r10k https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy
type DeploySettings struct {
	PurgeLevels              []string `yaml:"purge_levels"`
	PurgeWhitelist           []string `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist []string `yaml:"deployment_purge_whitelist"`
	WriteLock                string   `yaml:"write_lock"`
	GenerateTypes            bool     `yaml:"generate_types"`
	PuppetPath               string   `yaml:"puppet_path"`
	PurgeBlacklist           []string `yaml:"purge_blacklist"`
}

// Forge is a simple struct that contains the base URL of
// the Forge that g10k should use. Defaults to: https://forgeapi.puppetlabs.com
type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

// Git is a simple struct that contains the optional SSH private key to
// use for authentication
type Git struct {
	privateKey string `yaml:"private_key"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	forgeBaseURL      string
	forgeCacheTTL     time.Duration
	forgeModules      map[string]ForgeModule
	gitModules        map[string]GitModule
	privateKey        string
	source            string
	sourceBranch      string
	workDir           string
	moduleDirs        []string
	controlRepoBranch string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version      string
	name         string
	author       string
	md5sum       string
	fileSize     int64
	baseURL      string
	cacheTTL     time.Duration
	sha256sum    string
	moduleDir    string
	sourceBranch string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	tree              string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
	moduleDir         string
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

// DeployResult contains information about the Puppet environment which was deployed by g10k and tries to emulate the .r10k-deploy.json
type DeployResult struct {
	Name               string    `json:"name"`
	Signature          string    `json:"signature"`
	StartedAt          time.Time `json:"started_at"`
	FinishedAt         time.Time `json:"finished_at"`
	DeploySuccess      bool      `json:"deploy_success"`
	PuppetfileChecksum string    `json:"puppetfile_checksum"`
}

func init() {
	// initialize global maps
	needSyncEnvs = make(map[string]struct{})
	uniqueForgeModules = make(map[string]ForgeModule)
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update. Just the branch name, e.g. master, qa, dev")
	flag.StringVar(&environmentParam, "environment", "", "which Puppet environment to update. Source name inside the config + '_' + branch name, e.g. foo_master, foo_qa, foo_dev")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&validate, "validate", false, "only validate given configuration and exit")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.8.9 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		checkDirAndCreate(config.CacheDir, "cachedir configured value")
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			branchParam = ""
			resolvePuppetEnvironment(tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "./"}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			// default purge_levels
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			config.PurgeLevels = []string{"puppetfile"}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", "cmdlineparam", false, false)
			puppetfile.workDir = ""
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJSONParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJSONParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolve and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}

	checkForAndExecutePostrunCommand()
}
<|endoftext|>


<commit_before>package gamedayapi

import (
	"os/user"
	"log"
	"bytes"
	"os"
	"io/ioutil"
	"net/http"
	"encoding/xml"
	s "strings"
)

type Game struct {
	XMLName xml.Name `xml:"game"`
	GameType string `xml:"type,attr"`
	LocalGameTime string `xml:"local_game_time,attr"`
	Teams []Team `xml:"team"`
	Stadium Stadium `xml:"stadium"`
}

func Init(teamCode string, date string) {
	log.Println("Fetching game for " + teamCode + " on " + date)

	epgResp, err := http.Get(epgUrl(date))
	if err != nil {
		log.Fatal(err)
	}
	defer epgResp.Body.Close()
	epgBody, err := ioutil.ReadAll(epgResp.Body)
	if err != nil {
		log.Fatal(err)
	}

	var epg Epg
	xml.Unmarshal(epgBody, &epg)
	gid := epg.GidForTeam(teamCode)

	game := fetchGame(&gid)
	log.Println(game)
}

func fetchGame(gid *Gid) Game {
	var game Game
	gameFileName := "game.xml"
	cachedFileName := cachePath(gid) + cacheFileName(gid, gameFileName)

	if _, err := os.Stat(cachedFileName); os.IsNotExist(err) {
		log.Println("No cache hit - go get it")

		resp, err := http.Get(gameUrl(gid))
		if err != nil {
			log.Fatal(err)
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal(err)
		}

		xml.Unmarshal(body, &game)
		log.Println(resp.Status)
		log.Println(string(body))
		cacheResponse(gid, gameFileName, body)
	} else {
		log.Println("Cache hit - load it up")
		body, _ := ioutil.ReadFile(cachedFileName)
		log.Println(string(body))
		xml.Unmarshal(body, &game)
	}

	return game
}

func cacheResponse(gid *Gid, filename string, body []byte) {
	cachePath := cachePath(gid)
	os.MkdirAll(cachePath, (os.FileMode)(0775))
	f, err := os.Create(cachePath + cacheFileName(gid, filename))
	f.Write(body)
	check(err)
	defer f.Close()
}

func baseUrl() string {
	return "http://gd2.mlb.com/components/game/mlb/"
}

func dateUrl(date string) string {
	var buffer bytes.Buffer
	buffer.WriteString(baseUrl())
	buffer.WriteString(datePath(date))
	return buffer.String()
}

func epgUrl(date string) string {
	var buffer bytes.Buffer
	buffer.WriteString(dateUrl(date))
	buffer.WriteString("/epg.xml")
	return buffer.String()
}

func gameDirectoryUrl(gid *Gid) string {
	var buffer bytes.Buffer
	buffer.WriteString(baseUrl())
	buffer.WriteString(gid.DatePath())
	buffer.WriteString("/")
	buffer.WriteString(gid.String())
	buffer.WriteString("/")
	return buffer.String()
}

func gameUrl(gid *Gid) string {
	return gameDirectoryUrl(gid) + "game.xml"
}

func datePath(date string) string {
	// firx this to be date parsing, validating
	datePieces := s.Split(date, "-")
	var buffer bytes.Buffer
	buffer.WriteString("year_")
	buffer.WriteString(datePieces[0])
	buffer.WriteString("/month_")
	buffer.WriteString(datePieces[1])
	buffer.WriteString("/day_")
	buffer.WriteString(datePieces[2])
	return buffer.String()
}

func homeDir() string {
	usr, err := user.Current()
	if err != nil {
		log.Fatal( err )
	}
	return usr.HomeDir
}

func cachePath(gid *Gid) string {
	return homeDir() + "/go-gameday-cache/" + gid.Year + "/"
}

func cacheFileName(gid *Gid, filename string) string {
	return gid.String() + "-" + filename
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>Moving base URL into a contstant.<commit_after>package gamedayapi

import (
	"os/user"
	"log"
	"bytes"
	"os"
	"io/ioutil"
	"net/http"
	"encoding/xml"
	s "strings"
)

const (
	BaseUrl = "http://gd2.mlb.com/components/game/mlb/"
)

type Game struct {
	XMLName xml.Name `xml:"game"`
	GameType string `xml:"type,attr"`
	LocalGameTime string `xml:"local_game_time,attr"`
	Teams []Team `xml:"team"`
	Stadium Stadium `xml:"stadium"`
}

func Init(teamCode string, date string) {
	log.Println("Fetching game for " + teamCode + " on " + date)

	epgResp, err := http.Get(epgUrl(date))
	if err != nil {
		log.Fatal(err)
	}
	defer epgResp.Body.Close()
	epgBody, err := ioutil.ReadAll(epgResp.Body)
	if err != nil {
		log.Fatal(err)
	}

	var epg Epg
	xml.Unmarshal(epgBody, &epg)
	gid := epg.GidForTeam(teamCode)

	game := fetchGame(&gid)
	log.Println(game)
}

func fetchGame(gid *Gid) Game {
	var game Game
	gameFileName := "game.xml"
	cachedFileName := cachePath(gid) + cacheFileName(gid, gameFileName)

	if _, err := os.Stat(cachedFileName); os.IsNotExist(err) {
		log.Println("No cache hit - go get it")

		resp, err := http.Get(gameUrl(gid))
		if err != nil {
			log.Fatal(err)
		}
		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			log.Fatal(err)
		}

		xml.Unmarshal(body, &game)
		log.Println(resp.Status)
		log.Println(string(body))
		cacheResponse(gid, gameFileName, body)
	} else {
		log.Println("Cache hit - load it up")
		body, _ := ioutil.ReadFile(cachedFileName)
		log.Println(string(body))
		xml.Unmarshal(body, &game)
	}

	return game
}

func cacheResponse(gid *Gid, filename string, body []byte) {
	cachePath := cachePath(gid)
	os.MkdirAll(cachePath, (os.FileMode)(0775))
	f, err := os.Create(cachePath + cacheFileName(gid, filename))
	f.Write(body)
	check(err)
	defer f.Close()
}

func dateUrl(date string) string {
	var buffer bytes.Buffer
	buffer.WriteString(BaseUrl)
	buffer.WriteString(datePath(date))
	return buffer.String()
}

func epgUrl(date string) string {
	var buffer bytes.Buffer
	buffer.WriteString(dateUrl(date))
	buffer.WriteString("/epg.xml")
	return buffer.String()
}

func gameDirectoryUrl(gid *Gid) string {
	var buffer bytes.Buffer
	buffer.WriteString(BaseUrl)
	buffer.WriteString(gid.DatePath())
	buffer.WriteString("/")
	buffer.WriteString(gid.String())
	buffer.WriteString("/")
	return buffer.String()
}

func gameUrl(gid *Gid) string {
	return gameDirectoryUrl(gid) + "game.xml"
}

func datePath(date string) string {
	// firx this to be date parsing, validating
	datePieces := s.Split(date, "-")
	var buffer bytes.Buffer
	buffer.WriteString("year_")
	buffer.WriteString(datePieces[0])
	buffer.WriteString("/month_")
	buffer.WriteString(datePieces[1])
	buffer.WriteString("/day_")
	buffer.WriteString(datePieces[2])
	return buffer.String()
}

func homeDir() string {
	usr, err := user.Current()
	if err != nil {
		log.Fatal( err )
	}
	return usr.HomeDir
}

func cachePath(gid *Gid) string {
	return homeDir() + "/go-gameday-cache/" + gid.Year + "/"
}

func cacheFileName(gid *Gid, filename string) string {
	return gid.String() + "-" + filename
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/siddontang/go-mysql-elasticsearch/river"
)

var configFile = flag.String("config", "./etc/river.toml", "go-mysql-elasticsearch config file")
var my_addr = flag.String("my_addr", "", "MySQL addr")
var my_user = flag.String("my_user", "", "MySQL user")
var my_pass = flag.String("my_pass", "", "MySQL password")
var es_addr = flag.String("es_addr", "", "Elasticsearch addr")
var data_dir = flag.String("data_dir", "", "path for go-mysql-elasticsearch to save data")
var server_id = flag.Int("server_id", 0, "MySQL server id, as a pseudo slave")
var flavor = flag.String("flavor", "", "flavor: mysql or mariadb")
var execution = flag.String("exec", "", "mysqldump execution path")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		os.Kill,
		os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	cfg, err := river.NewConfigWithFile(*configFile)
	if err != nil {
		println(err.Error())
		return
	}

	if len(*my_addr) > 0 {
		cfg.MyAddr = *my_addr
	}

	if len(*my_user) > 0 {
		cfg.MyUser = *my_user
	}

	if len(*my_pass) > 0 {
		cfg.MyPassword = *my_pass
	}

	if *server_id > 0 {
		cfg.ServerID = uint32(*server_id)
	}

	if len(*es_addr) > 0 {
		cfg.ESAddr = *es_addr
	}

	if len(*data_dir) > 0 {
		cfg.DataDir = *data_dir
	}

	if len(*flavor) > 0 {
		cfg.Flavor = *flavor
	}

	if len(*execution) > 0 {
		cfg.DumpExec = *execution
	}

	r, err := river.NewRiver(cfg)
	if err != nil {
		println(err.Error())
		return
	}

	go func() {
		<-sc
		r.Close()
	}()

	r.Run()
}
<commit_msg>fix terminate when run<commit_after>package main

import (
	"flag"
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/siddontang/go-mysql-elasticsearch/river"
)

var configFile = flag.String("config", "./etc/river.toml", "go-mysql-elasticsearch config file")
var my_addr = flag.String("my_addr", "", "MySQL addr")
var my_user = flag.String("my_user", "", "MySQL user")
var my_pass = flag.String("my_pass", "", "MySQL password")
var es_addr = flag.String("es_addr", "", "Elasticsearch addr")
var data_dir = flag.String("data_dir", "", "path for go-mysql-elasticsearch to save data")
var server_id = flag.Int("server_id", 0, "MySQL server id, as a pseudo slave")
var flavor = flag.String("flavor", "", "flavor: mysql or mariadb")
var execution = flag.String("exec", "", "mysqldump execution path")

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	flag.Parse()

	sc := make(chan os.Signal, 1)
	signal.Notify(sc,
		os.Kill,
		os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)

	cfg, err := river.NewConfigWithFile(*configFile)
	if err != nil {
		println(err.Error())
		return
	}

	if len(*my_addr) > 0 {
		cfg.MyAddr = *my_addr
	}

	if len(*my_user) > 0 {
		cfg.MyUser = *my_user
	}

	if len(*my_pass) > 0 {
		cfg.MyPassword = *my_pass
	}

	if *server_id > 0 {
		cfg.ServerID = uint32(*server_id)
	}

	if len(*es_addr) > 0 {
		cfg.ESAddr = *es_addr
	}

	if len(*data_dir) > 0 {
		cfg.DataDir = *data_dir
	}

	if len(*flavor) > 0 {
		cfg.Flavor = *flavor
	}

	if len(*execution) > 0 {
		cfg.DumpExec = *execution
	}

	r, err := river.NewRiver(cfg)
	if err != nil {
		println(err.Error())
		return
	}

	r.Run()

	<-sc
	r.Close()
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build !solaris,!windows,!noupgrade

package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"bitbucket.org/kardianos/osext"
)

var GoArchExtra string // "", "v5", "v6", "v7"

func upgrade() error {
	if runtime.GOOS == "windows" {
		return errors.New("Upgrade currently unsupported on Windows")
	}

	path, err := osext.Executable()
	if err != nil {
		return err
	}

	rel, err := currentRelease()
	if err != nil {
		return err
	}

	switch compareVersions(rel.Tag, Version) {
	case -1:
		l.Okf("Current version %s is newer than latest release %s. Not upgrading.", Version, rel.Tag)
		return nil
	case 0:
		l.Okf("Already running the latest version, %s. Not upgrading.", Version)
		return nil
	default:
		l.Infof("Attempting upgrade to %s...", rel.Tag)
	}

	expectedRelease := fmt.Sprintf("syncthing-%s-%s%s-%s.", runtime.GOOS, runtime.GOARCH, GoArchExtra, rel.Tag)
	for _, asset := range rel.Assets {
		if strings.HasPrefix(asset.Name, expectedRelease) {
			if strings.HasSuffix(asset.Name, ".tar.gz") {
				l.Infof("Downloading %s...", asset.Name)
				fname, err := readTarGZ(asset.URL, filepath.Dir(path))
				if err != nil {
					return err
				}

				old := path + "." + Version
				err = os.Rename(path, old)
				if err != nil {
					return err
				}
				err = os.Rename(fname, path)
				if err != nil {
					return err
				}

				l.Okf("Upgraded %q to %s.", path, rel.Tag)
				l.Okf("Previous version saved in %q.", old)

				return nil
			}
		}
	}

	return fmt.Errorf("Found no asset for %q", expectedRelease)
}

func currentRelease() (githubRelease, error) {
	resp, err := http.Get("https://api.github.com/repos/calmh/syncthing/releases?per_page=10")
	if err != nil {
		return githubRelease{}, err
	}

	var rels []githubRelease
	json.NewDecoder(resp.Body).Decode(&rels)
	resp.Body.Close()

	if strings.Contains(Version, "-beta") {
		// We are a beta version. Use whatever we can find that is newer-or-equal than current.
		for _, rel := range rels {
			if compareVersions(rel.Tag, Version) >= 0 {
				return rel, nil
			}
		}
		// We found nothing. Return the latest release and let the next layer decide.
		return rels[0], nil
	} else {
		// We are a regular release. Only consider non-prerelease versions for upgrade.
		for _, rel := range rels {
			if !rel.Prerelease {
				return rel, nil
			}
		}
		return githubRelease{}, errors.New("no suitable release found")
	}
}

func readTarGZ(url string, dir string) (string, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", err
	}

	req.Header.Add("Accept", "application/octet-stream")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	gr, err := gzip.NewReader(resp.Body)
	if err != nil {
		return "", err
	}

	tr := tar.NewReader(gr)
	if err != nil {
		return "", err
	}

	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return "", err
		}

		if path.Base(hdr.Name) == "syncthing" {
			of, err := ioutil.TempFile(dir, "syncthing")
			if err != nil {
				return "", err
			}
			io.Copy(of, tr)
			err = of.Close()
			if err != nil {
				os.Remove(of.Name())
				return "", err
			}

			os.Chmod(of.Name(), os.FileMode(hdr.Mode))
			return of.Name(), nil
		}
	}

	return "", fmt.Errorf("No upgrade found")
}
<commit_msg>Don't log a panic when there are no releases<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// +build !solaris,!windows,!noupgrade

package main

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"bitbucket.org/kardianos/osext"
)

var GoArchExtra string // "", "v5", "v6", "v7"

func upgrade() error {
	if runtime.GOOS == "windows" {
		return errors.New("Upgrade currently unsupported on Windows")
	}

	path, err := osext.Executable()
	if err != nil {
		return err
	}

	rel, err := currentRelease()
	if err != nil {
		return err
	}

	switch compareVersions(rel.Tag, Version) {
	case -1:
		l.Okf("Current version %s is newer than latest release %s. Not upgrading.", Version, rel.Tag)
		return nil
	case 0:
		l.Okf("Already running the latest version, %s. Not upgrading.", Version)
		return nil
	default:
		l.Infof("Attempting upgrade to %s...", rel.Tag)
	}

	expectedRelease := fmt.Sprintf("syncthing-%s-%s%s-%s.", runtime.GOOS, runtime.GOARCH, GoArchExtra, rel.Tag)
	for _, asset := range rel.Assets {
		if strings.HasPrefix(asset.Name, expectedRelease) {
			if strings.HasSuffix(asset.Name, ".tar.gz") {
				l.Infof("Downloading %s...", asset.Name)
				fname, err := readTarGZ(asset.URL, filepath.Dir(path))
				if err != nil {
					return err
				}

				old := path + "." + Version
				err = os.Rename(path, old)
				if err != nil {
					return err
				}
				err = os.Rename(fname, path)
				if err != nil {
					return err
				}

				l.Okf("Upgraded %q to %s.", path, rel.Tag)
				l.Okf("Previous version saved in %q.", old)

				return nil
			}
		}
	}

	return fmt.Errorf("Found no asset for %q", expectedRelease)
}

func currentRelease() (githubRelease, error) {
	resp, err := http.Get("https://api.github.com/repos/calmh/syncthing/releases?per_page=10")
	if err != nil {
		return githubRelease{}, err
	}

	var rels []githubRelease
	json.NewDecoder(resp.Body).Decode(&rels)
	resp.Body.Close()

	if len(rels) == 0 {
		return githubRelease{}, errors.New("no releases found")
	}

	if strings.Contains(Version, "-beta") {
		// We are a beta version. Use whatever we can find that is newer-or-equal than current.
		for _, rel := range rels {
			if compareVersions(rel.Tag, Version) >= 0 {
				return rel, nil
			}
		}
		// We found nothing. Return the latest release and let the next layer decide.
		return rels[0], nil
	} else {
		// We are a regular release. Only consider non-prerelease versions for upgrade.
		for _, rel := range rels {
			if !rel.Prerelease {
				return rel, nil
			}
		}
		return githubRelease{}, errors.New("no suitable release found")
	}
}

func readTarGZ(url string, dir string) (string, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return "", err
	}

	req.Header.Add("Accept", "application/octet-stream")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	gr, err := gzip.NewReader(resp.Body)
	if err != nil {
		return "", err
	}

	tr := tar.NewReader(gr)
	if err != nil {
		return "", err
	}

	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			return "", err
		}

		if path.Base(hdr.Name) == "syncthing" {
			of, err := ioutil.TempFile(dir, "syncthing")
			if err != nil {
				return "", err
			}
			io.Copy(of, tr)
			err = of.Close()
			if err != nil {
				os.Remove(of.Name())
				return "", err
			}

			os.Chmod(of.Name(), os.FileMode(hdr.Mode))
			return of.Name(), nil
		}
	}

	return "", fmt.Errorf("No upgrade found")
}
<|endoftext|>


<commit_before>package options

import (
	"github.com/spf13/pflag"
)

type Config struct {
	Master                string
	KubeConfig            string
	ProviderName          string
	ClusterName           string
	LoadbalancerImageName string
}

func NewConfig() *Config {
	return &Config{
		Master:                "",
		KubeConfig:            "",
		ProviderName:          "",
		ClusterName:           "",
		LoadbalancerImageName: "appscode/haproxy:1.7.0-k8s",
	}
}

func (s *Config) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.KubeConfig, "kubeconfig", s.KubeConfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")

	fs.StringVarP(&s.ProviderName, "cloud-provider", "c", s.ProviderName, "Name of cloud provider")
	fs.StringVarP(&s.ClusterName, "cluster-name", "k", s.ClusterName, "Name of Kubernetes cluster")
	fs.StringVarP(&s.LoadbalancerImageName, "haproxy-image", "h", s.LoadbalancerImageName, "haproxy image name to be run")
}
<commit_msg>Set default HAproxy to 1.7.2-k8s<commit_after>package options

import (
	"github.com/spf13/pflag"
)

type Config struct {
	Master                string
	KubeConfig            string
	ProviderName          string
	ClusterName           string
	LoadbalancerImageName string
}

func NewConfig() *Config {
	return &Config{
		Master:                "",
		KubeConfig:            "",
		ProviderName:          "",
		ClusterName:           "",
		LoadbalancerImageName: "appscode/haproxy:1.7.2-k8s",
	}
}

func (s *Config) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.KubeConfig, "kubeconfig", s.KubeConfig, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")

	fs.StringVarP(&s.ProviderName, "cloud-provider", "c", s.ProviderName, "Name of cloud provider")
	fs.StringVarP(&s.ClusterName, "cluster-name", "k", s.ClusterName, "Name of Kubernetes cluster")
	fs.StringVarP(&s.LoadbalancerImageName, "haproxy-image", "h", s.LoadbalancerImageName, "haproxy image name to be run")
}
<|endoftext|>


<commit_before>// Copyright 2018 The go-daq Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package adc101x provides access to a 10-bit Analog-to-Digital converter.
//
// See:
//  http://www.ti.com/lit/ds/symlink/adc101c021.pdf
package adc101x

import (
	"encoding/binary"

	"github.com/go-daq/smbus"
)

const (
	DefaultI2CAddr uint8 = 0x50 // default I2C address of the ADC101x sensor.
)

// Device is a handle to an ADC101x device.
type Device struct {
	conn *smbus.Conn
	addr uint8
	bits uint8

	frange int     // ADC full range
	vdd    float64 // ADC full range
}

// Open opens a connection to an ADC101x device.
func Open(conn *smbus.Conn, addr uint8, frange int, vdd float64) (*Device, error) {
	dev := &Device{
		conn:   conn,
		addr:   addr,
		bits:   10,
		frange: frange,
		vdd:    vdd,
	}

	err := dev.conn.SetAddr(dev.addr)
	if err != nil {
		return nil, err
	}

	const (
		configRegister = 0x02
		autoConvMode   = 0x20
	)
	err = dev.conn.WriteReg(dev.addr, configRegister, autoConvMode)
	if err != nil {
		return nil, err
	}

	return dev, nil
}

func (dev *Device) ADC() (int, error) {
	var buf [2]byte
	err := dev.conn.ReadBlockData(dev.addr, 0x000, buf[:])
	if err != nil {
		return 0, err
	}

	raw := binary.BigEndian.Uint16(buf[:])

	// convert data to 10-bits
	adc := int(raw&0xFFF) >> (12 - dev.bits)
	return adc, nil
}

func (dev *Device) Voltage() (float64, error) {
	adc, err := dev.ADC()
	if err != nil {
		return 0, err
	}

	return dev.vdd * float64(adc) / float64(dev.frange), nil
}
<commit_msg>sensor/adc101x: add more context to errors in Open<commit_after>// Copyright 2018 The go-daq Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package adc101x provides access to a 10-bit Analog-to-Digital converter.
//
// See:
//  http://www.ti.com/lit/ds/symlink/adc101c021.pdf
package adc101x

import (
	"encoding/binary"
	"fmt"

	"github.com/go-daq/smbus"
)

const (
	DefaultI2CAddr uint8 = 0x50 // default I2C address of the ADC101x sensor.
)

// Device is a handle to an ADC101x device.
type Device struct {
	conn *smbus.Conn
	addr uint8
	bits uint8

	frange int     // ADC full range
	vdd    float64 // ADC full range
}

// Open opens a connection to an ADC101x device.
func Open(conn *smbus.Conn, addr uint8, frange int, vdd float64) (*Device, error) {
	dev := &Device{
		conn:   conn,
		addr:   addr,
		bits:   10,
		frange: frange,
		vdd:    vdd,
	}

	err := dev.conn.SetAddr(dev.addr)
	if err != nil {
		return nil, fmt.Errorf("adc101x: error in set-addr: %v", err)
	}

	const (
		configRegister = 0x02
		autoConvMode   = 0x20
	)
	err = dev.conn.WriteReg(dev.addr, configRegister, autoConvMode)
	if err != nil {
		return nil, fmt.Errorf("adc101x: error in write-reg: %v", err)
	}

	return dev, nil
}

func (dev *Device) ADC() (int, error) {
	var buf [2]byte
	err := dev.conn.ReadBlockData(dev.addr, 0x000, buf[:])
	if err != nil {
		return 0, fmt.Errorf("adc101x: error in read-block-data: %v", err)
	}

	raw := binary.BigEndian.Uint16(buf[:])

	// convert data to 10-bits
	adc := int(raw&0xFFF) >> (12 - dev.bits)
	return adc, nil
}

func (dev *Device) Voltage() (float64, error) {
	adc, err := dev.ADC()
	if err != nil {
		return 0, err
	}

	return dev.vdd * float64(adc) / float64(dev.frange), nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/service/ssoadmin"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
)

func dataSourceAwsSsoInstance() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsSsoInstanceRead,

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"identity_store_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsSsoInstanceRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ssoadminconn

	log.Printf("[DEBUG] Reading AWS SSO Instances")
	resp, err := conn.ListInstances(&ssoadmin.ListInstancesInput{})
	if err != nil {
		return fmt.Errorf("Error getting AWS SSO Instances: %s", err)
	}

	if resp == nil || len(resp.Instances) == 0 {
		log.Printf("[DEBUG] No AWS SSO Instance found")
		d.SetId("")
		return nil
	}

	if len(resp.Instances) > 1 {
		return fmt.Errorf("Found multiple AWS SSO Instances. Not sure which one to use. %s", resp.Instances)
	}

	instance := resp.Instances[0]
	log.Printf("[DEBUG] Received AWS SSO Instance: %s", instance)

	d.SetId(time.Now().UTC().String())
	d.Set("arn", instance.InstanceArn)
	d.Set("identity_store_id", instance.IdentityStoreId)

	return nil
}
<commit_msg>update to use paging with data.aws_sso_instance<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/service/ssoadmin"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
)

func dataSourceAwsSsoInstance() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsSsoInstanceRead,

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},

			"identity_store_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsSsoInstanceRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ssoadminconn

	log.Printf("[DEBUG] Reading AWS SSO Instances")
	instances := []*ssoadmin.InstanceMetadata{}
	err := conn.ListInstancesPages(&ssoadmin.ListInstancesInput{}, func(page *ssoadmin.ListInstancesOutput, lastPage bool) bool {
		if page != nil && page.Instances != nil && len(page.Instances) != 0 {
			instances = append(instances, page.Instances...)
		}
		return !lastPage
	})
	if err != nil {
		return fmt.Errorf("Error getting AWS SSO Instances: %s", err)
	}

	if instances == nil || len(instances) == 0 {
		log.Printf("[DEBUG] No AWS SSO Instance found")
		d.SetId("")
		return nil
	}

	if len(instances) > 1 {
		return fmt.Errorf("Found multiple AWS SSO Instances. Not sure which one to use. %s", instances)
	}

	instance := instances[0]
	log.Printf("[DEBUG] Received AWS SSO Instance: %s", instance)

	d.SetId(time.Now().UTC().String())
	d.Set("arn", instance.InstanceArn)
	d.Set("identity_store_id", instance.IdentityStoreId)

	return nil
}
<|endoftext|>


<commit_before>package gcnotifier

import (
	"io/ioutil"
	"runtime"
	"testing"
	"time"
)

func TestAfterGC(t *testing.T) {
	doneCh := make(chan struct{})

	go func() {
		M := &runtime.MemStats{}
		NumGC := uint32(0)
		gcn := New()
		for range gcn.AfterGC() {
			runtime.ReadMemStats(M)
			NumGC += 1
			if NumGC != M.NumGC {
				t.Fatal("Skipped a GC notification")
			}
			if NumGC >= 500 {
				gcn.Close()
				gcn.Close() // harmless, just for testing repeated Close()
			}
		}
		doneCh <- struct{}{}
	}()

	for {
		select {
		case <-time.After(1 * time.Millisecond):
			b := make([]byte, 1<<20)
			b[0] = 1
		case <-doneCh:
			return
		}
	}
}

func TestAutoclose(t *testing.T) {
	count := 10000
	done := make(chan struct{})
	go func() {
		for i := 0; i < count; i++ {
			gcn := New().AfterGC()
			go func() {
				for range gcn {
				}
				// to reach here autoclose() must have been called
				done <- struct{}{}
			}()
		}
	}()
	for i := 0; i < count; {
		select {
		case <-done:
			i++
		default:
			runtime.GC() // required to quickly trigger autoclose()
		}
	}
}

// Example implements a simple time-based buffering io.Writer: data sent over
// dataCh is buffered for up to 100ms, then flushed out in a single call to
// out.Write and the buffer is reused. If GC runs, the buffer is flushed and
// then discarded so that it can be collected during the next GC run. The
// example is necessarily simplistic, a real implementation would be more
// refined (e.g. on GC flush or resize the buffer based on a threshold,
// perform asynchronous flushes, properly signal completions and propagate
// errors, adaptively preallocate the buffer based on the previous capacity,
// etc.)
func Example() {
	dataCh := make(chan []byte)
	doneCh := make(chan struct{})

	out := ioutil.Discard

	go func() {
		var buf []byte
		var tick <-chan time.Time
		gcn := New()

		for {
			select {
			case data := <-dataCh:
				if tick == nil {
					tick = time.After(100 * time.Millisecond)
				}
				// received data to write to the buffer
				buf = append(buf, data...)
			case <-tick:
				// time to flush the buffer (but reuse it for the next writes)
				if len(buf) > 0 {
					out.Write(buf)
					buf = buf[:0]
				}
				tick = nil
			case <-gcn.AfterGC():
				// GC just ran: flush and then drop the buffer
				if len(buf) > 0 {
					out.Write(buf)
				}
				buf = nil
				tick = nil
			case <-doneCh:
				// close the writer: flush the buffer and return
				if len(buf) > 0 {
					out.Write(buf)
				}
				return
			}
		}
	}()

	for i := 0; i < 1<<20; i++ {
		dataCh <- make([]byte, 1024)
	}
	doneCh <- struct{}{}
}
<commit_msg>split double close test to a new test<commit_after>package gcnotifier

import (
	"io/ioutil"
	"runtime"
	"testing"
	"time"
)

func TestAfterGC(t *testing.T) {
	doneCh := make(chan struct{})

	go func() {
		M := &runtime.MemStats{}
		NumGC := uint32(0)
		gcn := New()
		for range gcn.AfterGC() {
			runtime.ReadMemStats(M)
			NumGC += 1
			if NumGC != M.NumGC {
				t.Fatal("Skipped a GC notification")
			}
			if NumGC >= 500 {
				gcn.Close()
			}
		}
		doneCh <- struct{}{}
	}()

	for {
		select {
		case <-time.After(1 * time.Millisecond):
			b := make([]byte, 1<<20)
			b[0] = 1
		case <-doneCh:
			return
		}
	}
}

func TestDoubleClose(t *testing.T) {
	gcn := New()
	gcn.Close()
	gcn.Close() // no-op
}

func TestAutoclose(t *testing.T) {
	count := 10000
	done := make(chan struct{})
	go func() {
		for i := 0; i < count; i++ {
			gcn := New().AfterGC()
			go func() {
				for range gcn {
				}
				// to reach here autoclose() must have been called
				done <- struct{}{}
			}()
		}
	}()
	for i := 0; i < count; {
		select {
		case <-done:
			i++
		default:
			runtime.GC() // required to quickly trigger autoclose()
		}
	}
}

// Example implements a simple time-based buffering io.Writer: data sent over
// dataCh is buffered for up to 100ms, then flushed out in a single call to
// out.Write and the buffer is reused. If GC runs, the buffer is flushed and
// then discarded so that it can be collected during the next GC run. The
// example is necessarily simplistic, a real implementation would be more
// refined (e.g. on GC flush or resize the buffer based on a threshold,
// perform asynchronous flushes, properly signal completions and propagate
// errors, adaptively preallocate the buffer based on the previous capacity,
// etc.)
func Example() {
	dataCh := make(chan []byte)
	doneCh := make(chan struct{})

	out := ioutil.Discard

	go func() {
		var buf []byte
		var tick <-chan time.Time
		gcn := New()

		for {
			select {
			case data := <-dataCh:
				if tick == nil {
					tick = time.After(100 * time.Millisecond)
				}
				// received data to write to the buffer
				buf = append(buf, data...)
			case <-tick:
				// time to flush the buffer (but reuse it for the next writes)
				if len(buf) > 0 {
					out.Write(buf)
					buf = buf[:0]
				}
				tick = nil
			case <-gcn.AfterGC():
				// GC just ran: flush and then drop the buffer
				if len(buf) > 0 {
					out.Write(buf)
				}
				buf = nil
				tick = nil
			case <-doneCh:
				// close the writer: flush the buffer and return
				if len(buf) > 0 {
					out.Write(buf)
				}
				return
			}
		}
	}()

	for i := 0; i < 1<<20; i++ {
		dataCh <- make([]byte, 1024)
	}
	doneCh <- struct{}{}
}
<|endoftext|>


<commit_before>package wal

import (
	"bufio"
	"encoding/gob"
	"fmt"
	"io"
	"math"
	"os"
)

type GlobalState struct {
	// used for creating index entries
	CurrentFileSuffix int
	CurrentFileOffset int64

	// keep track of the next request number
	LargestRequestNumber uint32

	// used for rollover
	FirstSuffix int

	// last seq number used
	ShardLastSequenceNumber map[uint32]uint64

	// committed request number per server
	ServerLastRequestNumber map[uint32]uint32

	// path to the state file
	path string
}

func newGlobalState(path string) (*GlobalState, error) {
	f, err := os.Open(path)
	state := &GlobalState{
		ServerLastRequestNumber: map[uint32]uint32{},
		ShardLastSequenceNumber: map[uint32]uint64{},
		path: path,
	}
	if os.IsNotExist(err) {
		return state, nil
	}
	if err != nil {
		return nil, err
	}
	if err := state.read(f); err != nil {
		return nil, err
	}
	state.path = path
	return state, nil
}

func (self *GlobalState) writeToFile() error {
	newFile, err := os.OpenFile(self.path+".new", os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}

	if _, err := newFile.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	if err := self.write(newFile); err != nil {
		return err
	}

	if err := newFile.Sync(); err != nil {
		return err
	}

	if err := newFile.Close(); err != nil {
		return err
	}

	os.Remove(self.path)
	return os.Rename(self.path+".new", self.path)
}

func (self *GlobalState) write(w io.Writer) error {
	fmt.Fprintf(w, "%d\n", 1) // write the version
	return gob.NewEncoder(w).Encode(self)
}

func (self *GlobalState) read(r io.Reader) error {
	// skip the version
	reader := bufio.NewReader(r)
	// read the version line
	_, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	return gob.NewDecoder(reader).Decode(self)
}

func (self *GlobalState) recover(shardId uint32, sequenceNumber uint64) {
	lastSequenceNumber := self.ShardLastSequenceNumber[shardId]

	if sequenceNumber > lastSequenceNumber {
		self.ShardLastSequenceNumber[shardId] = sequenceNumber
	}
}

func (self *GlobalState) getNextRequestNumber() uint32 {
	self.LargestRequestNumber++
	return self.LargestRequestNumber
}

func (self *GlobalState) getCurrentSequenceNumber(shardId uint32) uint64 {
	return self.ShardLastSequenceNumber[shardId]
}

func (self *GlobalState) setCurrentSequenceNumber(shardId uint32, sequenceNumber uint64) {
	self.ShardLastSequenceNumber[shardId] = sequenceNumber
}

func (self *GlobalState) commitRequestNumber(serverId, requestNumber uint32) {
	currentRequestNumber, ok := self.ServerLastRequestNumber[serverId]
	if nextRequestNumber := currentRequestNumber + 1; ok && nextRequestNumber != requestNumber {
		panic(fmt.Errorf("Expecting %d to equal %d", nextRequestNumber, requestNumber))
	}
	self.ServerLastRequestNumber[serverId] = requestNumber
}

func (self *GlobalState) LowestCommitedRequestNumber() uint32 {
	requestNumber := uint32(math.MaxUint32)
	for _, number := range self.ServerLastRequestNumber {
		if number < requestNumber {
			requestNumber = number
		}
	}
	return requestNumber
}
<commit_msg>Revert "panic if we ever receive commits out of order"<commit_after>package wal

import (
	"bufio"
	"encoding/gob"
	"fmt"
	"io"
	"math"
	"os"
)

type GlobalState struct {
	// used for creating index entries
	CurrentFileSuffix int
	CurrentFileOffset int64

	// keep track of the next request number
	LargestRequestNumber uint32

	// used for rollover
	FirstSuffix int

	// last seq number used
	ShardLastSequenceNumber map[uint32]uint64

	// committed request number per server
	ServerLastRequestNumber map[uint32]uint32

	// path to the state file
	path string
}

func newGlobalState(path string) (*GlobalState, error) {
	f, err := os.Open(path)
	state := &GlobalState{
		ServerLastRequestNumber: map[uint32]uint32{},
		ShardLastSequenceNumber: map[uint32]uint64{},
		path: path,
	}
	if os.IsNotExist(err) {
		return state, nil
	}
	if err != nil {
		return nil, err
	}
	if err := state.read(f); err != nil {
		return nil, err
	}
	state.path = path
	return state, nil
}

func (self *GlobalState) writeToFile() error {
	newFile, err := os.OpenFile(self.path+".new", os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}

	if _, err := newFile.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	if err := self.write(newFile); err != nil {
		return err
	}

	if err := newFile.Sync(); err != nil {
		return err
	}

	if err := newFile.Close(); err != nil {
		return err
	}

	os.Remove(self.path)
	return os.Rename(self.path+".new", self.path)
}

func (self *GlobalState) write(w io.Writer) error {
	fmt.Fprintf(w, "%d\n", 1) // write the version
	return gob.NewEncoder(w).Encode(self)
}

func (self *GlobalState) read(r io.Reader) error {
	// skip the version
	reader := bufio.NewReader(r)
	// read the version line
	_, err := reader.ReadString('\n')
	if err != nil {
		return err
	}
	return gob.NewDecoder(reader).Decode(self)
}

func (self *GlobalState) recover(shardId uint32, sequenceNumber uint64) {
	lastSequenceNumber := self.ShardLastSequenceNumber[shardId]

	if sequenceNumber > lastSequenceNumber {
		self.ShardLastSequenceNumber[shardId] = sequenceNumber
	}
}

func (self *GlobalState) getNextRequestNumber() uint32 {
	self.LargestRequestNumber++
	return self.LargestRequestNumber
}

func (self *GlobalState) getCurrentSequenceNumber(shardId uint32) uint64 {
	return self.ShardLastSequenceNumber[shardId]
}

func (self *GlobalState) setCurrentSequenceNumber(shardId uint32, sequenceNumber uint64) {
	self.ShardLastSequenceNumber[shardId] = sequenceNumber
}

func (self *GlobalState) commitRequestNumber(serverId, requestNumber uint32) {
	self.ServerLastRequestNumber[serverId] = requestNumber
}

func (self *GlobalState) LowestCommitedRequestNumber() uint32 {
	requestNumber := uint32(math.MaxUint32)
	for _, number := range self.ServerLastRequestNumber {
		if number < requestNumber {
			requestNumber = number
		}
	}
	return requestNumber
}
<|endoftext|>


<commit_before>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clock manages game timers.
package clock

import (
	"sync"
	"time"
)

var (
	// lastSystemTime is the last system time in the previous Update.
	lastSystemTime int64

	currentFPS  float64
	currentTPS  float64
	lastUpdated int64
	fpsCount    = 0
	tpsCount    = 0

	m sync.Mutex
)

func CurrentFPS() float64 {
	m.Lock()
	v := currentFPS
	m.Unlock()
	return v
}

func CurrentTPS() float64 {
	m.Lock()
	v := currentTPS
	m.Unlock()
	return v
}

func calcCountFromTPS(tps int64, now int64) int {
	if tps == 0 {
		return 0
	}
	if tps < 0 {
		panic("clock: tps must >= 0")
	}

	// Initialize lastSystemTime if needed.
	if lastSystemTime == 0 {
		lastSystemTime = now
	}

	diff := now - lastSystemTime
	if diff < 0 {
		// TODO: Should this panic?
		lastSystemTime = now
		return 0
	}

	count := 0
	syncWithSystemClock := false

	if diff > int64(time.Second)*5/60 {
		// The previous time is too old.
		// Let's force to sync the game time with the system clock.
		syncWithSystemClock = true
	} else {
		count = int(diff * tps / int64(time.Second))
	}

	// Stabilize FPS.
	// Without this adjustment, count can be unstable like 0, 2, 0, 2, ...
	if count == 0 && (int64(time.Second)/tps/2) < diff {
		count = 1
	}
	if count == 2 && (int64(time.Second)/tps*3/2) > diff {
		count = 1
	}

	if syncWithSystemClock {
		lastSystemTime = now
	} else {
		lastSystemTime += int64(count) * int64(time.Second) / tps
	}

	return count
}

func updateFPSAndTPS(now int64, count int) {
	if lastUpdated == 0 {
		lastUpdated = now
	}
	fpsCount++
	tpsCount += count
	if now < lastUpdated {
		// TODO: Should this panic?
		lastUpdated = now
		fpsCount = 0
		tpsCount = 0
		return
	}
	if time.Second > time.Duration(now-lastUpdated) {
		return
	}
	currentFPS = float64(fpsCount) * float64(time.Second) / float64(now-lastUpdated)
	currentTPS = float64(tpsCount) * float64(time.Second) / float64(now-lastUpdated)
	lastUpdated = now
	fpsCount = 0
	tpsCount = 0
}

const UncappedTPS = -1

// Update updates the inner clock state and returns an integer value
// indicating how many times the game should update based on given tps.
// tps represents TPS (ticks per second).
// If tps is UncappedTPS, Update always returns 1.
// If tps <= 0 and not UncappedTPS, Update always returns 0.
//
// Update is expected to be called per frame.
func Update(tps int) int {
	m.Lock()
	defer m.Unlock()

	n := now()

	c := 0
	if tps == UncappedTPS {
		c = 1
	} else if tps > 0 {
		c = calcCountFromTPS(int64(tps), n)
	}
	updateFPSAndTPS(n, c)

	return c
}
<commit_msg>clock: Assert that now() must be monotonic<commit_after>// Copyright 2017 The Ebiten Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package clock manages game timers.
package clock

import (
	"sync"
	"time"
)

var (
	lastNow int64

	// lastSystemTime is the last system time in the previous Update.
	// lastSystemTime indicates the logical time in the game, so this can be bigger than the curren time.
	lastSystemTime int64

	currentFPS  float64
	currentTPS  float64
	lastUpdated int64
	fpsCount    = 0
	tpsCount    = 0

	m sync.Mutex
)

func init() {
	n := now()
	lastNow = n
	lastSystemTime = n
	lastUpdated = n
}

func CurrentFPS() float64 {
	m.Lock()
	v := currentFPS
	m.Unlock()
	return v
}

func CurrentTPS() float64 {
	m.Lock()
	v := currentTPS
	m.Unlock()
	return v
}

func calcCountFromTPS(tps int64, now int64) int {
	if tps == 0 {
		return 0
	}
	if tps < 0 {
		panic("clock: tps must >= 0")
	}

	diff := now - lastSystemTime
	if diff < 0 {
		return 0
	}

	count := 0
	syncWithSystemClock := false

	if diff > int64(time.Second)*5/60 {
		// The previous time is too old.
		// Let's force to sync the game time with the system clock.
		syncWithSystemClock = true
	} else {
		count = int(diff * tps / int64(time.Second))
	}

	// Stabilize FPS.
	// Without this adjustment, count can be unstable like 0, 2, 0, 2, ...
	if count == 0 && (int64(time.Second)/tps/2) < diff {
		count = 1
	}
	if count == 2 && (int64(time.Second)/tps*3/2) > diff {
		count = 1
	}

	if syncWithSystemClock {
		lastSystemTime = now
	} else {
		lastSystemTime += int64(count) * int64(time.Second) / tps
	}

	return count
}

func updateFPSAndTPS(now int64, count int) {
	fpsCount++
	tpsCount += count
	if now < lastUpdated {
		panic("clock: lastUpdated must be older than now")
	}
	if time.Second > time.Duration(now-lastUpdated) {
		return
	}
	currentFPS = float64(fpsCount) * float64(time.Second) / float64(now-lastUpdated)
	currentTPS = float64(tpsCount) * float64(time.Second) / float64(now-lastUpdated)
	lastUpdated = now
	fpsCount = 0
	tpsCount = 0
}

const UncappedTPS = -1

// Update updates the inner clock state and returns an integer value
// indicating how many times the game should update based on given tps.
// tps represents TPS (ticks per second).
// If tps is UncappedTPS, Update always returns 1.
// If tps <= 0 and not UncappedTPS, Update always returns 0.
//
// Update is expected to be called per frame.
func Update(tps int) int {
	m.Lock()
	defer m.Unlock()

	n := now()
	if lastNow > n {
		// This ensures that now() must be monotonic (#875).
		panic("clock: lastNow must be older than n")
	}
	lastNow = n

	c := 0
	if tps == UncappedTPS {
		c = 1
	} else if tps > 0 {
		c = calcCountFromTPS(int64(tps), n)
	}
	updateFPSAndTPS(n, c)

	return c
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"encoding/base64"
	"encoding/hex"
	"testing"
)

// From http://en.wikipedia.org/wiki/Hash-based_message_authentication_code
// HMAC_SHA1("key", "The quick brown fox jumps over the lazy dog")
//     = 0xde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9
var message = "The quick brown fox jumps over the lazy dog"
var signingKey = base64.URLEncoding.EncodeToString([]byte("key"))
var signature = "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9"

func TestSigner(t *testing.T) {
	s, err := hex.DecodeString(signature)
	expected := base64.URLEncoding.EncodeToString(s)
	if err != nil {
		t.Errorf("Couldn't decode expected signature: %+v", err)
	}
	generated, err := GenerateSignature(signingKey, message)
	if err != nil {
		t.Errorf("Couldn't generate actual signature: %+v", err)
	}
	if expected != generated {
		t.Errorf("expected equal signature, was %s, expected %s", generated, expected)
	}
}
<commit_msg>Putting error check where it makes more sense<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"encoding/base64"
	"encoding/hex"
	"testing"
)

// From http://en.wikipedia.org/wiki/Hash-based_message_authentication_code
// HMAC_SHA1("key", "The quick brown fox jumps over the lazy dog")
//     = 0xde7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9
var message = "The quick brown fox jumps over the lazy dog"
var signingKey = base64.URLEncoding.EncodeToString([]byte("key"))
var signature = "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9"

func TestSigner(t *testing.T) {
	s, err := hex.DecodeString(signature)
	if err != nil {
		t.Errorf("Couldn't decode expected signature: %+v", err)
	}
	expected := base64.URLEncoding.EncodeToString(s)
	generated, err := GenerateSignature(signingKey, message)
	if err != nil {
		t.Errorf("Couldn't generate actual signature: %+v", err)
	}
	if expected != generated {
		t.Errorf("expected equal signature, was %s, expected %s", generated, expected)
	}
}
<|endoftext|>