<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

type FsckCallbackType int

const (
	// Process is just working through data (progress update)
	FsckWorking FsckCallbackType = iota
	// A file was missing and not recovered from shared store (desc = name)
	FsckMissing FsckCallbackType = iota
	// A content file had the wrong size (desc = name)
	// This file will be deleted if --delete was specified
	FsckWrongSize FsckCallbackType = iota
	// A binary is corrupt - either the metadata is invalid, or the SHA doesn't match the content (only detected with --deep) (desc = SHA)
	// The files will be deleted if --delete was specified
	FsckCorruptData FsckCallbackType = iota
)

// Collected callback data for a fsck operation
type FsckCallbackData struct {
	// What stage of the process this is for, preparing, transferring or skipping something
	Type FsckCallbackType
	// What SHA is being checked
	SHA string
	// Either a general message or an item name (e.g. file name)
	Desc string
	// The percentage complete
	PercentComplete int
}

// Fsck command line tool
func cmdFsck() int {

	// git-lob fsck [--deep] [--shared]

	// Validate custom options
	errorList := validateCustomOptions(GlobalOptions, nil, []string{"deep", "d", "shared", "s", "delete", "x"})
	if len(errorList) > 0 {
		LogConsoleError(strings.Join(errorList, "\n"))
		return 9
	}

	optDeep := GlobalOptions.BoolOpts.Contains("deep") || GlobalOptions.BoolOpts.Contains("d")
	optShared := GlobalOptions.BoolOpts.Contains("shared") || GlobalOptions.BoolOpts.Contains("s")
	optDelete := GlobalOptions.BoolOpts.Contains("delete") || GlobalOptions.BoolOpts.Contains("x")

	if optShared {
		// Check we have a shared store
		if GlobalOptions.SharedStore == "" {
			LogConsoleError("No shared store is configured for this repository, cannot use --shared")
			return 8
		}
		LogConsole("Checking shared store at", GlobalOptions.SharedStore)
	} else {
		LogConsole("Checking local binary store")
	}

	var shas []string
	if len(GlobalOptions.Args) > 0 {
		shas = GlobalOptions.Args
	}

	callback := func(data *FsckCallbackData) (quit bool) {
		// Ensure we clear previous progress
		LogConsolef("\r")
		switch data.Type {
		case FsckMissing:
			LogErrorf(" * %v: file is missing, try fetch/prune (%v)\n", data.SHA[:7], data.Desc)
		case FsckCorruptData:
			LogErrorf(" * %v: content is corrupt (deleted: %v)\n", data.SHA[:7], optDelete)
		case FsckWrongSize:
			LogErrorf(" * %v: file is wrong size (%v deleted: %v)\n", data.SHA[:7], data.Desc, optDelete)
		case FsckWorking:
			// Do nothing, just progress below
		}
		// Display progress always (fixed line width always large enough)
		LogConsoleOverwrite(fmt.Sprintf("Progress: %d%%", data.PercentComplete), 14)
		// Always continue
		return false
	}
	// Add newlines to messages since progress doesn't
	err := Fsck(optDeep, optShared, optDelete, shas, callback)
	if err != nil {
		LogConsoleError("\nError(s) in fsck, see above.")
		return 12
	}
	LogConsole("\nCompleted successfully, no problems found")
	return 0
}

// Validate local binary store
// deep = recaculate SHAs & check (takes longer) instead of just checking file size
// shared = check shared store instead of local store
// deleteBadFiles = delete files which are the wrong size or corrupted
// shas = specific list of binaries to check; if empty, checks entire store
// callback = for progress and file errors, return quit to abort (also skips deleting current item)
// The returned error will be nil if no files had any issues but the process will continue (with callbacks)
// even when missing/bad files are encountered until all have been checked
func Fsck(deep, shared, deleteBadFiles bool, shas []string, callback func(data *FsckCallbackData) (quit bool)) error {

	// When listing all LOBs it returns a set to eliminate dupes so use this across both
	// cheaper to construct a set from small number of arguments than a slice from potentially
	// unlimited number of items on the disk
	var shaSet StringSet
	if len(shas) == 0 {
		var err error
		if shared {
			shaSet, err = getAllSharedLOBSHAs()
		} else {
			shaSet, err = getAllLocalLOBSHAs()
		}
		if err != nil {
			return err
		}
	} else {
		shaSet = NewStringSetFromSlice(shas)
	}

	var basedir string
	if shared {
		basedir = GetSharedLOBRoot()
	} else {
		basedir = GetLocalLOBRoot()
	}
	i := 0
	var errorList []string
	for sha := range shaSet.Iter() {
		percent := int(float32(i+1) * 100 / float32(len(shaSet)))
		err := CheckLOBFilesForSHA(sha, basedir, deep)
		var quit bool
		if err != nil {
			switch e := err.(type) {
			case *NotFoundError:
				quit = callback(&FsckCallbackData{FsckMissing, sha, e.Path, percent})
			case *IntegrityError:
				quit = callback(&FsckCallbackData{FsckCorruptData, sha, sha, percent})
				if !quit && deleteBadFiles {
					// Delete all files for this LOB
					delerr := deleteLOBRelative(sha, basedir)
					if delerr != nil {
						// Log but don't abort for this
						LogErrorf("fsck error: Unable to delete bad LOB %v from %v: %v", sha, basedir, delerr.Error())
					}
				}
			case *WrongSizeError:
				quit = callback(&FsckCallbackData{FsckWrongSize, sha, e.Filename, percent})
				if !quit && deleteBadFiles {
					// in this case we only delete the single file which is bad (might just be one chunk of many)
					// others could be downloaded again later
					delerr := os.Remove(e.Filename)
					if delerr != nil {
						// Log but don't abort for this
						LogErrorf("fsck error: Unable to delete %v: %v", e.Filename, delerr.Error())
					}
				}
			default:
				// Something else, abort
				return fmt.Errorf("fsck aborted: %v", e.Error())
			}
			errorList = append(errorList, err.Error())
		} else {
			quit = callback(&FsckCallbackData{FsckWorking, sha, "", percent})
		}
		i++

		if quit {
			break
		}

	}

	if len(errorList) > 0 {
		return errors.New(strings.Join(errorList, "\n"))
	}
	return nil

}

func cmdFsckHelp() {
	LogConsole(`Usage: git-lob fsck [options] [SHA...]

  Validates that the local binary store is internally consistent.

  This utility command checks the contents of the local binary store to make
  sure that each binary stored there is complete & correct. The basic mode
  just ensures that all the required file components are there and are of the
  correct size, wheras --deep mode also checks every byte of the content
  to ensure it is correct (by checking that the SHA matches the content).

  If you're using a shared store across repos (see git-lob.sharedstore in
  'git lob help config') and a missing local file is available in that shared
  store, it will automatically be re-linked into your local repo to resolve the
  problem.

  The --delete option can be used to clean any files which are invalid.
  Partially downloaded binaries where some chunks are missing are not deleted
  so you can resume downloading, but invalid files such as corrupt metadata,
  incorrectly sized chunks, and content where the SHA doesn't agree (only
  checked with the --deep option) are deleted.

  This command doesn't check your working copy, use 'git lob status' for that,
  for example to help figure out why a binary file is still a placeholder.

Parameters:
  SHA...        If you supply one or more 40-character SHA arguments, only
                those binaries are checked rather than the entire store. The
                SHA is the identifier of the binary content itself, not a Git
                object.

Options:
  --deep, -d    Recalculates the SHA of each binary file to ensure the contents
                are correct. Without this option, just checks that all files
                are present and the correct size.
  --shared, -s  Checks the shared store instead of the local repo
  --delete, -x  Delete files which are invalid. Doesn't delete partial binaries
                where 1 or more chunks are missing, but deletes files which are
                internally inconsistent; e.g. invalid meta files, partial
                chunks, and all files where --deep is used and SHA doesn't
                agree with content.
  --quiet, -q   Print less output
  --verbose, -v Print more output

`)
}
<commit_msg>Refer to 'missing' command in fsck help<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
)

type FsckCallbackType int

const (
	// Process is just working through data (progress update)
	FsckWorking FsckCallbackType = iota
	// A file was missing and not recovered from shared store (desc = name)
	FsckMissing FsckCallbackType = iota
	// A content file had the wrong size (desc = name)
	// This file will be deleted if --delete was specified
	FsckWrongSize FsckCallbackType = iota
	// A binary is corrupt - either the metadata is invalid, or the SHA doesn't match the content (only detected with --deep) (desc = SHA)
	// The files will be deleted if --delete was specified
	FsckCorruptData FsckCallbackType = iota
)

// Collected callback data for a fsck operation
type FsckCallbackData struct {
	// What stage of the process this is for, preparing, transferring or skipping something
	Type FsckCallbackType
	// What SHA is being checked
	SHA string
	// Either a general message or an item name (e.g. file name)
	Desc string
	// The percentage complete
	PercentComplete int
}

// Fsck command line tool
func cmdFsck() int {

	// git-lob fsck [--deep] [--shared]

	// Validate custom options
	errorList := validateCustomOptions(GlobalOptions, nil, []string{"deep", "d", "shared", "s", "delete", "x"})
	if len(errorList) > 0 {
		LogConsoleError(strings.Join(errorList, "\n"))
		return 9
	}

	optDeep := GlobalOptions.BoolOpts.Contains("deep") || GlobalOptions.BoolOpts.Contains("d")
	optShared := GlobalOptions.BoolOpts.Contains("shared") || GlobalOptions.BoolOpts.Contains("s")
	optDelete := GlobalOptions.BoolOpts.Contains("delete") || GlobalOptions.BoolOpts.Contains("x")

	if optShared {
		// Check we have a shared store
		if GlobalOptions.SharedStore == "" {
			LogConsoleError("No shared store is configured for this repository, cannot use --shared")
			return 8
		}
		LogConsole("Checking shared store at", GlobalOptions.SharedStore)
	} else {
		LogConsole("Checking local binary store")
	}

	var shas []string
	if len(GlobalOptions.Args) > 0 {
		shas = GlobalOptions.Args
	}

	callback := func(data *FsckCallbackData) (quit bool) {
		// Ensure we clear previous progress
		LogConsolef("\r")
		switch data.Type {
		case FsckMissing:
			LogErrorf(" * %v: file is missing, try fetch/prune (%v)\n", data.SHA[:7], data.Desc)
		case FsckCorruptData:
			LogErrorf(" * %v: content is corrupt (deleted: %v)\n", data.SHA[:7], optDelete)
		case FsckWrongSize:
			LogErrorf(" * %v: file is wrong size (%v deleted: %v)\n", data.SHA[:7], data.Desc, optDelete)
		case FsckWorking:
			// Do nothing, just progress below
		}
		// Display progress always (fixed line width always large enough)
		LogConsoleOverwrite(fmt.Sprintf("Progress: %d%%", data.PercentComplete), 14)
		// Always continue
		return false
	}
	// Add newlines to messages since progress doesn't
	err := Fsck(optDeep, optShared, optDelete, shas, callback)
	if err != nil {
		LogConsoleError("\nError(s) in fsck, see above.")
		return 12
	}
	LogConsole("\nCompleted successfully, no problems found")
	return 0
}

// Validate local binary store
// deep = recaculate SHAs & check (takes longer) instead of just checking file size
// shared = check shared store instead of local store
// deleteBadFiles = delete files which are the wrong size or corrupted
// shas = specific list of binaries to check; if empty, checks entire store
// callback = for progress and file errors, return quit to abort (also skips deleting current item)
// The returned error will be nil if no files had any issues but the process will continue (with callbacks)
// even when missing/bad files are encountered until all have been checked
func Fsck(deep, shared, deleteBadFiles bool, shas []string, callback func(data *FsckCallbackData) (quit bool)) error {

	// When listing all LOBs it returns a set to eliminate dupes so use this across both
	// cheaper to construct a set from small number of arguments than a slice from potentially
	// unlimited number of items on the disk
	var shaSet StringSet
	if len(shas) == 0 {
		var err error
		if shared {
			shaSet, err = getAllSharedLOBSHAs()
		} else {
			shaSet, err = getAllLocalLOBSHAs()
		}
		if err != nil {
			return err
		}
	} else {
		shaSet = NewStringSetFromSlice(shas)
	}

	var basedir string
	if shared {
		basedir = GetSharedLOBRoot()
	} else {
		basedir = GetLocalLOBRoot()
	}
	i := 0
	var errorList []string
	for sha := range shaSet.Iter() {
		percent := int(float32(i+1) * 100 / float32(len(shaSet)))
		err := CheckLOBFilesForSHA(sha, basedir, deep)
		var quit bool
		if err != nil {
			switch e := err.(type) {
			case *NotFoundError:
				quit = callback(&FsckCallbackData{FsckMissing, sha, e.Path, percent})
			case *IntegrityError:
				quit = callback(&FsckCallbackData{FsckCorruptData, sha, sha, percent})
				if !quit && deleteBadFiles {
					// Delete all files for this LOB
					delerr := deleteLOBRelative(sha, basedir)
					if delerr != nil {
						// Log but don't abort for this
						LogErrorf("fsck error: Unable to delete bad LOB %v from %v: %v", sha, basedir, delerr.Error())
					}
				}
			case *WrongSizeError:
				quit = callback(&FsckCallbackData{FsckWrongSize, sha, e.Filename, percent})
				if !quit && deleteBadFiles {
					// in this case we only delete the single file which is bad (might just be one chunk of many)
					// others could be downloaded again later
					delerr := os.Remove(e.Filename)
					if delerr != nil {
						// Log but don't abort for this
						LogErrorf("fsck error: Unable to delete %v: %v", e.Filename, delerr.Error())
					}
				}
			default:
				// Something else, abort
				return fmt.Errorf("fsck aborted: %v", e.Error())
			}
			errorList = append(errorList, err.Error())
		} else {
			quit = callback(&FsckCallbackData{FsckWorking, sha, "", percent})
		}
		i++

		if quit {
			break
		}

	}

	if len(errorList) > 0 {
		return errors.New(strings.Join(errorList, "\n"))
	}
	return nil

}

func cmdFsckHelp() {
	LogConsole(`Usage: git-lob fsck [options] [SHA...]

  Validates that the local binary store is internally consistent.

  This utility command checks the contents of the local binary store to make
  sure that each binary stored there is complete & correct. The basic mode
  just ensures that all the required file components are there and are of the
  correct size, wheras --deep mode also checks every byte of the content
  to ensure it is correct (by checking that the SHA matches the content).

  If you're using a shared store across repos (see git-lob.sharedstore in
  'git lob help config') and a missing local file is available in that shared
  store, it will automatically be re-linked into your local repo to resolve the
  problem.

  The --delete option can be used to clean any files which are invalid.
  Partially downloaded binaries where some chunks are missing are not deleted
  so you can resume downloading, but invalid files such as corrupt metadata,
  incorrectly sized chunks, and content where the SHA doesn't agree (only
  checked with the --deep option) are deleted.

  This command doesn't check your working copy, use 'git lob missing' to check
  why a binary file is still a placeholder.

Parameters:
  SHA...        If you supply one or more 40-character SHA arguments, only
                those binaries are checked rather than the entire store. The
                SHA is the identifier of the binary content itself, not a Git
                object.

Options:
  --deep, -d    Recalculates the SHA of each binary file to ensure the contents
                are correct. Without this option, just checks that all files
                are present and the correct size.
  --shared, -s  Checks the shared store instead of the local repo
  --delete, -x  Delete files which are invalid. Doesn't delete partial binaries
                where 1 or more chunks are missing, but deletes files which are
                internally inconsistent; e.g. invalid meta files, partial
                chunks, and all files where --deep is used and SHA doesn't
                agree with content.
  --quiet, -q   Print less output
  --verbose, -v Print more output

`)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	validate                     bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	environmentParam             string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	needSyncDirs                 []string
	needSyncEnvs                 map[string]struct{}
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJSONParseTime           float64
	metadataJSONParseTime        float64
	gmetadataJSONParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
	desiredContent               []string
	unchangedModuleDirs          []string
	mapModulesRefsToPuppetEnv    map[string]string
)

// LatestForgeModules contains a map of unique Forge modules
// that should be the latest versions of them
type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int            `yaml:"timeout"`
	IgnoreUnreachableModules    bool           `yaml:"ignore_unreachable_modules"`
	Maxworker                   int            `yaml:"maxworker"`
	MaxExtractworker            int            `yaml:"maxextractworker"`
	UseCacheFallback            bool           `yaml:"use_cache_fallback"`
	RetryGitCommands            bool           `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool           `yaml:"git_object_syntax_not_supported"`
	PostRunCommand              []string       `yaml:"postrun"`
	Deploy                      DeploySettings `yaml:"deploy"`
	PurgeLevels                 []string       `yaml:"purge_levels"`
	PurgeWhitelist              []string       `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist    []string       `yaml:"deployment_purge_whitelist"`
	WriteLock                   string         `yaml:"write_lock"`
	GenerateTypes               bool           `yaml:"generate_types"`
	PuppetPath                  string         `yaml:"puppet_path"`
	PurgeBlacklist              []string       `yaml:"purge_blacklist"`
	CloneGitModules             bool           `yaml:"clone_git_modules"`
}

// DeploySettings is a struct for settings for controlling how g10k deploys behave.
// Trying to emulate r10k https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy
type DeploySettings struct {
	PurgeLevels              []string `yaml:"purge_levels"`
	PurgeWhitelist           []string `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist []string `yaml:"deployment_purge_whitelist"`
	WriteLock                string   `yaml:"write_lock"`
	GenerateTypes            bool     `yaml:"generate_types"`
	PuppetPath               string   `yaml:"puppet_path"`
	PurgeBlacklist           []string `yaml:"purge_blacklist"`
}

// Forge is a simple struct that contains the base URL of
// the Forge that g10k should use. Defaults to: https://forgeapi.puppetlabs.com
type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

// Git is a simple struct that contains the optional SSH private key to
// use for authentication
type Git struct {
	privateKey string `yaml:"private_key"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
	FilterCommand               string `yaml:"filter_command"`
	FilterRegex                 string `yaml:"filter_regex"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	forgeBaseURL      string
	forgeCacheTTL     time.Duration
	forgeModules      map[string]ForgeModule
	gitModules        map[string]GitModule
	privateKey        string
	source            string
	sourceBranch      string
	workDir           string
	moduleDirs        []string
	controlRepoBranch string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version      string
	name         string
	author       string
	md5sum       string
	fileSize     int64
	baseURL      string
	cacheTTL     time.Duration
	sha256sum    string
	moduleDir    string
	sourceBranch string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	tree              string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
	moduleDir         string
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

// DeployResult contains information about the Puppet environment which was deployed by g10k and tries to emulate the .r10k-deploy.json
type DeployResult struct {
	Name               string    `json:"name"`
	Signature          string    `json:"signature"`
	StartedAt          time.Time `json:"started_at"`
	FinishedAt         time.Time `json:"finished_at"`
	DeploySuccess      bool      `json:"deploy_success"`
	PuppetfileChecksum string    `json:"puppetfile_checksum"`
}

func init() {
	// initialize global maps
	needSyncEnvs = make(map[string]struct{})
	uniqueForgeModules = make(map[string]ForgeModule)
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update. Just the branch name, e.g. master, qa, dev")
	flag.StringVar(&environmentParam, "environment", "", "which Puppet environment to update. Source name inside the config + '_' + branch name, e.g. foo_master, foo_qa, foo_dev")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&validate, "validate", false, "only validate given configuration and exit")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.8.11 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		checkDirAndCreate(config.CacheDir, "cachedir configured value")
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			branchParam = ""
			resolvePuppetEnvironment(tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "./"}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			// default purge_levels
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			config.PurgeLevels = []string{"puppetfile"}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", "cmdlineparam", false, false)
			puppetfile.workDir = ""
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJSONParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJSONParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolve and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}

	checkForAndExecutePostrunCommand()
}
<commit_msg>add git_dir and git_url to .g10k-deploy.json<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	debug                        bool
	verbose                      bool
	info                         bool
	quiet                        bool
	force                        bool
	usemove                      bool
	usecacheFallback             bool
	retryGitCommands             bool
	pfMode                       bool
	pfLocation                   string
	dryRun                       bool
	validate                     bool
	check4update                 bool
	checkSum                     bool
	gitObjectSyntaxNotSupported  bool
	moduleDirParam               string
	cacheDirParam                string
	branchParam                  string
	environmentParam             string
	tags                         bool
	outputNameParam              string
	moduleParam                  string
	configFile                   string
	config                       ConfigSettings
	mutex                        sync.Mutex
	empty                        struct{}
	syncGitCount                 int
	syncForgeCount               int
	needSyncGitCount             int
	needSyncForgeCount           int
	needSyncDirs                 []string
	needSyncEnvs                 map[string]struct{}
	syncGitTime                  float64
	syncForgeTime                float64
	ioGitTime                    float64
	ioForgeTime                  float64
	forgeJSONParseTime           float64
	metadataJSONParseTime        float64
	gmetadataJSONParseTime       float64
	buildtime                    string
	uniqueForgeModules           map[string]ForgeModule
	latestForgeModules           LatestForgeModules
	maxworker                    int
	maxExtractworker             int
	forgeModuleDeprecationNotice string
	desiredContent               []string
	unchangedModuleDirs          []string
	mapModulesRefsToPuppetEnv    map[string]string
)

// LatestForgeModules contains a map of unique Forge modules
// that should be the latest versions of them
type LatestForgeModules struct {
	sync.RWMutex
	m map[string]string
}

// ConfigSettings contains the key value pairs from the g10k config file
type ConfigSettings struct {
	CacheDir                    string `yaml:"cachedir"`
	ForgeCacheDir               string
	ModulesCacheDir             string
	EnvCacheDir                 string
	Git                         Git
	Forge                       Forge
	Sources                     map[string]Source
	Timeout                     int            `yaml:"timeout"`
	IgnoreUnreachableModules    bool           `yaml:"ignore_unreachable_modules"`
	Maxworker                   int            `yaml:"maxworker"`
	MaxExtractworker            int            `yaml:"maxextractworker"`
	UseCacheFallback            bool           `yaml:"use_cache_fallback"`
	RetryGitCommands            bool           `yaml:"retry_git_commands"`
	GitObjectSyntaxNotSupported bool           `yaml:"git_object_syntax_not_supported"`
	PostRunCommand              []string       `yaml:"postrun"`
	Deploy                      DeploySettings `yaml:"deploy"`
	PurgeLevels                 []string       `yaml:"purge_levels"`
	PurgeWhitelist              []string       `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist    []string       `yaml:"deployment_purge_whitelist"`
	WriteLock                   string         `yaml:"write_lock"`
	GenerateTypes               bool           `yaml:"generate_types"`
	PuppetPath                  string         `yaml:"puppet_path"`
	PurgeBlacklist              []string       `yaml:"purge_blacklist"`
	CloneGitModules             bool           `yaml:"clone_git_modules"`
}

// DeploySettings is a struct for settings for controlling how g10k deploys behave.
// Trying to emulate r10k https://github.com/puppetlabs/r10k/blob/master/doc/dynamic-environments/configuration.mkd#deploy
type DeploySettings struct {
	PurgeLevels              []string `yaml:"purge_levels"`
	PurgeWhitelist           []string `yaml:"purge_whitelist"`
	DeploymentPurgeWhitelist []string `yaml:"deployment_purge_whitelist"`
	WriteLock                string   `yaml:"write_lock"`
	GenerateTypes            bool     `yaml:"generate_types"`
	PuppetPath               string   `yaml:"puppet_path"`
	PurgeBlacklist           []string `yaml:"purge_blacklist"`
}

// Forge is a simple struct that contains the base URL of
// the Forge that g10k should use. Defaults to: https://forgeapi.puppetlabs.com
type Forge struct {
	Baseurl string `yaml:"baseurl"`
}

// Git is a simple struct that contains the optional SSH private key to
// use for authentication
type Git struct {
	privateKey string `yaml:"private_key"`
}

// Source contains basic information about a Puppet environment repository
type Source struct {
	Remote                      string
	Basedir                     string
	Prefix                      string
	PrivateKey                  string `yaml:"private_key"`
	ForceForgeVersions          bool   `yaml:"force_forge_versions"`
	WarnMissingBranch           bool   `yaml:"warn_if_branch_is_missing"`
	ExitIfUnreachable           bool   `yaml:"exit_if_unreachable"`
	AutoCorrectEnvironmentNames string `yaml:"invalid_branches"`
	FilterCommand               string `yaml:"filter_command"`
	FilterRegex                 string `yaml:"filter_regex"`
}

// Puppetfile contains the key value pairs from the Puppetfile
type Puppetfile struct {
	forgeBaseURL      string
	forgeCacheTTL     time.Duration
	forgeModules      map[string]ForgeModule
	gitModules        map[string]GitModule
	privateKey        string
	source            string
	sourceBranch      string
	workDir           string
	gitDir            string
	gitURL            string
	moduleDirs        []string
	controlRepoBranch string
}

// ForgeModule contains information (Version, Name, Author, md5 checksum, file size of the tar.gz archive, Forge BaseURL if custom) about a Puppetlabs Forge module
type ForgeModule struct {
	version      string
	name         string
	author       string
	md5sum       string
	fileSize     int64
	baseURL      string
	cacheTTL     time.Duration
	sha256sum    string
	moduleDir    string
	sourceBranch string
}

// GitModule contains information about a Git Puppet module
type GitModule struct {
	privateKey        string
	git               string
	branch            string
	tag               string
	commit            string
	ref               string
	tree              string
	link              bool
	ignoreUnreachable bool
	fallback          []string
	installPath       string
	local             bool
	moduleDir         string
}

// ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded
type ForgeResult struct {
	needToGet     bool
	versionNumber string
	md5sum        string
	fileSize      int64
}

// ExecResult contains the exit code and output of an external command (e.g. git)
type ExecResult struct {
	returnCode int
	output     string
}

// DeployResult contains information about the Puppet environment which was deployed by g10k and tries to emulate the .r10k-deploy.json
type DeployResult struct {
	Name               string    `json:"name"`
	Signature          string    `json:"signature"`
	StartedAt          time.Time `json:"started_at"`
	FinishedAt         time.Time `json:"finished_at"`
	DeploySuccess      bool      `json:"deploy_success"`
	PuppetfileChecksum string    `json:"puppetfile_checksum"`
	GitDir             string    `json:"git_dir"`
	GitURL             string    `json:"git_url"`
}

func init() {
	// initialize global maps
	needSyncEnvs = make(map[string]struct{})
	uniqueForgeModules = make(map[string]ForgeModule)
}

func main() {

	var (
		configFileFlag = flag.String("config", "", "which config file to use")
		versionFlag    = flag.Bool("version", false, "show build time and version number")
	)
	flag.StringVar(&branchParam, "branch", "", "which git branch of the Puppet environment to update. Just the branch name, e.g. master, qa, dev")
	flag.StringVar(&environmentParam, "environment", "", "which Puppet environment to update. Source name inside the config + '_' + branch name, e.g. foo_master, foo_qa, foo_dev")
	flag.BoolVar(&tags, "tags", false, "to pull tags as well as branches")
	flag.StringVar(&outputNameParam, "outputname", "", "overwrite the environment name if -branch is specified")
	flag.StringVar(&moduleParam, "module", "", "which module of the Puppet environment to update, e.g. stdlib")
	flag.StringVar(&moduleDirParam, "moduledir", "", "allows overriding of Puppetfile specific moduledir setting, the folder in which Puppet modules will be extracted")
	flag.StringVar(&cacheDirParam, "cachedir", "", "allows overriding of the g10k config file cachedir setting, the folder in which g10k will download git repositories and Forge modules")
	flag.IntVar(&maxworker, "maxworker", 50, "how many Goroutines are allowed to run in parallel for Git and Forge module resolving")
	flag.IntVar(&maxExtractworker, "maxextractworker", 20, "how many Goroutines are allowed to run in parallel for local Git and Forge module extracting processes (git clone, untar and gunzip)")
	flag.BoolVar(&pfMode, "puppetfile", false, "install all modules from Puppetfile in cwd")
	flag.StringVar(&pfLocation, "puppetfilelocation", "./Puppetfile", "which Puppetfile to use in -puppetfile mode")
	flag.BoolVar(&force, "force", false, "purge the Puppet environment directory and do a full sync")
	flag.BoolVar(&dryRun, "dryrun", false, "do not modify anything, just print what would be changed")
	flag.BoolVar(&validate, "validate", false, "only validate given configuration and exit")
	flag.BoolVar(&usemove, "usemove", false, "do not use hardlinks to populate your Puppet environments with Puppetlabs Forge modules. Instead uses simple move commands and purges the Forge cache directory after each run! (Useful for g10k runs inside a Docker container)")
	flag.BoolVar(&check4update, "check4update", false, "only check if the is newer version of the Puppet module avaialable. Does implicitly set dryrun to true")
	flag.BoolVar(&checkSum, "checksum", false, "get the md5 check sum for each Puppetlabs Forge module and verify the integrity of the downloaded archive. Increases g10k run time!")
	flag.BoolVar(&debug, "debug", false, "log debug output, defaults to false")
	flag.BoolVar(&verbose, "verbose", false, "log verbose output, defaults to false")
	flag.BoolVar(&info, "info", false, "log info output, defaults to false")
	flag.BoolVar(&quiet, "quiet", false, "no output, defaults to false")
	flag.BoolVar(&usecacheFallback, "usecachefallback", false, "if g10k should try to use its cache for sources and modules instead of failing")
	flag.BoolVar(&retryGitCommands, "retrygitcommands", false, "if g10k should purge the local repository and retry a failed git command (clone or remote update) instead of failing")
	flag.BoolVar(&gitObjectSyntaxNotSupported, "gitobjectsyntaxnotsupported", false, "if your git version is too old to support reference syntax like master^{object} use this setting to revert to the older syntax")
	flag.Parse()

	configFile = *configFileFlag
	version := *versionFlag

	if version {
		fmt.Println("g10k version 0.8.11 Build time:", buildtime, "UTC")
		os.Exit(0)
	}

	if check4update {
		dryRun = true
	}

	// check for git executable dependency
	if _, err := exec.LookPath("git"); err != nil {
		Fatalf("Error: could not find 'git' executable in PATH")
	}

	target := ""
	before := time.Now()
	if len(configFile) > 0 {
		if usemove {
			Fatalf("Error: -usemove parameter is only allowed in -puppetfile mode!")
		}
		if pfMode {
			Fatalf("Error: -puppetfile parameter is not allowed with -config parameter!")
		}
		if (len(outputNameParam) > 0) && (len(branchParam) == 0) {
			Fatalf("Error: -outputname specified without -branch!")
		}
		if usecacheFallback {
			config.UseCacheFallback = true
		}
		Debugf("Using as config file: " + configFile)
		config = readConfigfile(configFile)
		checkDirAndCreate(config.CacheDir, "cachedir configured value")
		target = configFile
		if len(branchParam) > 0 {
			resolvePuppetEnvironment(tags, outputNameParam)
			target += " with branch " + branchParam
		} else {
			branchParam = ""
			resolvePuppetEnvironment(tags, "")
		}
	} else {
		if pfMode {
			Debugf("Trying to use as Puppetfile: " + pfLocation)
			sm := make(map[string]Source)
			sm["cmdlineparam"] = Source{Basedir: "./"}
			cachedir := "/tmp/g10k"
			if len(os.Getenv("g10k_cachedir")) > 0 {
				cachedir = os.Getenv("g10k_cachedir")
				cachedir = checkDirAndCreate(cachedir, "cachedir environment variable g10k_cachedir")
				Debugf("Found environment variable g10k_cachedir set to: " + cachedir)
			} else if len(cacheDirParam) > 0 {
				Debugf("Using -cachedir parameter set to : " + cacheDirParam)
				cachedir = checkDirAndCreate(cacheDirParam, "cachedir CLI param")
			} else {
				cachedir = checkDirAndCreate(cachedir, "cachedir default value")
			}
			// default purge_levels
			forgeDefaultSettings := Forge{Baseurl: "https://forgeapi.puppetlabs.com"}
			config = ConfigSettings{CacheDir: cachedir, ForgeCacheDir: cachedir, ModulesCacheDir: cachedir, EnvCacheDir: cachedir, Sources: sm, Forge: forgeDefaultSettings, Maxworker: maxworker, UseCacheFallback: usecacheFallback, MaxExtractworker: maxExtractworker, RetryGitCommands: retryGitCommands, GitObjectSyntaxNotSupported: gitObjectSyntaxNotSupported}
			config.PurgeLevels = []string{"puppetfile"}
			target = pfLocation
			puppetfile := readPuppetfile(target, "", "cmdlineparam", "cmdlineparam", false, false)
			puppetfile.workDir = ""
			pfm := make(map[string]Puppetfile)
			pfm["cmdlineparam"] = puppetfile
			resolvePuppetfile(pfm)
		} else {
			Fatalf("Error: you need to specify at least a config file or use the Puppetfile mode\nExample call: " + os.Args[0] + " -config test.yaml or " + os.Args[0] + " -puppetfile\n")
		}
	}

	if usemove {
		// we can not reuse the Forge cache at all when -usemove gets used, because we can not delete the -latest link for some reason
		defer purgeDir(config.ForgeCacheDir, "main() -puppetfile mode with -usemove parameter")
	}

	Debugf("Forge response JSON parsing took " + strconv.FormatFloat(forgeJSONParseTime, 'f', 4, 64) + " seconds")
	Debugf("Forge modules metadata.json parsing took " + strconv.FormatFloat(metadataJSONParseTime, 'f', 4, 64) + " seconds")

	if !check4update && !quiet {
		if len(forgeModuleDeprecationNotice) > 0 {
			Warnf(strings.TrimSuffix(forgeModuleDeprecationNotice, "\n"))
		}
		fmt.Println("Synced", target, "with", syncGitCount, "git repositories and", syncForgeCount, "Forge modules in "+strconv.FormatFloat(time.Since(before).Seconds(), 'f', 1, 64)+"s with git ("+strconv.FormatFloat(syncGitTime, 'f', 1, 64)+"s sync, I/O", strconv.FormatFloat(ioGitTime, 'f', 1, 64)+"s) and Forge ("+strconv.FormatFloat(syncForgeTime, 'f', 1, 64)+"s query+download, I/O", strconv.FormatFloat(ioForgeTime, 'f', 1, 64)+"s) using", strconv.Itoa(config.Maxworker), "resolve and", strconv.Itoa(config.MaxExtractworker), "extract workers")
	}
	if dryRun && (needSyncForgeCount > 0 || needSyncGitCount > 0) {
		os.Exit(1)
	}

	checkForAndExecutePostrunCommand()
}
<|endoftext|>


<commit_before>package npbbis

import (
	"github.com/PuerkitoBio/goquery"
	"strings"
)

type Game struct {
	Date     string
	Id       string
	Homeruns []*Homerun
}

func NewGame(date string, id string, homeruns []*Homerun) *Game {
	return &Game{
		Date:     date,
		Id:       id,
		Homeruns: homeruns,
	}
}

func GetGame(date string, id string) (*Game, error) {
	year := date[0:4]
	uri := "http://bis.npb.or.jp/" + year + "/games/s" + id + ".html"
	doc, err := newGoqueryDocument(uri)
	if err != nil {
		return nil, err
	}

	hrs := []*Homerun{}
	doc.Find("#gmdivhr .gmresults").Each(func(i int, s *goquery.Selection) {
		hrs = append(hrs, parseHomerun(s.Text())...)
	})
	return NewGame(date, id, hrs), nil
}

func GetGames(date string) ([]*Game, error) {
	uri := "http://bis.npb.or.jp/" + date[0:4] + "/games/gm" + date + ".html"
	doc, err := newGoqueryDocument(uri)
	if err != nil {
		return nil, err
	}
	games := []*Game{}
	doc.Find(".contentsgame .contentsinfo a").Each(func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if exists {
			id := href[1:14]
			var game *Game
			game, err = GetGame(date, id)
			games = append(games, game)
		}
	})
	return games, err
}

// parsers

func parseHomerun(line string) []*Homerun {
	hrs := []*Homerun{}
	for _, chunk := range strings.SplitAfter(line, "］ ")[1:] {
		lastBatter := ""
		for _, subchunk := range strings.Split(chunk, ",") {
			tokens := strings.Split(subchunk, " ")
			batter := tokens[0]
			number := tokens[1]
			scenario := tokens[3]
			pitcher := tokens[4]
			if len(batter) == 0 {
				batter = lastBatter
			}
			hrs = append(hrs, NewHomerun(batter, number, scenario, pitcher))
			lastBatter = batter
		}
	}
	return hrs
}
<commit_msg>Add game status<commit_after>package npbbis

import (
	"github.com/PuerkitoBio/goquery"
	"strings"
)

type Game struct {
	Date     string
	Id       string
	Status   string
	Homeruns []*Homerun
}

func NewGame(date string, id string, status string, homeruns []*Homerun) *Game {
	return &Game{
		Date:     date,
		Id:       id,
		Status:   status,
		Homeruns: homeruns,
	}
}

func GetGame(date string, id string) (*Game, error) {
	year := date[0:4]
	uri := "http://bis.npb.or.jp/" + year + "/games/s" + id + ".html"
	doc, err := newGoqueryDocument(uri)
	if err != nil {
		return nil, err
	}

	// Status
	var status string
	doc.Find("#gmdivinfo .gmout").Each(func(i int, s *goquery.Selection) {
		status = s.Text()
	})

	// Homeruns
	hrs := []*Homerun{}
	doc.Find("#gmdivhr .gmresults").Each(func(i int, s *goquery.Selection) {
		hrs = append(hrs, parseHomerun(s.Text())...)
	})
	return NewGame(date, id, status, hrs), nil
}

func GetGames(date string) ([]*Game, error) {
	uri := "http://bis.npb.or.jp/" + date[0:4] + "/games/gm" + date + ".html"
	doc, err := newGoqueryDocument(uri)
	if err != nil {
		return nil, err
	}
	games := []*Game{}
	doc.Find(".contentsgame .contentsinfo a").Each(func(i int, s *goquery.Selection) {
		href, exists := s.Attr("href")
		if exists {
			id := href[1:14]
			var game *Game
			game, err = GetGame(date, id)
			games = append(games, game)
		}
	})
	return games, err
}

// parsers

func parseHomerun(line string) []*Homerun {
	hrs := []*Homerun{}
	for _, chunk := range strings.SplitAfter(line, "］ ")[1:] {
		lastBatter := ""
		for _, subchunk := range strings.Split(chunk, ",") {
			tokens := strings.Split(subchunk, " ")
			batter := tokens[0]
			number := tokens[1]
			scenario := tokens[3]
			pitcher := tokens[4]
			if len(batter) == 0 {
				batter = lastBatter
			}
			hrs = append(hrs, NewHomerun(batter, number, scenario, pitcher))
			lastBatter = batter
		}
	}
	return hrs
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/codegangsta/cli"
	"github.com/kr/pty"
	"github.com/mattn/go-shellwords"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pkg/term"

	"github.com/cloudfoundry-incubator/garden"
	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
)

func handleComplete(c *cli.Context) {
	// This will complete if no args are passed
	if c.Args().Present() {
		return
	}

	containers, err := client(c).Containers(nil)
	failIf(err)

	for _, container := range containers {
		fmt.Println(container.Handle())
	}
}

func fail(err error) {
	fmt.Fprintln(os.Stderr, "failed:", err)
	os.Exit(1)
}

func failIf(err error) {
	if err != nil {
		fail(err)
	}
}

func client(c *cli.Context) garden.Client {
	target := c.GlobalString("target")
	return gclient.New(gconn.New("tcp", target))
}

func handle(c *cli.Context) string {
	if len(c.Args()) == 0 {
		fail(errors.New("must provide container handle"))
	}
	return c.Args().First()
}

func main() {
	app := cli.NewApp()
	app.Name = "gaol"
	app.Usage = "a cli for garden"
	app.Version = "0.0.1"
	app.Author = "Chris Brown"
	app.Email = "cbrown@pivotal.io"
	app.EnableBashCompletion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "target, t",
			Value:  "localhost:7777",
			Usage:  "server to which commands are sent",
			EnvVar: "GAOL_TARGET",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "ping",
			Usage: "check if the server is running",
			Action: func(c *cli.Context) {
				err := client(c).Ping()
				failIf(err)
			},
		},
		{
			Name:  "create",
			Usage: "create a container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "handle, n",
					Usage: "name to give container",
				},
				cli.StringFlag{
					Name:  "rootfs, r",
					Usage: "rootfs image with which to create the container",
				},
				cli.DurationFlag{
					Name:  "grace, g",
					Usage: "grace time (resetting ttl) of container",
					Value: 5 * time.Minute,
				},
				cli.BoolFlag{
					Name:  "privileged, p",
					Usage: "privileged user in container is privileged in host",
				},
			},
			Action: func(c *cli.Context) {
				handle := c.String("handle")
				grace := c.Duration("grace")
				rootfs := c.String("rootfs")
				privileged := c.Bool("privileged")

				container, err := client(c).Create(garden.ContainerSpec{
					Handle:     handle,
					GraceTime:  grace,
					RootFSPath: rootfs,
					Privileged: privileged,
				})
				failIf(err)

				fmt.Println(container.Handle())
			},
		},
		{
			Name:         "destroy",
			Usage:        "destroy a container",
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				client := client(c)
				handles := c.Args()

				for _, handle := range handles {
					err := client.Destroy(handle)
					failIf(err)
				}
			},
		},
		{
			Name:  "list",
			Usage: "get a list of running containers",
			Action: func(c *cli.Context) {
				containers, err := client(c).Containers(nil)
				failIf(err)

				for _, container := range containers {
					fmt.Println(container.Handle())
				}
			},
		},
		{
			Name:  "run",
			Usage: "run a command in a container",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "attach, a",
					Usage: "attach to the process after it is started",
				},
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "current working directory of process",
				},
				cli.StringFlag{
					Name:  "user, u",
					Usage: "user to run the process as",
				},
				cli.StringFlag{
					Name:  "command, c",
					Usage: "the command to run",
				},
				cli.BoolFlag{
					Name:  "privileged, p",
					Usage: "use privileged user in container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				attach := c.Bool("attach")
				dir := c.String("dir")
				user := c.String("user")
				privileged := c.Bool("privileged")
				command := c.String("command")

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				var processIo garden.ProcessIO
				if attach {
					processIo = garden.ProcessIO{
						Stdin:  os.Stdin,
						Stdout: os.Stdout,
						Stderr: os.Stderr,
					}
				} else {
					processIo = garden.ProcessIO{}
				}

				args, err := shellwords.Parse(command)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					Path:       args[0],
					Args:       args[1:],
					Dir:        dir,
					Privileged: privileged,
					User:       user,
				}, processIo)
				failIf(err)

				if attach {
					_, err = process.Wait()
					failIf(err)
				} else {
					fmt.Println(process.ID())
				}
			},
		},
		{
			Name:  "attach",
			Usage: "attach to command running in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "pid, p",
					Usage: "process id to connect to",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				pid := uint32(c.Int("pid"))
				if pid == 0 {
					err := errors.New("must specify pid to attach to")
					failIf(err)
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				process, err := container.Attach(pid, garden.ProcessIO{
					Stdin:  os.Stdin,
					Stdout: os.Stdout,
					Stderr: os.Stderr,
				})
				failIf(err)

				_, err = process.Wait()
				failIf(err)
			},
		},
		{
			Name:         "shell",
			Usage:        "open a shell inside the running container",
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				container, err := client(c).Lookup(handle(c))
				failIf(err)

				term, err := term.Open(os.Stdin.Name())
				failIf(err)

				err = term.SetRaw()
				failIf(err)

				rows, cols, err := pty.Getsize(os.Stdin)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					Path: "/bin/sh",
					Args: []string{"-l"},
					Env:  []string{"TERM=" + os.Getenv("TERM")},
					TTY: &garden.TTYSpec{
						WindowSize: &garden.WindowSize{
							Rows:    rows,
							Columns: cols,
						},
					},
					Privileged: true,
				}, garden.ProcessIO{
					Stdin:  term,
					Stdout: term,
					Stderr: term,
				})
				if err != nil {
					term.Restore()
					failIf(err)
				}

				resized := make(chan os.Signal, 10)
				signal.Notify(resized, syscall.SIGWINCH)

				go func() {
					for {
						<-resized

						rows, cols, err := pty.Getsize(os.Stdin)
						if err == nil {
							process.SetTTY(garden.TTYSpec{
								WindowSize: &garden.WindowSize{
									Rows:    rows,
									Columns: cols,
								},
							})
						}
					}
				}()

				process.Wait()
				term.Restore()
			},
		},
		{
			Name:  "stream-in",
			Usage: "stream data into the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "to-file, t",
					Usage: "destination path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				dst := c.String("to-file")
				if dst == "" {
					fail(errors.New("missing --to-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				// perform dance to get correct file names
				tmpDir, err := ioutil.TempDir("", "gaol")
				failIf(err)
				defer os.RemoveAll(tmpDir)

				tmp, err := os.Create(filepath.Join(tmpDir, filepath.Base(dst)))
				failIf(err)

				_, err = io.Copy(tmp, os.Stdin)
				failIf(err)

				err = tmp.Close()
				failIf(err)

				reader, writer := io.Pipe()
				go func(w io.WriteCloser) {
					err := compressor.WriteTar(tmp.Name(), w)
					failIf(err)
					w.Close()
				}(writer)

				err = container.StreamIn(filepath.Dir(dst), reader)
				failIf(err)
			},
		},
		{
			Name:  "stream-out",
			Usage: "stream data out of the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "from-file, f",
					Usage: "source path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				src := c.String("from-file")
				if src == "" {
					fail(errors.New("missing --from-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				output, err := container.StreamOut(src)
				failIf(err)

				tr := tar.NewReader(output)
				_, err = tr.Next()
				failIf(err)

				_, err = io.Copy(os.Stdout, tr)
				failIf(err)
			},
		},
		{
			Name:  "net-in",
			Usage: "map a port on the host to a port in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "port, p",
					Usage: "container port",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				target := c.GlobalString("target")
				requestedContainerPort := uint32(c.Int("port"))

				if target == "" {
					fail(errors.New("target must be set"))
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				hostPort, _, err := container.NetIn(0, requestedContainerPort)
				failIf(err)

				host, _, err := net.SplitHostPort(target)
				failIf(err)

				fmt.Println(net.JoinHostPort(host, fmt.Sprintf("%d", hostPort)))
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>allow setting env on create and run<commit_after>package main

import (
	"archive/tar"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"syscall"
	"time"

	"github.com/codegangsta/cli"
	"github.com/kr/pty"
	"github.com/mattn/go-shellwords"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/pkg/term"

	"github.com/cloudfoundry-incubator/garden"
	gclient "github.com/cloudfoundry-incubator/garden/client"
	gconn "github.com/cloudfoundry-incubator/garden/client/connection"
)

func handleComplete(c *cli.Context) {
	// This will complete if no args are passed
	if c.Args().Present() {
		return
	}

	containers, err := client(c).Containers(nil)
	failIf(err)

	for _, container := range containers {
		fmt.Println(container.Handle())
	}
}

func fail(err error) {
	fmt.Fprintln(os.Stderr, "failed:", err)
	os.Exit(1)
}

func failIf(err error) {
	if err != nil {
		fail(err)
	}
}

func client(c *cli.Context) garden.Client {
	target := c.GlobalString("target")
	return gclient.New(gconn.New("tcp", target))
}

func handle(c *cli.Context) string {
	if len(c.Args()) == 0 {
		fail(errors.New("must provide container handle"))
	}
	return c.Args().First()
}

func main() {
	app := cli.NewApp()
	app.Name = "gaol"
	app.Usage = "a cli for garden"
	app.Version = "0.0.1"
	app.Author = "Chris Brown"
	app.Email = "cbrown@pivotal.io"
	app.EnableBashCompletion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "target, t",
			Value:  "localhost:7777",
			Usage:  "server to which commands are sent",
			EnvVar: "GAOL_TARGET",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "ping",
			Usage: "check if the server is running",
			Action: func(c *cli.Context) {
				err := client(c).Ping()
				failIf(err)
			},
		},
		{
			Name:  "create",
			Usage: "create a container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "handle, n",
					Usage: "name to give container",
				},
				cli.StringFlag{
					Name:  "rootfs, r",
					Usage: "rootfs image with which to create the container",
				},
				cli.StringSliceFlag{
					Name:  "env, e",
					Usage: "set environment variables",
					Value: &cli.StringSlice{},
				},
				cli.DurationFlag{
					Name:  "grace, g",
					Usage: "grace time (resetting ttl) of container",
					Value: 5 * time.Minute,
				},
				cli.BoolFlag{
					Name:  "privileged, p",
					Usage: "privileged user in container is privileged in host",
				},
			},
			Action: func(c *cli.Context) {
				handle := c.String("handle")
				grace := c.Duration("grace")
				rootfs := c.String("rootfs")
				env := c.StringSlice("env")
				privileged := c.Bool("privileged")

				container, err := client(c).Create(garden.ContainerSpec{
					Handle:     handle,
					GraceTime:  grace,
					RootFSPath: rootfs,
					Privileged: privileged,
					Env:        env,
				})
				failIf(err)

				fmt.Println(container.Handle())
			},
		},
		{
			Name:         "destroy",
			Usage:        "destroy a container",
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				client := client(c)
				handles := c.Args()

				for _, handle := range handles {
					err := client.Destroy(handle)
					failIf(err)
				}
			},
		},
		{
			Name:  "list",
			Usage: "get a list of running containers",
			Action: func(c *cli.Context) {
				containers, err := client(c).Containers(nil)
				failIf(err)

				for _, container := range containers {
					fmt.Println(container.Handle())
				}
			},
		},
		{
			Name:  "run",
			Usage: "run a command in a container",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "attach, a",
					Usage: "attach to the process after it is started",
				},
				cli.StringFlag{
					Name:  "dir, d",
					Usage: "current working directory of process",
				},
				cli.StringFlag{
					Name:  "user, u",
					Usage: "user to run the process as",
				},
				cli.StringFlag{
					Name:  "command, c",
					Usage: "the command to run",
				},
				cli.StringSliceFlag{
					Name:  "env, e",
					Usage: "set environment variables",
					Value: &cli.StringSlice{},
				},
				cli.BoolFlag{
					Name:  "privileged, p",
					Usage: "use privileged user in container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				attach := c.Bool("attach")
				dir := c.String("dir")
				user := c.String("user")
				privileged := c.Bool("privileged")
				command := c.String("command")
				env := c.StringSlice("env")

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				var processIo garden.ProcessIO
				if attach {
					processIo = garden.ProcessIO{
						Stdin:  os.Stdin,
						Stdout: os.Stdout,
						Stderr: os.Stderr,
					}
				} else {
					processIo = garden.ProcessIO{}
				}

				args, err := shellwords.Parse(command)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					Path:       args[0],
					Args:       args[1:],
					Dir:        dir,
					Privileged: privileged,
					User:       user,
					Env:        env,
				}, processIo)
				failIf(err)

				if attach {
					_, err = process.Wait()
					failIf(err)
				} else {
					fmt.Println(process.ID())
				}
			},
		},
		{
			Name:  "attach",
			Usage: "attach to command running in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "pid, p",
					Usage: "process id to connect to",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				pid := uint32(c.Int("pid"))
				if pid == 0 {
					err := errors.New("must specify pid to attach to")
					failIf(err)
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				process, err := container.Attach(pid, garden.ProcessIO{
					Stdin:  os.Stdin,
					Stdout: os.Stdout,
					Stderr: os.Stderr,
				})
				failIf(err)

				_, err = process.Wait()
				failIf(err)
			},
		},
		{
			Name:         "shell",
			Usage:        "open a shell inside the running container",
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				container, err := client(c).Lookup(handle(c))
				failIf(err)

				term, err := term.Open(os.Stdin.Name())
				failIf(err)

				err = term.SetRaw()
				failIf(err)

				rows, cols, err := pty.Getsize(os.Stdin)
				failIf(err)

				process, err := container.Run(garden.ProcessSpec{
					Path: "/bin/sh",
					Args: []string{"-l"},
					Env:  []string{"TERM=" + os.Getenv("TERM")},
					TTY: &garden.TTYSpec{
						WindowSize: &garden.WindowSize{
							Rows:    rows,
							Columns: cols,
						},
					},
					Privileged: true,
				}, garden.ProcessIO{
					Stdin:  term,
					Stdout: term,
					Stderr: term,
				})
				if err != nil {
					term.Restore()
					failIf(err)
				}

				resized := make(chan os.Signal, 10)
				signal.Notify(resized, syscall.SIGWINCH)

				go func() {
					for {
						<-resized

						rows, cols, err := pty.Getsize(os.Stdin)
						if err == nil {
							process.SetTTY(garden.TTYSpec{
								WindowSize: &garden.WindowSize{
									Rows:    rows,
									Columns: cols,
								},
							})
						}
					}
				}()

				process.Wait()
				term.Restore()
			},
		},
		{
			Name:  "stream-in",
			Usage: "stream data into the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "to-file, t",
					Usage: "destination path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				dst := c.String("to-file")
				if dst == "" {
					fail(errors.New("missing --to-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				// perform dance to get correct file names
				tmpDir, err := ioutil.TempDir("", "gaol")
				failIf(err)
				defer os.RemoveAll(tmpDir)

				tmp, err := os.Create(filepath.Join(tmpDir, filepath.Base(dst)))
				failIf(err)

				_, err = io.Copy(tmp, os.Stdin)
				failIf(err)

				err = tmp.Close()
				failIf(err)

				reader, writer := io.Pipe()
				go func(w io.WriteCloser) {
					err := compressor.WriteTar(tmp.Name(), w)
					failIf(err)
					w.Close()
				}(writer)

				err = container.StreamIn(filepath.Dir(dst), reader)
				failIf(err)
			},
		},
		{
			Name:  "stream-out",
			Usage: "stream data out of the container",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "from-file, f",
					Usage: "source path in the container",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				handle := handle(c)

				src := c.String("from-file")
				if src == "" {
					fail(errors.New("missing --from-file argument"))
				}

				container, err := client(c).Lookup(handle)
				failIf(err)

				output, err := container.StreamOut(src)
				failIf(err)

				tr := tar.NewReader(output)
				_, err = tr.Next()
				failIf(err)

				_, err = io.Copy(os.Stdout, tr)
				failIf(err)
			},
		},
		{
			Name:  "net-in",
			Usage: "map a port on the host to a port in the container",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "port, p",
					Usage: "container port",
				},
			},
			BashComplete: handleComplete,
			Action: func(c *cli.Context) {
				target := c.GlobalString("target")
				requestedContainerPort := uint32(c.Int("port"))

				if target == "" {
					fail(errors.New("target must be set"))
				}

				handle := handle(c)
				container, err := client(c).Lookup(handle)
				failIf(err)

				hostPort, _, err := container.NetIn(0, requestedContainerPort)
				failIf(err)

				host, _, err := net.SplitHostPort(target)
				failIf(err)

				fmt.Println(net.JoinHostPort(host, fmt.Sprintf("%d", hostPort)))
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package gbdt

import (
	"fmt"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"
)

var _ = time.Now
var _ = os.Exit

type GBDT struct {
	trees      []*RegressionTree
	tree_count int
	shrinkage  float32
	bias       float32
}

func NewGBDT() (gbdt *GBDT) {
	gbdt = &GBDT{
		trees:      make([]*RegressionTree, 0),
		tree_count: Conf.Tree_count,
		shrinkage:  Conf.Shrinkage,
		bias:       0,
	}
	for i := 0; i < Conf.Tree_count; i++ {
		tree := NewRegressionTree()
		gbdt.trees = append(gbdt.trees, tree)
	}
	//fmt.Println("gbdt:",len(gbdt.trees))
	return
}

func (self *GBDT) Init(d *DataSet) {
	var s float32 = 0
	var c float32 = 0

	i := 0
	for _, sample := range d.samples {
		s += float32(sample.label) * (sample.weight)
		c += sample.weight
		i++
	}
	y_avg := s / c
	if Conf.Losstype == LEAST_SQUARE {
		self.bias = y_avg
	} else if Conf.Losstype == LOG_LIKEHOOD {
		self.bias = float32(math.Log(float64((1+y_avg)/(1-y_avg))) / 2.0)
	}
	//self.trees = make([]*RegressionTree, Conf.Tree_count)
}

func (self *GBDT) Train(d *DataSet) {

	var sample_number int = len(d.samples)
	if Conf.Data_sampling_ratio < 1 {
		sample_number = int(Conf.Data_sampling_ratio * float32(len(d.samples)))
	}
	self.Init(d)

	for i := 0; i < Conf.Tree_count; i++ {
		fmt.Printf("iteration:%d ", i)
		if Conf.Data_sampling_ratio < 1 {
			random_shuffle(d.samples, len(d.samples))
		}
		for j := 0; j < sample_number; j++ {
			p := self.Predict(d.samples[j], i)
			d.samples[j].target = FxGradient(d.samples[j].label, p)
		}
		if Conf.Debug {
			//cal auc

			//cal loss
			var s, c float64 = 0, 0
			for j := 0; j < len(d.samples); j++ {
				p := self.Predict(d.samples[j], i)
				s += float64(Float32Square(float32(d.samples[j].label)-p) * d.samples[j].weight)
				c += float64(d.samples[j].weight)
			}
			fmt.Println("rmse:", math.Sqrt(s/c))

		}
		start := time.Now()
		self.trees[i].Fit(d, sample_number)
		latency := time.Since(start)
		fmt.Println("latency:", latency)
	}

}

func (self *GBDT) Predict(sample *Sample, n int) float32 {
	if self.trees == nil {
		return UNKNOWN_VALUE
	}
	r := self.bias
	for i := 0; i < n; i++ {
		r += self.shrinkage * self.trees[i].Predict(sample)
	}
	return r
}

func (self *GBDT) Save() string {
	vs := make([]string, 0)
	vs = append(vs, strconv.FormatFloat(float64(self.shrinkage), 'f', 4, 32))
	vs = append(vs, strconv.FormatFloat(float64(self.bias), 'f', 4, 32))
	for i := 0; i < self.tree_count; i++ {
		vs = append(vs, self.trees[i].Save())
	}
	return strings.Join(vs, "\n;\n")
}

func (self *GBDT) Load(s string) {
	self.trees = nil
	vs := strings.Split(s, "\n;\n")
	self.tree_count = len(vs) - 2
	if tempshrinkage, err := strconv.ParseFloat(vs[0], 32); err != nil {
		log.Fatal("shrinkage", err)
	} else {
		self.shrinkage = float32(tempshrinkage)
	}

	if tempbias, err := strconv.ParseFloat(vs[1], 32); err != nil {
		log.Fatal("bias", err)
	} else {
		self.bias = float32(tempbias)
	}

	self.trees = make([]*RegressionTree, self.tree_count)

	for i := 0; i < self.tree_count; i++ {
		self.trees[i] = NewRegressionTree()
		self.trees[i].Load(vs[i+2])
	}

}
<commit_msg>add time monitor<commit_after>package gbdt

import (
	"fmt"
	"log"
	"math"
	"os"
	"strconv"
	"strings"
	"time"
)

var _ = time.Now
var _ = os.Exit

type GBDT struct {
	trees      []*RegressionTree
	tree_count int
	shrinkage  float32
	bias       float32
}

func NewGBDT() (gbdt *GBDT) {
	gbdt = &GBDT{
		trees:      make([]*RegressionTree, 0),
		tree_count: Conf.Tree_count,
		shrinkage:  Conf.Shrinkage,
		bias:       0,
	}
	for i := 0; i < Conf.Tree_count; i++ {
		tree := NewRegressionTree()
		gbdt.trees = append(gbdt.trees, tree)
	}
	//fmt.Println("gbdt:",len(gbdt.trees))
	return
}

func (self *GBDT) Init(d *DataSet) {
	var s float32 = 0
	var c float32 = 0

	i := 0
	for _, sample := range d.samples {
		s += float32(sample.label) * (sample.weight)
		c += sample.weight
		i++
	}
	y_avg := s / c
	if Conf.Losstype == LEAST_SQUARE {
		self.bias = y_avg
	} else if Conf.Losstype == LOG_LIKEHOOD {
		self.bias = float32(math.Log(float64((1+y_avg)/(1-y_avg))) / 2.0)
	}
	//self.trees = make([]*RegressionTree, Conf.Tree_count)
}

func (self *GBDT) Train(d *DataSet) {

	var sample_number int = len(d.samples)
	if Conf.Data_sampling_ratio < 1 {
		sample_number = int(Conf.Data_sampling_ratio * float32(len(d.samples)))
	}
	self.Init(d)

	for i := 0; i < Conf.Tree_count; i++ {
		fmt.Printf("iteration:%d ", i)
		start := time.Now()

		if Conf.Data_sampling_ratio < 1 {
			random_shuffle(d.samples, len(d.samples))
		}
		for j := 0; j < sample_number; j++ {
			p := self.Predict(d.samples[j], i)
			d.samples[j].target = FxGradient(d.samples[j].label, p)
		}
		if Conf.Debug {
			//cal auc

			//cal loss
			var s, c float64 = 0, 0
			for j := 0; j < len(d.samples); j++ {
				p := self.Predict(d.samples[j], i)
				s += float64(Float32Square(float32(d.samples[j].label)-p) * d.samples[j].weight)
				c += float64(d.samples[j].weight)
			}
			fmt.Println("rmse:", math.Sqrt(s/c))

		}
		self.trees[i].Fit(d, sample_number)
		latency := time.Since(start)
		fmt.Println("latency:", latency)
	}

}

func (self *GBDT) Predict(sample *Sample, n int) float32 {
	if self.trees == nil {
		return UNKNOWN_VALUE
	}
	r := self.bias
	for i := 0; i < n; i++ {
		r += self.shrinkage * self.trees[i].Predict(sample)
	}
	return r
}

func (self *GBDT) Save() string {
	vs := make([]string, 0)
	vs = append(vs, strconv.FormatFloat(float64(self.shrinkage), 'f', 4, 32))
	vs = append(vs, strconv.FormatFloat(float64(self.bias), 'f', 4, 32))
	for i := 0; i < self.tree_count; i++ {
		vs = append(vs, self.trees[i].Save())
	}
	return strings.Join(vs, "\n;\n")
}

func (self *GBDT) Load(s string) {
	self.trees = nil
	vs := strings.Split(s, "\n;\n")
	self.tree_count = len(vs) - 2
	if tempshrinkage, err := strconv.ParseFloat(vs[0], 32); err != nil {
		log.Fatal("shrinkage", err)
	} else {
		self.shrinkage = float32(tempshrinkage)
	}

	if tempbias, err := strconv.ParseFloat(vs[1], 32); err != nil {
		log.Fatal("bias", err)
	} else {
		self.bias = float32(tempbias)
	}

	self.trees = make([]*RegressionTree, self.tree_count)

	for i := 0; i < self.tree_count; i++ {
		self.trees[i] = NewRegressionTree()
		self.trees[i].Load(vs[i+2])
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blueprint

import (
	"fmt"
	"reflect"
	"sort"
)

type GlobPath struct {
	Pattern  string
	Excludes []string
	Files    []string
	Deps     []string
	Name     string
}

func verifyGlob(fileName, pattern string, excludes []string, g GlobPath) {
	if pattern != g.Pattern {
		panic(fmt.Errorf("Mismatched patterns %q and %q for glob file %q", pattern, g.Pattern, fileName))
	}
	if !reflect.DeepEqual(g.Excludes, excludes) {
		panic(fmt.Errorf("Mismatched excludes %v and %v for glob file %q", excludes, g.Excludes, fileName))
	}
}

func (c *Context) glob(pattern string, excludes []string) ([]string, error) {
	fileName := globToFileName(pattern, excludes)

	// Try to get existing glob from the stored results
	c.globLock.Lock()
	g, exists := c.globs[fileName]
	c.globLock.Unlock()

	if exists {
		// Glob has already been done, double check it is identical
		verifyGlob(fileName, pattern, excludes, g)
		return g.Files, nil
	}

	// Get a globbed file list
	files, deps, err := c.fs.Glob(pattern, excludes)
	if err != nil {
		return nil, err
	}

	// Store the results
	c.globLock.Lock()
	if g, exists = c.globs[fileName]; !exists {
		c.globs[fileName] = GlobPath{pattern, excludes, files, deps, fileName}
	}
	c.globLock.Unlock()

	// Getting the list raced with another goroutine, throw away the results and use theirs
	if exists {
		verifyGlob(fileName, pattern, excludes, g)
		return g.Files, nil
	}

	return files, nil
}

func (c *Context) Globs() []GlobPath {
	fileNames := make([]string, 0, len(c.globs))
	for k := range c.globs {
		fileNames = append(fileNames, k)
	}
	sort.Strings(fileNames)

	globs := make([]GlobPath, len(fileNames))
	for i, fileName := range fileNames {
		globs[i] = c.globs[fileName]
	}

	return globs
}

func globToString(pattern string) string {
	ret := ""
	for _, c := range pattern {
		switch {
		case c >= 'a' && c <= 'z',
			c >= 'A' && c <= 'Z',
			c >= '0' && c <= '9',
			c == '_', c == '-', c == '/':
			ret += string(c)
		default:
			ret += "_"
		}
	}

	return ret
}

func globToFileName(pattern string, excludes []string) string {
	ret := globToString(pattern)
	for _, e := range excludes {
		ret += "__" + globToString(e)
	}
	return ret + ".glob"
}
<commit_msg>Limit length of glob result file names<commit_after>// Copyright 2015 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blueprint

import (
	"crypto/md5"
	"fmt"
	"reflect"
	"sort"
	"strings"
)

type GlobPath struct {
	Pattern  string
	Excludes []string
	Files    []string
	Deps     []string
	Name     string
}

func verifyGlob(fileName, pattern string, excludes []string, g GlobPath) {
	if pattern != g.Pattern {
		panic(fmt.Errorf("Mismatched patterns %q and %q for glob file %q", pattern, g.Pattern, fileName))
	}
	if !reflect.DeepEqual(g.Excludes, excludes) {
		panic(fmt.Errorf("Mismatched excludes %v and %v for glob file %q", excludes, g.Excludes, fileName))
	}
}

func (c *Context) glob(pattern string, excludes []string) ([]string, error) {
	fileName := globToFileName(pattern, excludes)

	// Try to get existing glob from the stored results
	c.globLock.Lock()
	g, exists := c.globs[fileName]
	c.globLock.Unlock()

	if exists {
		// Glob has already been done, double check it is identical
		verifyGlob(fileName, pattern, excludes, g)
		return g.Files, nil
	}

	// Get a globbed file list
	files, deps, err := c.fs.Glob(pattern, excludes)
	if err != nil {
		return nil, err
	}

	// Store the results
	c.globLock.Lock()
	if g, exists = c.globs[fileName]; !exists {
		c.globs[fileName] = GlobPath{pattern, excludes, files, deps, fileName}
	}
	c.globLock.Unlock()

	// Getting the list raced with another goroutine, throw away the results and use theirs
	if exists {
		verifyGlob(fileName, pattern, excludes, g)
		return g.Files, nil
	}

	return files, nil
}

func (c *Context) Globs() []GlobPath {
	fileNames := make([]string, 0, len(c.globs))
	for k := range c.globs {
		fileNames = append(fileNames, k)
	}
	sort.Strings(fileNames)

	globs := make([]GlobPath, len(fileNames))
	for i, fileName := range fileNames {
		globs[i] = c.globs[fileName]
	}

	return globs
}

func globToString(pattern string) string {
	ret := ""
	for _, c := range pattern {
		switch {
		case c >= 'a' && c <= 'z',
			c >= 'A' && c <= 'Z',
			c >= '0' && c <= '9',
			c == '_', c == '-', c == '/':
			ret += string(c)
		default:
			ret += "_"
		}
	}

	return ret
}

func globToFileName(pattern string, excludes []string) string {
	name := globToString(pattern)
	excludeName := ""
	for _, e := range excludes {
		excludeName += "__" + globToString(e)
	}

	// Prevent file names from reaching ninja's path component limit
	if strings.Count(name, "/")+strings.Count(excludeName, "/") > 30 {
		excludeName = fmt.Sprintf("___%x", md5.Sum([]byte(excludeName)))
	}

	return name + excludeName + ".glob"
}
<|endoftext|>


<commit_before>package glob

import "strings"

// The character which is treated like a glob
const GLOB = "*"

// Glob will test a string pattern, potentially containing globs, against a
// subject string. The result is a simple true/false, determining whether or
// not the glob pattern matched the subject text.
func Glob(pattern, subj string) bool {
	// Shortcut the entire function if the pattern provided was empty. An empty
	// pattern can, however, match an empty subject.
	if pattern == "" {
		return subj == pattern
	}

	parts := strings.Split(pattern, GLOB)

	switch len(parts) {
	case 1:
		// No globs in pattern, so test for equality
		return subj == pattern
	default:
		leadingGlob := strings.HasPrefix(pattern, GLOB)
		trailingGlob := strings.HasSuffix(pattern, GLOB)
		end := len(parts) - 1

		for i, part := range parts {
			switch {
			case i == 0 && leadingGlob:
				continue
			case i == end:
				if len(subj) > 0 {
					return trailingGlob || strings.HasSuffix(subj, part)
				}
				return true
			default:
				if !strings.Contains(subj, part) {
					return false
				}
			}

			// Trim evaluated text from subj as we loop over the pattern.
			idx := strings.Index(subj, part) + len(part)
			subj = subj[idx:]
		}

		// All parts of the pattern matched
		return true
	}
}
<commit_msg>Check front of subject if no leading glob is found.<commit_after>package glob

import "strings"

// The character which is treated like a glob
const GLOB = "*"

// Glob will test a string pattern, potentially containing globs, against a
// subject string. The result is a simple true/false, determining whether or
// not the glob pattern matched the subject text.
func Glob(pattern, subj string) bool {
	// Shortcut the entire function if the pattern provided was empty. An empty
	// pattern can, however, match an empty subject.
	if pattern == "" {
		return subj == pattern
	}

	parts := strings.Split(pattern, GLOB)

	switch len(parts) {
	case 1:
		// No globs in pattern, so test for equality
		return subj == pattern
	default:
		leadingGlob := strings.HasPrefix(pattern, GLOB)
		trailingGlob := strings.HasSuffix(pattern, GLOB)
		end := len(parts) - 1

		for i, part := range parts {
			switch {
			case i == 0:
				if leadingGlob {
					continue
				}
				if !strings.HasPrefix(subj, part) {
					return false
				}
			case i == end:
				if len(subj) > 0 {
					return trailingGlob || strings.HasSuffix(subj, part)
				}
				return true
			default:
				if !strings.Contains(subj, part) {
					return false
				}
			}

			// Trim evaluated text from subj as we loop over the pattern.
			idx := strings.Index(subj, part) + len(part)
			subj = subj[idx:]
		}

		// All parts of the pattern matched
		return true
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"github.com/BurntSushi/xgb"
	"github.com/BurntSushi/xgb/screensaver"
	"github.com/BurntSushi/xgb/xproto"
	"html/template"
	"log"
	"net/http"
	"os"
	"sort"
	"time"
)

type Tracker map[Window]*Track

type Track struct {
	Start time.Time
	Spent time.Duration
}

type Window struct {
	Class string
	Name  string
}

type Xorg struct {
	X           *xgb.Conn
	root        xproto.Window
	activeAtom  *xproto.InternAtomReply
	netNameAtom *xproto.InternAtomReply
	nameAtom    *xproto.InternAtomReply
	classAtom   *xproto.InternAtomReply
}

func (t Track) String() string {
	return fmt.Sprint(t.Spent)
}

func (w Window) String() string {
	return fmt.Sprintf("%s %s", w.Class, w.Name)
}

func (x Xorg) atom(aname string) *xproto.InternAtomReply {
	a, err := xproto.InternAtom(x.X, true, uint16(len(aname)), aname).Reply()
	if err != nil {
		log.Fatal("atom: ", err)
	}
	return a
}

func (x Xorg) property(w xproto.Window, a *xproto.InternAtomReply) (*xproto.GetPropertyReply, error) {
	return xproto.GetProperty(x.X, false, w, a.Atom,
		xproto.GetPropertyTypeAny, 0, (1<<32)-1).Reply()
}

func (x Xorg) active() xproto.Window {
	p, err := x.property(x.root, x.activeAtom)
	if err != nil {
		log.Fatal("active: ", err)
	}
	return xproto.Window(xgb.Get32(p.Value))
}

func (x Xorg) name(w xproto.Window) string {
	name, err := x.property(w, x.netNameAtom)
	if err != nil {
		log.Fatal("net name: ", err)
	}
	if string(name.Value) != "" {
		return string(name.Value)
	}
	name, err = x.property(w, x.nameAtom)
	if err != nil {
		log.Fatal("wm name: ", err)
	}
	return string(name.Value)
}

func (x Xorg) class(w xproto.Window) string {
	class, err := x.property(w, x.classAtom)
	if err != nil {
		log.Fatal("class: ", err)
	}
	i := bytes.IndexByte(class.Value, 0)
	if i == -1 || string(class.Value[:i]) == "" {
		return "unknown"
	}
	return string(class.Value[:i])
}

func (x Xorg) winName() (Window, bool) {
	windowId := x.active()
	/* skip invalid window id */
	if windowId == 0 {
		return Window{}, false
	}
	x.spy(windowId)
	return Window{
		Class: x.class(windowId),
		Name:  x.name(windowId),
	}, true
}

func (x Xorg) spy(w xproto.Window) {
	xproto.ChangeWindowAttributes(x.X, w, xproto.CwEventMask,
		[]uint32{xproto.EventMaskPropertyChange})
}

func (x Xorg) Update(t Tracker) (prev *Track) {
	if win, ok := x.winName(); ok {
		if _, ok := t[win]; !ok {
			t[win] = new(Track)
		}
		t[win].Start = time.Now()
		prev = t[win]
	}
	return
}

func connect() Xorg {
	var x Xorg
	var err error

	x.X, err = xgb.NewConn()
	if err != nil {
		log.Fatal("xgb: ", err)
	}

	err = screensaver.Init(x.X)
	if err != nil {
		log.Fatal("screensaver: ", err)
	}

	setup := xproto.Setup(x.X)
	x.root = setup.DefaultScreen(x.X).Root

	drw := xproto.Drawable(x.root)
	screensaver.SelectInput(x.X, drw, screensaver.EventNotifyMask)

	x.activeAtom = x.atom("_NET_ACTIVE_WINDOW")
	x.netNameAtom = x.atom("_NET_WM_NAME")
	x.nameAtom = x.atom("WM_NAME")
	x.classAtom = x.atom("WM_CLASS")

	x.spy(x.root)

	return x
}

func (t Tracker) collect() {
	x := connect()
	defer x.X.Close()

	prev := x.Update(t)
	for {
		ev, everr := x.X.WaitForEvent()
		if everr != nil {
			log.Fatal("wait for event: ", everr)
		}
		switch event := ev.(type) {
		case xproto.PropertyNotifyEvent:
			if prev != nil {
				prev.Spent += time.Since(prev.Start)
			}
			prev = x.Update(t)
			zzz = false
		case screensaver.NotifyEvent:
			switch event.State {
			case screensaver.StateOn:
				fmt.Println("away from keyboard")
				prev = nil
				zzz = true
			}
		}
	}
}

func (t Tracker) cleanup(d time.Duration) {
	for k, v := range t {
		if time.Since(v.Start) > d {
			log.Println("removing", k)
			delete(t, k)
		}
	}
}

func (t Tracker) load(fname string) {
	dump, err := os.Open(fname)
	if err != nil {
		log.Println(err)
		return
	}
	defer dump.Close()
	dec := gob.NewDecoder(dump)
	err = dec.Decode(&t)
	if err != nil {
		log.Println(err)
	}
}

func (t Tracker) store(fname string) {
	dump, err := os.Create(fname)
	if err != nil {
		log.Println(err)
		return
	}
	defer dump.Close()
	enc := gob.NewEncoder(dump)
	err = enc.Encode(t)
	if err != nil {
		log.Println(err)
	}
}

type Index struct {
	Title  string
	Tracks Tracks
	Class  Tracks
	Total  time.Duration
	Zzz    bool
}

type Tracks []track

type track struct {
	Class string
	Name  string
	Time  time.Duration
}

func (t Tracks) Len() int           { return len(t) }
func (t Tracks) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t Tracks) Less(i, j int) bool { return t[i].Time < t[j].Time }

func indexHandler(w http.ResponseWriter, r *http.Request) {
	var i Index
	i.Title = "Gone Time Tracker"
	i.Zzz = zzz
	class := r.URL.Path[1:]

	classtotal := make(map[string]time.Duration)

	for k, v := range tracks {
		classtotal[k.Class] += v.Spent
		i.Total += v.Spent
		if class != "" && class != k.Class {
			continue
		}
		i.Tracks = append(i.Tracks, track{
			Class: k.Class,
			Name:  k.Name,
			Time:  v.Spent})
	}
	for k, v := range classtotal {
		i.Class = append(i.Class, track{Class: k, Time: v})
	}
	sort.Sort(sort.Reverse(i.Class))
	sort.Sort(sort.Reverse(i.Tracks))
	err := tmpl.Execute(w, i)
	if err != nil {
		log.Println(err)
	}
}

const (
	port = ":8001"
	file = "dump.gob"
)

var (
	tracks = make(Tracker)
	tmpl   = template.Must(template.ParseFiles("index.html"))
	zzz    bool
)

func main() {
	tracks.load(file)
	go tracks.collect()
	go func() {
		for {
			tracks.cleanup(12 * time.Hour)
			tracks.store(file)
			time.Sleep(time.Minute)
		}
	}()
	log.Println("listen on", port)
	http.HandleFunc("/", indexHandler)
	http.ListenAndServe(port, nil)
}
<commit_msg>turn off snooze in more logically way<commit_after>package main

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"github.com/BurntSushi/xgb"
	"github.com/BurntSushi/xgb/screensaver"
	"github.com/BurntSushi/xgb/xproto"
	"html/template"
	"log"
	"net/http"
	"os"
	"sort"
	"time"
)

type Tracker map[Window]*Track

type Track struct {
	Start time.Time
	Spent time.Duration
}

type Window struct {
	Class string
	Name  string
}

type Xorg struct {
	X           *xgb.Conn
	root        xproto.Window
	activeAtom  *xproto.InternAtomReply
	netNameAtom *xproto.InternAtomReply
	nameAtom    *xproto.InternAtomReply
	classAtom   *xproto.InternAtomReply
}

func (t Track) String() string {
	return fmt.Sprint(t.Spent)
}

func (w Window) String() string {
	return fmt.Sprintf("%s %s", w.Class, w.Name)
}

func (x Xorg) atom(aname string) *xproto.InternAtomReply {
	a, err := xproto.InternAtom(x.X, true, uint16(len(aname)), aname).Reply()
	if err != nil {
		log.Fatal("atom: ", err)
	}
	return a
}

func (x Xorg) property(w xproto.Window, a *xproto.InternAtomReply) (*xproto.GetPropertyReply, error) {
	return xproto.GetProperty(x.X, false, w, a.Atom,
		xproto.GetPropertyTypeAny, 0, (1<<32)-1).Reply()
}

func (x Xorg) active() xproto.Window {
	p, err := x.property(x.root, x.activeAtom)
	if err != nil {
		log.Fatal("active: ", err)
	}
	return xproto.Window(xgb.Get32(p.Value))
}

func (x Xorg) name(w xproto.Window) string {
	name, err := x.property(w, x.netNameAtom)
	if err != nil {
		log.Fatal("net name: ", err)
	}
	if string(name.Value) != "" {
		return string(name.Value)
	}
	name, err = x.property(w, x.nameAtom)
	if err != nil {
		log.Fatal("wm name: ", err)
	}
	return string(name.Value)
}

func (x Xorg) class(w xproto.Window) string {
	class, err := x.property(w, x.classAtom)
	if err != nil {
		log.Fatal("class: ", err)
	}
	i := bytes.IndexByte(class.Value, 0)
	if i == -1 || string(class.Value[:i]) == "" {
		return "unknown"
	}
	return string(class.Value[:i])
}

func (x Xorg) winName() (Window, bool) {
	windowId := x.active()
	/* skip invalid window id */
	if windowId == 0 {
		return Window{}, false
	}
	x.spy(windowId)
	return Window{
		Class: x.class(windowId),
		Name:  x.name(windowId),
	}, true
}

func (x Xorg) spy(w xproto.Window) {
	xproto.ChangeWindowAttributes(x.X, w, xproto.CwEventMask,
		[]uint32{xproto.EventMaskPropertyChange})
}

func (x Xorg) Update(t Tracker) (prev *Track) {
	if win, ok := x.winName(); ok {
		if _, ok := t[win]; !ok {
			t[win] = new(Track)
		}
		t[win].Start = time.Now()
		prev = t[win]
	}
	return
}

func connect() Xorg {
	var x Xorg
	var err error

	x.X, err = xgb.NewConn()
	if err != nil {
		log.Fatal("xgb: ", err)
	}

	err = screensaver.Init(x.X)
	if err != nil {
		log.Fatal("screensaver: ", err)
	}

	setup := xproto.Setup(x.X)
	x.root = setup.DefaultScreen(x.X).Root

	drw := xproto.Drawable(x.root)
	screensaver.SelectInput(x.X, drw, screensaver.EventNotifyMask)

	x.activeAtom = x.atom("_NET_ACTIVE_WINDOW")
	x.netNameAtom = x.atom("_NET_WM_NAME")
	x.nameAtom = x.atom("WM_NAME")
	x.classAtom = x.atom("WM_CLASS")

	x.spy(x.root)

	return x
}

func (t Tracker) collect() {
	x := connect()
	defer x.X.Close()

	prev := x.Update(t)
	for {
		ev, everr := x.X.WaitForEvent()
		if everr != nil {
			log.Fatal("wait for event: ", everr)
		}
		switch event := ev.(type) {
		case xproto.PropertyNotifyEvent:
			if prev != nil {
				prev.Spent += time.Since(prev.Start)
			}
			prev = x.Update(t)
		case screensaver.NotifyEvent:
			switch event.State {
			case screensaver.StateOn:
				fmt.Println("away from keyboard")
				prev = nil
				zzz = true
			default:
				zzz = false
			}
		}
	}
}

func (t Tracker) cleanup(d time.Duration) {
	for k, v := range t {
		if time.Since(v.Start) > d {
			log.Println("removing", k)
			delete(t, k)
		}
	}
}

func (t Tracker) load(fname string) {
	dump, err := os.Open(fname)
	if err != nil {
		log.Println(err)
		return
	}
	defer dump.Close()
	dec := gob.NewDecoder(dump)
	err = dec.Decode(&t)
	if err != nil {
		log.Println(err)
	}
}

func (t Tracker) store(fname string) {
	dump, err := os.Create(fname)
	if err != nil {
		log.Println(err)
		return
	}
	defer dump.Close()
	enc := gob.NewEncoder(dump)
	err = enc.Encode(t)
	if err != nil {
		log.Println(err)
	}
}

type Index struct {
	Title  string
	Tracks Tracks
	Class  Tracks
	Total  time.Duration
	Zzz    bool
}

type Tracks []track

type track struct {
	Class string
	Name  string
	Time  time.Duration
}

func (t Tracks) Len() int           { return len(t) }
func (t Tracks) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
func (t Tracks) Less(i, j int) bool { return t[i].Time < t[j].Time }

func indexHandler(w http.ResponseWriter, r *http.Request) {
	var i Index
	i.Title = "Gone Time Tracker"
	i.Zzz = zzz
	class := r.URL.Path[1:]

	classtotal := make(map[string]time.Duration)

	for k, v := range tracks {
		classtotal[k.Class] += v.Spent
		i.Total += v.Spent
		if class != "" && class != k.Class {
			continue
		}
		i.Tracks = append(i.Tracks, track{
			Class: k.Class,
			Name:  k.Name,
			Time:  v.Spent})
	}
	for k, v := range classtotal {
		i.Class = append(i.Class, track{Class: k, Time: v})
	}
	sort.Sort(sort.Reverse(i.Class))
	sort.Sort(sort.Reverse(i.Tracks))
	err := tmpl.Execute(w, i)
	if err != nil {
		log.Println(err)
	}
}

const (
	port = ":8001"
	file = "dump.gob"
)

var (
	tracks = make(Tracker)
	tmpl   = template.Must(template.ParseFiles("index.html"))
	zzz    bool
)

func main() {
	tracks.load(file)
	go tracks.collect()
	go func() {
		for {
			tracks.cleanup(12 * time.Hour)
			tracks.store(file)
			time.Sleep(time.Minute)
		}
	}()
	log.Println("listen on", port)
	http.HandleFunc("/", indexHandler)
	http.ListenAndServe(port, nil)
}
<|endoftext|>


<commit_before>package gore

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/mitchellh/go-homedir"
)

type gore struct {
	autoImport           bool
	extFiles             string
	packageName          string
	outWriter, errWriter io.Writer
}

func (g *gore) run() error {
	s, err := NewSession(g.outWriter, g.errWriter)
	defer s.Clear()
	if err != nil {
		return err
	}
	s.autoImport = g.autoImport

	fmt.Fprintf(os.Stderr, "gore version %s  :help for help\n", version)

	if g.extFiles != "" {
		extFiles := strings.Split(g.extFiles, ",")
		s.includeFiles(extFiles)
	}

	if g.packageName != "" {
		err := s.includePackage(g.packageName)
		if err != nil {
			errorf("-pkg: %s", err)
			os.Exit(1)
		}
	}

	rl := newContLiner()
	defer rl.Close()

	var historyFile string
	home, err := homeDir()
	if err != nil {
		errorf("home: %s", err)
	} else {
		historyFile = filepath.Join(home, "history")

		f, err := os.Open(historyFile)
		if err != nil {
			if !os.IsNotExist(err) {
				errorf("%s", err)
			}
		} else {
			_, err := rl.ReadHistory(f)
			if err != nil {
				errorf("while reading history: %s", err)
			}
			f.Close()
		}
	}

	rl.SetWordCompleter(s.completeWord)

	for {
		in, err := rl.Prompt()
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintf(os.Stderr, "fatal: %s", err)
			os.Exit(1)
		}

		if in == "" {
			continue
		}

		if err := rl.Reindent(); err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err)
			rl.Clear()
			continue
		}

		err = s.Eval(in)
		if err != nil {
			if err == ErrContinue {
				continue
			} else if err == ErrQuit {
				break
			} else if err != ErrCmdRun {
				rl.Clear()
				continue
			}
		}
		rl.Accepted()
	}

	if historyFile != "" {
		err := os.MkdirAll(filepath.Dir(historyFile), 0755)
		if err != nil {
			errorf("%s", err)
		} else {
			f, err := os.Create(historyFile)
			if err != nil {
				errorf("%s", err)
			} else {
				_, err := rl.WriteHistory(f)
				if err != nil {
					errorf("while saving history: %s", err)
				}
				f.Close()
			}
		}
	}

	return nil
}

func homeDir() (home string, err error) {
	home = os.Getenv("GORE_HOME")
	if home != "" {
		return
	}

	home, err = homedir.Dir()
	if err != nil {
		return
	}

	home = filepath.Join(home, ".gore")
	return
}
<commit_msg>use errWriter of gore instead of os.Stderr<commit_after>package gore

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/mitchellh/go-homedir"
)

type gore struct {
	autoImport           bool
	extFiles             string
	packageName          string
	outWriter, errWriter io.Writer
}

func (g *gore) run() error {
	s, err := NewSession(g.outWriter, g.errWriter)
	defer s.Clear()
	if err != nil {
		return err
	}
	s.autoImport = g.autoImport

	fmt.Fprintf(g.errWriter, "gore version %s  :help for help\n", version)

	if g.extFiles != "" {
		extFiles := strings.Split(g.extFiles, ",")
		s.includeFiles(extFiles)
	}

	if g.packageName != "" {
		err := s.includePackage(g.packageName)
		if err != nil {
			errorf("-pkg: %s", err)
			os.Exit(1)
		}
	}

	rl := newContLiner()
	defer rl.Close()

	var historyFile string
	home, err := homeDir()
	if err != nil {
		errorf("home: %s", err)
	} else {
		historyFile = filepath.Join(home, "history")

		f, err := os.Open(historyFile)
		if err != nil {
			if !os.IsNotExist(err) {
				errorf("%s", err)
			}
		} else {
			_, err := rl.ReadHistory(f)
			if err != nil {
				errorf("while reading history: %s", err)
			}
			f.Close()
		}
	}

	rl.SetWordCompleter(s.completeWord)

	for {
		in, err := rl.Prompt()
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintf(g.errWriter, "fatal: %s", err)
			os.Exit(1)
		}

		if in == "" {
			continue
		}

		if err := rl.Reindent(); err != nil {
			fmt.Fprintf(g.errWriter, "error: %s\n", err)
			rl.Clear()
			continue
		}

		err = s.Eval(in)
		if err != nil {
			if err == ErrContinue {
				continue
			} else if err == ErrQuit {
				break
			} else if err != ErrCmdRun {
				rl.Clear()
				continue
			}
		}
		rl.Accepted()
	}

	if historyFile != "" {
		err := os.MkdirAll(filepath.Dir(historyFile), 0755)
		if err != nil {
			errorf("%s", err)
		} else {
			f, err := os.Create(historyFile)
			if err != nil {
				errorf("%s", err)
			} else {
				_, err := rl.WriteHistory(f)
				if err != nil {
					errorf("while saving history: %s", err)
				}
				f.Close()
			}
		}
	}

	return nil
}

func homeDir() (home string, err error) {
	home = os.Getenv("GORE_HOME")
	if home != "" {
		return
	}

	home, err = homedir.Dir()
	if err != nil {
		return
	}

	home = filepath.Join(home, ".gore")
	return
}
<|endoftext|>