<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
)

// Push command line tool
func cmdPush() int {

	// git-lob push [--all] [--recheck] [--force] [<remote> [<ref>...]]

	// Validate custom options
	errorList := validateCustomOptions(GlobalOptions, nil, []string{"all", "recheck", "force"})
	if len(errorList) > 0 {
		fmt.Fprintf(os.Stderr, strings.Join(errorList, "\n"))
		return 9
	}

	optAll := GlobalOptions.BoolOpts.Contains("all")
	optRecheck := GlobalOptions.BoolOpts.Contains("recheck")
	optForce := GlobalOptions.BoolOpts.Contains("force")
	optDryRun := GlobalOptions.DryRun

	// Determine remote
	var remoteName string
	var refspecs []*GitRefSpec
	if len(GlobalOptions.Args) > 0 {
		// first parameter must be remote if there are arguments
		remoteName = GlobalOptions.Args[0]

		// Remaining args are refspecs
		if len(GlobalOptions.Args) > 1 {
			// Not valid if --all
			if optAll {
				fmt.Fprintf(os.Stderr, "git-lob: Too many arguments; cannot include refspec when using --all\n")
				return 7
			}
			for _, arg := range GlobalOptions.Args[1:] {
				r := ParseGitRefSpec(arg)
				refspecs = append(refspecs, r)
			}
		}

	} else {
		remoteName = GetGitDefaultRemote()
	}

	// check the remote config to make sure it's valid
	provider, err := GetProviderForRemote(remoteName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: %v\n", err)
		return 6
	}
	if err = provider.ValidateConfig(remoteName); err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: remote %v has configuration problems:\n%v\n", remoteName, err)
		return 6
	}

	if len(refspecs) == 0 {
		// No refspecs specified, so determine default
		if optAll {
			branches, err := GetGitLocalBranches()
			if err != nil {
				fmt.Fprintf(os.Stderr, "git-lob: unable to get local branch list - %v\n", err)
				return 7
			}
			for _, branch := range branches {
				refspecs = append(refspecs, &GitRefSpec{branch, "", ""})
			}

		} else {
			// Check remote.<remote>.push for default refspec
			def, ok := GlobalOptions.GitConfig[fmt.Sprintf("remote.%v.push", remoteName)]
			if ok {
				pushspecs := strings.Fields(def)
				for _, s := range pushspecs {
					refspecs = append(refspecs, &GitRefSpec{s, "", ""})
				}
			} else {
				// determine refspec from current branch & push settings
				branch := GetGitCurrentBranch()
				refspecs = append(refspecs, &GitRefSpec{branch, "", ""})
			}
		}
	}

	progress := func(fileInProgress string, isSkipped bool, filepercent, overallpercent int, rate string) (abort bool) {
		// Note, no newline, just carriage return to overwrite single line in place
		if GlobalOptions.Verbose {
			// report file name too
			if filepercent == 100 {
				// in verbose mode we include newline after reporting file name at 100%
				if isSkipped {
					fmt.Printf("\rSkipped: %v (Up to date)\n", fileInProgress)
				} else {
					fmt.Printf("\rPushed: %v 100%%\n", fileInProgress)
				}
			} else {
				fmt.Printf("\rPushing: %v %d%%\tOverall: %d%%\t(%v)", fileInProgress, filepercent, overallpercent, rate)
			}
		} else if !GlobalOptions.Quiet {
			fmt.Printf("\rPushing: %d%%\t(%v)", overallpercent, rate)
		}

		return false

	}

	if !GlobalOptions.Quiet {
		fmt.Println("Pushing binaries for", refspecs, "to", remoteName)
	}

	switch p := provider.(type) {
	case BasicSyncProvider:
		PushBasic(p, remoteName, refspecs, optDryRun, optForce, optRecheck, progress)
	case SmartSyncProvider:
		PushSmart(p, remoteName, refspecs, optDryRun, optForce, optRecheck, progress)
	}

	// Because no newlines in progress reporting
	if !GlobalOptions.Quiet {
		fmt.Println()
	}

	return 0
}

func PushBasic(provider BasicSyncProvider, remoteName string, refspecs []*GitRefSpec, dryRun, force, recheck bool,
	callback func(fileInProgress string, isSkipped bool, filepercent, overallpercent int, rate string) (abort bool)) {
	// TODO
}
func PushSmart(provider SmartSyncProvider, remoteName string, refspecs []*GitRefSpec, dryRun, force, recheck bool,
	callback func(fileInProgress string, isSkipped bool, filepercent, overallpercent int, rate string) (abort bool)) {
	// TODO
}

func cmdPushHelp() {
	fmt.Println(`Usage: git-lob push [options] [<remote> [<ref>...]]

  Uploads binaries to a remote, sending only binaries required to ensure
  that remote has the binary resources referenced at a set of commits.

  Behaves much like 'git push' except there are no destination refs, only
  supporting binary files.

Parameters:
  <remote>: The destination to upload to. This should correspond to the
            name of a remote (no direct URLs permitted) which is configured
            in .git/config. See REMOTES below for more details, additional
            config parameters are required in the remote.

            If no remote is specified, branch.*.remote configuration for the
            current branch is consulted to determine where to push. If the
            configuration is missing, it defaults to origin.
     <ref>: Which local reference(s) up to which we should make sure binaries
            are uploaded for. You can specify zero, one, or many local refs.
            There is no destination ref as in git push.

            If no ref is specified, and --all is not used, then
            remote.<remotename>.push is used if present, otherwise the current
            branch is used.

            COMMIT RANGES

            You can also specify a range of refs in the form <ref1>..<ref2> to
            force git-lob push to check a specific range of commits for
            binaries, instead of using its own records of which commits it
            thinks are already up to date on this remote.
            See HISTORY CHECKING below.

Options:
  --all         Push all branches; cannot be used with other refs.
  --recheck     Re-check entire commit history to each ref instead of only
                back to last commit we believe is already pushed.
                See HISTORY CHECKING below for more details.
  --force       Always upload files even if the provider believes the file is
                already present on the remote. You shouldn't need this.
  --quiet, -q   Print less output
  --verbose, -v Print more output
  --dry-run     Don't actually delete anything, just report

REMOTES

Binaries in git-lob are not stored in the regular git repo, but a corresponding
binary store must always exist to provide the actual binary content. A remote
in git usually only gives you the real git repo, so git-lob needs to expand
the configuration parameters to git remotes to specify the location of the
corresponding remote binary store.

The <remote> parameter refers to a named remote in .git/config (plain URLs
cannot be supported). The remote entry itself is the same as any normal git
remote, except that it requires additional git-lob specific parameters. The
parameters depend on the type of binary storage ('provider') being used; see
'git-lob providers' for a list of available providers and
'git-lob provider <provider>' for specific details of one provider.

As an example, let's take the 'filesystem' provider, which simply uses the OS's
file system as a remote transport (obviously very simplistic):

[remote "origin"]
    # these 2 lines are standard git
    url = ssh://git@bitbucket.org/something/somthing.git
    fetch = +refs/heads/*:refs/remotes/origin/*
    # these next 2 lines are required to configure the remote binary store
    git-lob-provider = filesystem
    git-lob-path = /Volumes/shared/something/something/binary/store

Other providers may require other parameters. It's important to note that you
can share a binary store among multiple remote repos if you wish, much like
the local git-lob.sharedstore option, since binaries are stored by SHA.
Identical file content in multiple repos can be stored only once this way.
Of course, access control may be an issue to consider here though.

HISTORY CHECKING

When pushing binaries for a given ref, git-lob performs a search for commits
which reference git-lob binaries from that ref backwards, before checking
which of those binaries it needs to upload. This is so that we only upload
binaries that are actually referenced by the ref you're choosing to push,
and don't waste time on binaries in unpublished feature branches etc.

Because searching the whole of the git history can be slow on large
repositories, git-lob speeds this search up by keeping a record of which
commits it believes the remote already has all binaries for.

These records are updated whenever you git-lob push/pull. We do not use git's
own remote branch refs to track this, because pushing commits can be done
completely separately from binaries so we can't rely on that information.
So pushing and pulling branches in git has no effect on this state, only
git-lob push/pull.

If for some reason these records are wrong, and you need to push binaries
for a bigger range of commits, you can do this 2 ways:

1. Use the --recheck option. This is the 'nuclear option'; it will scan the
   entire history of the repo again to make 100% sure everything is correct.
   Can take a while on large repos.

2. Use a commit range for <ref>, i.e. <ref1>..<ref2>. git-lob will check that
   entire range of commits for binary references which will then be checked
   with the remote.

There are not many circumstances where you need to manually override the commit
range that is checked for binaries. Even if you edit commits, rebase etc,
git-lob should not miss any binaries, because the commit SHAs would change and
it would know to check any referenced binaries again. The main reason why
you would need to override the history checking is if the remote changed, for
example if someone manually deleted the remote binary store, or you moved to
a new URL without copying the data and needed to re-populate it from your local
repo.

`)
}
<commit_msg>Change the callback structure of push utilities to allow for preparation callbacks & less ambiguity<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
)

// Push command line tool
func cmdPush() int {

	// git-lob push [--all] [--recheck] [--force] [<remote> [<ref>...]]

	// Validate custom options
	errorList := validateCustomOptions(GlobalOptions, nil, []string{"all", "recheck", "force"})
	if len(errorList) > 0 {
		fmt.Fprintf(os.Stderr, strings.Join(errorList, "\n"))
		return 9
	}

	optAll := GlobalOptions.BoolOpts.Contains("all")
	optRecheck := GlobalOptions.BoolOpts.Contains("recheck")
	optForce := GlobalOptions.BoolOpts.Contains("force")
	optDryRun := GlobalOptions.DryRun

	// Determine remote
	var remoteName string
	var refspecs []*GitRefSpec
	if len(GlobalOptions.Args) > 0 {
		// first parameter must be remote if there are arguments
		remoteName = GlobalOptions.Args[0]

		// Remaining args are refspecs
		if len(GlobalOptions.Args) > 1 {
			// Not valid if --all
			if optAll {
				fmt.Fprintf(os.Stderr, "git-lob: Too many arguments; cannot include refspec when using --all\n")
				return 7
			}
			for _, arg := range GlobalOptions.Args[1:] {
				r := ParseGitRefSpec(arg)
				refspecs = append(refspecs, r)
			}
		}

	} else {
		remoteName = GetGitDefaultRemote()
	}

	// check the remote config to make sure it's valid
	provider, err := GetProviderForRemote(remoteName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: %v\n", err)
		return 6
	}
	if err = provider.ValidateConfig(remoteName); err != nil {
		fmt.Fprintf(os.Stderr, "git-lob: remote %v has configuration problems:\n%v\n", remoteName, err)
		return 6
	}

	if len(refspecs) == 0 {
		// No refspecs specified, so determine default
		if optAll {
			branches, err := GetGitLocalBranches()
			if err != nil {
				fmt.Fprintf(os.Stderr, "git-lob: unable to get local branch list - %v\n", err)
				return 7
			}
			for _, branch := range branches {
				refspecs = append(refspecs, &GitRefSpec{branch, "", ""})
			}

		} else {
			// Check remote.<remote>.push for default refspec
			def, ok := GlobalOptions.GitConfig[fmt.Sprintf("remote.%v.push", remoteName)]
			if ok {
				pushspecs := strings.Fields(def)
				for _, s := range pushspecs {
					refspecs = append(refspecs, &GitRefSpec{s, "", ""})
				}
			} else {
				// determine refspec from current branch & push settings
				branch := GetGitCurrentBranch()
				refspecs = append(refspecs, &GitRefSpec{branch, "", ""})
			}
		}
	}

	progress := func(t PushCallbackType, desc string, itempercent, overallpercent int, rate string) (abort bool) {
		switch t {
		case PushCallbackCalculate:
			if !GlobalOptions.Quiet {
				fmt.Println(desc)
			}
		case PushCallbackSkip:
			if GlobalOptions.Verbose {
				fmt.Println("Skipped:", desc, "(Up to date)")
			}
		case PushCallbackUpload:
			// Re-use the same line for each of these updates, except in verbose mode when we newline on completion
			if GlobalOptions.Verbose {
				// report file name too
				if itempercent == 100 {
					fmt.Printf("\rPushed: %v 100%%\n", desc)
				} else {
					fmt.Printf("\rPushing: %v %d%%\tOverall: %d%%\t(%v)", desc, itempercent, overallpercent, rate)
				}
			} else if !GlobalOptions.Quiet {
				fmt.Printf("\rPushing: %d%%\t(%v)", overallpercent, rate)
			}

		}

		return false

	}

	if !GlobalOptions.Quiet {
		fmt.Println("Pushing binaries for", refspecs, "to", remoteName)
	}

	var pusherr error
	switch p := provider.(type) {
	case BasicSyncProvider:
		pusherr = PushBasic(p, remoteName, refspecs, optDryRun, optForce, optRecheck, progress)
	case SmartSyncProvider:
		pusherr = PushSmart(p, remoteName, refspecs, optDryRun, optForce, optRecheck, progress)
	}

	if pusherr != nil {
		fmt.Fprintf(os.Stderr, "git-lob: push error - %v", err.Error())
		return 12
	}
	// Because no newlines in progress reporting
	if !GlobalOptions.Quiet {
		fmt.Println()
	}

	return 0
}

type PushCallbackType int

const (
	// Push process is figuring out what to push
	PushCallbackCalculate PushCallbackType = iota
	// Push process is transferring data
	PushCallbackUpload PushCallbackType = iota
	// Push process is skipping data because it's already up to date
	PushCallbackSkip PushCallbackType = iota
)

// Callback when progress is made during push
// t: what stage of the push process this is for, preparing, uploading or skipping something
// desc: either a general message or an item name (e.g. file name in upload stage)
// itempercent: if applicable, what percent of this item is done
// overallpercent: what overall percent of the process is done
// rate: upload rate for info if applicable
// return true to abort the (entire) process
type PushCallback func(t PushCallbackType, desc string, itempercent, overallpercent int, rate string) (abort bool)

func PushBasic(provider BasicSyncProvider, remoteName string, refspecs []*GitRefSpec, dryRun, force, recheck bool,
	callback PushCallback) error {
	return nil
}
func PushSmart(provider SmartSyncProvider, remoteName string, refspecs []*GitRefSpec, dryRun, force, recheck bool,
	callback PushCallback) error {
	// TODO
	return nil
}

func cmdPushHelp() {
	fmt.Println(`Usage: git-lob push [options] [<remote> [<ref>...]]

  Uploads binaries to a remote, sending only binaries required to ensure
  that remote has the binary resources referenced at a set of commits.

  Behaves much like 'git push' except there are no destination refs, only
  supporting binary files.

Parameters:
  <remote>: The destination to upload to. This should correspond to the
            name of a remote (no direct URLs permitted) which is configured
            in .git/config. See REMOTES below for more details, additional
            config parameters are required in the remote.

            If no remote is specified, branch.*.remote configuration for the
            current branch is consulted to determine where to push. If the
            configuration is missing, it defaults to origin.
     <ref>: Which local reference(s) up to which we should make sure binaries
            are uploaded for. You can specify zero, one, or many local refs.
            There is no destination ref as in git push.

            If no ref is specified, and --all is not used, then
            remote.<remotename>.push is used if present, otherwise the current
            branch is used.

            COMMIT RANGES

            You can also specify a range of refs in the form <ref1>..<ref2> to
            force git-lob push to check a specific range of commits for
            binaries, instead of using its own records of which commits it
            thinks are already up to date on this remote.
            See HISTORY CHECKING below.

Options:
  --all         Push all branches; cannot be used with other refs.
  --recheck     Re-check entire commit history to each ref instead of only
                back to last commit we believe is already pushed.
                See HISTORY CHECKING below for more details.
  --force       Always upload files even if the provider believes the file is
                already present on the remote. You shouldn't need this.
  --quiet, -q   Print less output
  --verbose, -v Print more output
  --dry-run     Don't actually delete anything, just report

REMOTES

Binaries in git-lob are not stored in the regular git repo, but a corresponding
binary store must always exist to provide the actual binary content. A remote
in git usually only gives you the real git repo, so git-lob needs to expand
the configuration parameters to git remotes to specify the location of the
corresponding remote binary store.

The <remote> parameter refers to a named remote in .git/config (plain URLs
cannot be supported). The remote entry itself is the same as any normal git
remote, except that it requires additional git-lob specific parameters. The
parameters depend on the type of binary storage ('provider') being used; see
'git-lob providers' for a list of available providers and
'git-lob provider <provider>' for specific details of one provider.

As an example, let's take the 'filesystem' provider, which simply uses the OS's
file system as a remote transport (obviously very simplistic):

[remote "origin"]
    # these 2 lines are standard git
    url = ssh://git@bitbucket.org/something/somthing.git
    fetch = +refs/heads/*:refs/remotes/origin/*
    # these next 2 lines are required to configure the remote binary store
    git-lob-provider = filesystem
    git-lob-path = /Volumes/shared/something/something/binary/store

Other providers may require other parameters. It's important to note that you
can share a binary store among multiple remote repos if you wish, much like
the local git-lob.sharedstore option, since binaries are stored by SHA.
Identical file content in multiple repos can be stored only once this way.
Of course, access control may be an issue to consider here though.

HISTORY CHECKING

When pushing binaries for a given ref, git-lob performs a search for commits
which reference git-lob binaries from that ref backwards, before checking
which of those binaries it needs to upload. This is so that we only upload
binaries that are actually referenced by the ref you're choosing to push,
and don't waste time on binaries in unpublished feature branches etc.

Because searching the whole of the git history can be slow on large
repositories, git-lob speeds this search up by keeping a record of which
commits it believes the remote already has all binaries for.

These records are updated whenever you git-lob push/pull. We do not use git's
own remote branch refs to track this, because pushing commits can be done
completely separately from binaries so we can't rely on that information.
So pushing and pulling branches in git has no effect on this state, only
git-lob push/pull.

If for some reason these records are wrong, and you need to push binaries
for a bigger range of commits, you can do this 2 ways:

1. Use the --recheck option. This is the 'nuclear option'; it will scan the
   entire history of the repo again to make 100% sure everything is correct.
   Can take a while on large repos.

2. Use a commit range for <ref>, i.e. <ref1>..<ref2>. git-lob will check that
   entire range of commits for binary references which will then be checked
   with the remote.

There are not many circumstances where you need to manually override the commit
range that is checked for binaries. Even if you edit commits, rebase etc,
git-lob should not miss any binaries, because the commit SHAs would change and
it would know to check any referenced binaries again. The main reason why
you would need to override the history checking is if the remote changed, for
example if someone manually deleted the remote binary store, or you moved to
a new URL without copying the data and needed to re-populate it from your local
repo.

`)
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	log "github.com/Sirupsen/logrus"
)

type testpair struct {
	value  string
	result string
}

func TestRawTextExtraction(t *testing.T) {
	log.SetLevel(log.DebugLevel)

	var tests = []testpair{
		{
			`An undocumented(test)`,
			`An undocumented(test)`,
		},
		{
			`An undocumented `,
			`An undocumented `,
		},
		{
			`Adventurer's clothes`,
			`Adventurer's clothes`,
		},
		{
			`An undocumented if`,
			`An undocumented if`,
		},
		{
			`An undocumented if()`,
			`An undocumented `,
		},
		{
			`if(v[178] \\>= 40)`,
			``,
		},
		{
			`踊れ if(v[178] \\>= 40)`,
			`踊れ `,
		},
		{
			`\\\>\\C[14]…今は使用できません。`,
			`…今は使用できません。`,
		},
	}

	for _, pair := range tests {
		v := getOnlyText(pair.value)
		if v != pair.result {
			t.Error(
				"For", pair.value,
				"expected", pair.result,
				"got", v,
			)
		}
	}

}

func TestTranslatableTextExtraction(t *testing.T) {
	log.SetLevel(log.DebugLevel)

	var tests = []testpair{
		{
			`if(v[178] \\>= 40)`,
			``,
		},
		{
			`踊れ if(v[178] \\>= 40)`,
			`踊れ `,
		},
		{
			`\\\>\\C[14]…今は使用できません。`,
			`今は使用できません`,
		},
		{
			`\\\>…「\\C[14]\\N[2]\\C[0]」は、\\\>　カナーン村の酒場へ帰っていった。`,
			`は、カナーン村の酒場へ帰っていった`,
		},
		{
			`\\\>…「\\C[14]\\N[2]\\C[0]」の同伴時間が\\C[3] 10 \\C[0]を超えていますので、\\\>　「\\C[14]\\N[2]\\C[0]」の欲情度が\\C[3] 100 \\C[0]未満の場合、\\\>　時間経過で上昇していきます。`,
			`の同伴時間が 10 を超えていますので、の欲情度が 100 未満の場合、時間経過で上昇していきます`,
		},
		{
			`%sを %s 回復した！`,
			`を  回復した！`,
		},
		{
			`"お金を %s\\\\G 手に入れた！"`,
			``,
		},
		{
			`'\\.'`,
			`''`,
		},
		{
			`【\\C[14]\\N[2]\\C[0]】　\\{アハァァーーーンッ！！`,
			`アハァァーーーンッ！！`,
		},
		{
			`\\\>…「\\C[10]バッポウ\\C[0]」再出現カウント： \\C[3] \\V[491] \\C[0]\\\>\\\>　\\C[14]※出現場所：キータニ平原\\\>　\\C[14]※カウントが 0 になると再挑戦可能になります。`,
			`バッポウ再出現カウント：   ※出現場所：キータニ平原※カウントが 0 になると再挑戦可能になります`,
		},
		{
			`【\\C[14]骨董屋\\C[0]】　「謎の防具」か。　ほぅ、複数持っているようだな。　一気に鑑定するかい？\\C[3] \\V[982] G\\C[0] 頂くけどな。\\$`,
			`骨董屋謎の防具かほぅ、複数持っているようだな一気に鑑定するかい？  G 頂くけどな`,
		},
		{
			`【\\C[14]ザウナー\\C[0]】　\\}…待て待て、この場を乗り切るための詭弁さ。\\{　\\}すまないが、我慢して様子を見ててくれ。\\{　\\}必ず上手くいくさ。`,
			`ザウナー待て待て、この場を乗り切るための詭弁さすまないが、我慢して様子を見ててくれ必ず上手くいくさ`,
		},
		{
			`\\c[3]宿屋の主人\\c[0]ほほ～、シスターとは珍しい！\\lこんな辺鄙な島で布教活動かね？言っとくが、ワシは神など信じないぞ。`,
			`宿屋の主人ほほ、シスターとは珍しい！こんな辺鄙な島で布教活動かね？言っとくが、ワシは神など信じないぞ`,
		},
		{
			`0\\G 手に入れた！`,
			`0 手に入れた！`,
		},
		{
			`\\\>牡丹の命が５回復した。\\|\\.\\^`,
			`牡丹の命が５回復した`,
		},
		{
			`氷結水　(残\\V[29]) en(s[28])`,
			`氷結水残 `,
		},
		{
			`疾風苦無(消費1)　en(v[25] \>= 1)`,
			`疾風苦無消費1`,
		},
	}

	for _, pair := range tests {
		items := parseText(pair.value)

		var v string

		for _, item := range items {
			if item.typ == itemText {
				v += item.val
			}
		}

		if v != pair.result {
			t.Error(
				"For", pair.value,
				"expected", pair.result,
				"got", v,
			)
		}
	}
}
<commit_msg>Disable debug logging during tests<commit_after>package main

import "testing"

type testpair struct {
	value  string
	result string
}

func TestRawTextExtraction(t *testing.T) {
	var tests = []testpair{
		{
			`An undocumented(test)`,
			`An undocumented(test)`,
		},
		{
			`An undocumented `,
			`An undocumented `,
		},
		{
			`Adventurer's clothes`,
			`Adventurer's clothes`,
		},
		{
			`An undocumented if`,
			`An undocumented if`,
		},
		{
			`An undocumented if()`,
			`An undocumented `,
		},
		{
			`if(v[178] \\>= 40)`,
			``,
		},
		{
			`踊れ if(v[178] \\>= 40)`,
			`踊れ `,
		},
		{
			`\\\>\\C[14]…今は使用できません。`,
			`…今は使用できません。`,
		},
	}

	for _, pair := range tests {
		v := getOnlyText(pair.value)
		if v != pair.result {
			t.Error(
				"For", pair.value,
				"expected", pair.result,
				"got", v,
			)
		}
	}

}

func TestTranslatableTextExtraction(t *testing.T) {
	var tests = []testpair{
		{
			`if(v[178] \\>= 40)`,
			``,
		},
		{
			`踊れ if(v[178] \\>= 40)`,
			`踊れ `,
		},
		{
			`\\\>\\C[14]…今は使用できません。`,
			`今は使用できません`,
		},
		{
			`\\\>…「\\C[14]\\N[2]\\C[0]」は、\\\>　カナーン村の酒場へ帰っていった。`,
			`は、カナーン村の酒場へ帰っていった`,
		},
		{
			`\\\>…「\\C[14]\\N[2]\\C[0]」の同伴時間が\\C[3] 10 \\C[0]を超えていますので、\\\>　「\\C[14]\\N[2]\\C[0]」の欲情度が\\C[3] 100 \\C[0]未満の場合、\\\>　時間経過で上昇していきます。`,
			`の同伴時間が 10 を超えていますので、の欲情度が 100 未満の場合、時間経過で上昇していきます`,
		},
		{
			`%sを %s 回復した！`,
			`を  回復した！`,
		},
		{
			`"お金を %s\\\\G 手に入れた！"`,
			``,
		},
		{
			`'\\.'`,
			`''`,
		},
		{
			`【\\C[14]\\N[2]\\C[0]】　\\{アハァァーーーンッ！！`,
			`アハァァーーーンッ！！`,
		},
		{
			`\\\>…「\\C[10]バッポウ\\C[0]」再出現カウント： \\C[3] \\V[491] \\C[0]\\\>\\\>　\\C[14]※出現場所：キータニ平原\\\>　\\C[14]※カウントが 0 になると再挑戦可能になります。`,
			`バッポウ再出現カウント：   ※出現場所：キータニ平原※カウントが 0 になると再挑戦可能になります`,
		},
		{
			`【\\C[14]骨董屋\\C[0]】　「謎の防具」か。　ほぅ、複数持っているようだな。　一気に鑑定するかい？\\C[3] \\V[982] G\\C[0] 頂くけどな。\\$`,
			`骨董屋謎の防具かほぅ、複数持っているようだな一気に鑑定するかい？  G 頂くけどな`,
		},
		{
			`【\\C[14]ザウナー\\C[0]】　\\}…待て待て、この場を乗り切るための詭弁さ。\\{　\\}すまないが、我慢して様子を見ててくれ。\\{　\\}必ず上手くいくさ。`,
			`ザウナー待て待て、この場を乗り切るための詭弁さすまないが、我慢して様子を見ててくれ必ず上手くいくさ`,
		},
		{
			`\\c[3]宿屋の主人\\c[0]ほほ～、シスターとは珍しい！\\lこんな辺鄙な島で布教活動かね？言っとくが、ワシは神など信じないぞ。`,
			`宿屋の主人ほほ、シスターとは珍しい！こんな辺鄙な島で布教活動かね？言っとくが、ワシは神など信じないぞ`,
		},
		{
			`0\\G 手に入れた！`,
			`0 手に入れた！`,
		},
		{
			`\\\>牡丹の命が５回復した。\\|\\.\\^`,
			`牡丹の命が５回復した`,
		},
		{
			`氷結水　(残\\V[29]) en(s[28])`,
			`氷結水残 `,
		},
		{
			`疾風苦無(消費1)　en(v[25] \>= 1)`,
			`疾風苦無消費1`,
		},
	}

	for _, pair := range tests {
		items := parseText(pair.value)

		var v string

		for _, item := range items {
			if item.typ == itemText {
				v += item.val
			}
		}

		if v != pair.result {
			t.Error(
				"For", pair.value,
				"expected", pair.result,
				"got", v,
			)
		}
	}
}
<|endoftext|>


<commit_before>package ssh

import (
	"testing"

	"github.com/gliderlabs/ssh"
)

func TestKeysEqual(t *testing.T) {
	defer func() {
		if r := recover(); r != nil {
			t.Errorf("The code did panic")
		}
	}()

	if ssh.KeysEqual(nil, nil) {
		t.Error("two nil keys should not return true")
	}
}
<commit_msg>fix test<commit_after>package ssh

import (
	"testing"
)

func TestKeysEqual(t *testing.T) {
	defer func() {
		if r := recover(); r != nil {
			t.Errorf("The code did panic")
		}
	}()

	if KeysEqual(nil, nil) {
		t.Error("two nil keys should not return true")
	}
}
<|endoftext|>


<commit_before>package log

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"text/template"
	"time"
)

type record struct {
	Start, End string
	Date, Time string
	Tag        string
	Level      string
	File       string
	Line       int
	Message    string
	Stack      []byte
}

// Standard 日志输出基本实现
type Standard struct {
	mu  sync.Mutex    // ensures atomic writes; protects the following fields
	out *bufio.Writer // destination for output

	format    string
	pattern   string
	colorized bool

	tpl       *template.Template
	prefixLen int
	dateFmt   string
	timeFmt   string
}

// NewStandard 返回标准实现
func NewStandard(w io.Writer, format string) *Standard {
	std := &Standard{out: bufio.NewWriter(w), colorized: true}

	// hack 如果用户不调用 SetFormat，直接用，那么也能找到主函数（main，实际是 init 函数）的所在的文件
	std.prefixLen = -5

	std.SetFormat(format)
	return std
}

// SetWriter 改变输出流
func (s *Standard) SetWriter(w io.Writer) {
	s.mu.Lock()
	s.out = bufio.NewWriter(w)
	s.mu.Unlock()
}

// Colorized 输出日志是否着色，默认着色
func (s *Standard) Colorized(c bool) {
	// 没改变
	if c == s.colorized {
		return
	}

	s.colorized = c

	s.mu.Lock()
	defer s.mu.Unlock()

	p := s.pattern
	if s.colorized {
		p = "{{.Start}}" + p + "{{.End}}"
	}
	s.tpl = template.Must(template.New("record").Parse(p))
}

// SetFormat 改变日志格式
func (s *Standard) SetFormat(format string) {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.format = format

	skip := 3
	if s.prefixLen == -5 {
		skip = 5
	}
	s.prefixLen = CalculatePrefixLen(format, skip)

	s.dateFmt, s.timeFmt = ExtactDateTime(format)

	p := parseFormat(format, s.prefixLen, s.dateFmt, s.timeFmt)

	s.pattern = p
	if s.colorized {
		p = "{{.Start}}" + p + "{{.End}}"
	}
	s.tpl = template.Must(template.New("record").Parse(p))
}

// Tprintf 打印日志
func (s *Standard) Tprintf(l Level, tag string, format string, m ...interface{}) {
	if v > l {
		return
	}

	if tag == "" {
		tag = "-"
	}
	r := record{
		Level: l.String(),
		Tag:   tag,
	}

	if s.dateFmt != "" {
		now := time.Now() // get this early.
		r.Date = now.Format(s.dateFmt)
		if s.timeFmt != "" {
			r.Time = now.Format(s.timeFmt)
		}
	}

	if s.prefixLen > -1 {
		var ok bool
		_, r.File, r.Line, ok = runtime.Caller(2) // expensive
		if ok && s.prefixLen < len(r.File) {
			r.File = r.File[s.prefixLen:]
		} else {
			r.File = "???"
		}
	}

	if format == "" {
		for _, x := range m {
			if _, ok := x.([]byte); ok {
				format += ", %s"
			} else {
				format += ", %v"
			}
		}
		if len(format) > 2 {
			format = format[2:]
		}
	}
	r.Message = fmt.Sprintf(format, m...)
	r.Message = strings.TrimSpace(r.Message)

	if l == Lstack {
		r.Stack = make([]byte, 1024*1024)
		n := runtime.Stack(r.Stack, true)
		r.Stack = r.Stack[:n]
	}

	if s.colorized {
		r.Start, r.End = calculateColor(l)
	}

	s.mu.Lock()
	defer func() {
		s.mu.Unlock()

		if l == Lpanic {
			panic(m)
		}

		if l == Lfatal {
			os.Exit(-1)
		}
	}()

	s.tpl.Execute(s.out, r)
	s.out.WriteByte('\n')

	if l == Lstack {
		s.out.Write(r.Stack)
	}

	s.out.Flush()
}

// 格式解析，把格式串替换成 token 串
func parseFormat(format string, prefixLen int, dateFmt, timeFmt string) (pattern string) {
	// 顺序最好不要变，从最长的开始匹配
	pattern = strings.Replace(format, PathToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, PackageToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, ProjectToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, FileToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, TagToken, "{{ .Tag }}", -1)
	pattern = strings.Replace(pattern, LevelToken, "{{ .Level }}", -1)
	pattern = strings.Replace(pattern, strconv.Itoa(LineToken), "{{ .Line }}", -1)
	pattern = strings.Replace(pattern, MessageToken, "{{ .Message }}", -1)

	// 提取出日期和时间的格式化模式字符串
	if dateFmt != "" {
		pattern = strings.Replace(pattern, dateFmt, "{{ .Date }}", -1)
	}
	if timeFmt != "" {
		pattern = strings.Replace(pattern, timeFmt, "{{ .Time }}", -1)
	}
	return pattern
}

func calculateColor(l Level) (start, end string) {
	// all, trace, debug,          info,   warn,   error,  panic,  fatal, print, stack
	colors := []string{"", "", "", "0;32", "0;34", "0;31", "0;35", "0;35", "", ""}
	if colors[l] != "" {
		start = "\033[" + colors[l] + "m"
		end = "\033[0m"
	}
	return start, end
}

/*

Bash Shell定义文字颜色有三个参数：Style，Frontground和Background，每个参数有7个值，意义如下：

0：黑色
1：蓝色
2：绿色
3：青色
4：红色
5：洋红色
6：黄色
7：白色
其中，+30表示前景色，+40表示背景色
这里提供一段代码可以打印颜色表：

#/bin/bash
for STYLE in 0 1 2 3 4 5 6 7; do
  for FG in 30 31 32 33 34 35 36 37; do
    for BG in 40 41 42 43 44 45 46 47; do
      CTRL="\033[${STYLE};${FG};${BG}m"
      echo -en "${CTRL}"
      echo -n " ${STYLE};${FG};${BG} "
      echo -en "\033[0m"
    done
    echo
  done
  echo
done
# Reset
echo -e "\033[0m"

代码               意义
 -------------------------
 0                 OFF
 1                  高亮显示
 4                 underline
 5                  闪烁
 7                  反 白显示
 8                  不可见
*/
<commit_msg>fix warn color<commit_after>package log

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"text/template"
	"time"
)

type record struct {
	Start, End string
	Date, Time string
	Tag        string
	Level      string
	File       string
	Line       int
	Message    string
	Stack      []byte
}

// Standard 日志输出基本实现
type Standard struct {
	mu  sync.Mutex    // ensures atomic writes; protects the following fields
	out *bufio.Writer // destination for output

	format    string
	pattern   string
	colorized bool

	tpl       *template.Template
	prefixLen int
	dateFmt   string
	timeFmt   string
}

// NewStandard 返回标准实现
func NewStandard(w io.Writer, format string) *Standard {
	std := &Standard{out: bufio.NewWriter(w), colorized: true}

	// hack 如果用户不调用 SetFormat，直接用，那么也能找到主函数（main，实际是 init 函数）的所在的文件
	std.prefixLen = -5

	std.SetFormat(format)
	return std
}

// SetWriter 改变输出流
func (s *Standard) SetWriter(w io.Writer) {
	s.mu.Lock()
	s.out = bufio.NewWriter(w)
	s.mu.Unlock()
}

// Colorized 输出日志是否着色，默认着色
func (s *Standard) Colorized(c bool) {
	// 没改变
	if c == s.colorized {
		return
	}

	s.colorized = c

	s.mu.Lock()
	defer s.mu.Unlock()

	p := s.pattern
	if s.colorized {
		p = "{{.Start}}" + p + "{{.End}}"
	}
	s.tpl = template.Must(template.New("record").Parse(p))
}

// SetFormat 改变日志格式
func (s *Standard) SetFormat(format string) {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.format = format

	skip := 3
	if s.prefixLen == -5 {
		skip = 5
	}
	s.prefixLen = CalculatePrefixLen(format, skip)

	s.dateFmt, s.timeFmt = ExtactDateTime(format)

	p := parseFormat(format, s.prefixLen, s.dateFmt, s.timeFmt)

	s.pattern = p
	if s.colorized {
		p = "{{.Start}}" + p + "{{.End}}"
	}
	s.tpl = template.Must(template.New("record").Parse(p))
}

// Tprintf 打印日志
func (s *Standard) Tprintf(l Level, tag string, format string, m ...interface{}) {
	if v > l {
		return
	}

	if tag == "" {
		tag = "-"
	}
	r := record{
		Level: l.String(),
		Tag:   tag,
	}

	if s.dateFmt != "" {
		now := time.Now() // get this early.
		r.Date = now.Format(s.dateFmt)
		if s.timeFmt != "" {
			r.Time = now.Format(s.timeFmt)
		}
	}

	if s.prefixLen > -1 {
		var ok bool
		_, r.File, r.Line, ok = runtime.Caller(2) // expensive
		if ok && s.prefixLen < len(r.File) {
			r.File = r.File[s.prefixLen:]
		} else {
			r.File = "???"
		}
	}

	if format == "" {
		for _, x := range m {
			if _, ok := x.([]byte); ok {
				format += ", %s"
			} else {
				format += ", %v"
			}
		}
		if len(format) > 2 {
			format = format[2:]
		}
	}
	r.Message = fmt.Sprintf(format, m...)
	r.Message = strings.TrimSpace(r.Message)

	if l == Lstack {
		r.Stack = make([]byte, 1024*1024)
		n := runtime.Stack(r.Stack, true)
		r.Stack = r.Stack[:n]
	}

	if s.colorized {
		r.Start, r.End = calculateColor(l)
	}

	s.mu.Lock()
	defer func() {
		s.mu.Unlock()

		if l == Lpanic {
			panic(m)
		}

		if l == Lfatal {
			os.Exit(-1)
		}
	}()

	s.tpl.Execute(s.out, r)
	s.out.WriteByte('\n')

	if l == Lstack {
		s.out.Write(r.Stack)
	}

	s.out.Flush()
}

// 格式解析，把格式串替换成 token 串
func parseFormat(format string, prefixLen int, dateFmt, timeFmt string) (pattern string) {
	// 顺序最好不要变，从最长的开始匹配
	pattern = strings.Replace(format, PathToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, PackageToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, ProjectToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, FileToken, "{{ .File }}", -1)
	pattern = strings.Replace(pattern, TagToken, "{{ .Tag }}", -1)
	pattern = strings.Replace(pattern, LevelToken, "{{ .Level }}", -1)
	pattern = strings.Replace(pattern, strconv.Itoa(LineToken), "{{ .Line }}", -1)
	pattern = strings.Replace(pattern, MessageToken, "{{ .Message }}", -1)

	// 提取出日期和时间的格式化模式字符串
	if dateFmt != "" {
		pattern = strings.Replace(pattern, dateFmt, "{{ .Date }}", -1)
	}
	if timeFmt != "" {
		pattern = strings.Replace(pattern, timeFmt, "{{ .Time }}", -1)
	}
	return pattern
}

func calculateColor(l Level) (start, end string) {
	// all, trace, debug,          info,   warn,   error,  panic,  fatal, print, stack
	colors := []string{"", "", "", "0;32", "0;33", "0;31", "0;35", "0;35", "", ""}
	if colors[l] != "" {
		start = "\033[" + colors[l] + "m"
		end = "\033[0m"
	}
	return start, end
}

/*

Bash Shell定义文字颜色有三个参数：Style，Frontground和Background，每个参数有7个值，意义如下：

0：黑色
1：蓝色
2：绿色
3：青色
4：红色
5：洋红色
6：黄色
7：白色
其中，+30表示前景色，+40表示背景色
这里提供一段代码可以打印颜色表：

#/bin/bash
for STYLE in 0 1 2 3 4 5 6 7; do
  for FG in 30 31 32 33 34 35 36 37; do
    for BG in 40 41 42 43 44 45 46 47; do
      CTRL="\033[${STYLE};${FG};${BG}m"
      echo -en "${CTRL}"
      echo -n " ${STYLE};${FG};${BG} "
      echo -en "\033[0m"
    done
    echo
  done
  echo
done
# Reset
echo -e "\033[0m"

代码               意义
 -------------------------
 0                 OFF
 1                  高亮显示
 4                 underline
 5                  闪烁
 7                  反 白显示
 8                  不可见
*/
<|endoftext|>


<commit_before>package docker

import (
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/iron-io/titan/runner/agent"
	"golang.org/x/net/context"
)

// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
// bake in retries we can use internally, could contribute it at some point, would
// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
// also adds timeouts to many operations, varying by operation
// TODO could generate this, maybe not worth it, may not change often
type dockerClient interface {
	// Each of these are github.com/fsouza/go-dockerclient methods

	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
	WaitContainer(ctx context.Context, id string) (int, error)
	StartContainer(id string, hostConfig *docker.HostConfig) error
	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
	RemoveContainer(opts docker.RemoveContainerOptions) error
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(name string) (*docker.Image, error)
	InspectContainer(id string) (*docker.Container, error)
	StopContainer(id string, timeout uint) error
	Stats(opts docker.StatsOptions) error
}

// TODO: switch to github.com/docker/engine-api
func newClient() dockerClient {
	// docker, err := docker.NewClient(conf.Docker)
	client, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	// NOTE add granularity to things like pull, should not effect
	// hijacked / streaming endpoints
	client.SetTimeout(120 * time.Second)

	return &dockerWrap{client}
}

type dockerWrap struct {
	docker *docker.Client
}

func retry(f func() error) {
	var b agent.Backoff
	then := time.Now()
	limit := 10 * time.Minute
	for time.Since(then) < limit {
		err := f()
		if agent.IsTemporary(err) || isDocker500(err) {
			logrus.WithError(err).Warn("docker temporary error, retrying")
			b.Sleep()
			continue
		}
		return
	}
	logrus.Warnf("retrying on docker errors exceeded %s, restart docker or rotate this instance?", limit)
}

func isDocker500(err error) bool {
	derr, ok := err.(*docker.Error)
	return ok && derr.Status >= 500
}

func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
	retry(func() error {
		w, err = d.docker.AttachToContainerNonBlocking(opts)
		return err
	})
	return w, err
}

func (d *dockerWrap) WaitContainer(ctx context.Context, id string) (code int, err error) {
	// special one, since fsouza doesn't have context on this one and tasks can
	// take longer than 20 minutes
	for {
		// backup bail mechanism so this doesn't sit here forever
		select {
		case <-ctx.Done():
		default:
		}

		retry(func() error {
			code, err = d.docker.WaitContainer(id)
			return err
		})
		err = filterNoSuchContainer(err)
		if err == nil {
			break
		}
		logrus.WithError(err).Warn("retrying wait container (this is ok)")
	}
	return code, err
}

func filterNoSuchContainer(err error) error {
	if err == nil {
		return nil
	}
	_, containerNotFound := err.(*docker.NoSuchContainer)
	dockerErr, ok := err.(*docker.Error)
	if containerNotFound || (ok && dockerErr.Status == 404) {
		return nil
	}
	return err
}

func filterNotRunning(err error) error {
	if err == nil {
		return nil
	}

	_, containerNotRunning := err.(*docker.ContainerNotRunning)
	dockerErr, ok := err.(*docker.Error)
	if containerNotRunning || (ok && dockerErr.Status == 304) {
		return nil
	}

	return err
}

func (d *dockerWrap) StartContainer(id string, hostConfig *docker.HostConfig) (err error) {
	retry(func() error {
		err = d.docker.StartContainer(id, hostConfig)
		return err
	})
	return err
}

func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.CreateContainer(opts)
		return err
	})
	return c, err
}

func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
	retry(func() error {
		err = d.docker.RemoveContainer(opts)
		return err
	})
	return filterNoSuchContainer(err)
}

func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
	retry(func() error {
		err = d.docker.PullImage(opts, auth)
		return err
	})
	return err
}

func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
	retry(func() error {
		i, err = d.docker.InspectImage(name)
		return err
	})
	return i, err
}

func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.InspectContainer(id)
		return err
	})
	return c, err
}

func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
	retry(func() error {
		err = d.docker.StopContainer(id, timeout)
		return err
	})
	return filterNotRunning(filterNoSuchContainer(err))
}

func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
	retry(func() error {
		err = d.docker.Stats(opts)
		return err
	})
	return err
}
<commit_msg>Fix rebase conflict.<commit_after>package docker

import (
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/iron-io/worker/runner/agent"
	"golang.org/x/net/context"
)

// wrap docker client calls so we can retry 500s, kind of sucks but fsouza doesn't
// bake in retries we can use internally, could contribute it at some point, would
// be much more convenient if we didn't have to do this, but it's better than ad hoc retries.
// also adds timeouts to many operations, varying by operation
// TODO could generate this, maybe not worth it, may not change often
type dockerClient interface {
	// Each of these are github.com/fsouza/go-dockerclient methods

	AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (docker.CloseWaiter, error)
	WaitContainer(ctx context.Context, id string) (int, error)
	StartContainer(id string, hostConfig *docker.HostConfig) error
	CreateContainer(opts docker.CreateContainerOptions) (*docker.Container, error)
	RemoveContainer(opts docker.RemoveContainerOptions) error
	PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error
	InspectImage(name string) (*docker.Image, error)
	InspectContainer(id string) (*docker.Container, error)
	StopContainer(id string, timeout uint) error
	Stats(opts docker.StatsOptions) error
}

// TODO: switch to github.com/docker/engine-api
func newClient() dockerClient {
	// docker, err := docker.NewClient(conf.Docker)
	client, err := docker.NewClientFromEnv()
	if err != nil {
		logrus.WithError(err).Fatal("couldn't create docker client")
	}

	// NOTE add granularity to things like pull, should not effect
	// hijacked / streaming endpoints
	client.SetTimeout(120 * time.Second)

	return &dockerWrap{client}
}

type dockerWrap struct {
	docker *docker.Client
}

func retry(f func() error) {
	var b agent.Backoff
	then := time.Now()
	limit := 10 * time.Minute
	for time.Since(then) < limit {
		err := f()
		if agent.IsTemporary(err) || isDocker500(err) {
			logrus.WithError(err).Warn("docker temporary error, retrying")
			b.Sleep()
			continue
		}
		return
	}
	logrus.Warnf("retrying on docker errors exceeded %s, restart docker or rotate this instance?", limit)
}

func isDocker500(err error) bool {
	derr, ok := err.(*docker.Error)
	return ok && derr.Status >= 500
}

func (d *dockerWrap) AttachToContainerNonBlocking(opts docker.AttachToContainerOptions) (w docker.CloseWaiter, err error) {
	retry(func() error {
		w, err = d.docker.AttachToContainerNonBlocking(opts)
		return err
	})
	return w, err
}

func (d *dockerWrap) WaitContainer(ctx context.Context, id string) (code int, err error) {
	// special one, since fsouza doesn't have context on this one and tasks can
	// take longer than 20 minutes
	for {
		// backup bail mechanism so this doesn't sit here forever
		select {
		case <-ctx.Done():
		default:
		}

		retry(func() error {
			code, err = d.docker.WaitContainer(id)
			return err
		})
		err = filterNoSuchContainer(err)
		if err == nil {
			break
		}
		logrus.WithError(err).Warn("retrying wait container (this is ok)")
	}
	return code, err
}

func filterNoSuchContainer(err error) error {
	if err == nil {
		return nil
	}
	_, containerNotFound := err.(*docker.NoSuchContainer)
	dockerErr, ok := err.(*docker.Error)
	if containerNotFound || (ok && dockerErr.Status == 404) {
		return nil
	}
	return err
}

func filterNotRunning(err error) error {
	if err == nil {
		return nil
	}

	_, containerNotRunning := err.(*docker.ContainerNotRunning)
	dockerErr, ok := err.(*docker.Error)
	if containerNotRunning || (ok && dockerErr.Status == 304) {
		return nil
	}

	return err
}

func (d *dockerWrap) StartContainer(id string, hostConfig *docker.HostConfig) (err error) {
	retry(func() error {
		err = d.docker.StartContainer(id, hostConfig)
		return err
	})
	return err
}

func (d *dockerWrap) CreateContainer(opts docker.CreateContainerOptions) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.CreateContainer(opts)
		return err
	})
	return c, err
}

func (d *dockerWrap) RemoveContainer(opts docker.RemoveContainerOptions) (err error) {
	retry(func() error {
		err = d.docker.RemoveContainer(opts)
		return err
	})
	return filterNoSuchContainer(err)
}

func (d *dockerWrap) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) (err error) {
	retry(func() error {
		err = d.docker.PullImage(opts, auth)
		return err
	})
	return err
}

func (d *dockerWrap) InspectImage(name string) (i *docker.Image, err error) {
	retry(func() error {
		i, err = d.docker.InspectImage(name)
		return err
	})
	return i, err
}

func (d *dockerWrap) InspectContainer(id string) (c *docker.Container, err error) {
	retry(func() error {
		c, err = d.docker.InspectContainer(id)
		return err
	})
	return c, err
}

func (d *dockerWrap) StopContainer(id string, timeout uint) (err error) {
	retry(func() error {
		err = d.docker.StopContainer(id, timeout)
		return err
	})
	return filterNotRunning(filterNoSuchContainer(err))
}

func (d *dockerWrap) Stats(opts docker.StatsOptions) (err error) {
	retry(func() error {
		err = d.docker.Stats(opts)
		return err
	})
	return err
}
<|endoftext|>


<commit_before>package conio

import (
	"bufio"
	"bytes"
	"os"
	"unicode"
)

var stdOut *bufio.Writer = bufio.NewWriter(os.Stdout)

var hasCache = map[rune]bool{}

func PutRep(ch rune, n int) {
	if n <= 0 {
		return
	}
	if hasCache[ch] {
		stdOut.WriteRune(ch)
	} else {
		stdOut.Flush()
		pre_x, pre_y := GetLocate()
		stdOut.WriteRune(ch)
		stdOut.Flush()
		post_x, post_y := GetLocate()
		if post_y == pre_y && post_x > pre_x {
			hasCache[ch] = true
			SetCharWidth(ch, post_x-pre_x)
		}
	}
	for i := 1; i < n; i++ {
		stdOut.WriteRune(ch)
	}
}

func Backspace(n int) {
	if n > 0 {
		stdOut.Flush()
		x, y := GetLocate()
		Locate(x-n, y)
	}
}

func shineCursor() {
	x, y := GetLocate()
	Locate(x, y)
}

type Buffer struct {
	Buffer     []rune
	Length     int
	Cursor     int
	Unicode    rune
	Keycode    uint16
	ShiftState uint32
	ViewStart  int
	ViewWidth  int
	Prompt     func() int
}

func (this *Buffer) Insert(pos int, c []rune) {
	n := len(c)
	for this.Length+n >= len(this.Buffer) {
		tmp := make([]rune, len(this.Buffer)*2)
		copy(tmp, this.Buffer)
		this.Buffer = tmp
	}
	for i := this.Length - 1; i >= pos; i-- {
		this.Buffer[i+n] = this.Buffer[i]
	}
	for i := 0; i < n; i++ {
		this.Buffer[pos+i] = c[i]
	}
	this.Length += n
}

// Insert String :s at :pos
// returns
//    count of rune
func (this *Buffer) InsertString(pos int, s string) int {
	list := []rune(s)
	this.Insert(pos, list)
	return len(list)
}

func (this *Buffer) Delete(pos int, n int) int {
	if n <= 0 || this.Length < pos+n {
		return 0
	}
	delw := this.GetWidthBetween(pos, pos+n)
	for i := pos; i < this.Length-n; i++ {
		this.Buffer[i] = this.Buffer[i+n]
	}
	this.Length -= n
	return delw
}

func (this *Buffer) InsertAndRepaint(str string) {
	this.ReplaceAndRepaint(this.Cursor, str)
}

func (this *Buffer) ReplaceAndRepaint(pos int, str string) {
	// Cursor rewind
	Backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))

	// Replace Buffer
	this.Delete(pos, this.Cursor-pos)

	// Define ViewStart , Cursor
	this.Cursor = pos + this.InsertString(pos, str)
	this.ViewStart = 0
	w := 0
	for i := 0; i < this.Cursor; i++ {
		w += GetCharWidth(this.Buffer[i])
		for w >= this.ViewWidth {
			w -= GetCharWidth(this.Buffer[this.ViewStart])
			this.ViewStart++
		}
	}

	// Repaint
	w = 0
	for i := this.ViewStart; i < this.Cursor; i++ {
		PutRep(this.Buffer[i], 1)
		w += GetCharWidth(this.Buffer[i])
	}
	bs := 0
	for i := this.Cursor; i < this.Length; i++ {
		w1 := GetCharWidth(this.Buffer[i])
		if w+w1 >= this.ViewWidth {
			break
		}
		PutRep(this.Buffer[i], 1)
		w += w1
		bs += w1
	}
	if bs > 0 {
		Backspace(bs)
	}
}

func (this *Buffer) GetWidthBetween(from int, to int) int {
	width := 0
	for i := from; i < to; i++ {
		width += GetCharWidth(this.Buffer[i])
	}
	return width
}

func (this *Buffer) Repaint(pos int, del int) {
	bs := 0
	vp := this.GetWidthBetween(this.ViewStart, pos)

	for i := pos; i < this.Length; i++ {
		w1 := GetCharWidth(this.Buffer[i])
		vp += w1
		if vp >= this.ViewWidth {
			break
		}
		PutRep(this.Buffer[i], 1)
		bs += w1
	}
	PutRep(' ', del)
	Backspace(bs + del)
}

func (this *Buffer) RepaintAll() {
	if this.Prompt != nil {
		this.Prompt()
	}
	for i := this.ViewStart; i < this.Cursor; i++ {
		PutRep(this.Buffer[i], 1)
	}
	this.Repaint(this.Cursor, 0)
}

func (this Buffer) String() string {
	var result bytes.Buffer
	for i := 0; i < this.Length; i++ {
		result.WriteRune(this.Buffer[i])
	}
	return result.String()
}

func (this *Buffer) CurrentWordTop() (wordTop int) {
	wordTop = -1
	isQuoted := false
	for i := 0; i < this.Cursor; i++ {
		if this.Buffer[i] == '"' {
			isQuoted = !isQuoted
		}
		if unicode.IsSpace(this.Buffer[i]) && !isQuoted {
			wordTop = -1
		} else if wordTop < 0 {
			wordTop = i
		}
	}
	if wordTop < 0 {
		return this.Cursor
	} else {
		return wordTop
	}
}

func (this *Buffer) CurrentWord() (string, int) {
	var buffer bytes.Buffer
	start := this.CurrentWordTop()
	for i := start; i < this.Cursor; i++ {
		if this.Buffer[i] != '"' {
			buffer.WriteRune(this.Buffer[i])
		}
	}
	return buffer.String(), start
}
<commit_msg>Fix: conio.ReadLine left trash at replacing string<commit_after>package conio

import (
	"bufio"
	"bytes"
	"os"
	"unicode"
)

var stdOut *bufio.Writer = bufio.NewWriter(os.Stdout)

var hasCache = map[rune]bool{}

func PutRep(ch rune, n int) {
	if n <= 0 {
		return
	}
	if hasCache[ch] {
		stdOut.WriteRune(ch)
	} else {
		stdOut.Flush()
		pre_x, pre_y := GetLocate()
		stdOut.WriteRune(ch)
		stdOut.Flush()
		post_x, post_y := GetLocate()
		if post_y == pre_y && post_x > pre_x {
			hasCache[ch] = true
			SetCharWidth(ch, post_x-pre_x)
		}
	}
	for i := 1; i < n; i++ {
		stdOut.WriteRune(ch)
	}
}

func Backspace(n int) {
	if n > 0 {
		stdOut.Flush()
		x, y := GetLocate()
		Locate(x-n, y)
	}
}

func shineCursor() {
	x, y := GetLocate()
	Locate(x, y)
}

type Buffer struct {
	Buffer     []rune
	Length     int
	Cursor     int
	Unicode    rune
	Keycode    uint16
	ShiftState uint32
	ViewStart  int
	ViewWidth  int
	Prompt     func() int
}

func (this *Buffer) Insert(pos int, c []rune) {
	n := len(c)
	for this.Length+n >= len(this.Buffer) {
		tmp := make([]rune, len(this.Buffer)*2)
		copy(tmp, this.Buffer)
		this.Buffer = tmp
	}
	for i := this.Length - 1; i >= pos; i-- {
		this.Buffer[i+n] = this.Buffer[i]
	}
	for i := 0; i < n; i++ {
		this.Buffer[pos+i] = c[i]
	}
	this.Length += n
}

// Insert String :s at :pos
// returns
//    count of rune
func (this *Buffer) InsertString(pos int, s string) int {
	list := []rune(s)
	this.Insert(pos, list)
	return len(list)
}

func (this *Buffer) Delete(pos int, n int) int {
	if n <= 0 || this.Length < pos+n {
		return 0
	}
	delw := this.GetWidthBetween(pos, pos+n)
	for i := pos; i < this.Length-n; i++ {
		this.Buffer[i] = this.Buffer[i+n]
	}
	this.Length -= n
	return delw
}

func (this *Buffer) InsertAndRepaint(str string) {
	this.ReplaceAndRepaint(this.Cursor, str)
}

func (this *Buffer) ReplaceAndRepaint(pos int, str string) {
	// Cursor rewind
	Backspace(this.GetWidthBetween(this.ViewStart, this.Cursor))

	// Replace Buffer
	this.Delete(pos, this.Cursor-pos)

	// Define ViewStart , Cursor
	this.Cursor = pos + this.InsertString(pos, str)
	this.ViewStart = 0
	w := 0
	for i := 0; i < this.Cursor; i++ {
		w += GetCharWidth(this.Buffer[i])
		for w >= this.ViewWidth {
			w -= GetCharWidth(this.Buffer[this.ViewStart])
			this.ViewStart++
		}
	}

	// Repaint
	w = 0
	for i := this.ViewStart; i < this.Cursor; i++ {
		PutRep(this.Buffer[i], 1)
		w += GetCharWidth(this.Buffer[i])
	}
	bs := 0
	for i := this.Cursor; i < this.Length; i++ {
		w1 := GetCharWidth(this.Buffer[i])
		if w+w1 >= this.ViewWidth {
			break
		}
		PutRep(this.Buffer[i], 1)
		w += w1
		bs += w1
	}
	for w+1 < this.ViewWidth {
		PutRep(' ', 1)
		bs++
		w++
	}
	if bs > 0 {
		Backspace(bs)
	}
}

func (this *Buffer) GetWidthBetween(from int, to int) int {
	width := 0
	for i := from; i < to; i++ {
		width += GetCharWidth(this.Buffer[i])
	}
	return width
}

func (this *Buffer) Repaint(pos int, del int) {
	bs := 0
	vp := this.GetWidthBetween(this.ViewStart, pos)

	for i := pos; i < this.Length; i++ {
		w1 := GetCharWidth(this.Buffer[i])
		vp += w1
		if vp >= this.ViewWidth {
			break
		}
		PutRep(this.Buffer[i], 1)
		bs += w1
	}
	PutRep(' ', del)
	Backspace(bs + del)
}

func (this *Buffer) RepaintAll() {
	if this.Prompt != nil {
		this.Prompt()
	}
	for i := this.ViewStart; i < this.Cursor; i++ {
		PutRep(this.Buffer[i], 1)
	}
	this.Repaint(this.Cursor, 0)
}

func (this Buffer) String() string {
	var result bytes.Buffer
	for i := 0; i < this.Length; i++ {
		result.WriteRune(this.Buffer[i])
	}
	return result.String()
}

func (this *Buffer) CurrentWordTop() (wordTop int) {
	wordTop = -1
	isQuoted := false
	for i := 0; i < this.Cursor; i++ {
		if this.Buffer[i] == '"' {
			isQuoted = !isQuoted
		}
		if unicode.IsSpace(this.Buffer[i]) && !isQuoted {
			wordTop = -1
		} else if wordTop < 0 {
			wordTop = i
		}
	}
	if wordTop < 0 {
		return this.Cursor
	} else {
		return wordTop
	}
}

func (this *Buffer) CurrentWord() (string, int) {
	var buffer bytes.Buffer
	start := this.CurrentWordTop()
	for i := start; i < this.Cursor; i++ {
		if this.Buffer[i] != '"' {
			buffer.WriteRune(this.Buffer[i])
		}
	}
	return buffer.String(), start
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"
)

const purgeInterval = (15 * time.Minute)

func remove(path string) {
	err := os.Remove(path)
	if err != nil {
		log.Printf("Error when removing %s: %v\n", path, err)
	}
}

func purgeOldEntries(directory string, limit time.Time) {
	infos, err := ioutil.ReadDir(directory)
	if err != nil {
		log.Printf("Error when purging directory %s: %v\n", directory, err)
		return
	}

	for _, entry := range infos {
		if !entry.IsDir() && entry.ModTime().Before(limit) {
			path := path.Join(directory, entry.Name())
			remove(path)
		}
	}
}

func purgeWorker(directory string, purgeOlder time.Duration) {
	for {
		limit := time.Now().Add(-purgeOlder)
		purgeOldEntries(directory, limit)
		time.Sleep(purgeInterval)

	}
}

func initPurge(directory string, purgeOlder time.Duration) {
	go purgeWorker(directory, purgeOlder)
}
<commit_msg>Ignore hidden files (filenames starting with a .) in data/ when purging<commit_after>package main

import (
	"io/ioutil"
	"log"
	"os"
	"path"
	"time"
)

const purgeInterval = (15 * time.Minute)

func remove(path string) {
	err := os.Remove(path)
	if err != nil {
		log.Printf("Error when removing %s: %v\n", path, err)
	}
}

func isHiddenFile(name string) bool {
	return len(name) > 0 && name[0] == '.'
}
func purgeOldEntries(directory string, limit time.Time) {
	infos, err := ioutil.ReadDir(directory)
	if err != nil {
		log.Printf("Error when purging directory %s: %v\n", directory, err)
		return
	}

	for _, entry := range infos {
		if !entry.IsDir() && !isHiddenFile(entry.name) && entry.ModTime().Before(limit) {
			path := path.Join(directory, entry.Name())
			remove(path)
		}
	}
}

func purgeWorker(directory string, purgeOlder time.Duration) {
	for {
		limit := time.Now().Add(-purgeOlder)
		purgeOldEntries(directory, limit)
		time.Sleep(purgeInterval)

	}
}

func initPurge(directory string, purgeOlder time.Duration) {
	go purgeWorker(directory, purgeOlder)
}
<|endoftext|>


<commit_before>package libkb

import (
	"fmt"

	keybase1 "github.com/keybase/client/protocol/go"
	jsonw "github.com/keybase/go-jsonw"
)

type LoadUserArg struct {
	Uid               keybase1.UID
	Name              string
	PublicKeyOptional bool
	NoCacheResult     bool // currently ignore
	Self              bool
	ForceReload       bool
	AllKeys           bool
	LoginContext      LoginContext
	Contextified
}

func LoadMe(arg LoadUserArg) (ret *User, err error) {
	arg.Self = true
	return LoadUser(arg)
}

func LoadUser(arg LoadUserArg) (ret *User, err error) {
	G.Log.Debug("LoadUser: %+v", arg)

	// Whatever the reply is, pass along our desired global context
	defer func() {
		if ret != nil {
			ret.SetGlobalContext(arg.G())
		}
	}()

	if arg.Uid.Exists() {
		// noop
	} else if len(arg.Name) == 0 && !arg.Self {
		err = fmt.Errorf("no username given to LoadUser")
	} else if len(arg.Name) > 0 && arg.Self {
		err = fmt.Errorf("If loading self, can't provide a username")
	} else if !arg.Self {
		// noop
	} else if arg.Uid = myUID(arg.G(), arg.LoginContext); arg.Uid.IsNil() {
		arg.Name = arg.G().Env.GetUsername()
	}

	if err != nil {
		return
	}

	G.Log.Debug("+ LoadUser(uid=%v, name=%v)", arg.Uid, arg.Name)

	var rres ResolveResult
	var uid keybase1.UID
	if arg.Uid.Exists() {
		uid = arg.Uid
	} else if len(arg.Name) == 0 {
		err = LoadUserError{"we don't know the current user's UID or name"}
		return
	} else if rres = ResolveUid(arg.Name); rres.err != nil {
		err = rres.err
		return
	} else if rres.uid.IsNil() {
		err = fmt.Errorf("No resolution for name=%s", arg.Name)
		return
	} else {
		arg.Uid = rres.uid
		uid = arg.Uid
	}

	if !arg.Self {
		if myuid := myUID(G, arg.LoginContext); myuid.Exists() && arg.Uid.Exists() && myuid.Equal(arg.Uid) {
			arg.Self = true
		}
	}

	G.Log.Debug("| resolved to %s", uid)

	var local, remote *User

	if local, err = LoadUserFromLocalStorage(uid); err != nil {
		G.Log.Warning("Failed to load %s from storage: %s", uid, err.Error())
	}

	leaf, err := LookupMerkleLeaf(uid, local)
	if err != nil {
		return
	}

	var f1, load_remote bool

	if local == nil {
		G.Log.Debug("| No local user stored for %s", uid)
		load_remote = true
	} else if f1, err = local.CheckBasicsFreshness(leaf.idVersion); err != nil {
		return
	} else {
		load_remote = !f1
	}

	G.Log.Debug("| Freshness: basics=%v; for %s", f1, uid)

	if !load_remote && !arg.ForceReload {
		ret = local
	} else if remote, err = LoadUserFromServer(arg, rres.body); err != nil {
		return
	} else {
		ret = remote
	}

	if ret == nil {
		return
	}

	ret.leaf = *leaf

	// If the user was looked-up via a keybase username, then
	// we should go ahead and check that the username matches the UID
	if len(rres.kbUsername) > 0 {
		if err = leaf.MatchUser(ret, arg.Uid, rres.kbUsername); err != nil {
			return
		}
	}

	if err = ret.LoadSigChains(arg.AllKeys, leaf, arg.Self); err != nil {
		return
	}

	if ret.sigHints, err = LoadAndRefreshSigHints(ret.id); err != nil {
		return
	}

	// Proactively cache fetches from remote server to local storage
	if e2 := ret.Store(); e2 != nil {
		G.Log.Warning("Problem storing user %s: %s", ret.GetName(), e2.Error())
	}

	if ret.HasActiveKey() {
		if err = ret.MakeIDTable(); err != nil {
			return
		}

		// Check that the user has self-signed only after we
		// consider revocations. See: https://github.com/keybase/go/issues/43
		if err = ret.VerifySelfSig(); err != nil {
			return
		}

	} else if !arg.PublicKeyOptional {

		var emsg string
		if arg.Self {
			emsg = "You don't have a public key; try `keybase push` if you have a key; or `keybase gen` if you don't"
		}
		err = NoKeyError{emsg}
	}

	return
}

func LoadUserFromLocalStorage(uid keybase1.UID) (u *User, err error) {
	G.Log.Debug("+ LoadUserFromLocalStorage(%s)", uid)
	jw, err := G.LocalDb.Get(DbKeyUID(DB_USER, uid))
	if err != nil {
		return nil, err
	}

	if jw == nil {
		G.Log.Debug("- LoadUserFromLocalStorage(%s): Not found", uid)
		return nil, nil
	}

	G.Log.Debug("| Loaded successfully")

	if u, err = NewUserFromLocalStorage(jw); err != nil {
		return nil, err
	}

	if u.id.NotEqual(uid) {
		err = fmt.Errorf("Bad lookup; uid mismatch: %s != %s", uid, u.id)
	}

	G.Log.Debug("| Loaded username %s (uid=%s)", u.name, uid)
	G.Log.Debug("- LoadUserFromLocalStorage(%s,%s)", u.name, uid)

	return
}

func LoadUserFromServer(arg LoadUserArg, body *jsonw.Wrapper) (u *User, err error) {
	G.Log.Debug("+ Load User from server: %s", arg.Uid)

	// Res.body might already have been preloaded a a result of a Resolve call earlier.
	if body == nil {
		res, err := G.API.Get(ApiArg{
			Endpoint:    "user/lookup",
			NeedSession: false,
			Args: HttpArgs{
				"uid": UIDArg(arg.Uid),
			},
		})

		if err != nil {
			return nil, err
		}
		body = res.Body.AtKey("them")
	} else {
		G.Log.Debug("| Skipped load; got user object previously")
	}

	if u, err = NewUserFromServer(body); err != nil {
		return
	}
	G.Log.Debug("- Load user from server: %s -> %s", arg.Uid, ErrToOk(err))

	return
}

func myUID(g *GlobalContext, lctx LoginContext) keybase1.UID {
	if lctx != nil {
		return lctx.LocalSession().GetUID()
	}
	return g.GetMyUID()
}

func LookupMerkleLeaf(uid keybase1.UID, local *User) (f *MerkleUserLeaf, err error) {
	if uid.IsNil() {
		err = fmt.Errorf("uid parameter for LookupMerkleLeaf empty")
		return
	}
	q := NewHttpArgs()
	q.Add("uid", UIDArg(uid))

	f, err = G.MerkleClient.LookupUser(q)
	if err == nil && f == nil && local != nil {
		err = fmt.Errorf("User not found in server Merkle tree")
	}
	return
}
<commit_msg>Check User against Merkle Leaf every time.<commit_after>package libkb

import (
	"fmt"

	keybase1 "github.com/keybase/client/protocol/go"
	jsonw "github.com/keybase/go-jsonw"
)

type LoadUserArg struct {
	Uid               keybase1.UID
	Name              string
	PublicKeyOptional bool
	NoCacheResult     bool // currently ignore
	Self              bool
	ForceReload       bool
	AllKeys           bool
	LoginContext      LoginContext
	Contextified
}

func LoadMe(arg LoadUserArg) (ret *User, err error) {
	arg.Self = true
	return LoadUser(arg)
}

func LoadUser(arg LoadUserArg) (ret *User, err error) {
	G.Log.Debug("LoadUser: %+v", arg)

	// Whatever the reply is, pass along our desired global context
	defer func() {
		if ret != nil {
			ret.SetGlobalContext(arg.G())
		}
	}()

	if arg.Uid.Exists() {
		// noop
	} else if len(arg.Name) == 0 && !arg.Self {
		err = fmt.Errorf("no username given to LoadUser")
	} else if len(arg.Name) > 0 && arg.Self {
		err = fmt.Errorf("If loading self, can't provide a username")
	} else if !arg.Self {
		// noop
	} else if arg.Uid = myUID(arg.G(), arg.LoginContext); arg.Uid.IsNil() {
		arg.Name = arg.G().Env.GetUsername()
	}

	if err != nil {
		return
	}

	G.Log.Debug("+ LoadUser(uid=%v, name=%v)", arg.Uid, arg.Name)

	var rres ResolveResult
	var uid keybase1.UID
	if arg.Uid.Exists() {
		uid = arg.Uid
	} else if len(arg.Name) == 0 {
		err = LoadUserError{"we don't know the current user's UID or name"}
		return
	} else if rres = ResolveUid(arg.Name); rres.err != nil {
		err = rres.err
		return
	} else if rres.uid.IsNil() {
		err = fmt.Errorf("No resolution for name=%s", arg.Name)
		return
	} else {
		arg.Uid = rres.uid
		uid = arg.Uid
	}

	if !arg.Self {
		if myuid := myUID(G, arg.LoginContext); myuid.Exists() && arg.Uid.Exists() && myuid.Equal(arg.Uid) {
			arg.Self = true
		}
	}

	G.Log.Debug("| resolved to %s", uid)

	var local, remote *User

	if local, err = LoadUserFromLocalStorage(uid); err != nil {
		G.Log.Warning("Failed to load %s from storage: %s", uid, err.Error())
	}

	leaf, err := LookupMerkleLeaf(uid, local)
	if err != nil {
		return
	}

	var f1, load_remote bool

	if local == nil {
		G.Log.Debug("| No local user stored for %s", uid)
		load_remote = true
	} else if f1, err = local.CheckBasicsFreshness(leaf.idVersion); err != nil {
		return
	} else {
		load_remote = !f1
	}

	G.Log.Debug("| Freshness: basics=%v; for %s", f1, uid)

	if !load_remote && !arg.ForceReload {
		ret = local
	} else if remote, err = LoadUserFromServer(arg, rres.body); err != nil {
		return
	} else {
		ret = remote
	}

	if ret == nil {
		return
	}

	ret.leaf = *leaf

	// Match the returned User object to the Merkle tree. Also make sure
	// that the username queried for matches the User returned (if it
	// was indeed queried for)
	if err = leaf.MatchUser(ret, arg.Uid, rres.kbUsername); err != nil {
		return
	}

	if err = ret.LoadSigChains(arg.AllKeys, leaf, arg.Self); err != nil {
		return
	}

	if ret.sigHints, err = LoadAndRefreshSigHints(ret.id); err != nil {
		return
	}

	// Proactively cache fetches from remote server to local storage
	if e2 := ret.Store(); e2 != nil {
		G.Log.Warning("Problem storing user %s: %s", ret.GetName(), e2.Error())
	}

	if ret.HasActiveKey() {
		if err = ret.MakeIDTable(); err != nil {
			return
		}

		// Check that the user has self-signed only after we
		// consider revocations. See: https://github.com/keybase/go/issues/43
		if err = ret.VerifySelfSig(); err != nil {
			return
		}

	} else if !arg.PublicKeyOptional {

		var emsg string
		if arg.Self {
			emsg = "You don't have a public key; try `keybase push` if you have a key; or `keybase gen` if you don't"
		}
		err = NoKeyError{emsg}
	}

	return
}

func LoadUserFromLocalStorage(uid keybase1.UID) (u *User, err error) {
	G.Log.Debug("+ LoadUserFromLocalStorage(%s)", uid)
	jw, err := G.LocalDb.Get(DbKeyUID(DB_USER, uid))
	if err != nil {
		return nil, err
	}

	if jw == nil {
		G.Log.Debug("- LoadUserFromLocalStorage(%s): Not found", uid)
		return nil, nil
	}

	G.Log.Debug("| Loaded successfully")

	if u, err = NewUserFromLocalStorage(jw); err != nil {
		return nil, err
	}

	if u.id.NotEqual(uid) {
		err = fmt.Errorf("Bad lookup; uid mismatch: %s != %s", uid, u.id)
	}

	G.Log.Debug("| Loaded username %s (uid=%s)", u.name, uid)
	G.Log.Debug("- LoadUserFromLocalStorage(%s,%s)", u.name, uid)

	return
}

func LoadUserFromServer(arg LoadUserArg, body *jsonw.Wrapper) (u *User, err error) {
	G.Log.Debug("+ Load User from server: %s", arg.Uid)

	// Res.body might already have been preloaded a a result of a Resolve call earlier.
	if body == nil {
		res, err := G.API.Get(ApiArg{
			Endpoint:    "user/lookup",
			NeedSession: false,
			Args: HttpArgs{
				"uid": UIDArg(arg.Uid),
			},
		})

		if err != nil {
			return nil, err
		}
		body = res.Body.AtKey("them")
	} else {
		G.Log.Debug("| Skipped load; got user object previously")
	}

	if u, err = NewUserFromServer(body); err != nil {
		return
	}
	G.Log.Debug("- Load user from server: %s -> %s", arg.Uid, ErrToOk(err))

	return
}

func myUID(g *GlobalContext, lctx LoginContext) keybase1.UID {
	if lctx != nil {
		return lctx.LocalSession().GetUID()
	}
	return g.GetMyUID()
}

func LookupMerkleLeaf(uid keybase1.UID, local *User) (f *MerkleUserLeaf, err error) {
	if uid.IsNil() {
		err = fmt.Errorf("uid parameter for LookupMerkleLeaf empty")
		return
	}
	q := NewHttpArgs()
	q.Add("uid", UIDArg(uid))

	f, err = G.MerkleClient.LookupUser(q)
	if err == nil && f == nil && local != nil {
		err = fmt.Errorf("User not found in server Merkle tree")
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/keybase/client/go/logger"
)

// Logs is the struct to specify the path of log files
type Logs struct {
	Desktop string
	Kbfs    string
	Service string
	Updater string
	Start   string
	Install string
	System  string
	Git     string
}

// LogSendContext for LogSend
type LogSendContext struct {
	Contextified
	Logs Logs
}

func addFile(mpart *multipart.Writer, param, filename, data string) error {
	if len(data) == 0 {
		return nil
	}

	part, err := mpart.CreateFormFile(param, filename)
	if err != nil {
		return err
	}
	gz := gzip.NewWriter(part)
	if _, err := gz.Write([]byte(data)); err != nil {
		return err
	}
	return gz.Close()
}

func (l *LogSendContext) post(status, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog string) (string, error) {
	l.G().Log.Debug("sending status + logs to keybase")

	var body bytes.Buffer
	mpart := multipart.NewWriter(&body)

	if feedback != "" {
		mpart.WriteField("feedback", feedback)
	}

	if err := addFile(mpart, "status_gz", "status.gz", status); err != nil {
		return "", err
	}
	if err := addFile(mpart, "kbfs_log_gz", "kbfs_log.gz", kbfsLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "keybase_log_gz", "keybase_log.gz", svcLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "updater_log_gz", "updater_log.gz", updaterLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "gui_log_gz", "gui_log.gz", desktopLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "start_log_gz", "start_log.gz", startLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "install_log_gz", "install_log.gz", installLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "system_log_gz", "system_log.gz", systemLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "git_log_gz", "git_log.gz", gitLog); err != nil {
		return "", err
	}

	if err := mpart.Close(); err != nil {
		return "", err
	}

	l.G().Log.Debug("body size: %d\n", body.Len())

	arg := APIArg{
		Endpoint:    "logdump/send",
		SessionType: APISessionTypeOPTIONAL,
	}

	// Get the login session, if any
	l.G().LoginState().LoggedInLoad()

	resp, err := l.G().API.PostRaw(arg, mpart.FormDataContentType(), &body)
	if err != nil {
		l.G().Log.Debug("post error: %s", err)
		return "", err
	}

	id, err := resp.Body.AtKey("logdump_id").GetString()
	if err != nil {
		return "", err
	}

	return id, nil
}

// tail the logs that start with the stem `stem`, which are of type `which`.
// Get the most recent `numBytes` from the concatenation of the files.
func tail(log logger.Logger, which string, stem string, numBytes int) (ret string) {

	numFiles := 0

	log.Debug("+ tailing %s file with stem %q", which, stem)
	defer func() {
		log.Debug("- collected %d bytes from %d files", len(ret), numFiles)
	}()

	if len(stem) == 0 {
		log.Debug("| skipping %s logs (since no stem given)", which)
		return
	}

	lognames := listLogFiles(log, stem)
	var parts []string
	remaining := numBytes

	// Keep reading logs in reverse chronological order until we've read nothing
	// more, or we've filled up numBytes worth of buffer, or we didn't have to read
	// the whole file.
	for _, logname := range lognames {
		data, seeked := tailFile(log, which, logname, remaining)
		if len(data) == 0 {
			break
		}
		parts = append(parts, data)
		numFiles++
		remaining -= len(data)
		if remaining <= 0 || seeked {
			break
		}
	}

	// Reverse the array; took this one-line from StackOverflow answer
	for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
		parts[i], parts[j] = parts[j], parts[i]
	}

	return strings.Join(parts, "")
}

type nameAndMTime struct {
	name  string
	mtime time.Time
}

type nameAndMTimes []nameAndMTime

func (a nameAndMTimes) Len() int           { return len(a) }
func (a nameAndMTimes) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a nameAndMTimes) Less(i, j int) bool { return a[i].mtime.After(a[j].mtime) }

// List logfiles that match the glob filename*, and return then in reverse chronological order.
// We'll need to read the first, and maybe the second
func listLogFiles(log logger.Logger, stem string) (ret []string) {
	stem = filepath.Clean(stem)
	dir := filepath.Dir(stem)
	base := filepath.Base(stem)
	files, err := ioutil.ReadDir(dir)

	defer func() {
		log.Debug("listLogFiles(%q) -> %v", stem, ret)
	}()

	// In the worst case, just return the stem in an array
	defret := []string{stem}

	if err != nil {
		log.Debug("failed to list directory %q: %s", dir, err)
		return defret
	}

	var tmp []nameAndMTime
	for _, d := range files {
		fullname := filepath.Clean(filepath.Join(dir, d.Name()))
		// Use the stat on the file (and not from the directory read)
		// since the latter doesn't work reliably on Windows
		finfo, err := os.Stat(fullname)
		if err != nil {
			log.Debug("Cannot stat %q: %s", fullname, err)
			continue
		}
		if strings.HasPrefix(d.Name(), base) {
			tmp = append(tmp, nameAndMTime{fullname, finfo.ModTime()})
		}
	}
	if len(tmp) == 0 {
		log.Debug("no files found matching \"%s*\"; falling back to default glob", stem)
		return defret
	}

	// Sort the files in reverse chronological mtime order. We should get the raw stem first.
	sort.Sort(nameAndMTimes(tmp))
	log.Debug("Sorted file list: %+v", tmp)

	for _, f := range tmp {
		ret = append(ret, f.name)
	}

	// If we didn't get the raw stem first, then we have a problem, so just use only the
	// raw stem and ignore the rest of our work.
	if stem != ret[0] {
		log.Debug("Expected to get %q first, but got %q instead! Falling back to one log only", stem, ret[0])
		return defret
	}
	return ret
}

// findFirstNewline first the first newline in the given byte array, and then returns the
// rest of the byte array. Should be safe to use on utf-8 strings.
func findFirstNewline(b []byte) []byte {
	index := bytes.IndexByte(b, '\n')
	if index < 0 || index == len(b)-1 {
		return nil
	}
	return b[(index + 1):]
}

func appendError(collected []byte, err error) []byte {
	return append(collected, []byte(fmt.Sprintf("\nERROR READING LOGS:\n%#v\n", err))...)
}

// If systemd is running, we look for logs in the systemd journal for our 3
// main services.
func tailSystemdJournal(log logger.Logger, which string, numBytes int) (ret string) {
	log.Debug("+ tailing journalctl for %s (%d bytes)", which, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	// Journalctl doesn't provide a "last N bytes" flag directly, so instead we
	// use "last N lines". Large log files in practice seem to be about 150
	// bits per line. We'll request lines on that assumption, but if we get
	// more than 2x as many bytes as we expected, we'll stop reading and
	// include a big error.
	guessedLines := numBytes / 150
	maxBytes := numBytes * 2

	journalCmd := exec.Command("journalctl", "--user", "--unit", which, "--lines", strconv.Itoa(guessedLines))
	journalCmd.Stderr = os.Stderr
	stdout, err := journalCmd.StdoutPipe()
	if err != nil {
		msg := fmt.Sprintf("Failed to open a pipe for journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}
	err = journalCmd.Start()
	if err != nil {
		msg := fmt.Sprintf("Failed to run journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}

	// Once we start reading output, don't short-circuit on errors. Just log
	// them, and return whatever we got.
	stdoutLimited := io.LimitReader(stdout, int64(maxBytes))
	output, err := ioutil.ReadAll(stdoutLimited)
	if err != nil {
		log.Errorf("Error reading from journalctl pipe: %s", err)
		output = appendError(output, err)
	}
	err = journalCmd.Wait()
	if err != nil {
		log.Errorf("Journalctl exited with an error: %s", err)
		output = appendError(output, err)
	}
	if len(output) >= maxBytes {
		truncation := "\n###\n### WARNING: Journal lines longer than expected. Logs truncated.\n###\n"
		output = append(output, truncation...)
	}
	return string(output)
}

// tailFile takes the last n bytes, but advances to the first newline. Returns the log (as a string)
// and a bool, indicating if we read the full log, or we had to advance into the log to find the newline.
func tailFile(log logger.Logger, which string, filename string, numBytes int) (ret string, seeked bool) {

	log.Debug("+ tailing %s log %q (%d bytes)", which, filename, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	seeked = false
	if filename == "" {
		return ret, seeked
	}
	finfo, err := os.Stat(filename)
	if os.IsNotExist(err) {
		log.Debug("log %q doesn't exist", filename)
		return ret, seeked
	}
	f, err := os.Open(filename)
	if err != nil {
		log.Errorf("error opening log %q: %s", filename, err)
		return ret, seeked
	}
	defer f.Close()
	if finfo.Size() > int64(numBytes) {
		seeked = true
		_, err = f.Seek(int64(-numBytes), io.SeekEnd)
		if err != nil {
			log.Errorf("Can't seek log %q: %s", filename, err)
			return ret, seeked
		}
	}
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		log.Errorf("Failure in reading file %q: %s", filename, err)
		return ret, seeked
	}
	if seeked {
		buf = findFirstNewline(buf)
	}
	return string(buf), seeked
}

// LogSend sends the the tails of log files to kb
func (l *LogSendContext) LogSend(statusJSON, feedback string, sendLogs bool, numBytes int) (string, error) {
	logs := l.Logs
	var kbfsLog string
	var svcLog string
	var desktopLog string
	var updaterLog string
	var startLog string
	var installLog string
	var systemLog string
	var gitLog string

	if sendLogs {
		if l.G().Env.WantsSystemd() {
			svcLog = tailSystemdJournal(l.G().Log, "keybase.service", numBytes)
			kbfsLog = tailSystemdJournal(l.G().Log, "kbfs.service", numBytes)
			desktopLog = tailSystemdJournal(l.G().Log, "keybase.gui.service", numBytes)
		} else {
			svcLog = tail(l.G().Log, "service", logs.Service, numBytes)
			kbfsLog = tail(l.G().Log, "kbfs", logs.Kbfs, numBytes)
			desktopLog = tail(l.G().Log, "desktop", logs.Desktop, numBytes)
		}
		updaterLog = tail(l.G().Log, "updater", logs.Updater, numBytes)
		startLog = tail(l.G().Log, "start", logs.Start, numBytes)
		installLog = tail(l.G().Log, "install", logs.Install, numBytes)
		systemLog = tail(l.G().Log, "system", logs.System, numBytes)
		gitLog = tail(l.G().Log, "git", logs.Git, numBytes)
	} else {
		kbfsLog = ""
		svcLog = ""
		desktopLog = ""
		updaterLog = ""
		startLog = ""
		installLog = ""
		systemLog = ""
		gitLog = ""
	}

	return l.post(statusJSON, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog)
}
<commit_msg>add --output=cat to journalctl, to avoid double timestamps<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/keybase/client/go/logger"
)

// Logs is the struct to specify the path of log files
type Logs struct {
	Desktop string
	Kbfs    string
	Service string
	Updater string
	Start   string
	Install string
	System  string
	Git     string
}

// LogSendContext for LogSend
type LogSendContext struct {
	Contextified
	Logs Logs
}

func addFile(mpart *multipart.Writer, param, filename, data string) error {
	if len(data) == 0 {
		return nil
	}

	part, err := mpart.CreateFormFile(param, filename)
	if err != nil {
		return err
	}
	gz := gzip.NewWriter(part)
	if _, err := gz.Write([]byte(data)); err != nil {
		return err
	}
	return gz.Close()
}

func (l *LogSendContext) post(status, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog string) (string, error) {
	l.G().Log.Debug("sending status + logs to keybase")

	var body bytes.Buffer
	mpart := multipart.NewWriter(&body)

	if feedback != "" {
		mpart.WriteField("feedback", feedback)
	}

	if err := addFile(mpart, "status_gz", "status.gz", status); err != nil {
		return "", err
	}
	if err := addFile(mpart, "kbfs_log_gz", "kbfs_log.gz", kbfsLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "keybase_log_gz", "keybase_log.gz", svcLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "updater_log_gz", "updater_log.gz", updaterLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "gui_log_gz", "gui_log.gz", desktopLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "start_log_gz", "start_log.gz", startLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "install_log_gz", "install_log.gz", installLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "system_log_gz", "system_log.gz", systemLog); err != nil {
		return "", err
	}
	if err := addFile(mpart, "git_log_gz", "git_log.gz", gitLog); err != nil {
		return "", err
	}

	if err := mpart.Close(); err != nil {
		return "", err
	}

	l.G().Log.Debug("body size: %d\n", body.Len())

	arg := APIArg{
		Endpoint:    "logdump/send",
		SessionType: APISessionTypeOPTIONAL,
	}

	// Get the login session, if any
	l.G().LoginState().LoggedInLoad()

	resp, err := l.G().API.PostRaw(arg, mpart.FormDataContentType(), &body)
	if err != nil {
		l.G().Log.Debug("post error: %s", err)
		return "", err
	}

	id, err := resp.Body.AtKey("logdump_id").GetString()
	if err != nil {
		return "", err
	}

	return id, nil
}

// tail the logs that start with the stem `stem`, which are of type `which`.
// Get the most recent `numBytes` from the concatenation of the files.
func tail(log logger.Logger, which string, stem string, numBytes int) (ret string) {

	numFiles := 0

	log.Debug("+ tailing %s file with stem %q", which, stem)
	defer func() {
		log.Debug("- collected %d bytes from %d files", len(ret), numFiles)
	}()

	if len(stem) == 0 {
		log.Debug("| skipping %s logs (since no stem given)", which)
		return
	}

	lognames := listLogFiles(log, stem)
	var parts []string
	remaining := numBytes

	// Keep reading logs in reverse chronological order until we've read nothing
	// more, or we've filled up numBytes worth of buffer, or we didn't have to read
	// the whole file.
	for _, logname := range lognames {
		data, seeked := tailFile(log, which, logname, remaining)
		if len(data) == 0 {
			break
		}
		parts = append(parts, data)
		numFiles++
		remaining -= len(data)
		if remaining <= 0 || seeked {
			break
		}
	}

	// Reverse the array; took this one-line from StackOverflow answer
	for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
		parts[i], parts[j] = parts[j], parts[i]
	}

	return strings.Join(parts, "")
}

type nameAndMTime struct {
	name  string
	mtime time.Time
}

type nameAndMTimes []nameAndMTime

func (a nameAndMTimes) Len() int           { return len(a) }
func (a nameAndMTimes) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a nameAndMTimes) Less(i, j int) bool { return a[i].mtime.After(a[j].mtime) }

// List logfiles that match the glob filename*, and return then in reverse chronological order.
// We'll need to read the first, and maybe the second
func listLogFiles(log logger.Logger, stem string) (ret []string) {
	stem = filepath.Clean(stem)
	dir := filepath.Dir(stem)
	base := filepath.Base(stem)
	files, err := ioutil.ReadDir(dir)

	defer func() {
		log.Debug("listLogFiles(%q) -> %v", stem, ret)
	}()

	// In the worst case, just return the stem in an array
	defret := []string{stem}

	if err != nil {
		log.Debug("failed to list directory %q: %s", dir, err)
		return defret
	}

	var tmp []nameAndMTime
	for _, d := range files {
		fullname := filepath.Clean(filepath.Join(dir, d.Name()))
		// Use the stat on the file (and not from the directory read)
		// since the latter doesn't work reliably on Windows
		finfo, err := os.Stat(fullname)
		if err != nil {
			log.Debug("Cannot stat %q: %s", fullname, err)
			continue
		}
		if strings.HasPrefix(d.Name(), base) {
			tmp = append(tmp, nameAndMTime{fullname, finfo.ModTime()})
		}
	}
	if len(tmp) == 0 {
		log.Debug("no files found matching \"%s*\"; falling back to default glob", stem)
		return defret
	}

	// Sort the files in reverse chronological mtime order. We should get the raw stem first.
	sort.Sort(nameAndMTimes(tmp))
	log.Debug("Sorted file list: %+v", tmp)

	for _, f := range tmp {
		ret = append(ret, f.name)
	}

	// If we didn't get the raw stem first, then we have a problem, so just use only the
	// raw stem and ignore the rest of our work.
	if stem != ret[0] {
		log.Debug("Expected to get %q first, but got %q instead! Falling back to one log only", stem, ret[0])
		return defret
	}
	return ret
}

// findFirstNewline first the first newline in the given byte array, and then returns the
// rest of the byte array. Should be safe to use on utf-8 strings.
func findFirstNewline(b []byte) []byte {
	index := bytes.IndexByte(b, '\n')
	if index < 0 || index == len(b)-1 {
		return nil
	}
	return b[(index + 1):]
}

func appendError(collected []byte, err error) []byte {
	return append(collected, []byte(fmt.Sprintf("\nERROR READING LOGS:\n%#v\n", err))...)
}

// If systemd is running, we look for logs in the systemd journal for our 3
// main services.
func tailSystemdJournal(log logger.Logger, which string, numBytes int) (ret string) {
	log.Debug("+ tailing journalctl for %s (%d bytes)", which, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	// Journalctl doesn't provide a "last N bytes" flag directly, so instead we
	// use "last N lines". Large log files in practice seem to be about 150
	// bits per line. We'll request lines on that assumption, but if we get
	// more than 2x as many bytes as we expected, we'll stop reading and
	// include a big error.
	guessedLines := numBytes / 150
	maxBytes := numBytes * 2

	journalCmd := exec.Command(
		"journalctl",
		"--user",
		"--unit="+which,
		"--lines="+strconv.Itoa(guessedLines),
		"--output=cat",
	)
	journalCmd.Stderr = os.Stderr
	stdout, err := journalCmd.StdoutPipe()
	if err != nil {
		msg := fmt.Sprintf("Failed to open a pipe for journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}
	err = journalCmd.Start()
	if err != nil {
		msg := fmt.Sprintf("Failed to run journalctl: %s", err)
		log.Errorf(msg)
		return msg
	}

	// Once we start reading output, don't short-circuit on errors. Just log
	// them, and return whatever we got.
	stdoutLimited := io.LimitReader(stdout, int64(maxBytes))
	output, err := ioutil.ReadAll(stdoutLimited)
	if err != nil {
		log.Errorf("Error reading from journalctl pipe: %s", err)
		output = appendError(output, err)
	}
	err = journalCmd.Wait()
	if err != nil {
		log.Errorf("Journalctl exited with an error: %s", err)
		output = appendError(output, err)
	}
	if len(output) >= maxBytes {
		truncation := "\n###\n### WARNING: Journal lines longer than expected. Logs truncated.\n###\n"
		output = append(output, truncation...)
	}
	return string(output)
}

// tailFile takes the last n bytes, but advances to the first newline. Returns the log (as a string)
// and a bool, indicating if we read the full log, or we had to advance into the log to find the newline.
func tailFile(log logger.Logger, which string, filename string, numBytes int) (ret string, seeked bool) {

	log.Debug("+ tailing %s log %q (%d bytes)", which, filename, numBytes)
	defer func() {
		log.Debug("- scanned %d bytes", len(ret))
	}()

	seeked = false
	if filename == "" {
		return ret, seeked
	}
	finfo, err := os.Stat(filename)
	if os.IsNotExist(err) {
		log.Debug("log %q doesn't exist", filename)
		return ret, seeked
	}
	f, err := os.Open(filename)
	if err != nil {
		log.Errorf("error opening log %q: %s", filename, err)
		return ret, seeked
	}
	defer f.Close()
	if finfo.Size() > int64(numBytes) {
		seeked = true
		_, err = f.Seek(int64(-numBytes), io.SeekEnd)
		if err != nil {
			log.Errorf("Can't seek log %q: %s", filename, err)
			return ret, seeked
		}
	}
	buf, err := ioutil.ReadAll(f)
	if err != nil {
		log.Errorf("Failure in reading file %q: %s", filename, err)
		return ret, seeked
	}
	if seeked {
		buf = findFirstNewline(buf)
	}
	return string(buf), seeked
}

// LogSend sends the the tails of log files to kb
func (l *LogSendContext) LogSend(statusJSON, feedback string, sendLogs bool, numBytes int) (string, error) {
	logs := l.Logs
	var kbfsLog string
	var svcLog string
	var desktopLog string
	var updaterLog string
	var startLog string
	var installLog string
	var systemLog string
	var gitLog string

	if sendLogs {
		if l.G().Env.WantsSystemd() {
			svcLog = tailSystemdJournal(l.G().Log, "keybase.service", numBytes)
			kbfsLog = tailSystemdJournal(l.G().Log, "kbfs.service", numBytes)
			desktopLog = tailSystemdJournal(l.G().Log, "keybase.gui.service", numBytes)
		} else {
			svcLog = tail(l.G().Log, "service", logs.Service, numBytes)
			kbfsLog = tail(l.G().Log, "kbfs", logs.Kbfs, numBytes)
			desktopLog = tail(l.G().Log, "desktop", logs.Desktop, numBytes)
		}
		updaterLog = tail(l.G().Log, "updater", logs.Updater, numBytes)
		startLog = tail(l.G().Log, "start", logs.Start, numBytes)
		installLog = tail(l.G().Log, "install", logs.Install, numBytes)
		systemLog = tail(l.G().Log, "system", logs.System, numBytes)
		gitLog = tail(l.G().Log, "git", logs.Git, numBytes)
	} else {
		kbfsLog = ""
		svcLog = ""
		desktopLog = ""
		updaterLog = ""
		startLog = ""
		installLog = ""
		systemLog = ""
		gitLog = ""
	}

	return l.post(statusJSON, feedback, kbfsLog, svcLog, desktopLog, updaterLog, startLog, installLog, systemLog, gitLog)
}
<|endoftext|>