<commit_before>package release

import (
	"bytes"
	"errors"
	"fmt"
	mo_path2 "github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/common/model/mo_string"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn_impl"
	"github.com/watermint/toolbox/domain/github/api/gh_conn"
	"github.com/watermint/toolbox/domain/github/api/gh_context"
	"github.com/watermint/toolbox/domain/github/api/gh_context_impl"
	"github.com/watermint/toolbox/domain/github/service/sv_release"
	"github.com/watermint/toolbox/domain/github/service/sv_release_asset"
	"github.com/watermint/toolbox/domain/github/service/sv_tag"
	"github.com/watermint/toolbox/infra/api/api_auth"
	"github.com/watermint/toolbox/infra/api/api_auth_impl"
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/ui/app_lang"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/ui/app_ui"
	"github.com/watermint/toolbox/infra/util/ut_filehash"
	"github.com/watermint/toolbox/quality/infra/qt_endtoend"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"github.com/watermint/toolbox/quality/infra/qt_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_runtime"
	"github.com/watermint/toolbox/recipe/dev/test"
	"go.uber.org/zap"
	"golang.org/x/text/language/display"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	ErrorBuildIsNotReadyForRelease = errors.New("the build does not satisfy release criteria")
	ErrorNoArtifactFound           = errors.New("no artifact found")
)

type Publish struct {
	TestResource              string
	Branch                    string
	ArtifactPath              mo_path2.FileSystemPath
	ConnGithub                gh_conn.ConnGithubRepo
	HeadingReleaseTheme       app_msg.Message
	HeadingChanges            app_msg.Message
	ListSpecChange            app_msg.Message
	HeadingDocument           app_msg.Message
	ListReadme                app_msg.Message
	HeadingBinary             app_msg.Message
	BinaryTableHeaderFilename app_msg.Message
	BinaryTableHeaderSize     app_msg.Message
	BinaryTableHeaderMD5      app_msg.Message
	BinaryTableHeaderSHA256   app_msg.Message
	TagCommitMessage          app_msg.Message
	ReleaseName               app_msg.Message
}

type ArtifactSum struct {
	Filename string
	Size     int64
	MD5      string
	SHA256   string
}

func (z *Publish) Preset() {
	z.TestResource = defaultTestResource
	z.Branch = "master"
}

func (z *Publish) artifactAssets(c app_control.Control) (paths []string, sizes map[string]int64, err error) {
	l := c.Log()

	entries, err := ioutil.ReadDir(z.ArtifactPath.Path())
	if err != nil {
		return nil, nil, err
	}
	paths = make([]string, 0)
	sizes = make(map[string]int64)
	for _, e := range entries {
		if !strings.HasPrefix(e.Name(), "tbx-"+app.Version) || !strings.HasSuffix(e.Name(), ".zip") {
			l.Debug("Ignore non artifact file", zap.Any("file", e))
			continue
		}
		path := filepath.Join(z.ArtifactPath.Path(), e.Name())
		paths = append(paths, path)
		sizes[path] = e.Size()
	}
	return paths, sizes, nil
}

func (z *Publish) verifyArtifacts(c app_control.Control) (a []*ArtifactSum, err error) {
	l := c.Log()
	a = make([]*ArtifactSum, 0)

	assets, assetSize, err := z.artifactAssets(c)

	h := ut_filehash.NewHash(l)
	for _, p := range assets {
		sum := &ArtifactSum{
			Filename: filepath.Base(p),
			Size:     assetSize[p],
		}
		sum.MD5, err = h.MD5(p)
		if err != nil {
			l.Debug("Unable to calc MD5", zap.Error(err))
			return nil, err
		}
		sum.SHA256, err = h.SHA256(p)
		if err != nil {
			l.Debug("Unable to calc SHA256", zap.Error(err))
			return nil, err
		}
		a = append(a, sum)
	}
	if len(a) < 1 {
		return nil, ErrorNoArtifactFound
	}
	return a, nil
}

func (z *Publish) releaseNotes(c app_control.Control, sum []*ArtifactSum) (relNote string, err error) {
	l := c.Log()
	baseUrl := "https://github.com/watermint/toolbox/blob/" + app.Version

	var buf bytes.Buffer
	mui := app_ui.NewMarkdown(c.Messages(), &buf, true)
	mui.Header(z.HeadingReleaseTheme)
	mui.Break()

	mui.Header(z.HeadingChanges)

	for _, lang := range app_lang.SupportedLanguages {
		mui.Info(z.ListSpecChange.
			With("Link", baseUrl+"/doc/generated"+app_lang.PathSuffix(lang)+"/changes.md").
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingDocument)

	for _, lang := range app_lang.SupportedLanguages {
		name := "README" + app_lang.PathSuffix(lang) + ".md"
		mui.Info(z.ListReadme.
			With("Name", name).
			With("Link", baseUrl+"/"+name).
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingBinary)
	mit := mui.InfoTable("Binaries")
	mit.Header(z.BinaryTableHeaderFilename, z.BinaryTableHeaderSize, z.BinaryTableHeaderMD5, z.BinaryTableHeaderSHA256)

	for _, s := range sum {
		mit.RowRaw(s.Filename, fmt.Sprintf("%d", s.Size), s.MD5, s.SHA256)
	}
	mit.Flush()

	relNotesPath := filepath.Join(c.Workspace().Report(), "release_notes.md")
	err = ioutil.WriteFile(relNotesPath, buf.Bytes(), 0644)
	if err != nil {
		l.Debug("Unable to write release notes", zap.Error(err), zap.String("path", relNotesPath))
		return "", err
	}
	l.Info("Release note created", zap.String("path", relNotesPath))

	return buf.String(), nil
}

func (z *Publish) endToEndTest(c app_control.Control) error {
	l := c.Log()
	if c.Feature().IsTest() {
		l.Info("Skip tests")
		return nil
	}

	if c.Feature().IsProduction() {
		l.Info("Prepare resources")
		if !api_auth_impl.IsCacheAvailable(c, qt_endtoend.EndToEndPeer, []string{
			api_auth.DropboxTokenFull,
			api_auth.DropboxTokenBusinessAudit,
			api_auth.DropboxTokenBusinessManagement,
			api_auth.DropboxTokenBusinessFile,
			api_auth.DropboxTokenBusinessInfo,
		}) {
			return qt_errors.ErrorNotEnoughResource
		}
	}

	l.Info("Ensure end to end resource availability")
	if !dbx_conn_impl.IsEndToEndTokenAllAvailable(c) {
		l.Error("At least one of end to end resource is not available.")
		return errors.New("end to end resource is not available")
	}

	l.Info("Testing all end to end test")
	err := rc_exec.Exec(c, &test.Recipe{}, func(r rc_recipe.Recipe) {
		m := r.(*test.Recipe)
		m.All = true
		_, err := os.Lstat(z.TestResource)
		if err == nil {
			m.Resource = mo_string.NewOptional(z.TestResource)
		} else {
			l.Warn("Unable to read test resource", zap.String("path", z.TestResource), zap.Error(err))
		}
	})
	return err
}

func (z *Publish) ghCtx(c app_control.Control) gh_context.Context {
	if c.Feature().IsTest() {
		return gh_context_impl.NewMock()
	} else {
		return z.ConnGithub.Context()
	}
}

func (z *Publish) createTag(c app_control.Control) error {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	ui := c.UI()
	svt := sv_tag.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	l.Debug("Create tag")
	tag, err := svt.Create(
		app.Version,
		ui.Text(z.TagCommitMessage.With("Version", app.Version)),
		app.Hash,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create tag", zap.Error(err))
		return err
	}
	if err == qt_errors.ErrorMock {
		return nil
	}
	l.Info("The tag created", zap.Any("tag", tag))
	return nil
}

func (z *Publish) createReleaseDraft(c app_control.Control, relNote string) error {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	ui := c.UI()
	svr := sv_release.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	rel, err := svr.CreateDraft(
		app.Version,
		ui.Text(z.ReleaseName.With("Version", app.Version)),
		relNote,
		z.Branch,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create release draft", zap.Error(err))
		return err
	}
	if err == qt_errors.ErrorMock {
		return nil
	}
	l.Info("Release created", zap.Any("rel", rel))
	return nil
}

func (z *Publish) uploadAssets(c app_control.Control) error {
	l := c.Log()
	assets, _, err := z.artifactAssets(c)
	if err != nil {
		return err
	}

	sva := sv_release_asset.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName, app.Version)
	for _, p := range assets {
		l.Info("Uploading asset", zap.String("path", p))
		a, err := sva.Upload(mo_path2.NewExistingFileSystemPath(p))
		if err != nil && err != qt_errors.ErrorMock {
			return err
		}
		if err == qt_errors.ErrorMock {
			continue
		}
		l.Info("Uploaded", zap.Any("asset", a))
	}
	return nil
}

func (z *Publish) Exec(c app_control.Control) error {
	l := c.Log()
	ready := true

	if app.IsProduction() {
		l.Info("Verify embedded resources")
		qt_runtime.Suite(c)
	} else {
		l.Info("Run as dev mode")
		ready = false
	}

	err := z.endToEndTest(c)
	if err != nil {
		return err
	}

	sum, err := z.verifyArtifacts(c)
	if err != nil {
		return err
	}

	relNote, err := z.releaseNotes(c, sum)
	if err != nil {
		return err
	}

	if err := z.createReleaseDraft(c, relNote); err != nil {
		return err
	}

	if err := z.uploadAssets(c); err != nil {
		return err
	}

	if !ready {
		l.Warn("The build does not satisfy release criteria")
		return ErrorBuildIsNotReadyForRelease
	}
	l.Info("The build is ready to publish")
	return nil
}

func (z *Publish) Test(c app_control.Control) error {
	d, err := qt_file.MakeTestFolder("release-publish", false)
	if err != nil {
		return err
	}

	platforms := []string{"linux", "mac", "win"}
	for _, platform := range platforms {
		app.Version = "dev-test"
		err = ioutil.WriteFile(filepath.Join(d, "tbx-"+app.Version+"-"+platform+".zip"), []byte("Test artifact"), 0644)
		if err != nil {
			c.Log().Warn("Unable to create test artifact", zap.Error(err))
			return err
		}
	}
	defer os.RemoveAll(d)

	err = rc_exec.ExecMock(c, &Publish{}, func(r rc_recipe.Recipe) {
		m := r.(*Publish)
		m.ArtifactPath = mo_path2.NewFileSystemPath(d)
	})
	if err, _ = qt_recipe.RecipeError(c.Log(), err); err != ErrorBuildIsNotReadyForRelease && err != nil {
		return err
	}
	return nil
}
<commit_msg>#333 : release publish command<commit_after>package release

import (
	"bytes"
	"errors"
	"fmt"
	mo_path2 "github.com/watermint/toolbox/domain/common/model/mo_path"
	"github.com/watermint/toolbox/domain/common/model/mo_string"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn_impl"
	"github.com/watermint/toolbox/domain/github/api/gh_conn"
	"github.com/watermint/toolbox/domain/github/api/gh_context"
	"github.com/watermint/toolbox/domain/github/api/gh_context_impl"
	"github.com/watermint/toolbox/domain/github/service/sv_release"
	"github.com/watermint/toolbox/domain/github/service/sv_release_asset"
	"github.com/watermint/toolbox/domain/github/service/sv_tag"
	"github.com/watermint/toolbox/infra/api/api_auth"
	"github.com/watermint/toolbox/infra/api/api_auth_impl"
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/ui/app_lang"
	"github.com/watermint/toolbox/infra/ui/app_msg"
	"github.com/watermint/toolbox/infra/ui/app_ui"
	"github.com/watermint/toolbox/infra/util/ut_filehash"
	"github.com/watermint/toolbox/quality/infra/qt_endtoend"
	"github.com/watermint/toolbox/quality/infra/qt_errors"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"github.com/watermint/toolbox/quality/infra/qt_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_runtime"
	"github.com/watermint/toolbox/recipe/dev/test"
	"go.uber.org/zap"
	"golang.org/x/text/language/display"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

var (
	ErrorBuildIsNotReadyForRelease = errors.New("the build does not satisfy release criteria")
	ErrorNoArtifactFound           = errors.New("no artifact found")
)

type Publish struct {
	TestResource              string
	Branch                    string
	ArtifactPath              mo_path2.FileSystemPath
	ConnGithub                gh_conn.ConnGithubRepo
	HeadingReleaseTheme       app_msg.Message
	HeadingChanges            app_msg.Message
	ListSpecChange            app_msg.Message
	HeadingDocument           app_msg.Message
	ListReadme                app_msg.Message
	HeadingBinary             app_msg.Message
	BinaryTableHeaderFilename app_msg.Message
	BinaryTableHeaderSize     app_msg.Message
	BinaryTableHeaderMD5      app_msg.Message
	BinaryTableHeaderSHA256   app_msg.Message
	TagCommitMessage          app_msg.Message
	ReleaseName               app_msg.Message
}

type ArtifactSum struct {
	Filename string
	Size     int64
	MD5      string
	SHA256   string
}

func (z *Publish) Preset() {
	z.TestResource = defaultTestResource
	z.Branch = "master"
}

func (z *Publish) artifactAssets(c app_control.Control) (paths []string, sizes map[string]int64, err error) {
	l := c.Log()

	entries, err := ioutil.ReadDir(z.ArtifactPath.Path())
	if err != nil {
		return nil, nil, err
	}
	paths = make([]string, 0)
	sizes = make(map[string]int64)
	for _, e := range entries {
		if !strings.HasPrefix(e.Name(), "tbx-"+app.Version) || !strings.HasSuffix(e.Name(), ".zip") {
			l.Debug("Ignore non artifact file", zap.Any("file", e))
			continue
		}
		path := filepath.Join(z.ArtifactPath.Path(), e.Name())
		paths = append(paths, path)
		sizes[path] = e.Size()
	}
	return paths, sizes, nil
}

func (z *Publish) verifyArtifacts(c app_control.Control) (a []*ArtifactSum, err error) {
	l := c.Log()
	a = make([]*ArtifactSum, 0)

	assets, assetSize, err := z.artifactAssets(c)

	h := ut_filehash.NewHash(l)
	for _, p := range assets {
		sum := &ArtifactSum{
			Filename: filepath.Base(p),
			Size:     assetSize[p],
		}
		sum.MD5, err = h.MD5(p)
		if err != nil {
			l.Debug("Unable to calc MD5", zap.Error(err))
			return nil, err
		}
		sum.SHA256, err = h.SHA256(p)
		if err != nil {
			l.Debug("Unable to calc SHA256", zap.Error(err))
			return nil, err
		}
		a = append(a, sum)
	}
	if len(a) < 1 {
		return nil, ErrorNoArtifactFound
	}
	return a, nil
}

func (z *Publish) releaseNotes(c app_control.Control, sum []*ArtifactSum) (relNote string, err error) {
	l := c.Log()
	baseUrl := "https://github.com/watermint/toolbox/blob/" + app.Version

	var buf bytes.Buffer
	mui := app_ui.NewMarkdown(c.Messages(), &buf, true)
	mui.Header(z.HeadingReleaseTheme)
	mui.Break()

	mui.Header(z.HeadingChanges)

	for _, lang := range app_lang.SupportedLanguages {
		mui.Info(z.ListSpecChange.
			With("Link", baseUrl+"/doc/generated"+app_lang.PathSuffix(lang)+"/changes.md").
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingDocument)

	for _, lang := range app_lang.SupportedLanguages {
		name := "README" + app_lang.PathSuffix(lang) + ".md"
		mui.Info(z.ListReadme.
			With("Name", name).
			With("Link", baseUrl+"/"+name).
			With("Lang", display.Self.Name(lang)),
		)
	}

	mui.Break()
	mui.Header(z.HeadingBinary)
	mit := mui.InfoTable("Binaries")
	mit.Header(z.BinaryTableHeaderFilename, z.BinaryTableHeaderSize, z.BinaryTableHeaderMD5, z.BinaryTableHeaderSHA256)

	for _, s := range sum {
		mit.RowRaw(s.Filename, fmt.Sprintf("%d", s.Size), s.MD5, s.SHA256)
	}
	mit.Flush()

	relNotesPath := filepath.Join(c.Workspace().Report(), "release_notes.md")
	err = ioutil.WriteFile(relNotesPath, buf.Bytes(), 0644)
	if err != nil {
		l.Debug("Unable to write release notes", zap.Error(err), zap.String("path", relNotesPath))
		return "", err
	}
	l.Info("Release note created", zap.String("path", relNotesPath))

	return buf.String(), nil
}

func (z *Publish) endToEndTest(c app_control.Control) error {
	l := c.Log()
	if c.Feature().IsTest() {
		l.Info("Skip tests")
		return nil
	}

	if c.Feature().IsProduction() {
		l.Info("Prepare resources")
		if !api_auth_impl.IsCacheAvailable(c, qt_endtoend.EndToEndPeer, []string{
			api_auth.DropboxTokenFull,
			api_auth.DropboxTokenBusinessAudit,
			api_auth.DropboxTokenBusinessManagement,
			api_auth.DropboxTokenBusinessFile,
			api_auth.DropboxTokenBusinessInfo,
		}) {
			return qt_errors.ErrorNotEnoughResource
		}
	}

	l.Info("Ensure end to end resource availability")
	if !dbx_conn_impl.IsEndToEndTokenAllAvailable(c) {
		l.Error("At least one of end to end resource is not available.")
		return errors.New("end to end resource is not available")
	}

	l.Info("Testing all end to end test")
	err := rc_exec.Exec(c, &test.Recipe{}, func(r rc_recipe.Recipe) {
		m := r.(*test.Recipe)
		m.All = true
		_, err := os.Lstat(z.TestResource)
		if err == nil {
			m.Resource = mo_string.NewOptional(z.TestResource)
		} else {
			l.Warn("Unable to read test resource", zap.String("path", z.TestResource), zap.Error(err))
		}
	})
	return err
}

func (z *Publish) ghCtx(c app_control.Control) gh_context.Context {
	if c.Feature().IsTest() {
		return gh_context_impl.NewMock()
	} else {
		return z.ConnGithub.Context()
	}
}

func (z *Publish) createTag(c app_control.Control) error {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	ui := c.UI()
	svt := sv_tag.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	l.Debug("Create tag")
	tag, err := svt.Create(
		app.Version,
		ui.Text(z.TagCommitMessage.With("Version", app.Version)),
		app.Hash,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create tag", zap.Error(err))
		return err
	}
	if err == qt_errors.ErrorMock {
		return nil
	}
	l.Info("The tag created", zap.Any("tag", tag))
	return nil
}

func (z *Publish) createReleaseDraft(c app_control.Control, relNote string) error {
	l := c.Log().With(
		zap.String("owner", app.RepositoryOwner),
		zap.String("repository", app.RepositoryName),
		zap.String("version", app.Version),
		zap.String("hash", app.Hash))
	ui := c.UI()
	svr := sv_release.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName)
	rel, err := svr.CreateDraft(
		app.Version,
		ui.Text(z.ReleaseName.With("Version", app.Version)),
		relNote,
		z.Branch,
	)
	if err != nil && err != qt_errors.ErrorMock {
		l.Debug("Unable to create release draft", zap.Error(err))
		return err
	}
	if err == qt_errors.ErrorMock {
		return nil
	}
	l.Info("Release created", zap.Any("rel", rel))
	return nil
}

func (z *Publish) uploadAssets(c app_control.Control) error {
	l := c.Log()
	assets, _, err := z.artifactAssets(c)
	if err != nil {
		return err
	}

	sva := sv_release_asset.New(z.ghCtx(c), app.RepositoryOwner, app.RepositoryName, app.Version)
	for _, p := range assets {
		l.Info("Uploading asset", zap.String("path", p))
		a, err := sva.Upload(mo_path2.NewExistingFileSystemPath(p))
		if err != nil && err != qt_errors.ErrorMock {
			return err
		}
		if err == qt_errors.ErrorMock {
			continue
		}
		l.Info("Uploaded", zap.Any("asset", a))
	}
	return nil
}

func (z *Publish) Exec(c app_control.Control) error {
	l := c.Log()
	ready := true

	if app.IsProduction() {
		l.Info("Verify embedded resources")
		qt_runtime.Suite(c)
	} else {
		l.Info("Run as dev mode")
		ready = false
	}

	err := z.endToEndTest(c)
	if err != nil {
		return err
	}

	sum, err := z.verifyArtifacts(c)
	if err != nil {
		return err
	}

	relNote, err := z.releaseNotes(c, sum)
	if err != nil {
		return err
	}

	if err := z.createTag(c); err != nil {
		return nil
	}

	if err := z.createReleaseDraft(c, relNote); err != nil {
		return err
	}

	if err := z.uploadAssets(c); err != nil {
		return err
	}

	if !ready {
		l.Warn("The build does not satisfy release criteria")
		return ErrorBuildIsNotReadyForRelease
	}
	l.Info("The build is ready to publish")
	return nil
}

func (z *Publish) Test(c app_control.Control) error {
	d, err := qt_file.MakeTestFolder("release-publish", false)
	if err != nil {
		return err
	}

	platforms := []string{"linux", "mac", "win"}
	for _, platform := range platforms {
		app.Version = "dev-test"
		err = ioutil.WriteFile(filepath.Join(d, "tbx-"+app.Version+"-"+platform+".zip"), []byte("Test artifact"), 0644)
		if err != nil {
			c.Log().Warn("Unable to create test artifact", zap.Error(err))
			return err
		}
	}
	defer os.RemoveAll(d)

	err = rc_exec.ExecMock(c, &Publish{}, func(r rc_recipe.Recipe) {
		m := r.(*Publish)
		m.ArtifactPath = mo_path2.NewFileSystemPath(d)
	})
	if err, _ = qt_recipe.RecipeError(c.Log(), err); err != ErrorBuildIsNotReadyForRelease && err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package streak

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGetContributions(t *testing.T) {
	assert := assert.New(t)
	contributions, err := GetContributions("hico-horiuchi")

	assert.Nil(err)
	assert.NotNil(contributions)
}
<commit_msg>:leftwards_arrow_with_hook: fix line break<commit_after>package streak

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestGetContributions(t *testing.T) {
	assert := assert.New(t)

	contributions, err := GetContributions("hico-horiuchi")
	assert.Nil(err)
	assert.NotNil(contributions)
}
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Cong Ding <dinggnu@gmail.com>

package stun

import (
	"errors"
	"net"
	"strconv"
)

var (
	serverAddr   string
	softwareName string
)

func init() {
	SetSoftwareName(DefaultSoftwareName)
}

// SetServerHost allows user to set the STUN hostname and port.
func SetServerHost(host string, port int) error {
	ips, err := net.LookupHost(host)
	if err != nil {
		return err
	}
	if len(ips) == 0 {
		return errors.New("Failed to get IP address of " + host)
	}
	serverAddr = net.JoinHostPort(ips[0], strconv.Itoa(port))
	return nil
}

// SetServerAddr allows user to set the transport layer STUN server address.
func SetServerAddr(address string) {
	serverAddr = address
}

// SetSoftwareName allows user to set the name of her software.
func SetSoftwareName(name string) {
	softwareName = name
}

// Discover contacts the STUN server and gets the response of NAT type, host
// for UDP punching.
func Discover() (int, *Host, error) {
	if serverAddr == "" {
		err := SetServerHost(DefaultServerHost, DefaultServerPort)
		if err != nil {
			return NAT_ERROR, nil, err
		}
	}
	return discover()
}
<commit_msg>update comments<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Author: Cong Ding <dinggnu@gmail.com>

package stun

import (
	"errors"
	"net"
	"strconv"
)

var (
	serverAddr   string
	softwareName string
)

func init() {
	SetSoftwareName(DefaultSoftwareName)
}

// SetServerHost allows user to set the STUN hostname and port.
func SetServerHost(host string, port int) error {
	ips, err := net.LookupHost(host)
	if err != nil {
		return err
	}
	if len(ips) == 0 {
		return errors.New("Failed to get IP address of " + host)
	}
	serverAddr = net.JoinHostPort(ips[0], strconv.Itoa(port))
	return nil
}

// SetServerAddr allows user to set the transport layer STUN server address.
func SetServerAddr(address string) {
	serverAddr = address
}

// SetSoftwareName allows user to set the name of the software, which is used
// for logging purpose (NOT used in the current implementation).
func SetSoftwareName(name string) {
	softwareName = name
}

// Discover contacts the STUN server and gets the response of NAT type, host
// for UDP punching.
func Discover() (int, *Host, error) {
	if serverAddr == "" {
		err := SetServerHost(DefaultServerHost, DefaultServerPort)
		if err != nil {
			return NAT_ERROR, nil, err
		}
	}
	return discover()
}
<|endoftext|>


<commit_before>package pluginutil

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/base64"
	"errors"
	"fmt"
	"math/big"
	mathrand "math/rand"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/SermoDigital/jose/jws"
	"github.com/hashicorp/errwrap"
	uuid "github.com/hashicorp/go-uuid"
	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/logical"
)

var (
	PluginUnwrapTokenEnv = "VAULT_WRAP_TOKEN"
)

// GenerateCACert returns a CA cert used to later sign the certificates for the
// plugin client and server.
func GenerateCACert() ([]byte, *x509.Certificate, *ecdsa.PrivateKey, error) {
	key, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
	if err != nil {
		return nil, nil, nil, err
	}

	host, err := uuid.GenerateUUID()
	if err != nil {
		return nil, nil, nil, err
	}
	host = "localhost"
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: host,
		},
		DNSNames: []string{host},
		ExtKeyUsage: []x509.ExtKeyUsage{
			x509.ExtKeyUsageServerAuth,
			x509.ExtKeyUsageClientAuth,
		},
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement | x509.KeyUsageCertSign,
		SerialNumber: big.NewInt(mathrand.Int63()),
		NotBefore:    time.Now().Add(-30 * time.Second),
		// 30 years of single-active uptime ought to be enough for anybody
		NotAfter:              time.Now().Add(262980 * time.Hour),
		BasicConstraintsValid: true,
		IsCA: true,
	}

	certBytes, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("unable to generate replicated cluster certificate: %v", err)
	}

	caCert, err := x509.ParseCertificate(certBytes)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("error parsing generated replication certificate: %v", err)
	}

	return certBytes, caCert, key, nil
}

// generateSignedCert is used internally to create certificates for the plugin
// client and server. These certs are signed by the given CA Cert and Key.
func generateSignedCert(CACert *x509.Certificate, CAKey *ecdsa.PrivateKey) ([]byte, *x509.Certificate, *ecdsa.PrivateKey, error) {
	host, err := uuid.GenerateUUID()
	if err != nil {
		return nil, nil, nil, err
	}
	host = "localhost"
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: host,
		},
		DNSNames: []string{host},
		ExtKeyUsage: []x509.ExtKeyUsage{
			x509.ExtKeyUsageClientAuth,
			x509.ExtKeyUsageServerAuth,
		},
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement,
		SerialNumber: big.NewInt(mathrand.Int63()),
		NotBefore:    time.Now().Add(-30 * time.Second),
		NotAfter:     time.Now().Add(262980 * time.Hour),
	}

	clientKey, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
	if err != nil {
		return nil, nil, nil, errwrap.Wrapf("error generating client key: {{err}}", err)
	}

	certBytes, err := x509.CreateCertificate(rand.Reader, template, CACert, clientKey.Public(), CAKey)
	if err != nil {
		return nil, nil, nil, errwrap.Wrapf("unable to generate client certificate: {{err}}", err)
	}

	clientCert, err := x509.ParseCertificate(certBytes)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("error parsing generated replication certificate: %v", err)
	}

	return certBytes, clientCert, clientKey, nil
}

// CreateClientTLSConfig creates a signed certificate and returns a configured
// TLS config.
func CreateClientTLSConfig(CACert *x509.Certificate, CAKey *ecdsa.PrivateKey) (*tls.Config, error) {
	clientCertBytes, clientCert, clientKey, err := generateSignedCert(CACert, CAKey)
	if err != nil {
		return nil, err
	}

	cert := tls.Certificate{
		Certificate: [][]byte{clientCertBytes},
		PrivateKey:  clientKey,
		Leaf:        clientCert,
	}

	clientCertPool := x509.NewCertPool()
	clientCertPool.AddCert(CACert)

	tlsConfig := &tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      clientCertPool,
		ClientCAs:    clientCertPool,
		ServerName:   CACert.Subject.CommonName,
		MinVersion:   tls.VersionTLS12,
	}

	tlsConfig.BuildNameToCertificate()

	return tlsConfig, nil
}

// WrapServerConfig is used to create a server certificate and private key, then
// wrap them in an unwrap token for later retrieval by the plugin.
func WrapServerConfig(sys logical.SystemView, CACertBytes []byte, CACert *x509.Certificate, CAKey *ecdsa.PrivateKey) (string, error) {
	serverCertBytes, _, serverKey, err := generateSignedCert(CACert, CAKey)
	if err != nil {
		return "", err
	}
	rawKey, err := x509.MarshalECPrivateKey(serverKey)
	if err != nil {
		return "", err
	}

	wrapToken, err := sys.ResponseWrapData(map[string]interface{}{
		"CACert":     CACertBytes,
		"ServerCert": serverCertBytes,
		"ServerKey":  rawKey,
	}, time.Second*10, true)

	return wrapToken, err
}

// VaultPluginTLSProvider is run inside a plugin and retrives the response
// wrapped TLS certificate from vault. It returns a configured TLS Config.
func VaultPluginTLSProvider() (*tls.Config, error) {
	unwrapToken := os.Getenv(PluginUnwrapTokenEnv)

	// Ensure unwrap token is a JWT
	if strings.Count(unwrapToken, ".") != 2 {
		return nil, errors.New("Could not parse unwraptoken")
	}

	// Parse the JWT and retrieve the vault address
	wt, err := jws.ParseJWT([]byte(unwrapToken))
	if err != nil {
		return nil, errors.New(fmt.Sprintf("error decoding token: %s", err))
	}
	if wt == nil {
		return nil, errors.New("nil decoded token")
	}

	addrRaw := wt.Claims().Get("addr")
	if addrRaw == nil {
		return nil, errors.New("decoded token does not contain primary cluster address")
	}
	vaultAddr, ok := addrRaw.(string)
	if !ok {
		return nil, errors.New("decoded token's address not valid")
	}
	if vaultAddr == "" {
		return nil, errors.New(`no address for the vault found`)
	}

	// Sanity check the value
	if _, err := url.Parse(vaultAddr); err != nil {
		return nil, errors.New(fmt.Sprintf("error parsing the vault address: %s", err))
	}

	// Unwrap the token
	clientConf := api.DefaultConfig()
	clientConf.Address = vaultAddr
	client, err := api.NewClient(clientConf)
	if err != nil {
		return nil, errwrap.Wrapf("error during token unwrap request: {{err}}", err)
	}

	secret, err := client.Logical().Unwrap(unwrapToken)
	if err != nil {
		return nil, errwrap.Wrapf("error during token unwrap request: {{err}}", err)
	}

	// Retrieve and parse the CA Certificate
	CABytesRaw, ok := secret.Data["CACert"].(string)
	if !ok {
		return nil, errors.New("error unmarshalling CA certificate")
	}

	CABytes, err := base64.StdEncoding.DecodeString(CABytesRaw)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	CACert, err := x509.ParseCertificate(CABytes)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	// Retrieve and parse the server's certificate
	serverCertBytesRaw, ok := secret.Data["ServerCert"].(string)
	if !ok {
		return nil, errors.New("error unmarshalling certificate")
	}

	serverCertBytes, err := base64.StdEncoding.DecodeString(serverCertBytesRaw)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	serverCert, err := x509.ParseCertificate(serverCertBytes)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	// Retrieve and parse the server's private key
	serverKeyB64, ok := secret.Data["ServerKey"].(string)
	if !ok {
		return nil, errors.New("error unmarshalling certificate")
	}

	serverKeyRaw, err := base64.StdEncoding.DecodeString(serverKeyB64)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	serverKey, err := x509.ParseECPrivateKey(serverKeyRaw)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	// Add CA cert to the cert pool
	caCertPool := x509.NewCertPool()
	caCertPool.AddCert(CACert)

	// Build a certificate object out of the server's cert and private key.
	cert := tls.Certificate{
		Certificate: [][]byte{serverCertBytes},
		PrivateKey:  serverKey,
		Leaf:        serverCert,
	}

	// Setup TLS config
	tlsConfig := &tls.Config{
		ClientCAs:  caCertPool,
		RootCAs:    caCertPool,
		ClientAuth: tls.RequireAndVerifyClientCert,
		// TLS 1.2 minimum
		MinVersion:   tls.VersionTLS12,
		Certificates: []tls.Certificate{cert},
	}
	tlsConfig.BuildNameToCertificate()

	return tlsConfig, nil
}
<commit_msg>Update the name of PluginUnwrapTokenEnv<commit_after>package pluginutil

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/base64"
	"errors"
	"fmt"
	"math/big"
	mathrand "math/rand"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/SermoDigital/jose/jws"
	"github.com/hashicorp/errwrap"
	uuid "github.com/hashicorp/go-uuid"
	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/logical"
)

var (
	// PluginUnwrapTokenEnv is the ENV name used to pass unwrap tokens to the
	// plugin.
	PluginUnwrapTokenEnv = "VAULT_UNWRAP_TOKEN"
)

// GenerateCACert returns a CA cert used to later sign the certificates for the
// plugin client and server.
func GenerateCACert() ([]byte, *x509.Certificate, *ecdsa.PrivateKey, error) {
	key, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
	if err != nil {
		return nil, nil, nil, err
	}

	host, err := uuid.GenerateUUID()
	if err != nil {
		return nil, nil, nil, err
	}
	host = "localhost"
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: host,
		},
		DNSNames: []string{host},
		ExtKeyUsage: []x509.ExtKeyUsage{
			x509.ExtKeyUsageServerAuth,
			x509.ExtKeyUsageClientAuth,
		},
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement | x509.KeyUsageCertSign,
		SerialNumber: big.NewInt(mathrand.Int63()),
		NotBefore:    time.Now().Add(-30 * time.Second),
		// 30 years of single-active uptime ought to be enough for anybody
		NotAfter:              time.Now().Add(262980 * time.Hour),
		BasicConstraintsValid: true,
		IsCA: true,
	}

	certBytes, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("unable to generate replicated cluster certificate: %v", err)
	}

	caCert, err := x509.ParseCertificate(certBytes)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("error parsing generated replication certificate: %v", err)
	}

	return certBytes, caCert, key, nil
}

// generateSignedCert is used internally to create certificates for the plugin
// client and server. These certs are signed by the given CA Cert and Key.
func generateSignedCert(CACert *x509.Certificate, CAKey *ecdsa.PrivateKey) ([]byte, *x509.Certificate, *ecdsa.PrivateKey, error) {
	host, err := uuid.GenerateUUID()
	if err != nil {
		return nil, nil, nil, err
	}
	host = "localhost"
	template := &x509.Certificate{
		Subject: pkix.Name{
			CommonName: host,
		},
		DNSNames: []string{host},
		ExtKeyUsage: []x509.ExtKeyUsage{
			x509.ExtKeyUsageClientAuth,
			x509.ExtKeyUsageServerAuth,
		},
		KeyUsage:     x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement,
		SerialNumber: big.NewInt(mathrand.Int63()),
		NotBefore:    time.Now().Add(-30 * time.Second),
		NotAfter:     time.Now().Add(262980 * time.Hour),
	}

	clientKey, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
	if err != nil {
		return nil, nil, nil, errwrap.Wrapf("error generating client key: {{err}}", err)
	}

	certBytes, err := x509.CreateCertificate(rand.Reader, template, CACert, clientKey.Public(), CAKey)
	if err != nil {
		return nil, nil, nil, errwrap.Wrapf("unable to generate client certificate: {{err}}", err)
	}

	clientCert, err := x509.ParseCertificate(certBytes)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("error parsing generated replication certificate: %v", err)
	}

	return certBytes, clientCert, clientKey, nil
}

// CreateClientTLSConfig creates a signed certificate and returns a configured
// TLS config.
func CreateClientTLSConfig(CACert *x509.Certificate, CAKey *ecdsa.PrivateKey) (*tls.Config, error) {
	clientCertBytes, clientCert, clientKey, err := generateSignedCert(CACert, CAKey)
	if err != nil {
		return nil, err
	}

	cert := tls.Certificate{
		Certificate: [][]byte{clientCertBytes},
		PrivateKey:  clientKey,
		Leaf:        clientCert,
	}

	clientCertPool := x509.NewCertPool()
	clientCertPool.AddCert(CACert)

	tlsConfig := &tls.Config{
		Certificates: []tls.Certificate{cert},
		RootCAs:      clientCertPool,
		ClientCAs:    clientCertPool,
		ServerName:   CACert.Subject.CommonName,
		MinVersion:   tls.VersionTLS12,
	}

	tlsConfig.BuildNameToCertificate()

	return tlsConfig, nil
}

// WrapServerConfig is used to create a server certificate and private key, then
// wrap them in an unwrap token for later retrieval by the plugin.
func WrapServerConfig(sys logical.SystemView, CACertBytes []byte, CACert *x509.Certificate, CAKey *ecdsa.PrivateKey) (string, error) {
	serverCertBytes, _, serverKey, err := generateSignedCert(CACert, CAKey)
	if err != nil {
		return "", err
	}
	rawKey, err := x509.MarshalECPrivateKey(serverKey)
	if err != nil {
		return "", err
	}

	wrapToken, err := sys.ResponseWrapData(map[string]interface{}{
		"CACert":     CACertBytes,
		"ServerCert": serverCertBytes,
		"ServerKey":  rawKey,
	}, time.Second*10, true)

	return wrapToken, err
}

// VaultPluginTLSProvider is run inside a plugin and retrives the response
// wrapped TLS certificate from vault. It returns a configured TLS Config.
func VaultPluginTLSProvider() (*tls.Config, error) {
	unwrapToken := os.Getenv(PluginUnwrapTokenEnv)

	// Ensure unwrap token is a JWT
	if strings.Count(unwrapToken, ".") != 2 {
		return nil, errors.New("Could not parse unwraptoken")
	}

	// Parse the JWT and retrieve the vault address
	wt, err := jws.ParseJWT([]byte(unwrapToken))
	if err != nil {
		return nil, errors.New(fmt.Sprintf("error decoding token: %s", err))
	}
	if wt == nil {
		return nil, errors.New("nil decoded token")
	}

	addrRaw := wt.Claims().Get("addr")
	if addrRaw == nil {
		return nil, errors.New("decoded token does not contain primary cluster address")
	}
	vaultAddr, ok := addrRaw.(string)
	if !ok {
		return nil, errors.New("decoded token's address not valid")
	}
	if vaultAddr == "" {
		return nil, errors.New(`no address for the vault found`)
	}

	// Sanity check the value
	if _, err := url.Parse(vaultAddr); err != nil {
		return nil, errors.New(fmt.Sprintf("error parsing the vault address: %s", err))
	}

	// Unwrap the token
	clientConf := api.DefaultConfig()
	clientConf.Address = vaultAddr
	client, err := api.NewClient(clientConf)
	if err != nil {
		return nil, errwrap.Wrapf("error during token unwrap request: {{err}}", err)
	}

	secret, err := client.Logical().Unwrap(unwrapToken)
	if err != nil {
		return nil, errwrap.Wrapf("error during token unwrap request: {{err}}", err)
	}

	// Retrieve and parse the CA Certificate
	CABytesRaw, ok := secret.Data["CACert"].(string)
	if !ok {
		return nil, errors.New("error unmarshalling CA certificate")
	}

	CABytes, err := base64.StdEncoding.DecodeString(CABytesRaw)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	CACert, err := x509.ParseCertificate(CABytes)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	// Retrieve and parse the server's certificate
	serverCertBytesRaw, ok := secret.Data["ServerCert"].(string)
	if !ok {
		return nil, errors.New("error unmarshalling certificate")
	}

	serverCertBytes, err := base64.StdEncoding.DecodeString(serverCertBytesRaw)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	serverCert, err := x509.ParseCertificate(serverCertBytes)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	// Retrieve and parse the server's private key
	serverKeyB64, ok := secret.Data["ServerKey"].(string)
	if !ok {
		return nil, errors.New("error unmarshalling certificate")
	}

	serverKeyRaw, err := base64.StdEncoding.DecodeString(serverKeyB64)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	serverKey, err := x509.ParseECPrivateKey(serverKeyRaw)
	if err != nil {
		return nil, fmt.Errorf("error parsing certificate: %v", err)
	}

	// Add CA cert to the cert pool
	caCertPool := x509.NewCertPool()
	caCertPool.AddCert(CACert)

	// Build a certificate object out of the server's cert and private key.
	cert := tls.Certificate{
		Certificate: [][]byte{serverCertBytes},
		PrivateKey:  serverKey,
		Leaf:        serverCert,
	}

	// Setup TLS config
	tlsConfig := &tls.Config{
		ClientCAs:  caCertPool,
		RootCAs:    caCertPool,
		ClientAuth: tls.RequireAndVerifyClientCert,
		// TLS 1.2 minimum
		MinVersion:   tls.VersionTLS12,
		Certificates: []tls.Certificate{cert},
	}
	tlsConfig.BuildNameToCertificate()

	return tlsConfig, nil
}
<|endoftext|>


<commit_before>package async

import (
	"errors"
	"fmt"
	"sync"
	"time"
)

const Unlimit = 0

type Method func(args ...interface{}) (interface{}, error)
type LambdaMethod func() (interface{}, error)

var panicHandler func(interface{})

func SetPanicHandler(hanlder func(interface{})) {
	panicHandler = hanlder
}

func Safety(method func() (interface{}, error)) (res interface{}, err error) {
	defer func() {
		if e := recover(); e != nil {
			_, ok := e.(error)
			if ok {
				err = e.(error)
			} else {
				err = errors.New(fmt.Sprintf("%v", e))
			}
			if panicHandler != nil {
				panicHandler(err)
			}
		}
	}()
	return method()
}

func Retry(method func() (interface{}, error), maxCount int, interval time.Duration) (interface{}, error) {
	count := 0
	for {
		res, err := Lambda(method, Unlimit)
		if err == nil {
			return res, err
		}
		count = count + 1
		if count >= maxCount {
			return nil, err
		}
		<-time.After(time.Duration(interval * count))
	}
}

func Lambda(method func() (interface{}, error), timeout time.Duration) (interface{}, error) {
	output := make(chan interface{}, 1)
	go func() {
		defer close(output)
		defer func() {
			if e := recover(); e != nil {
				err := errors.New(fmt.Sprintf("%s", e))
				if panicHandler != nil {
					panicHandler(err)
				}
				output <- err
			}
		}()
		res, err := method()
		if err != nil {
			output <- err
		} else {
			output <- res
		}
	}()
	if timeout > 0 {
		timer := time.NewTimer(timeout)
		defer timer.Stop()
		select {
		case res := <-output:
			{
				switch err := res.(type) {
				case error:
					{
						return nil, err
					}
				default:
					{
						return res, nil
					}
				}
			}
		case <-timer.C:
			{
				return nil, errors.New("Async_Timeout")
			}
		}
	} else {
		res := <-output
		switch err := res.(type) {
		case error:
			{
				return nil, err
			}
		default:
			{
				return res, nil
			}
		}
	}
}

func Call(m Method, timeout time.Duration, args ...interface{}) (interface{}, error) {
	return Lambda(func() (interface{}, error) {
		return m(args...)
	}, timeout)
}

func All(methods []LambdaMethod, timeout time.Duration) []interface{} {
	var wg sync.WaitGroup
	result := make([]interface{}, len(methods))
	for i, m := range methods {
		wg.Add(1)
		go func(index int, method LambdaMethod) {
			defer wg.Done()
			res, err := Lambda(method, timeout)
			if err != nil {
				result[index] = err
			} else {
				result[index] = res
			}
		}(i, m)
	}
	wg.Wait()
	return result
}

func Serise(methods []LambdaMethod, timeout time.Duration) []interface{} {
	result := make([]interface{}, 0)
	for _, m := range methods {
		res, err := Lambda(m, timeout)
		if err != nil {
			result = append(result, err)
			return result
		} else {
			result = append(result, res)
		}
	}
	return result
}

func Flow(enter Method, args []interface{}, methods []Method, timeout time.Duration) (interface{}, error) {
	var (
		res interface{}
		err error
	)
	res, err = Call(enter, timeout, args...)
	if err != nil {
		return nil, err
	}
	for _, m := range methods {
		res, err = Call(m, timeout, res)
		if err != nil {
			return nil, err
		}
	}
	return res, nil
}

func Any(methods []LambdaMethod, timeout time.Duration) ([]interface{}, error) {
	resChan := make(chan []interface{}, 1)
	errChan := make(chan error, len(methods))
	go func() {
		defer func() {
			close(resChan)
			close(errChan)
		}()
		var wg sync.WaitGroup
		result := make([]interface{}, len(methods))
		for i, m := range methods {
			wg.Add(1)
			go func(index int, method LambdaMethod) {
				defer wg.Done()
				res, err := Lambda(method, timeout)
				if err != nil {
					errChan <- err
				} else {
					result[index] = res
				}
			}(i, m)
		}
		wg.Wait()
		resChan <- result
	}()
	select {
	case err := <-errChan:
		return nil, err
	case res := <-resChan:
		return res, nil
	}
}

func AnyOne(methods []LambdaMethod, timeout time.Duration) (interface{}, []error) {
	resChan := make(chan interface{}, len(methods))
	errChan := make(chan []error, 1)
	go func() {
		defer func() {
			close(resChan)
			close(errChan)
		}()
		var wg sync.WaitGroup
		errs := make([]error, len(methods))
		for i, m := range methods {
			wg.Add(1)
			go func(index int, method LambdaMethod) {
				defer wg.Done()
				res, err := Lambda(method, timeout)
				if err != nil {
					errs[index] = err
				} else {
					resChan <- res
				}
			}(i, m)
		}
		wg.Wait()
		errChan <- errs
	}()
	select {
	case errs := <-errChan:
		return nil, errs
	case res := <-resChan:
		return res, nil
	}
}

func Parallel(methods []LambdaMethod, maxCount int) []interface{} {
	if maxCount == Unlimit {
		maxCount = 64
	}
	var wg sync.WaitGroup
	workers := make(chan bool, maxCount)
	results := make([]interface{}, len(methods))
	for index, method := range methods {
		workers <- true
		wg.Add(1)
		go func(i int, m LambdaMethod) {
			defer func() {
				<-workers
				wg.Done()
			}()
			res, err := Lambda(m, 0)
			if err != nil {
				results[i] = err
			} else {
				results[i] = res
			}
		}(index, method)
	}
	wg.Wait()
	close(workers)
	return results
}

func Foreach(objs []interface{}, method func(int) (interface{}, error), maxConcurrent int) []interface{} {
	if maxConcurrent == Unlimit {
		maxConcurrent = 64
	}
	var wg sync.WaitGroup
	workers := make(chan bool, maxConcurrent)
	results := make([]interface{}, len(objs))
	for index, _ := range objs {
		workers <- true
		wg.Add(1)
		go func(i int, method func(int) (interface{}, error)) {
			defer func() {
				<-workers
				wg.Done()
			}()
			res, err := Safety(func() (interface{}, error) {
				return method(i)
			})
			if err != nil {
				results[i] = err
			} else {
				results[i] = res
			}
		}(index, method)
	}
	wg.Wait()
	close(workers)
	return results
}

func For(count int, method func(int) (interface{}, error), maxConcurrent int) []interface{} {
	if maxConcurrent == Unlimit {
		maxConcurrent = 64
	}
	var wg sync.WaitGroup
	workers := make(chan bool, maxConcurrent)
	results := make([]interface{}, count)
	for index := 0; index < count; index++ {
		workers <- true
		wg.Add(1)
		go func(i int, method func(int) (interface{}, error)) {
			defer func() {
				<-workers
				wg.Done()
			}()
			res, err := Safety(func() (interface{}, error) {
				return method(i)
			})
			if err != nil {
				results[i] = err
			} else {
				results[i] = res
			}
		}(index, method)
	}
	wg.Wait()
	close(workers)
	return results
}
<commit_msg>fix bug<commit_after>package async

import (
	"errors"
	"fmt"
	"sync"
	"time"
)

const Unlimit = 0

type Method func(args ...interface{}) (interface{}, error)
type LambdaMethod func() (interface{}, error)

var panicHandler func(interface{})

func SetPanicHandler(hanlder func(interface{})) {
	panicHandler = hanlder
}

func Safety(method func() (interface{}, error)) (res interface{}, err error) {
	defer func() {
		if e := recover(); e != nil {
			_, ok := e.(error)
			if ok {
				err = e.(error)
			} else {
				err = errors.New(fmt.Sprintf("%v", e))
			}
			if panicHandler != nil {
				panicHandler(err)
			}
		}
	}()
	return method()
}

func Retry(method func() (interface{}, error), maxCount int, interval time.Duration) (interface{}, error) {
	count := 0
	for {
		res, err := Lambda(method, Unlimit)
		if err == nil {
			return res, err
		}
		count = count + 1
		if count >= maxCount {
			return nil, err
		}
		<-time.After(time.Duration(count) * interval)
	}
}

func Lambda(method func() (interface{}, error), timeout time.Duration) (interface{}, error) {
	output := make(chan interface{}, 1)
	go func() {
		defer close(output)
		defer func() {
			if e := recover(); e != nil {
				err := errors.New(fmt.Sprintf("%s", e))
				if panicHandler != nil {
					panicHandler(err)
				}
				output <- err
			}
		}()
		res, err := method()
		if err != nil {
			output <- err
		} else {
			output <- res
		}
	}()
	if timeout > 0 {
		timer := time.NewTimer(timeout)
		defer timer.Stop()
		select {
		case res := <-output:
			{
				switch err := res.(type) {
				case error:
					{
						return nil, err
					}
				default:
					{
						return res, nil
					}
				}
			}
		case <-timer.C:
			{
				return nil, errors.New("Async_Timeout")
			}
		}
	} else {
		res := <-output
		switch err := res.(type) {
		case error:
			{
				return nil, err
			}
		default:
			{
				return res, nil
			}
		}
	}
}

func Call(m Method, timeout time.Duration, args ...interface{}) (interface{}, error) {
	return Lambda(func() (interface{}, error) {
		return m(args...)
	}, timeout)
}

func All(methods []LambdaMethod, timeout time.Duration) []interface{} {
	var wg sync.WaitGroup
	result := make([]interface{}, len(methods))
	for i, m := range methods {
		wg.Add(1)
		go func(index int, method LambdaMethod) {
			defer wg.Done()
			res, err := Lambda(method, timeout)
			if err != nil {
				result[index] = err
			} else {
				result[index] = res
			}
		}(i, m)
	}
	wg.Wait()
	return result
}

func Serise(methods []LambdaMethod, timeout time.Duration) []interface{} {
	result := make([]interface{}, 0)
	for _, m := range methods {
		res, err := Lambda(m, timeout)
		if err != nil {
			result = append(result, err)
			return result
		} else {
			result = append(result, res)
		}
	}
	return result
}

func Flow(enter Method, args []interface{}, methods []Method, timeout time.Duration) (interface{}, error) {
	var (
		res interface{}
		err error
	)
	res, err = Call(enter, timeout, args...)
	if err != nil {
		return nil, err
	}
	for _, m := range methods {
		res, err = Call(m, timeout, res)
		if err != nil {
			return nil, err
		}
	}
	return res, nil
}

func Any(methods []LambdaMethod, timeout time.Duration) ([]interface{}, error) {
	resChan := make(chan []interface{}, 1)
	errChan := make(chan error, len(methods))
	go func() {
		defer func() {
			close(resChan)
			close(errChan)
		}()
		var wg sync.WaitGroup
		result := make([]interface{}, len(methods))
		for i, m := range methods {
			wg.Add(1)
			go func(index int, method LambdaMethod) {
				defer wg.Done()
				res, err := Lambda(method, timeout)
				if err != nil {
					errChan <- err
				} else {
					result[index] = res
				}
			}(i, m)
		}
		wg.Wait()
		resChan <- result
	}()
	select {
	case err := <-errChan:
		return nil, err
	case res := <-resChan:
		return res, nil
	}
}

func AnyOne(methods []LambdaMethod, timeout time.Duration) (interface{}, []error) {
	resChan := make(chan interface{}, len(methods))
	errChan := make(chan []error, 1)
	go func() {
		defer func() {
			close(resChan)
			close(errChan)
		}()
		var wg sync.WaitGroup
		errs := make([]error, len(methods))
		for i, m := range methods {
			wg.Add(1)
			go func(index int, method LambdaMethod) {
				defer wg.Done()
				res, err := Lambda(method, timeout)
				if err != nil {
					errs[index] = err
				} else {
					resChan <- res
				}
			}(i, m)
		}
		wg.Wait()
		errChan <- errs
	}()
	select {
	case errs := <-errChan:
		return nil, errs
	case res := <-resChan:
		return res, nil
	}
}

func Parallel(methods []LambdaMethod, maxCount int) []interface{} {
	if maxCount == Unlimit {
		maxCount = 64
	}
	var wg sync.WaitGroup
	workers := make(chan bool, maxCount)
	results := make([]interface{}, len(methods))
	for index, method := range methods {
		workers <- true
		wg.Add(1)
		go func(i int, m LambdaMethod) {
			defer func() {
				<-workers
				wg.Done()
			}()
			res, err := Lambda(m, 0)
			if err != nil {
				results[i] = err
			} else {
				results[i] = res
			}
		}(index, method)
	}
	wg.Wait()
	close(workers)
	return results
}

func Foreach(objs []interface{}, method func(int) (interface{}, error), maxConcurrent int) []interface{} {
	if maxConcurrent == Unlimit {
		maxConcurrent = 64
	}
	var wg sync.WaitGroup
	workers := make(chan bool, maxConcurrent)
	results := make([]interface{}, len(objs))
	for index, _ := range objs {
		workers <- true
		wg.Add(1)
		go func(i int, method func(int) (interface{}, error)) {
			defer func() {
				<-workers
				wg.Done()
			}()
			res, err := Safety(func() (interface{}, error) {
				return method(i)
			})
			if err != nil {
				results[i] = err
			} else {
				results[i] = res
			}
		}(index, method)
	}
	wg.Wait()
	close(workers)
	return results
}

func For(count int, method func(int) (interface{}, error), maxConcurrent int) []interface{} {
	if maxConcurrent == Unlimit {
		maxConcurrent = 64
	}
	var wg sync.WaitGroup
	workers := make(chan bool, maxConcurrent)
	results := make([]interface{}, count)
	for index := 0; index < count; index++ {
		workers <- true
		wg.Add(1)
		go func(i int, method func(int) (interface{}, error)) {
			defer func() {
				<-workers
				wg.Done()
			}()
			res, err := Safety(func() (interface{}, error) {
				return method(i)
			})
			if err != nil {
				results[i] = err
			} else {
				results[i] = res
			}
		}(index, method)
	}
	wg.Wait()
	close(workers)
	return results
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"testing"

	"knative.dev/eventing/test/e2e/helpers"
)

/*
TestEventTransformationForSubscriptiop tests the following scenario:

             1            2                 5            6                  7
EventSource ---> Channel ---> Subscription ---> Channel ---> Subscription ----> Service(Logger)
                                   |  ^
                                 3 |  | 4
                                   |  |
                                   |  ---------
                                   -----------> Service(Transformation)
*/
func TestEventTransformationForSubscription(t *testing.T) {
	helpers.EventTransformationForSubscriptionTestHelper(context.Background(), t, helpers.SubscriptionV1beta1, channelTestRunner)
}

func TestEventTransformationForSubscriptionV1(t *testing.T) {
	helpers.EventTransformationForSubscriptionTestHelper(context.Background(), t, helpers.SubscriptionV1, channelTestRunner)
}
<commit_msg>:lipstick: reflect versions on test func (#4082)<commit_after>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"context"
	"testing"

	"knative.dev/eventing/test/e2e/helpers"
)

/*
TestEventTransformationForSubscriptiop tests the following scenario:

             1            2                 5            6                  7
EventSource ---> Channel ---> Subscription ---> Channel ---> Subscription ----> Service(Logger)
                                   |  ^
                                 3 |  | 4
                                   |  |
                                   |  ---------
                                   -----------> Service(Transformation)
*/
func TestEventTransformationForSubscriptionV1Beta1(t *testing.T) {
	helpers.EventTransformationForSubscriptionTestHelper(context.Background(), t, helpers.SubscriptionV1beta1, channelTestRunner)
}

func TestEventTransformationForSubscriptionV1(t *testing.T) {
	helpers.EventTransformationForSubscriptionTestHelper(context.Background(), t, helpers.SubscriptionV1, channelTestRunner)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"bufio"
	"bytes"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	. "github.com/onsi/gomega"
	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/util/system"
)

const (
	resourceDataGatheringPeriod = 60 * time.Second
	probeDuration               = 15 * time.Second
)

type ResourceConstraint struct {
	CPUConstraint    float64
	MemoryConstraint uint64
}

type SingleContainerSummary struct {
	Name string
	Cpu  float64
	Mem  uint64
}

// we can't have int here, as JSON does not accept integer keys.
type ResourceUsageSummary map[string][]SingleContainerSummary

func (s *ResourceUsageSummary) PrintHumanReadable() string {
	buf := &bytes.Buffer{}
	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
	for perc, summaries := range *s {
		buf.WriteString(fmt.Sprintf("%v percentile:\n", perc))
		fmt.Fprintf(w, "container\tcpu(cores)\tmemory(MB)\n")
		for _, summary := range summaries {
			fmt.Fprintf(w, "%q\t%.3f\t%.2f\n", summary.Name, summary.Cpu, float64(summary.Mem)/(1024*1024))
		}
		w.Flush()
	}
	return buf.String()
}

func (s *ResourceUsageSummary) PrintJSON() string {
	return PrettyPrintJSON(*s)
}

func computePercentiles(timeSeries []ResourceUsagePerContainer, percentilesToCompute []int) map[int]ResourceUsagePerContainer {
	if len(timeSeries) == 0 {
		return make(map[int]ResourceUsagePerContainer)
	}
	dataMap := make(map[string]*usageDataPerContainer)
	for i := range timeSeries {
		for name, data := range timeSeries[i] {
			if dataMap[name] == nil {
				dataMap[name] = &usageDataPerContainer{
					cpuData:        make([]float64, len(timeSeries)),
					memUseData:     make([]uint64, len(timeSeries)),
					memWorkSetData: make([]uint64, len(timeSeries)),
				}
			}
			dataMap[name].cpuData = append(dataMap[name].cpuData, data.CPUUsageInCores)
			dataMap[name].memUseData = append(dataMap[name].memUseData, data.MemoryUsageInBytes)
			dataMap[name].memWorkSetData = append(dataMap[name].memWorkSetData, data.MemoryWorkingSetInBytes)
		}
	}
	for _, v := range dataMap {
		sort.Float64s(v.cpuData)
		sort.Sort(uint64arr(v.memUseData))
		sort.Sort(uint64arr(v.memWorkSetData))
	}

	result := make(map[int]ResourceUsagePerContainer)
	for _, perc := range percentilesToCompute {
		data := make(ResourceUsagePerContainer)
		for k, v := range dataMap {
			percentileIndex := int(math.Ceil(float64(len(v.cpuData)*perc)/100)) - 1
			data[k] = &ContainerResourceUsage{
				Name:                    k,
				CPUUsageInCores:         v.cpuData[percentileIndex],
				MemoryUsageInBytes:      v.memUseData[percentileIndex],
				MemoryWorkingSetInBytes: v.memWorkSetData[percentileIndex],
			}
		}
		result[perc] = data
	}
	return result
}

func leftMergeData(left, right map[int]ResourceUsagePerContainer) map[int]ResourceUsagePerContainer {
	result := make(map[int]ResourceUsagePerContainer)
	for percentile, data := range left {
		result[percentile] = data
		if _, ok := right[percentile]; !ok {
			continue
		}
		for k, v := range right[percentile] {
			result[percentile][k] = v
		}
	}
	return result
}

type resourceGatherWorker struct {
	c                    *client.Client
	nodeName             string
	wg                   *sync.WaitGroup
	containerIDToNameMap map[string]string
	containerIDs         []string
	stopCh               chan struct{}
	dataSeries           []ResourceUsagePerContainer
	finished             bool
	inKubemark           bool
}

func (w *resourceGatherWorker) singleProbe() {
	var data ResourceUsagePerContainer
	if w.inKubemark {
		data = getKubemarkMasterComponentsResourceUsage()
		if data == nil {
			return
		}
	} else {
		data = make(ResourceUsagePerContainer)
		nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, probeDuration, func() []string { return w.containerIDs }, true)
		if err != nil {
			Logf("Error while reading data from %v: %v", w.nodeName, err)
			return
		}
		for k, v := range nodeUsage {
			data[w.containerIDToNameMap[k]] = v
		}
	}
	w.dataSeries = append(w.dataSeries, data)
}

func (w *resourceGatherWorker) gather(initialSleep time.Duration) {
	defer utilruntime.HandleCrash()
	defer w.wg.Done()
	defer Logf("Closing worker for %v", w.nodeName)
	defer func() { w.finished = true }()
	select {
	case <-time.After(initialSleep):
		w.singleProbe()
		for {
			select {
			case <-time.After(resourceDataGatheringPeriod):
				w.singleProbe()
			case <-w.stopCh:
				return
			}
		}
	case <-w.stopCh:
		return
	}
}

func getKubemarkMasterComponentsResourceUsage() ResourceUsagePerContainer {
	result := make(ResourceUsagePerContainer)
	sshResult, err := SSH("ps ax -o %cpu,rss,command | tail -n +2 | grep kube", GetMasterHost()+":22", TestContext.Provider)
	if err != nil {
		Logf("Error when trying to SSH to master machine. Skipping probe")
		return nil
	}
	scanner := bufio.NewScanner(strings.NewReader(sshResult.Stdout))
	for scanner.Scan() {
		var cpu float64
		var mem uint64
		var name string
		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d kubernetes/server/bin/%s", &cpu, &mem, &name)
		if name != "" {
			// Gatherer expects pod_name/container_name format
			fullName := name + "/" + name
			result[fullName] = &ContainerResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
		}
	}
	return result
}

func (g *containerResourceGatherer) getKubeSystemContainersResourceUsage(c *client.Client) {
	delayPeriod := resourceDataGatheringPeriod / time.Duration(len(g.workers))
	delay := time.Duration(0)
	for i := range g.workers {
		go g.workers[i].gather(delay)
		delay += delayPeriod
	}
	g.workerWg.Wait()
}

type containerResourceGatherer struct {
	client               *client.Client
	stopCh               chan struct{}
	workers              []resourceGatherWorker
	workerWg             sync.WaitGroup
	containerIDToNameMap map[string]string
	containerIDs         []string
	options              ResourceGathererOptions
}

type ResourceGathererOptions struct {
	inKubemark bool
	masterOnly bool
}

func NewResourceUsageGatherer(c *client.Client, options ResourceGathererOptions) (*containerResourceGatherer, error) {
	g := containerResourceGatherer{
		client:               c,
		stopCh:               make(chan struct{}),
		containerIDToNameMap: make(map[string]string),
		containerIDs:         make([]string, 0),
		options:              options,
	}

	if options.inKubemark {
		g.workerWg.Add(1)
		g.workers = append(g.workers, resourceGatherWorker{
			inKubemark: true,
			stopCh:     g.stopCh,
			wg:         &g.workerWg,
			finished:   false,
		})
	} else {
		pods, err := c.Pods("kube-system").List(api.ListOptions{})
		if err != nil {
			Logf("Error while listing Pods: %v", err)
			return nil, err
		}
		for _, pod := range pods.Items {
			for _, container := range pod.Status.ContainerStatuses {
				containerID := strings.TrimPrefix(container.ContainerID, "docker:/")
				g.containerIDToNameMap[containerID] = pod.Name + "/" + container.Name
				g.containerIDs = append(g.containerIDs, containerID)
			}
		}
		nodeList, err := c.Nodes().List(api.ListOptions{})
		if err != nil {
			Logf("Error while listing Nodes: %v", err)
			return nil, err
		}

		for _, node := range nodeList.Items {
			if !options.masterOnly || system.IsMasterNode(&node) {
				g.workerWg.Add(1)
				g.workers = append(g.workers, resourceGatherWorker{
					c:                    c,
					nodeName:             node.Name,
					wg:                   &g.workerWg,
					containerIDToNameMap: g.containerIDToNameMap,
					containerIDs:         g.containerIDs,
					stopCh:               g.stopCh,
					finished:             false,
					inKubemark:           false,
				})
				break
			}
		}
	}
	return &g, nil
}

// startGatheringData blocks until stopAndSummarize is called.
func (g *containerResourceGatherer) startGatheringData() {
	g.getKubeSystemContainersResourceUsage(g.client)
}

func (g *containerResourceGatherer) stopAndSummarize(percentiles []int, constraints map[string]ResourceConstraint) *ResourceUsageSummary {
	close(g.stopCh)
	Logf("Closed stop channel. Waiting for %v workers", len(g.workers))
	finished := make(chan struct{})
	go func() {
		g.workerWg.Wait()
		finished <- struct{}{}
	}()
	select {
	case <-finished:
		Logf("Waitgroup finished.")
	case <-time.After(2 * time.Minute):
		unfinished := make([]string, 0)
		for i := range g.workers {
			if !g.workers[i].finished {
				unfinished = append(unfinished, g.workers[i].nodeName)
			}
		}
		Logf("Timed out while waiting for waitgroup, some workers failed to finish: %v", unfinished)
	}

	if len(percentiles) == 0 {
		Logf("Warning! Empty percentile list for stopAndPrintData.")
		return &ResourceUsageSummary{}
	}
	data := make(map[int]ResourceUsagePerContainer)
	for i := range g.workers {
		if g.workers[i].finished {
			stats := computePercentiles(g.workers[i].dataSeries, percentiles)
			data = leftMergeData(stats, data)
		}
	}

	// Workers has been stopped. We need to gather data stored in them.
	sortedKeys := []string{}
	for name := range data[percentiles[0]] {
		sortedKeys = append(sortedKeys, name)
	}
	sort.Strings(sortedKeys)
	violatedConstraints := make([]string, 0)
	summary := make(ResourceUsageSummary)
	for _, perc := range percentiles {
		for _, name := range sortedKeys {
			usage := data[perc][name]
			summary[strconv.Itoa(perc)] = append(summary[strconv.Itoa(perc)], SingleContainerSummary{
				Name: name,
				Cpu:  usage.CPUUsageInCores,
				Mem:  usage.MemoryWorkingSetInBytes,
			})
			// Verifying 99th percentile of resource usage
			if perc == 99 {
				// Name has a form: <pod_name>/<container_name>
				containerName := strings.Split(name, "/")[1]
				if constraint, ok := constraints[containerName]; ok {
					if usage.CPUUsageInCores > constraint.CPUConstraint {
						violatedConstraints = append(
							violatedConstraints,
							fmt.Sprintf("Container %v is using %v/%v CPU",
								name,
								usage.CPUUsageInCores,
								constraint.CPUConstraint,
							),
						)
					}
					if usage.MemoryWorkingSetInBytes > constraint.MemoryConstraint {
						violatedConstraints = append(
							violatedConstraints,
							fmt.Sprintf("Container %v is using %v/%v MB of memory",
								name,
								float64(usage.MemoryWorkingSetInBytes)/(1024*1024),
								float64(constraint.MemoryConstraint)/(1024*1024),
							),
						)
					}
				}
			}
		}
	}
	Expect(violatedConstraints).To(BeEmpty())
	return &summary
}
<commit_msg>Prevent division by 0 error in resource-gatherer<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"bufio"
	"bytes"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
	"sync"
	"text/tabwriter"
	"time"

	. "github.com/onsi/gomega"
	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
	"k8s.io/kubernetes/pkg/util/system"
)

const (
	resourceDataGatheringPeriod = 60 * time.Second
	probeDuration               = 15 * time.Second
)

type ResourceConstraint struct {
	CPUConstraint    float64
	MemoryConstraint uint64
}

type SingleContainerSummary struct {
	Name string
	Cpu  float64
	Mem  uint64
}

// we can't have int here, as JSON does not accept integer keys.
type ResourceUsageSummary map[string][]SingleContainerSummary

func (s *ResourceUsageSummary) PrintHumanReadable() string {
	buf := &bytes.Buffer{}
	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
	for perc, summaries := range *s {
		buf.WriteString(fmt.Sprintf("%v percentile:\n", perc))
		fmt.Fprintf(w, "container\tcpu(cores)\tmemory(MB)\n")
		for _, summary := range summaries {
			fmt.Fprintf(w, "%q\t%.3f\t%.2f\n", summary.Name, summary.Cpu, float64(summary.Mem)/(1024*1024))
		}
		w.Flush()
	}
	return buf.String()
}

func (s *ResourceUsageSummary) PrintJSON() string {
	return PrettyPrintJSON(*s)
}

func computePercentiles(timeSeries []ResourceUsagePerContainer, percentilesToCompute []int) map[int]ResourceUsagePerContainer {
	if len(timeSeries) == 0 {
		return make(map[int]ResourceUsagePerContainer)
	}
	dataMap := make(map[string]*usageDataPerContainer)
	for i := range timeSeries {
		for name, data := range timeSeries[i] {
			if dataMap[name] == nil {
				dataMap[name] = &usageDataPerContainer{
					cpuData:        make([]float64, len(timeSeries)),
					memUseData:     make([]uint64, len(timeSeries)),
					memWorkSetData: make([]uint64, len(timeSeries)),
				}
			}
			dataMap[name].cpuData = append(dataMap[name].cpuData, data.CPUUsageInCores)
			dataMap[name].memUseData = append(dataMap[name].memUseData, data.MemoryUsageInBytes)
			dataMap[name].memWorkSetData = append(dataMap[name].memWorkSetData, data.MemoryWorkingSetInBytes)
		}
	}
	for _, v := range dataMap {
		sort.Float64s(v.cpuData)
		sort.Sort(uint64arr(v.memUseData))
		sort.Sort(uint64arr(v.memWorkSetData))
	}

	result := make(map[int]ResourceUsagePerContainer)
	for _, perc := range percentilesToCompute {
		data := make(ResourceUsagePerContainer)
		for k, v := range dataMap {
			percentileIndex := int(math.Ceil(float64(len(v.cpuData)*perc)/100)) - 1
			data[k] = &ContainerResourceUsage{
				Name:                    k,
				CPUUsageInCores:         v.cpuData[percentileIndex],
				MemoryUsageInBytes:      v.memUseData[percentileIndex],
				MemoryWorkingSetInBytes: v.memWorkSetData[percentileIndex],
			}
		}
		result[perc] = data
	}
	return result
}

func leftMergeData(left, right map[int]ResourceUsagePerContainer) map[int]ResourceUsagePerContainer {
	result := make(map[int]ResourceUsagePerContainer)
	for percentile, data := range left {
		result[percentile] = data
		if _, ok := right[percentile]; !ok {
			continue
		}
		for k, v := range right[percentile] {
			result[percentile][k] = v
		}
	}
	return result
}

type resourceGatherWorker struct {
	c                    *client.Client
	nodeName             string
	wg                   *sync.WaitGroup
	containerIDToNameMap map[string]string
	containerIDs         []string
	stopCh               chan struct{}
	dataSeries           []ResourceUsagePerContainer
	finished             bool
	inKubemark           bool
}

func (w *resourceGatherWorker) singleProbe() {
	var data ResourceUsagePerContainer
	if w.inKubemark {
		data = getKubemarkMasterComponentsResourceUsage()
		if data == nil {
			return
		}
	} else {
		data = make(ResourceUsagePerContainer)
		nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, probeDuration, func() []string { return w.containerIDs }, true)
		if err != nil {
			Logf("Error while reading data from %v: %v", w.nodeName, err)
			return
		}
		for k, v := range nodeUsage {
			data[w.containerIDToNameMap[k]] = v
		}
	}
	w.dataSeries = append(w.dataSeries, data)
}

func (w *resourceGatherWorker) gather(initialSleep time.Duration) {
	defer utilruntime.HandleCrash()
	defer w.wg.Done()
	defer Logf("Closing worker for %v", w.nodeName)
	defer func() { w.finished = true }()
	select {
	case <-time.After(initialSleep):
		w.singleProbe()
		for {
			select {
			case <-time.After(resourceDataGatheringPeriod):
				w.singleProbe()
			case <-w.stopCh:
				return
			}
		}
	case <-w.stopCh:
		return
	}
}

func getKubemarkMasterComponentsResourceUsage() ResourceUsagePerContainer {
	result := make(ResourceUsagePerContainer)
	sshResult, err := SSH("ps ax -o %cpu,rss,command | tail -n +2 | grep kube", GetMasterHost()+":22", TestContext.Provider)
	if err != nil {
		Logf("Error when trying to SSH to master machine. Skipping probe")
		return nil
	}
	scanner := bufio.NewScanner(strings.NewReader(sshResult.Stdout))
	for scanner.Scan() {
		var cpu float64
		var mem uint64
		var name string
		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d kubernetes/server/bin/%s", &cpu, &mem, &name)
		if name != "" {
			// Gatherer expects pod_name/container_name format
			fullName := name + "/" + name
			result[fullName] = &ContainerResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
		}
	}
	return result
}

func (g *containerResourceGatherer) getKubeSystemContainersResourceUsage(c *client.Client) {
	if len(g.workers) == 0 {
		return
	}
	delayPeriod := resourceDataGatheringPeriod / time.Duration(len(g.workers))
	delay := time.Duration(0)
	for i := range g.workers {
		go g.workers[i].gather(delay)
		delay += delayPeriod
	}
	g.workerWg.Wait()
}

type containerResourceGatherer struct {
	client               *client.Client
	stopCh               chan struct{}
	workers              []resourceGatherWorker
	workerWg             sync.WaitGroup
	containerIDToNameMap map[string]string
	containerIDs         []string
	options              ResourceGathererOptions
}

type ResourceGathererOptions struct {
	inKubemark bool
	masterOnly bool
}

func NewResourceUsageGatherer(c *client.Client, options ResourceGathererOptions) (*containerResourceGatherer, error) {
	g := containerResourceGatherer{
		client:               c,
		stopCh:               make(chan struct{}),
		containerIDToNameMap: make(map[string]string),
		containerIDs:         make([]string, 0),
		options:              options,
	}

	if options.inKubemark {
		g.workerWg.Add(1)
		g.workers = append(g.workers, resourceGatherWorker{
			inKubemark: true,
			stopCh:     g.stopCh,
			wg:         &g.workerWg,
			finished:   false,
		})
	} else {
		pods, err := c.Pods("kube-system").List(api.ListOptions{})
		if err != nil {
			Logf("Error while listing Pods: %v", err)
			return nil, err
		}
		for _, pod := range pods.Items {
			for _, container := range pod.Status.ContainerStatuses {
				containerID := strings.TrimPrefix(container.ContainerID, "docker:/")
				g.containerIDToNameMap[containerID] = pod.Name + "/" + container.Name
				g.containerIDs = append(g.containerIDs, containerID)
			}
		}
		nodeList, err := c.Nodes().List(api.ListOptions{})
		if err != nil {
			Logf("Error while listing Nodes: %v", err)
			return nil, err
		}

		for _, node := range nodeList.Items {
			if !options.masterOnly || system.IsMasterNode(&node) {
				g.workerWg.Add(1)
				g.workers = append(g.workers, resourceGatherWorker{
					c:                    c,
					nodeName:             node.Name,
					wg:                   &g.workerWg,
					containerIDToNameMap: g.containerIDToNameMap,
					containerIDs:         g.containerIDs,
					stopCh:               g.stopCh,
					finished:             false,
					inKubemark:           false,
				})
				break
			}
		}
	}
	return &g, nil
}

// startGatheringData blocks until stopAndSummarize is called.
func (g *containerResourceGatherer) startGatheringData() {
	g.getKubeSystemContainersResourceUsage(g.client)
}

func (g *containerResourceGatherer) stopAndSummarize(percentiles []int, constraints map[string]ResourceConstraint) *ResourceUsageSummary {
	close(g.stopCh)
	Logf("Closed stop channel. Waiting for %v workers", len(g.workers))
	finished := make(chan struct{})
	go func() {
		g.workerWg.Wait()
		finished <- struct{}{}
	}()
	select {
	case <-finished:
		Logf("Waitgroup finished.")
	case <-time.After(2 * time.Minute):
		unfinished := make([]string, 0)
		for i := range g.workers {
			if !g.workers[i].finished {
				unfinished = append(unfinished, g.workers[i].nodeName)
			}
		}
		Logf("Timed out while waiting for waitgroup, some workers failed to finish: %v", unfinished)
	}

	if len(percentiles) == 0 {
		Logf("Warning! Empty percentile list for stopAndPrintData.")
		return &ResourceUsageSummary{}
	}
	data := make(map[int]ResourceUsagePerContainer)
	for i := range g.workers {
		if g.workers[i].finished {
			stats := computePercentiles(g.workers[i].dataSeries, percentiles)
			data = leftMergeData(stats, data)
		}
	}

	// Workers has been stopped. We need to gather data stored in them.
	sortedKeys := []string{}
	for name := range data[percentiles[0]] {
		sortedKeys = append(sortedKeys, name)
	}
	sort.Strings(sortedKeys)
	violatedConstraints := make([]string, 0)
	summary := make(ResourceUsageSummary)
	for _, perc := range percentiles {
		for _, name := range sortedKeys {
			usage := data[perc][name]
			summary[strconv.Itoa(perc)] = append(summary[strconv.Itoa(perc)], SingleContainerSummary{
				Name: name,
				Cpu:  usage.CPUUsageInCores,
				Mem:  usage.MemoryWorkingSetInBytes,
			})
			// Verifying 99th percentile of resource usage
			if perc == 99 {
				// Name has a form: <pod_name>/<container_name>
				containerName := strings.Split(name, "/")[1]
				if constraint, ok := constraints[containerName]; ok {
					if usage.CPUUsageInCores > constraint.CPUConstraint {
						violatedConstraints = append(
							violatedConstraints,
							fmt.Sprintf("Container %v is using %v/%v CPU",
								name,
								usage.CPUUsageInCores,
								constraint.CPUConstraint,
							),
						)
					}
					if usage.MemoryWorkingSetInBytes > constraint.MemoryConstraint {
						violatedConstraints = append(
							violatedConstraints,
							fmt.Sprintf("Container %v is using %v/%v MB of memory",
								name,
								float64(usage.MemoryWorkingSetInBytes)/(1024*1024),
								float64(constraint.MemoryConstraint)/(1024*1024),
							),
						)
					}
				}
			}
		}
	}
	Expect(violatedConstraints).To(BeEmpty())
	return &summary
}
<|endoftext|>


<commit_before>package flag

import (
	"fmt"
	"os"
	"reflect"
)

var envParser = os.Getenv

type resolver struct {
	LastSet *FlagSet
}

func (r *resolver) fromDefault(f *Flag) []string {
	if !isSlicePtr(f.Ptr) {
		return []string{fmt.Sprint(f.Default)}
	}

	refval := reflect.ValueOf(f.Default)
	vals := make([]string, 0, refval.Len())
	for i, l := 0, refval.Len(); i < l; i++ {
		val := fmt.Sprint(refval.Index(i).Interface())
		if val != "" {
			vals = append(vals, val)
		}
	}
	return vals
}

func (r *resolver) fromEnv(f *Flag) []string {
	val := envParser(f.Env)
	if val == "" {
		return nil
	}

	var vals []string
	if isSlicePtr(f.Ptr) {
		vals = splitAndTrimSpace(val, f.ValSep)
	} else {
		vals = []string{val}
	}
	return vals
}

func (r *resolver) applyVals(f *Flag, vals ...string) error {
	for _, val := range vals {
		err := applyValToPtr(f.Names, f.Ptr, val, f.Selects)
		if err != nil {
			return err
		}
	}
	return nil
}

func (r *resolver) applyEnvAndDefault(f *FlagSet, applied map[*Flag]bool) error {
	for i := range f.flags {
		flag := &f.flags[i]
		if applied[flag] {
			continue
		}
		applied[flag] = true

		var vals []string
		if flag.Env != "" {
			vals = r.fromEnv(flag)
		}
		if len(vals) == 0 && flag.Default != nil {
			vals = r.fromDefault(flag)
		}
		err := r.applyVals(flag, vals...)
		if err != nil {
			return err
		}
	}
	return nil
}

func (r *resolver) resolveFlags(f *FlagSet, context []string, args []argument) error {
	var positional []*Flag
	for i := range f.flags {
		if f.flags[i].Names == flagNamePositional {
			positional = append(positional, &f.flags[i])
		}
	}
	var (
		applied = make(map[*Flag]bool)
		flag    *Flag
		err     error

		positionalIndex int
		applyValue      = func(flag *Flag, val string) error {
			applied[flag] = true
			return r.applyVals(flag, val)
		}
		applyLastFlag = func() error {
			if flag == nil {
				return nil
			}
			return newErrorf(errFlagValueNotProvided, "flag value is not provided: %v.%s", context, flag.Names)
		}
		hasFlag = func(args []argument) bool {
			for i := range args {
				if args[i].Type == argumentFlag {
					return true
				}
			}
			return false
		}
		appendNonFlagArg = func(arg argument, args []argument) error {
			if f.self.ArgsPtr == nil || (!f.self.ArgsAnywhere && hasFlag(args[1:])) {
				return newErrorf(errNonFlagValue, "unexpected non-flag value: %v %s", context, arg.Value)
			}
			if positionalIndex < len(positional) {
				err = applyValue(positional[positionalIndex], arg.Value)
				if err != nil {
					return err
				}
				positionalIndex++
				return nil
			}

			slice := *f.self.ArgsPtr
			slice = append(slice, arg.Value)
			*f.self.ArgsPtr = slice
			return nil
		}
	)

	for i, arg := range args {
		switch arg.Type {
		case argumentFlag:
			err = applyLastFlag()
			if err != nil {
				return err
			}

			flag = f.searchFlag(arg.Value)
			if flag == nil {
				return newErrorf(errFlagNotFound, "unsupported flag: %v.%s", context, arg.Value)
			}
			if applied[flag] && !isSlicePtr(flag.Ptr) {
				return newErrorf(errDuplicateFlagParsed, "duplicated flag: %v.%s", context, flag.Names)
			}

			if arg.AttachValid {
				// directly consume flag attached value
				err = applyValue(flag, arg.Attached)
				if err != nil {
					return err
				}
				flag = nil
			} else if isBoolPtr(flag.Ptr) {
				// bool flag should not consume next value to not affect positional or non flag parsing
				err = applyValue(flag, "true")
				if err != nil {
					return err
				}
				flag = nil
			}
		case argumentValue:
			if flag == nil {
				err = appendNonFlagArg(args[i], args[i:])
				if err != nil {
					return err
				}
			} else {
				err = applyValue(flag, arg.Value)
				if err != nil {
					return err
				}
				flag = nil
			}
		default:
			panic("unreachable")
		}
	}
	err = applyLastFlag()
	if err != nil {
		return err
	}
	if positionalIndex < len(positional) {
		var names []string
		for i := positionalIndex; i < len(positional); i++ {
			names = append(names, positional[i].Arglist)
		}
		return newErrorf(errPositionalFlagNotProvided, "flag not provided: %v.%v", context, names)
	}

	return r.applyEnvAndDefault(f, applied)
}

func (r *resolver) resolveSet(f *FlagSet, context []string, args *scanArgs) (lastSubset *FlagSet, err error) {
	context = append(context, f.self.Names)
	err = r.resolveFlags(f, context, args.Flags[1:])
	if err != nil {
		return nil, err
	}
	for sub, subArgs := range args.Sets {
		set := &f.subsets[f.subsetIndexes[sub]]
		err = r.applyVals(&set.self, "true")
		if err != nil {
			return nil, err
		}

		last, err := r.resolveSet(set, context, subArgs)
		if err != nil {
			return nil, err
		}
		if sub == args.FirstSubset {
			lastSubset = last
			if lastSubset == nil {
				lastSubset = set
			}
		}
	}

	if lastSubset == nil {
		lastSubset = f
	}
	return lastSubset, nil
}

func (r *resolver) resolve(f *FlagSet, args *scanArgs) error {
	var err error
	r.LastSet, err = r.resolveSet(f, nil, args)
	return err
}

func (r *resolver) reset(f *FlagSet) {
	if f.self.ArgsPtr != nil {
		*f.self.ArgsPtr = nil
	}
	resetPtrVal(f.self.Ptr)
	for i := range f.flags {
		resetPtrVal(f.flags[i].Ptr)
	}
	for i := range f.subsets {
		r.reset(&f.subsets[i])
	}
}
<commit_msg>don't enforce positional flag must appeared all.<commit_after>package flag

import (
	"fmt"
	"os"
	"reflect"
)

var envParser = os.Getenv

type resolver struct {
	LastSet *FlagSet
}

func (r *resolver) fromDefault(f *Flag) []string {
	if !isSlicePtr(f.Ptr) {
		return []string{fmt.Sprint(f.Default)}
	}

	refval := reflect.ValueOf(f.Default)
	vals := make([]string, 0, refval.Len())
	for i, l := 0, refval.Len(); i < l; i++ {
		val := fmt.Sprint(refval.Index(i).Interface())
		if val != "" {
			vals = append(vals, val)
		}
	}
	return vals
}

func (r *resolver) fromEnv(f *Flag) []string {
	val := envParser(f.Env)
	if val == "" {
		return nil
	}

	var vals []string
	if isSlicePtr(f.Ptr) {
		vals = splitAndTrimSpace(val, f.ValSep)
	} else {
		vals = []string{val}
	}
	return vals
}

func (r *resolver) applyVals(f *Flag, vals ...string) error {
	for _, val := range vals {
		err := applyValToPtr(f.Names, f.Ptr, val, f.Selects)
		if err != nil {
			return err
		}
	}
	return nil
}

func (r *resolver) applyEnvAndDefault(f *FlagSet, applied map[*Flag]bool) error {
	for i := range f.flags {
		flag := &f.flags[i]
		if applied[flag] {
			continue
		}
		applied[flag] = true

		var vals []string
		if flag.Env != "" {
			vals = r.fromEnv(flag)
		}
		if len(vals) == 0 && flag.Default != nil {
			vals = r.fromDefault(flag)
		}
		err := r.applyVals(flag, vals...)
		if err != nil {
			return err
		}
	}
	return nil
}

func (r *resolver) resolveFlags(f *FlagSet, context []string, args []argument) error {
	var positional []*Flag
	for i := range f.flags {
		if f.flags[i].Names == flagNamePositional {
			positional = append(positional, &f.flags[i])
		}
	}
	var (
		applied = make(map[*Flag]bool)
		flag    *Flag
		err     error

		positionalIndex int
		applyValue      = func(flag *Flag, val string) error {
			applied[flag] = true
			return r.applyVals(flag, val)
		}
		applyLastFlag = func() error {
			if flag == nil {
				return nil
			}
			return newErrorf(errFlagValueNotProvided, "flag value is not provided: %v.%s", context, flag.Names)
		}
		hasFlag = func(args []argument) bool {
			for i := range args {
				if args[i].Type == argumentFlag {
					return true
				}
			}
			return false
		}
		appendNonFlagArg = func(arg argument, args []argument) error {
			if f.self.ArgsPtr == nil || (!f.self.ArgsAnywhere && hasFlag(args[1:])) {
				return newErrorf(errNonFlagValue, "unexpected non-flag value: %v %s", context, arg.Value)
			}
			if positionalIndex < len(positional) {
				err = applyValue(positional[positionalIndex], arg.Value)
				if err != nil {
					return err
				}
				positionalIndex++
				return nil
			}

			slice := *f.self.ArgsPtr
			slice = append(slice, arg.Value)
			*f.self.ArgsPtr = slice
			return nil
		}
	)

	for i, arg := range args {
		switch arg.Type {
		case argumentFlag:
			err = applyLastFlag()
			if err != nil {
				return err
			}

			flag = f.searchFlag(arg.Value)
			if flag == nil {
				return newErrorf(errFlagNotFound, "unsupported flag: %v.%s", context, arg.Value)
			}
			if applied[flag] && !isSlicePtr(flag.Ptr) {
				return newErrorf(errDuplicateFlagParsed, "duplicated flag: %v.%s", context, flag.Names)
			}

			if arg.AttachValid {
				// directly consume flag attached value
				err = applyValue(flag, arg.Attached)
				if err != nil {
					return err
				}
				flag = nil
			} else if isBoolPtr(flag.Ptr) {
				// bool flag should not consume next value to not affect positional or non flag parsing
				err = applyValue(flag, "true")
				if err != nil {
					return err
				}
				flag = nil
			}
		case argumentValue:
			if flag == nil {
				err = appendNonFlagArg(args[i], args[i:])
				if err != nil {
					return err
				}
			} else {
				err = applyValue(flag, arg.Value)
				if err != nil {
					return err
				}
				flag = nil
			}
		default:
			panic("unreachable")
		}
	}
	err = applyLastFlag()
	if err != nil {
		return err
	}
	//if positionalIndex < len(positional) {
	//	var names []string
	//	for i := positionalIndex; i < len(positional); i++ {
	//		names = append(names, positional[i].Arglist)
	//	}
	//	return newErrorf(errPositionalFlagNotProvided, "flag not provided: %v.%v", context, names)
	//}

	return r.applyEnvAndDefault(f, applied)
}

func (r *resolver) resolveSet(f *FlagSet, context []string, args *scanArgs) (lastSubset *FlagSet, err error) {
	context = append(context, f.self.Names)
	err = r.resolveFlags(f, context, args.Flags[1:])
	if err != nil {
		return nil, err
	}
	for sub, subArgs := range args.Sets {
		set := &f.subsets[f.subsetIndexes[sub]]
		err = r.applyVals(&set.self, "true")
		if err != nil {
			return nil, err
		}

		last, err := r.resolveSet(set, context, subArgs)
		if err != nil {
			return nil, err
		}
		if sub == args.FirstSubset {
			lastSubset = last
			if lastSubset == nil {
				lastSubset = set
			}
		}
	}

	if lastSubset == nil {
		lastSubset = f
	}
	return lastSubset, nil
}

func (r *resolver) resolve(f *FlagSet, args *scanArgs) error {
	var err error
	r.LastSet, err = r.resolveSet(f, nil, args)
	return err
}

func (r *resolver) reset(f *FlagSet) {
	if f.self.ArgsPtr != nil {
		*f.self.ArgsPtr = nil
	}
	resetPtrVal(f.self.Ptr)
	for i := range f.flags {
		resetPtrVal(f.flags[i].Ptr)
	}
	for i := range f.subsets {
		r.reset(&f.subsets[i])
	}
}
<|endoftext|>