<commit_before>package commands

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

func migrateExportCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr)
	defer l.Close()

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	rewriter := getHistoryRewriter(cmd, db, l)

	filter := rewriter.Filter()
	if len(filter.Include()) <= 0 {
		ExitWithError(errors.Errorf("fatal: one or more files must be specified with --include"))
	}

	tracked := trackedFromExportFilter(filter)
	gitfilter := lfs.NewGitFilter(cfg)

	opts := &githistory.RewriteOptions{
		Verbose:           migrateVerbose,
		ObjectMapFilePath: objectMapFilePath,
		BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
			if filepath.Base(path) == ".gitattributes" {
				return b, nil
			}

			var buf bytes.Buffer

			if _, err := smudge(gitfilter, &buf, b.Contents, path, false, rewriter.Filter()); err != nil {
				return nil, err
			}

			return &odb.Blob{
				Contents: &buf, Size: int64(buf.Len()),
			}, nil
		},

		TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
			if path != "/" {
				// Ignore non-root trees.
				return t, nil
			}

			ours := tracked
			theirs, err := trackedFromAttrs(db, t)
			if err != nil {
				return nil, err
			}

			// Create a blob of the attributes that are optionally
			// present in the "t" tree's .gitattributes blob, and
			// union in the patterns that we've tracked.
			//
			// Perform this Union() operation each time we visit a
			// root tree such that if the underlying .gitattributes
			// is present and has a diff between commits in the
			// range of commits to migrate, those changes are
			// preserved.
			blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
			if err != nil {
				return nil, err
			}

			// Finally, return a copy of the tree "t" that has the
			// new .gitattributes file included/replaced.
			return t.Merge(&odb.TreeEntry{
				Name:     ".gitattributes",
				Filemode: 0100644,
				Oid:      blob,
			}), nil
		},

		UpdateRefs: true,
	}

	requireInRepo()

	opts, err = rewriteOptions(args, opts, l)
	if err != nil {
		ExitWithError(err)
	}

	// If we have a valid remote, pre-download all objects using the Transfer Queue
	if remoteURL := getAPIClient().Endpoints.RemoteEndpoint("download", cfg.Remote()).Url; remoteURL != "" {
		q := newDownloadQueue(getTransferManifestOperationRemote("Download", cfg.Remote()), cfg.Remote())
		if err := rewriter.ScanForPointers(q, opts, gitfilter); err != nil {
			ExitWithError(err)
		}

		q.Wait()

		for _, err := range q.Errors() {
			if err != nil {
				ExitWithError(err)
			}
		}
	}

	// Perform the rewrite
	if _, err := rewriter.Rewrite(opts); err != nil {
		ExitWithError(err)
	}

	// Only perform `git-checkout(1) -f` if the repository is
	// non-bare.
	if bare, _ := git.IsBare(); !bare {
		t := l.Waiter("migrate: checkout")
		err := git.Checkout("", nil, true)
		t.Complete()

		if err != nil {
			ExitWithError(err)
		}
	}
}

// trackedFromExportFilter returns an ordered set of strings where each entry is a
// line in the .gitattributes file. It adds/removes the fiter/diff/merge=lfs
// attributes based on patterns included/excluded in the given filter. Since
// `migrate export` removes files from Git LFS, it will remove attributes for included
// files, and add attributes for excluded files
func trackedFromExportFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
	tracked := tools.NewOrderedSet()

	for _, include := range filter.Include() {
		tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", escapeAttrPattern(include)))
	}

	for _, exclude := range filter.Exclude() {
		tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(exclude)))
	}

	return tracked
}
<commit_msg>commands: fix comment strings<commit_after>package commands

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/git/odb"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/spf13/cobra"
)

func migrateExportCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr)
	defer l.Close()

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	rewriter := getHistoryRewriter(cmd, db, l)

	filter := rewriter.Filter()
	if len(filter.Include()) <= 0 {
		ExitWithError(errors.Errorf("fatal: one or more files must be specified with --include"))
	}

	tracked := trackedFromExportFilter(filter)
	gitfilter := lfs.NewGitFilter(cfg)

	opts := &githistory.RewriteOptions{
		Verbose:           migrateVerbose,
		ObjectMapFilePath: objectMapFilePath,
		BlobFn: func(path string, b *odb.Blob) (*odb.Blob, error) {
			if filepath.Base(path) == ".gitattributes" {
				return b, nil
			}

			var buf bytes.Buffer

			if _, err := smudge(gitfilter, &buf, b.Contents, path, false, rewriter.Filter()); err != nil {
				return nil, err
			}

			return &odb.Blob{
				Contents: &buf, Size: int64(buf.Len()),
			}, nil
		},

		TreeCallbackFn: func(path string, t *odb.Tree) (*odb.Tree, error) {
			if path != "/" {
				// Ignore non-root trees.
				return t, nil
			}

			ours := tracked
			theirs, err := trackedFromAttrs(db, t)
			if err != nil {
				return nil, err
			}

			// Create a blob of the attributes that are optionally
			// present in the "t" tree's .gitattributes blob, and
			// union in the patterns that we've tracked.
			//
			// Perform this Union() operation each time we visit a
			// root tree such that if the underlying .gitattributes
			// is present and has a diff between commits in the
			// range of commits to migrate, those changes are
			// preserved.
			blob, err := trackedToBlob(db, theirs.Clone().Union(ours))
			if err != nil {
				return nil, err
			}

			// Finally, return a copy of the tree "t" that has the
			// new .gitattributes file included/replaced.
			return t.Merge(&odb.TreeEntry{
				Name:     ".gitattributes",
				Filemode: 0100644,
				Oid:      blob,
			}), nil
		},

		UpdateRefs: true,
	}

	requireInRepo()

	opts, err = rewriteOptions(args, opts, l)
	if err != nil {
		ExitWithError(err)
	}

	// If we have a valid remote, pre-download all objects using the Transfer Queue
	if remoteURL := getAPIClient().Endpoints.RemoteEndpoint("download", cfg.Remote()).Url; remoteURL != "" {
		q := newDownloadQueue(getTransferManifestOperationRemote("Download", cfg.Remote()), cfg.Remote())
		if err := rewriter.ScanForPointers(q, opts, gitfilter); err != nil {
			ExitWithError(err)
		}

		q.Wait()

		for _, err := range q.Errors() {
			if err != nil {
				ExitWithError(err)
			}
		}
	}

	// Perform the rewrite
	if _, err := rewriter.Rewrite(opts); err != nil {
		ExitWithError(err)
	}

	// Only perform `git-checkout(1) -f` if the repository is non-bare.
	if bare, _ := git.IsBare(); !bare {
		t := l.Waiter("migrate: checkout")
		err := git.Checkout("", nil, true)
		t.Complete()

		if err != nil {
			ExitWithError(err)
		}
	}
}

// trackedFromExportFilter returns an ordered set of strings where each entry
// is a line we intend to place in the .gitattributes file. It adds/removes the
// filter/diff/merge=lfs attributes based on patterns included/excluded in the
// given filter. Since `migrate export` removes files from Git LFS, it will
// remove attributes for included files, and add attributes for excluded files
func trackedFromExportFilter(filter *filepathfilter.Filter) *tools.OrderedSet {
	tracked := tools.NewOrderedSet()

	for _, include := range filter.Include() {
		tracked.Add(fmt.Sprintf("%s text -filter -merge -diff", escapeAttrPattern(include)))
	}

	for _, exclude := range filter.Exclude() {
		tracked.Add(fmt.Sprintf("%s filter=lfs diff=lfs merge=lfs -text", escapeAttrPattern(exclude)))
	}

	return tracked
}
<|endoftext|>


<commit_before>package bootstrapCmd

import (
	// Stdlib
	"errors"
	"fmt"
	"os"
	"sort"

	// Internal
	"github.com/salsaflow/salsaflow/action"
	"github.com/salsaflow/salsaflow/app"
	"github.com/salsaflow/salsaflow/app/appflags"
	"github.com/salsaflow/salsaflow/config"
	"github.com/salsaflow/salsaflow/config/loader"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules"
	"github.com/salsaflow/salsaflow/prompt"

	// Other
	"gopkg.in/tchap/gocli.v2"
)

var Command = &gocli.Command{
	UsageLine: `
  bootstrap -skeleton=SKELETON [-skeleton_only]

  bootstrap -no_skeleton`,
	Short: "bootstrap repository for SalsaFlow",
	Long: `
  Bootstrap the repository for SalsaFlow.

  This command should be used to set up the local configuration directory
  for SalsaFlow (the directory that is then committed into the repository).

  The user is prompted for all necessary data.

  The -skeleton flag can be used to specify the repository to be used
  for custom scripts. It expects a string of "$OWNER/$REPO" and then uses
  the repository located at github.com/$OWNER/$REPO. It clones the repository
  and copies the content into the local configuration directory.

  In case no skeleton is to be used to bootstrap the repository,
  -no_skeleton must be specified explicitly.

  In case the repository is bootstrapped, but the skeleton is missing,
  it can be added by specifying -skeleton=SKELETON -skeleton_only.
  That will skip the configuration file generation step.
	`,
	Action: run,
}

var (
	flagNoSkeleton   bool
	flagSkeleton     string
	flagSkeletonOnly bool
)

func init() {
	// Register flags.
	Command.Flags.BoolVar(&flagNoSkeleton, "no_skeleton", flagNoSkeleton,
		"do not use any skeleton to bootstrap the repository")
	Command.Flags.StringVar(&flagSkeleton, "skeleton", flagSkeleton,
		"skeleton to be used to bootstrap the repository")
	Command.Flags.BoolVar(&flagSkeletonOnly, "skeleton_only", flagSkeletonOnly,
		"skip the config dialog and only install the skeleton")

	// Register global flags.
	appflags.RegisterGlobalFlags(&Command.Flags)
}

func run(cmd *gocli.Command, args []string) {
	if len(args) != 0 {
		cmd.Usage()
		os.Exit(2)
	}

	app.InitLogging()

	defer prompt.RecoverCancel()

	if err := runMain(cmd); err != nil {
		errs.Fatal(err)
	}
}

func runMain(cmd *gocli.Command) (err error) {
	// Validate CL flags.
	task := "Check the command line flags"
	switch {
	case flagSkeleton == "" && !flagNoSkeleton:
		cmd.Usage()
		return errs.NewError(
			task, errors.New("-no_skeleton must be specified when no skeleton is given"))

	case flagSkeletonOnly && flagSkeleton == "":
		cmd.Usage()
		return errs.NewError(
			task, errors.New("-skeleton must be specified when -skeleton_only is set"))
	}

	// Make sure the local config directory exists.
	act, err := ensureLocalConfigDirectoryExists()
	if err != nil {
		return err
	}
	action.RollbackOnError(&err, act)

	// Set up the global and local configuration file unless -skeleton_only.
	if !flagSkeletonOnly {
		if err := assembleAndWriteConfig(); err != nil {
			return err
		}
	}

	// Install the skeleton into the local config directory if desired.
	if skeleton := flagSkeleton; skeleton != "" {
		if err := getAndPourSkeleton(skeleton); err != nil {
			return err
		}
	}

	fmt.Println()
	log.Log("Successfully bootstrapped the repository for SalsaFlow")
	log.NewLine("Do not forget to commit modified configuration files!")
	return nil
}

func ensureLocalConfigDirectoryExists() (action.Action, error) {
	task := "Make sure the local configuration directory exists"

	// Get the directory absolute path.
	localConfigDir, err := config.LocalConfigDirectoryAbsolutePath()
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	// In case the path exists, make sure it is a directory.
	info, err := os.Stat(localConfigDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, errs.NewError(task, err)
		}
	} else {
		if !info.IsDir() {
			return nil, errs.NewError(task, fmt.Errorf("not a directory: %v", localConfigDir))
		}
		return action.Noop, nil
	}

	// Otherwise create the directory.
	if err := os.MkdirAll(localConfigDir, 0755); err != nil {
		return nil, errs.NewError(task, err)
	}

	// Return the rollback function.
	act := action.ActionFunc(func() error {
		// Delete the directory.
		log.Rollback(task)
		task := "Delete the local configuration directory"
		if err := os.RemoveAll(localConfigDir); err != nil {
			return errs.NewError(task, err)
		}
		return nil
	})
	return act, nil
}

func assembleAndWriteConfig() error {
	// Group available modules by kind.
	var (
		issueTrackingModules []loader.Module
		codeReviewModules    []loader.Module
		releaseNotesModules  []loader.Module
	)
	groups := groupModulesByKind(modules.AvailableModules())
	for _, group := range groups {
		switch group[0].Kind() {
		case loader.ModuleKindIssueTracking:
			issueTrackingModules = group
		case loader.ModuleKindCodeReview:
			codeReviewModules = group
		case loader.ModuleKindReleaseNotes:
			releaseNotesModules = group
		}
	}

	sort.Sort(commonModules(issueTrackingModules))
	sort.Sort(commonModules(codeReviewModules))
	sort.Sort(commonModules(releaseNotesModules))

	// Run the common dialog.
	task := "Run the core configuration dialog"
	if err := loader.RunCommonBootstrapDialog(); err != nil {
		return errs.NewError(task, err)
	}

	// Run the dialog.
	task = "Run the modules configuration dialog"
	err := loader.RunModuleBootstrapDialog(
		&loader.ModuleDialogSection{issueTrackingModules, false},
		&loader.ModuleDialogSection{codeReviewModules, false},
		&loader.ModuleDialogSection{releaseNotesModules, true},
	)
	if err != nil {
		return errs.NewError(task, err)
	}

	return nil
}

func getAndPourSkeleton(skeleton string) error {
	// Get or update given skeleton.
	task := fmt.Sprintf("Get or update skeleton '%v'", skeleton)
	log.Run(task)
	if err := getOrUpdateSkeleton(flagSkeleton); err != nil {
		return errs.NewError(task, err)
	}

	// Move the skeleton files into place.
	task = "Copy the skeleton into the configuration directory"
	log.Go(task)

	localConfigDir, err := config.LocalConfigDirectoryAbsolutePath()
	if err != nil {
		return errs.NewError(task, err)
	}

	log.NewLine("")
	if err := pourSkeleton(flagSkeleton, localConfigDir); err != nil {
		return errs.NewError(task, err)
	}
	log.NewLine("")
	log.Ok(task)

	return nil
}
<commit_msg>repo bootstrap: Fix multiple sorting<commit_after>package bootstrapCmd

import (
	// Stdlib
	"errors"
	"fmt"
	"os"

	// Internal
	"github.com/salsaflow/salsaflow/action"
	"github.com/salsaflow/salsaflow/app"
	"github.com/salsaflow/salsaflow/app/appflags"
	"github.com/salsaflow/salsaflow/config"
	"github.com/salsaflow/salsaflow/config/loader"
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/log"
	"github.com/salsaflow/salsaflow/modules"
	"github.com/salsaflow/salsaflow/prompt"

	// Other
	"gopkg.in/tchap/gocli.v2"
)

var Command = &gocli.Command{
	UsageLine: `
  bootstrap -skeleton=SKELETON [-skeleton_only]

  bootstrap -no_skeleton`,
	Short: "bootstrap repository for SalsaFlow",
	Long: `
  Bootstrap the repository for SalsaFlow.

  This command should be used to set up the local configuration directory
  for SalsaFlow (the directory that is then committed into the repository).

  The user is prompted for all necessary data.

  The -skeleton flag can be used to specify the repository to be used
  for custom scripts. It expects a string of "$OWNER/$REPO" and then uses
  the repository located at github.com/$OWNER/$REPO. It clones the repository
  and copies the content into the local configuration directory.

  In case no skeleton is to be used to bootstrap the repository,
  -no_skeleton must be specified explicitly.

  In case the repository is bootstrapped, but the skeleton is missing,
  it can be added by specifying -skeleton=SKELETON -skeleton_only.
  That will skip the configuration file generation step.
	`,
	Action: run,
}

var (
	flagNoSkeleton   bool
	flagSkeleton     string
	flagSkeletonOnly bool
)

func init() {
	// Register flags.
	Command.Flags.BoolVar(&flagNoSkeleton, "no_skeleton", flagNoSkeleton,
		"do not use any skeleton to bootstrap the repository")
	Command.Flags.StringVar(&flagSkeleton, "skeleton", flagSkeleton,
		"skeleton to be used to bootstrap the repository")
	Command.Flags.BoolVar(&flagSkeletonOnly, "skeleton_only", flagSkeletonOnly,
		"skip the config dialog and only install the skeleton")

	// Register global flags.
	appflags.RegisterGlobalFlags(&Command.Flags)
}

func run(cmd *gocli.Command, args []string) {
	if len(args) != 0 {
		cmd.Usage()
		os.Exit(2)
	}

	app.InitLogging()

	defer prompt.RecoverCancel()

	if err := runMain(cmd); err != nil {
		errs.Fatal(err)
	}
}

func runMain(cmd *gocli.Command) (err error) {
	// Validate CL flags.
	task := "Check the command line flags"
	switch {
	case flagSkeleton == "" && !flagNoSkeleton:
		cmd.Usage()
		return errs.NewError(
			task, errors.New("-no_skeleton must be specified when no skeleton is given"))

	case flagSkeletonOnly && flagSkeleton == "":
		cmd.Usage()
		return errs.NewError(
			task, errors.New("-skeleton must be specified when -skeleton_only is set"))
	}

	// Make sure the local config directory exists.
	act, err := ensureLocalConfigDirectoryExists()
	if err != nil {
		return err
	}
	action.RollbackOnError(&err, act)

	// Set up the global and local configuration file unless -skeleton_only.
	if !flagSkeletonOnly {
		if err := assembleAndWriteConfig(); err != nil {
			return err
		}
	}

	// Install the skeleton into the local config directory if desired.
	if skeleton := flagSkeleton; skeleton != "" {
		if err := getAndPourSkeleton(skeleton); err != nil {
			return err
		}
	}

	fmt.Println()
	log.Log("Successfully bootstrapped the repository for SalsaFlow")
	log.NewLine("Do not forget to commit modified configuration files!")
	return nil
}

func ensureLocalConfigDirectoryExists() (action.Action, error) {
	task := "Make sure the local configuration directory exists"

	// Get the directory absolute path.
	localConfigDir, err := config.LocalConfigDirectoryAbsolutePath()
	if err != nil {
		return nil, errs.NewError(task, err)
	}

	// In case the path exists, make sure it is a directory.
	info, err := os.Stat(localConfigDir)
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, errs.NewError(task, err)
		}
	} else {
		if !info.IsDir() {
			return nil, errs.NewError(task, fmt.Errorf("not a directory: %v", localConfigDir))
		}
		return action.Noop, nil
	}

	// Otherwise create the directory.
	if err := os.MkdirAll(localConfigDir, 0755); err != nil {
		return nil, errs.NewError(task, err)
	}

	// Return the rollback function.
	act := action.ActionFunc(func() error {
		// Delete the directory.
		log.Rollback(task)
		task := "Delete the local configuration directory"
		if err := os.RemoveAll(localConfigDir); err != nil {
			return errs.NewError(task, err)
		}
		return nil
	})
	return act, nil
}

func assembleAndWriteConfig() error {
	// Group available modules by kind.
	var (
		issueTrackingModules []loader.Module
		codeReviewModules    []loader.Module
		releaseNotesModules  []loader.Module
	)
	groups := groupModulesByKind(modules.AvailableModules())
	for _, group := range groups {
		switch group[0].Kind() {
		case loader.ModuleKindIssueTracking:
			issueTrackingModules = group
		case loader.ModuleKindCodeReview:
			codeReviewModules = group
		case loader.ModuleKindReleaseNotes:
			releaseNotesModules = group
		}
	}

	// Run the common dialog.
	task := "Run the core configuration dialog"
	if err := loader.RunCommonBootstrapDialog(); err != nil {
		return errs.NewError(task, err)
	}

	// Run the dialog.
	task = "Run the modules configuration dialog"
	err := loader.RunModuleBootstrapDialog(
		&loader.ModuleDialogSection{issueTrackingModules, false},
		&loader.ModuleDialogSection{codeReviewModules, false},
		&loader.ModuleDialogSection{releaseNotesModules, true},
	)
	if err != nil {
		return errs.NewError(task, err)
	}

	return nil
}

func getAndPourSkeleton(skeleton string) error {
	// Get or update given skeleton.
	task := fmt.Sprintf("Get or update skeleton '%v'", skeleton)
	log.Run(task)
	if err := getOrUpdateSkeleton(flagSkeleton); err != nil {
		return errs.NewError(task, err)
	}

	// Move the skeleton files into place.
	task = "Copy the skeleton into the configuration directory"
	log.Go(task)

	localConfigDir, err := config.LocalConfigDirectoryAbsolutePath()
	if err != nil {
		return errs.NewError(task, err)
	}

	log.NewLine("")
	if err := pourSkeleton(flagSkeleton, localConfigDir); err != nil {
		return errs.NewError(task, err)
	}
	log.NewLine("")
	log.Ok(task)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/clems4ever/authelia/internal/utils"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var arch string
var supportedArch = []string{"amd64", "arm32v7", "arm64v8"}
var defaultArch = "amd64"

func init() {
	DockerBuildCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))
	DockerPushCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))

}

func checkArchIsSupported(arch string) {
	for _, a := range supportedArch {
		if arch == a {
			return
		}
	}
	log.Fatal("Architecture is not supported. Please select one of " + strings.Join(supportedArch, ", ") + ".")
}

func dockerBuildOfficialImage(arch string) error {
	docker := &Docker{}
	// Set default Architecture Dockerfile to amd64
	dockerfile := "Dockerfile"

	// If not the default value
	if arch != defaultArch {
		dockerfile = fmt.Sprintf("%s.%s", dockerfile, arch)
	}

	if arch == "arm32v7" {
		err := utils.CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = utils.CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-arm-static -O ./qemu-arm-static && chmod +x ./qemu-arm-static").Run()

		if err != nil {
			panic(err)
		}
	} else if arch == "arm64v8" {
		err := utils.CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = utils.CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-aarch64-static -O ./qemu-aarch64-static && chmod +x ./qemu-aarch64-static").Run()

		if err != nil {
			panic(err)
		}
	}

	return docker.Build(IntermediateDockerImageName, dockerfile, ".")
}

// DockerBuildCmd Command for building docker image of Authelia.
var DockerBuildCmd = &cobra.Command{
	Use:   "build",
	Short: "Build the docker image of Authelia",
	Run: func(cmd *cobra.Command, args []string) {
		log.Infof("Building Docker image %s...", DockerImageName)
		checkArchIsSupported(arch)
		err := dockerBuildOfficialImage(arch)

		if err != nil {
			log.Fatal(err)
		}

		docker := &Docker{}
		err = docker.Tag(IntermediateDockerImageName, DockerImageName)

		if err != nil {
			panic(err)
		}
	},
}

// DockerPushCmd Command for pushing Authelia docker image to Dockerhub
var DockerPushCmd = &cobra.Command{
	Use:   "push-image",
	Short: "Publish Authelia docker image to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		log.Infof("Pushing Docker image %s to dockerhub...", DockerImageName)
		checkArchIsSupported(arch)
		publishDockerImage(arch)
	},
}

// DockerManifestCmd Command for pushing Authelia docker manifest to Dockerhub
var DockerManifestCmd = &cobra.Command{
	Use:   "push-manifest",
	Short: "Publish Authelia docker manifest to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		log.Infof("Pushing Docker manifest of %s to dockerhub...", DockerImageName)
		publishDockerManifest()
	},
}

func login(docker *Docker) {
	username := os.Getenv("DOCKER_USERNAME")
	password := os.Getenv("DOCKER_PASSWORD")

	if username == "" {
		log.Fatal(errors.New("DOCKER_USERNAME is empty"))
	}

	if password == "" {
		log.Fatal(errors.New("DOCKER_PASSWORD is empty"))
	}

	log.Debug("Login to dockerhub as " + username)
	err := docker.Login(username, password)

	if err != nil {
		log.Fatal("Login to dockerhub failed", err)
	}
}

func deploy(docker *Docker, tag string) {
	imageWithTag := DockerImageName + ":" + tag

	log.Debug("Docker image " + imageWithTag + " will be deployed on Dockerhub.")

	if err := docker.Tag(DockerImageName, imageWithTag); err != nil {
		log.Fatal(err)
	}

	if err := docker.Push(imageWithTag); err != nil {
		log.Fatal(err)
	}
}

func deployManifest(docker *Docker, tag string, amd64tag string, arm32v7tag string, arm64v8tag string) {
	dockerImagePrefix := DockerImageName + ":"

	log.Debug("Docker manifest " + dockerImagePrefix + tag + " will be deployed on Dockerhub.")

	err := docker.Manifest(dockerImagePrefix+tag, dockerImagePrefix+amd64tag, dockerImagePrefix+arm32v7tag, dockerImagePrefix+arm64v8tag)

	if err != nil {
		log.Fatal(err)
	}

	tags := []string{amd64tag, arm32v7tag, arm64v8tag}
	for _, t := range tags {
		log.Debug("Docker removing tag for " + dockerImagePrefix + t + " on Dockerhub.")

		if err := docker.CleanTag(t); err != nil {
			panic(err)
		}
	}

	log.Debug("Docker pushing README.md to Dockerhub.")

	if err := docker.PublishReadme(); err != nil {
		log.Fatal(err)
	}
}

func publishDockerImage(arch string) {
	docker := &Docker{}

	travisBranch := os.Getenv("TRAVIS_BRANCH")
	travisPullRequest := os.Getenv("TRAVIS_PULL_REQUEST")
	travisTag := os.Getenv("TRAVIS_TAG")

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deploy(docker, "master-"+arch)
	} else if travisTag != "" {
		login(docker)
		deploy(docker, travisTag+"-"+arch)
		deploy(docker, "latest-"+arch)
	} else {
		log.Info("Docker image will not be published")
	}
}

func publishDockerManifest() {
	docker := &Docker{}

	travisBranch := os.Getenv("TRAVIS_BRANCH")
	travisPullRequest := os.Getenv("TRAVIS_PULL_REQUEST")
	travisTag := os.Getenv("TRAVIS_TAG")
	ignoredSuffixes := regexp.MustCompile("alpha|beta")

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8")
	} else if travisTag != "" {
		login(docker)
		deployManifest(docker, travisTag, travisTag+"-amd64", travisTag+"-arm32v7", travisTag+"-arm64v8")
		if !ignoredSuffixes.MatchString(travisTag) {
			deployManifest(docker, "latest", "latest-amd64", "latest-arm32v7", "latest-arm64v8")
		}
	} else {
		fmt.Println("Docker manifest will not be published")
	}
}
<commit_msg>Publish additional minor and major tags on DockerHub<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/clems4ever/authelia/internal/utils"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var arch string
var supportedArch = []string{"amd64", "arm32v7", "arm64v8"}
var defaultArch = "amd64"
var travisBranch = os.Getenv("TRAVIS_BRANCH")
var travisPullRequest = os.Getenv("TRAVIS_PULL_REQUEST")
var travisTag = os.Getenv("TRAVIS_TAG")
var dockerTags = regexp.MustCompile(`(?P<Minor>(?P<Major>v\d+)\.\d+)\.\d+.*`)
var ignoredSuffixes = regexp.MustCompile("alpha|beta")
var tags = dockerTags.FindStringSubmatch(travisTag)

func init() {
	DockerBuildCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))
	DockerPushCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))

}

func checkArchIsSupported(arch string) {
	for _, a := range supportedArch {
		if arch == a {
			return
		}
	}
	log.Fatal("Architecture is not supported. Please select one of " + strings.Join(supportedArch, ", ") + ".")
}

func dockerBuildOfficialImage(arch string) error {
	docker := &Docker{}
	// Set default Architecture Dockerfile to amd64
	dockerfile := "Dockerfile"

	// If not the default value
	if arch != defaultArch {
		dockerfile = fmt.Sprintf("%s.%s", dockerfile, arch)
	}

	if arch == "arm32v7" {
		err := utils.CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = utils.CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-arm-static -O ./qemu-arm-static && chmod +x ./qemu-arm-static").Run()

		if err != nil {
			panic(err)
		}
	} else if arch == "arm64v8" {
		err := utils.CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = utils.CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-aarch64-static -O ./qemu-aarch64-static && chmod +x ./qemu-aarch64-static").Run()

		if err != nil {
			panic(err)
		}
	}

	return docker.Build(IntermediateDockerImageName, dockerfile, ".")
}

// DockerBuildCmd Command for building docker image of Authelia.
var DockerBuildCmd = &cobra.Command{
	Use:   "build",
	Short: "Build the docker image of Authelia",
	Run: func(cmd *cobra.Command, args []string) {
		log.Infof("Building Docker image %s...", DockerImageName)
		checkArchIsSupported(arch)
		err := dockerBuildOfficialImage(arch)

		if err != nil {
			log.Fatal(err)
		}

		docker := &Docker{}
		err = docker.Tag(IntermediateDockerImageName, DockerImageName)

		if err != nil {
			panic(err)
		}
	},
}

// DockerPushCmd Command for pushing Authelia docker image to Dockerhub
var DockerPushCmd = &cobra.Command{
	Use:   "push-image",
	Short: "Publish Authelia docker image to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		log.Infof("Pushing Docker image %s to dockerhub...", DockerImageName)
		checkArchIsSupported(arch)
		publishDockerImage(arch)
	},
}

// DockerManifestCmd Command for pushing Authelia docker manifest to Dockerhub
var DockerManifestCmd = &cobra.Command{
	Use:   "push-manifest",
	Short: "Publish Authelia docker manifest to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		log.Infof("Pushing Docker manifest of %s to dockerhub...", DockerImageName)
		publishDockerManifest()
	},
}

func login(docker *Docker) {
	username := os.Getenv("DOCKER_USERNAME")
	password := os.Getenv("DOCKER_PASSWORD")

	if username == "" {
		log.Fatal(errors.New("DOCKER_USERNAME is empty"))
	}

	if password == "" {
		log.Fatal(errors.New("DOCKER_PASSWORD is empty"))
	}

	log.Debug("Login to dockerhub as " + username)
	err := docker.Login(username, password)

	if err != nil {
		log.Fatal("Login to dockerhub failed", err)
	}
}

func deploy(docker *Docker, tag string) {
	imageWithTag := DockerImageName + ":" + tag

	log.Debug("Docker image " + imageWithTag + " will be deployed on Dockerhub")

	if err := docker.Tag(DockerImageName, imageWithTag); err != nil {
		log.Fatal(err)
	}

	if err := docker.Push(imageWithTag); err != nil {
		log.Fatal(err)
	}
}

func deployManifest(docker *Docker, tag string, amd64tag string, arm32v7tag string, arm64v8tag string) {
	dockerImagePrefix := DockerImageName + ":"

	log.Debug("Docker manifest " + dockerImagePrefix + tag + " will be deployed on Dockerhub")

	err := docker.Manifest(dockerImagePrefix+tag, dockerImagePrefix+amd64tag, dockerImagePrefix+arm32v7tag, dockerImagePrefix+arm64v8tag)

	if err != nil {
		log.Fatal(err)
	}

	tags := []string{amd64tag, arm32v7tag, arm64v8tag}
	for _, t := range tags {
		log.Debug("Docker removing tag for " + dockerImagePrefix + t + " on Dockerhub.")

		if err := docker.CleanTag(t); err != nil {
			panic(err)
		}
	}

	log.Debug("Docker pushing README.md to Dockerhub")

	if err := docker.PublishReadme(); err != nil {
		log.Fatal(err)
	}
}

func publishDockerImage(arch string) {
	docker := &Docker{}

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deploy(docker, "master-"+arch)
	} else if travisTag != "" {
		if len(tags) == 3 {
			login(docker)
			deploy(docker, tags[0]+"-"+arch)
		} else {
			log.Fatal("Docker image will not be published, the specified tag does not conform to the standard")
		}
		if !ignoredSuffixes.MatchString(travisTag) {
			deploy(docker, tags[1]+"-"+arch)
			deploy(docker, tags[2]+"-"+arch)
			deploy(docker, "latest-"+arch)
		}
	} else {
		log.Info("Docker image will not be published")
	}
}

func publishDockerManifest() {
	docker := &Docker{}

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8")
	} else if travisTag != "" {
		if len(tags) == 3 {
			login(docker)
			deployManifest(docker, tags[0], tags[0]+"-amd64", tags[0]+"-arm32v7", tags[0]+"-arm64v8")
		} else {
			log.Fatal("Docker manifest will not be published, the specified tag does not conform to the standard")
		}
		if !ignoredSuffixes.MatchString(travisTag) {
			deployManifest(docker, tags[1], tags[1]+"-amd64", tags[1]+"-arm32v7", tags[1]+"-arm64v8")
			deployManifest(docker, tags[2], tags[2]+"-amd64", tags[2]+"-arm32v7", tags[2]+"-arm64v8")
			deployManifest(docker, "latest", "latest-amd64", "latest-arm32v7", "latest-arm64v8")
		}
	} else {
		fmt.Println("Docker manifest will not be published")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
// existing IDL tags on the Go struct.
package protobuf

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/parser"
	"k8s.io/gengo/types"

	flag "github.com/spf13/pflag"
)

type Generator struct {
	Common               args.GeneratorArgs
	APIMachineryPackages string
	Packages             string
	OutputBase           string
	VendorOutputBase     string
	ProtoImport          []string
	Conditional          string
	Clean                bool
	OnlyIDL              bool
	KeepGogoproto        bool
	SkipGeneratedRewrite bool
	DropEmbeddedFields   string
}

func New() *Generator {
	sourceTree := args.DefaultSourceTree()
	common := args.GeneratorArgs{
		OutputBase:       sourceTree,
		GoHeaderFilePath: filepath.Join(sourceTree, "k8s.io/kubernetes/hack/boilerplate/boilerplate.go.txt"),
	}
	defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "vendor", "github.com", "gogo", "protobuf", "protobuf")
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Cannot get current directory.")
	}
	return &Generator{
		Common:           common,
		OutputBase:       sourceTree,
		VendorOutputBase: filepath.Join(cwd, "vendor"),
		ProtoImport:      []string{defaultProtoImport},
		APIMachineryPackages: strings.Join([]string{
			`+k8s.io/apimachinery/pkg/util/intstr`,
			`+k8s.io/apimachinery/pkg/api/resource`,
			`+k8s.io/apimachinery/pkg/runtime/schema`,
			`+k8s.io/apimachinery/pkg/runtime`,
			`k8s.io/apimachinery/pkg/apis/meta/v1`,
			`k8s.io/apimachinery/pkg/apis/meta/v1alpha1`,
		}, ","),
		Packages:           "",
		DropEmbeddedFields: "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta",
	}
}

func (g *Generator) BindFlags(flag *flag.FlagSet) {
	flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
	flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.")
	flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVar(&g.APIMachineryPackages, "apimachinery-packages", g.APIMachineryPackages, "comma-separated list of directories to get apimachinery input types from which are needed by any API. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/")
	flag.StringVar(&g.VendorOutputBase, "vendor-output-base", g.VendorOutputBase, "The vendor/ directory to look for packages in; defaults to $PWD/vendor/.")
	flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required; defaults $GOPATH/src/k8s.io/kubernetes/vendor/github.com/gogo/protobuf/protobuf.")
	flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
	flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
	flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
	flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
	flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
	flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}

func Run(g *Generator) {
	if g.Common.VerifyOnly {
		g.OnlyIDL = true
		g.Clean = false
	}

	b := parser.New()
	b.AddBuildTags("proto")

	omitTypes := map[types.Name]struct{}{}
	for _, t := range strings.Split(g.DropEmbeddedFields, ",") {
		name := types.Name{}
		if i := strings.LastIndex(t, "."); i != -1 {
			name.Package, name.Name = t[:i], t[i+1:]
		} else {
			name.Name = t
		}
		if len(name.Name) == 0 {
			log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t)
		}
		omitTypes[name] = struct{}{}
	}

	boilerplate, err := g.Common.LoadGoBoilerplate()
	if err != nil {
		log.Fatalf("Failed loading boilerplate (consider using the go-header-file flag): %v", err)
	}

	protobufNames := NewProtobufNamer()
	outputPackages := generator.Packages{}
	nonOutputPackages := map[string]struct{}{}

	var packages []string
	if len(g.APIMachineryPackages) != 0 {
		packages = append(packages, strings.Split(g.APIMachineryPackages, ",")...)
	}
	if len(g.Packages) != 0 {
		packages = append(packages, strings.Split(g.Packages, ",")...)
	}
	if len(packages) == 0 {
		log.Fatalf("Both apimachinery-packages and packages are empty. At least one package must be specified.")
	}

	for _, d := range packages {
		generateAllTypes, outputPackage := true, true
		switch {
		case strings.HasPrefix(d, "+"):
			d = d[1:]
			generateAllTypes = false
		case strings.HasPrefix(d, "-"):
			d = d[1:]
			outputPackage = false
		}
		name := protoSafePackage(d)
		parts := strings.SplitN(d, "=", 2)
		if len(parts) > 1 {
			d = parts[0]
			name = parts[1]
		}
		p := newProtobufPackage(d, name, generateAllTypes, omitTypes)
		header := append([]byte{}, boilerplate...)
		header = append(header, p.HeaderText...)
		p.HeaderText = header
		protobufNames.Add(p)
		if outputPackage {
			outputPackages = append(outputPackages, p)
		} else {
			nonOutputPackages[name] = struct{}{}
		}
	}

	if !g.Common.VerifyOnly {
		for _, p := range outputPackages {
			if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil {
				log.Fatalf("Unable to clean package %s: %v", p.Name(), err)
			}
		}
	}

	if g.Clean {
		return
	}

	for _, p := range protobufNames.List() {
		if err := b.AddDir(p.Path()); err != nil {
			log.Fatalf("Unable to add directory %q: %v", p.Path(), err)
		}
	}

	c, err := generator.NewContext(
		b,
		namer.NameSystems{
			"public": namer.NewPublicNamer(3),
			"proto":  protobufNames,
		},
		"public",
	)
	if err != nil {
		log.Fatalf("Failed making a context: %v", err)
	}

	c.Verify = g.Common.VerifyOnly
	c.FileTypes["protoidl"] = NewProtoFile()

	var vendoredOutputPackages, localOutputPackages generator.Packages
	for _, p := range protobufNames.packages {
		if _, ok := nonOutputPackages[p.Name()]; ok {
			// if we're not outputting the package, don't include it in either package list
			continue
		}
		p.Vendored = strings.Contains(c.Universe[p.PackagePath].SourcePath, "/vendor/")
		if p.Vendored {
			vendoredOutputPackages = append(vendoredOutputPackages, p)
		} else {
			localOutputPackages = append(localOutputPackages, p)
		}
	}

	if err := protobufNames.AssignTypesToPackages(c); err != nil {
		log.Fatalf("Failed to identify Common types: %v", err)
	}

	if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
		log.Fatalf("Failed executing vendor generator: %v", err)
	}
	if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
		log.Fatalf("Failed executing local generator: %v", err)
	}

	if g.OnlyIDL {
		return
	}

	if _, err := exec.LookPath("protoc"); err != nil {
		log.Fatalf("Unable to find 'protoc': %v", err)
	}

	searchArgs := []string{"-I", ".", "-I", g.OutputBase}
	if len(g.ProtoImport) != 0 {
		for _, s := range g.ProtoImport {
			searchArgs = append(searchArgs, "-I", s)
		}
	}
	args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase))

	buf := &bytes.Buffer{}
	if len(g.Conditional) > 0 {
		fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional)
	}
	buf.Write(boilerplate)

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		path := filepath.Join(g.OutputBase, p.ImportPath())
		outputPath := filepath.Join(g.OutputBase, p.OutputPath())
		if p.Vendored {
			path = filepath.Join(g.VendorOutputBase, p.ImportPath())
			outputPath = filepath.Join(g.VendorOutputBase, p.OutputPath())
		}

		// generate the gogoprotobuf protoc
		cmd := exec.Command("protoc", append(args, path)...)
		out, err := cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
		}

		if g.SkipGeneratedRewrite {
			continue
		}

		// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
		// package statement to match the desired package name
		if err := RewriteGeneratedGogoProtobufFile(outputPath, p.ExtractGeneratedType, p.OptionalTypeName, buf.Bytes()); err != nil {
			log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
		}

		// sort imports
		cmd = exec.Command("goimports", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
		}

		// format and simplify the generated file
		cmd = exec.Command("gofmt", "-s", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
		}
	}

	if g.SkipGeneratedRewrite {
		return
	}

	if !g.KeepGogoproto {
		// generate, but do so without gogoprotobuf extensions
		for _, outputPackage := range outputPackages {
			p := outputPackage.(*protobufPackage)
			p.OmitGogo = true
		}
		if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
			log.Fatalf("Failed executing vendor generator: %v", err)
		}
		if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
			log.Fatalf("Failed executing local generator: %v", err)
		}
	}

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		if len(p.StructTags) == 0 {
			continue
		}

		pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
		if p.Vendored {
			pattern = filepath.Join(g.VendorOutputBase, p.PackagePath, "*.go")
		}
		files, err := filepath.Glob(pattern)
		if err != nil {
			log.Fatalf("Can't glob pattern %q: %v", pattern, err)
		}

		for _, s := range files {
			if strings.HasSuffix(s, "_test.go") {
				continue
			}
			if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
				log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
			}
		}
	}
}
<commit_msg>Promote v1alpha1 meta to v1beta1<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
// existing IDL tags on the Go struct.
package protobuf

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"k8s.io/gengo/args"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/parser"
	"k8s.io/gengo/types"

	flag "github.com/spf13/pflag"
)

type Generator struct {
	Common               args.GeneratorArgs
	APIMachineryPackages string
	Packages             string
	OutputBase           string
	VendorOutputBase     string
	ProtoImport          []string
	Conditional          string
	Clean                bool
	OnlyIDL              bool
	KeepGogoproto        bool
	SkipGeneratedRewrite bool
	DropEmbeddedFields   string
}

func New() *Generator {
	sourceTree := args.DefaultSourceTree()
	common := args.GeneratorArgs{
		OutputBase:       sourceTree,
		GoHeaderFilePath: filepath.Join(sourceTree, "k8s.io/kubernetes/hack/boilerplate/boilerplate.go.txt"),
	}
	defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "vendor", "github.com", "gogo", "protobuf", "protobuf")
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Cannot get current directory.")
	}
	return &Generator{
		Common:           common,
		OutputBase:       sourceTree,
		VendorOutputBase: filepath.Join(cwd, "vendor"),
		ProtoImport:      []string{defaultProtoImport},
		APIMachineryPackages: strings.Join([]string{
			`+k8s.io/apimachinery/pkg/util/intstr`,
			`+k8s.io/apimachinery/pkg/api/resource`,
			`+k8s.io/apimachinery/pkg/runtime/schema`,
			`+k8s.io/apimachinery/pkg/runtime`,
			`k8s.io/apimachinery/pkg/apis/meta/v1`,
			`k8s.io/apimachinery/pkg/apis/meta/v1beta1`,
		}, ","),
		Packages:           "",
		DropEmbeddedFields: "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta",
	}
}

func (g *Generator) BindFlags(flag *flag.FlagSet) {
	flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
	flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.")
	flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVar(&g.APIMachineryPackages, "apimachinery-packages", g.APIMachineryPackages, "comma-separated list of directories to get apimachinery input types from which are needed by any API. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
	flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/")
	flag.StringVar(&g.VendorOutputBase, "vendor-output-base", g.VendorOutputBase, "The vendor/ directory to look for packages in; defaults to $PWD/vendor/.")
	flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required; defaults $GOPATH/src/k8s.io/kubernetes/vendor/github.com/gogo/protobuf/protobuf.")
	flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
	flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
	flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
	flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
	flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
	flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}

func Run(g *Generator) {
	if g.Common.VerifyOnly {
		g.OnlyIDL = true
		g.Clean = false
	}

	b := parser.New()
	b.AddBuildTags("proto")

	omitTypes := map[types.Name]struct{}{}
	for _, t := range strings.Split(g.DropEmbeddedFields, ",") {
		name := types.Name{}
		if i := strings.LastIndex(t, "."); i != -1 {
			name.Package, name.Name = t[:i], t[i+1:]
		} else {
			name.Name = t
		}
		if len(name.Name) == 0 {
			log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t)
		}
		omitTypes[name] = struct{}{}
	}

	boilerplate, err := g.Common.LoadGoBoilerplate()
	if err != nil {
		log.Fatalf("Failed loading boilerplate (consider using the go-header-file flag): %v", err)
	}

	protobufNames := NewProtobufNamer()
	outputPackages := generator.Packages{}
	nonOutputPackages := map[string]struct{}{}

	var packages []string
	if len(g.APIMachineryPackages) != 0 {
		packages = append(packages, strings.Split(g.APIMachineryPackages, ",")...)
	}
	if len(g.Packages) != 0 {
		packages = append(packages, strings.Split(g.Packages, ",")...)
	}
	if len(packages) == 0 {
		log.Fatalf("Both apimachinery-packages and packages are empty. At least one package must be specified.")
	}

	for _, d := range packages {
		generateAllTypes, outputPackage := true, true
		switch {
		case strings.HasPrefix(d, "+"):
			d = d[1:]
			generateAllTypes = false
		case strings.HasPrefix(d, "-"):
			d = d[1:]
			outputPackage = false
		}
		name := protoSafePackage(d)
		parts := strings.SplitN(d, "=", 2)
		if len(parts) > 1 {
			d = parts[0]
			name = parts[1]
		}
		p := newProtobufPackage(d, name, generateAllTypes, omitTypes)
		header := append([]byte{}, boilerplate...)
		header = append(header, p.HeaderText...)
		p.HeaderText = header
		protobufNames.Add(p)
		if outputPackage {
			outputPackages = append(outputPackages, p)
		} else {
			nonOutputPackages[name] = struct{}{}
		}
	}

	if !g.Common.VerifyOnly {
		for _, p := range outputPackages {
			if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil {
				log.Fatalf("Unable to clean package %s: %v", p.Name(), err)
			}
		}
	}

	if g.Clean {
		return
	}

	for _, p := range protobufNames.List() {
		if err := b.AddDir(p.Path()); err != nil {
			log.Fatalf("Unable to add directory %q: %v", p.Path(), err)
		}
	}

	c, err := generator.NewContext(
		b,
		namer.NameSystems{
			"public": namer.NewPublicNamer(3),
			"proto":  protobufNames,
		},
		"public",
	)
	if err != nil {
		log.Fatalf("Failed making a context: %v", err)
	}

	c.Verify = g.Common.VerifyOnly
	c.FileTypes["protoidl"] = NewProtoFile()

	var vendoredOutputPackages, localOutputPackages generator.Packages
	for _, p := range protobufNames.packages {
		if _, ok := nonOutputPackages[p.Name()]; ok {
			// if we're not outputting the package, don't include it in either package list
			continue
		}
		p.Vendored = strings.Contains(c.Universe[p.PackagePath].SourcePath, "/vendor/")
		if p.Vendored {
			vendoredOutputPackages = append(vendoredOutputPackages, p)
		} else {
			localOutputPackages = append(localOutputPackages, p)
		}
	}

	if err := protobufNames.AssignTypesToPackages(c); err != nil {
		log.Fatalf("Failed to identify Common types: %v", err)
	}

	if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
		log.Fatalf("Failed executing vendor generator: %v", err)
	}
	if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
		log.Fatalf("Failed executing local generator: %v", err)
	}

	if g.OnlyIDL {
		return
	}

	if _, err := exec.LookPath("protoc"); err != nil {
		log.Fatalf("Unable to find 'protoc': %v", err)
	}

	searchArgs := []string{"-I", ".", "-I", g.OutputBase}
	if len(g.ProtoImport) != 0 {
		for _, s := range g.ProtoImport {
			searchArgs = append(searchArgs, "-I", s)
		}
	}
	args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase))

	buf := &bytes.Buffer{}
	if len(g.Conditional) > 0 {
		fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional)
	}
	buf.Write(boilerplate)

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		path := filepath.Join(g.OutputBase, p.ImportPath())
		outputPath := filepath.Join(g.OutputBase, p.OutputPath())
		if p.Vendored {
			path = filepath.Join(g.VendorOutputBase, p.ImportPath())
			outputPath = filepath.Join(g.VendorOutputBase, p.OutputPath())
		}

		// generate the gogoprotobuf protoc
		cmd := exec.Command("protoc", append(args, path)...)
		out, err := cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
		}

		if g.SkipGeneratedRewrite {
			continue
		}

		// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
		// package statement to match the desired package name
		if err := RewriteGeneratedGogoProtobufFile(outputPath, p.ExtractGeneratedType, p.OptionalTypeName, buf.Bytes()); err != nil {
			log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
		}

		// sort imports
		cmd = exec.Command("goimports", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
		}

		// format and simplify the generated file
		cmd = exec.Command("gofmt", "-s", "-w", outputPath)
		out, err = cmd.CombinedOutput()
		if len(out) > 0 {
			log.Printf(string(out))
		}
		if err != nil {
			log.Println(strings.Join(cmd.Args, " "))
			log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
		}
	}

	if g.SkipGeneratedRewrite {
		return
	}

	if !g.KeepGogoproto {
		// generate, but do so without gogoprotobuf extensions
		for _, outputPackage := range outputPackages {
			p := outputPackage.(*protobufPackage)
			p.OmitGogo = true
		}
		if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
			log.Fatalf("Failed executing vendor generator: %v", err)
		}
		if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
			log.Fatalf("Failed executing local generator: %v", err)
		}
	}

	for _, outputPackage := range outputPackages {
		p := outputPackage.(*protobufPackage)

		if len(p.StructTags) == 0 {
			continue
		}

		pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
		if p.Vendored {
			pattern = filepath.Join(g.VendorOutputBase, p.PackagePath, "*.go")
		}
		files, err := filepath.Glob(pattern)
		if err != nil {
			log.Fatalf("Can't glob pattern %q: %v", pattern, err)
		}

		for _, s := range files {
			if strings.HasSuffix(s, "_test.go") {
				continue
			}
			if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
				log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Square, Inc
// +build linux

package osmain

import (
	"fmt"
	"log"
	"path/filepath"
	"sort"
	"time"

	"github.com/square/inspect/metrics"
	"github.com/square/inspect/os/cpustat"
	"github.com/square/inspect/os/diskstat"
	"github.com/square/inspect/os/entropystat"
	"github.com/square/inspect/os/fsstat"
	"github.com/square/inspect/os/interfacestat"
	"github.com/square/inspect/os/loadstat"
	"github.com/square/inspect/os/memstat"
	"github.com/square/inspect/os/misc"
	"github.com/square/inspect/os/netstat"
	"github.com/square/inspect/os/uptimestat"
)

type linuxStats struct {
	osind       *Stats
	dstat       *diskstat.DiskStat
	fsstat      *fsstat.FSStat
	ifstat      *interfacestat.InterfaceStat
	netstat     *netstat.NetStat
	cgMem       *memstat.CgroupStat
	cgCPU       *cpustat.CgroupStat
	loadstat    *loadstat.LoadStat
	uptimestat  *uptimestat.UptimeStat
	entropystat *entropystat.EntropyStat
}

// RegisterOsSpecific registers OS dependent statistics
func registerOsSpecific(m *metrics.MetricContext, step time.Duration,
	osind *Stats) *linuxStats {
	s := new(linuxStats)
	s.osind = osind
	s.dstat = diskstat.New(m, step)
	s.fsstat = fsstat.New(m, step)
	s.ifstat = interfacestat.New(m, step)
	s.netstat = netstat.New(m, step)
	s.loadstat = loadstat.New(m, step)
	s.uptimestat = uptimestat.New(m, step)
	s.cgMem = memstat.NewCgroupStat(m, step)
	s.cgCPU = cpustat.NewCgroupStat(m, step)
	s.entropystat = entropystat.New(m, step)
	return s
}

// PrintOsSpecific prints OS dependent statistics
func printOsSpecific(batchmode bool, layout *DisplayWidgets, v interface{}) {
	stats, ok := v.(*linuxStats)
	if !ok {
		log.Fatalf("Type assertion failed on printOsSpecific")
	}
	// Top N processes sorted by IO usage - requires root
	procsByUsage := stats.osind.ProcessStat.ByIOUsage()
	n := MaxEntries
	if len(procsByUsage) < n {
		n = len(procsByUsage)
	}
	var io []string
	for i := 0; i < n; i++ {
		io = append(io, fmt.Sprintf("%8s/s %10s %10s %8s",
			misc.ByteSize(procsByUsage[i].IOUsage()),
			truncate(procsByUsage[i].Comm(), 10),
			truncate(procsByUsage[i].User(), 10),
			procsByUsage[i].Pid()))
	}
	for i := n; i < MaxEntries; i++ {
		io = append(io, fmt.Sprintf("%8s/s %10s %10s %8s", "-", "-", "-", "-"))
	}
	displayList(batchmode, "io", layout, io)
	// Print top-N diskIO usage
	// disk stats
	diskIOByUsage := stats.dstat.ByUsage()
	var diskio []string
	// TODO(syamp): remove magic number
	for i := 0; i < 5; i++ {
		diskName := "-"
		diskIO := 0.0
		if len(diskIOByUsage) > i {
			d := diskIOByUsage[i]
			diskName = d.Name
			diskIO = d.Usage()
		}
		diskio = append(diskio, fmt.Sprintf("%6s %5s", diskName, fmt.Sprintf("%3.1f%% ", diskIO)))
	}
	displayList(batchmode, "diskio", layout, diskio)
	// Print top-N File system  usage
	// disk stats
	fsByUsage := stats.fsstat.ByUsage()
	var fs []string
	// TODO(syamp): remove magic number
	for i := 0; i < 5; i++ {
		fsName := "-"
		fsUsage := 0.0
		fsInodes := 0.0
		if len(fsByUsage) > i {
			f := fsByUsage[i]
			fsName = f.Name
			fsUsage = f.Usage()
			fsInodes = f.FileUsage()
		}
		fs = append(fs, fmt.Sprintf("%20s %6s i:%6s", truncate(fsName, 20),
			fmt.Sprintf("%3.1f%%", fsUsage),
			fmt.Sprintf("%3.1f%%", fsInodes)))
	}
	displayList(batchmode, "filesystem", layout, fs)
	// Detect potential problems for disk/fs
	for _, d := range diskIOByUsage {
		if d.Usage() > 75.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("Disk IO usage on (%v): %3.1f%%", d.Name, d.Usage()))
		}
	}
	for _, fs := range fsByUsage {
		if fs.Usage() > 90.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("FS block usage on (%v): %3.1f%%", fs.Name, fs.Usage()))
		}
		if fs.FileUsage() > 90.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("FS inode usage on (%v): %3.1f%%", fs.Name, fs.FileUsage()))
		}
	}
	// Interface usage statistics
	var interfaces []string
	interfaceByUsage := stats.ifstat.ByUsage()
	for i := 0; i < 5; i++ {
		name := "-"
		var rx misc.BitSize
		var tx misc.BitSize
		if len(interfaceByUsage) > i {
			iface := interfaceByUsage[i]
			name = truncate(iface.Name, 10)
			rx = misc.BitSize(iface.RXBandwidth())
			tx = misc.BitSize(iface.TXBandwidth())
		}
		interfaces = append(interfaces, fmt.Sprintf("%10s r:%8s t:%8s", name, rx, tx))
	}
	for _, iface := range interfaceByUsage {
		if iface.TXBandwidthUsage() > 75.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("TX bandwidth usage on (%v): %3.1f%%",
					iface.Name, iface.TXBandwidthUsage()))
		}
		if iface.RXBandwidthUsage() > 75.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("RX bandwidth usage on (%v): %3.1f%%",
					iface.Name, iface.RXBandwidthUsage()))
		}
	}
	displayList(batchmode, "interface", layout, interfaces)
	// CPU stats by cgroup
	// TODO(syamp): should be sorted by quota usage
	var cgcpu, keys []string
	for name := range stats.cgCPU.Cgroups {
		keys = append(keys, name)
	}
	sort.Strings(keys)
	for _, name := range keys {
		v, ok := stats.cgCPU.Cgroups[name]
		if ok {
			name, _ = filepath.Rel(stats.cgCPU.Mountpoint, name)
			cpuUsagePct := (v.Usage() / stats.osind.CPUStat.Total()) * 100
			cpuQuotaPct := (v.Usage() / v.Quota()) * 100
			cpuThrottle := v.Throttle() * 100
			cgcpu = append(cgcpu, fmt.Sprintf("%20s %5s %6s %5s",
				truncate(name, 20),
				fmt.Sprintf("%3.1f%%", cpuUsagePct),
				fmt.Sprintf("q:%3.1f", v.Quota()),
				fmt.Sprintf("qpct:%3.1f%%", cpuQuotaPct)))
			if cpuThrottle > 0.1 {
				stats.osind.Problems =
					append(stats.osind.Problems, fmt.Sprintf(
						"CPU throttling on cgroup(%s): %3.1f%%",
						name, cpuThrottle))
			}
		}
	}
	displayList(batchmode, "cpu(cgroup)", layout, cgcpu)

	// Memory stats by cgroup
	// TODO(syamp): should be sorted by usage
	var cgmem []string
	keys = keys[:0]
	for name := range stats.cgMem.Cgroups {
		keys = append(keys, name)
	}
	sort.Strings(keys)
	for _, name := range keys {
		v, ok := stats.cgMem.Cgroups[name]
		if ok {
			name, _ = filepath.Rel(stats.cgMem.Mountpoint, name)
			memUsagePct := (v.Usage() / stats.osind.MemStat.Total()) * 100
			memQuota := v.SoftLimit()
			if memQuota > stats.osind.MemStat.Total() {
				memQuota = stats.osind.MemStat.Total()
			}
			memQuotaPct := (v.Usage() / v.SoftLimit()) * 100
			cgmem = append(cgmem, fmt.Sprintf("%20s %5s %10s %5s",
				truncate(name, 20),
				fmt.Sprintf("%3.1f%%", memUsagePct),
				fmt.Sprintf("q:%8s", misc.ByteSize(memQuota)),
				fmt.Sprintf("qpct:%3.1f%%", memQuotaPct)))
			if memQuotaPct > 75 {
				stats.osind.Problems =
					append(stats.osind.Problems, fmt.Sprintf(
						"Memory close to quota on cgroup(%s): %3.1f%%",
						name, memQuotaPct))
			}
		}
	}
	displayList(batchmode, "memory(cgroup)", layout, cgmem)
	entropy := fmt.Sprintf("%10.0f", stats.entropystat.Available.Get())
	displayList(batchmode, "entropy", layout, []string{entropy})
}
<commit_msg>Increase shown cgroup path characters<commit_after>// Copyright (c) 2014 Square, Inc
// +build linux

package osmain

import (
	"fmt"
	"log"
	"path/filepath"
	"sort"
	"time"

	"github.com/square/inspect/metrics"
	"github.com/square/inspect/os/cpustat"
	"github.com/square/inspect/os/diskstat"
	"github.com/square/inspect/os/entropystat"
	"github.com/square/inspect/os/fsstat"
	"github.com/square/inspect/os/interfacestat"
	"github.com/square/inspect/os/loadstat"
	"github.com/square/inspect/os/memstat"
	"github.com/square/inspect/os/misc"
	"github.com/square/inspect/os/netstat"
	"github.com/square/inspect/os/uptimestat"
)

type linuxStats struct {
	osind       *Stats
	dstat       *diskstat.DiskStat
	fsstat      *fsstat.FSStat
	ifstat      *interfacestat.InterfaceStat
	netstat     *netstat.NetStat
	cgMem       *memstat.CgroupStat
	cgCPU       *cpustat.CgroupStat
	loadstat    *loadstat.LoadStat
	uptimestat  *uptimestat.UptimeStat
	entropystat *entropystat.EntropyStat
}

// RegisterOsSpecific registers OS dependent statistics
func registerOsSpecific(m *metrics.MetricContext, step time.Duration,
	osind *Stats) *linuxStats {
	s := new(linuxStats)
	s.osind = osind
	s.dstat = diskstat.New(m, step)
	s.fsstat = fsstat.New(m, step)
	s.ifstat = interfacestat.New(m, step)
	s.netstat = netstat.New(m, step)
	s.loadstat = loadstat.New(m, step)
	s.uptimestat = uptimestat.New(m, step)
	s.cgMem = memstat.NewCgroupStat(m, step)
	s.cgCPU = cpustat.NewCgroupStat(m, step)
	s.entropystat = entropystat.New(m, step)
	return s
}

// PrintOsSpecific prints OS dependent statistics
func printOsSpecific(batchmode bool, layout *DisplayWidgets, v interface{}) {
	stats, ok := v.(*linuxStats)
	if !ok {
		log.Fatalf("Type assertion failed on printOsSpecific")
	}
	// Top N processes sorted by IO usage - requires root
	procsByUsage := stats.osind.ProcessStat.ByIOUsage()
	n := MaxEntries
	if len(procsByUsage) < n {
		n = len(procsByUsage)
	}
	var io []string
	for i := 0; i < n; i++ {
		io = append(io, fmt.Sprintf("%8s/s %10s %10s %8s",
			misc.ByteSize(procsByUsage[i].IOUsage()),
			truncate(procsByUsage[i].Comm(), 10),
			truncate(procsByUsage[i].User(), 10),
			procsByUsage[i].Pid()))
	}
	for i := n; i < MaxEntries; i++ {
		io = append(io, fmt.Sprintf("%8s/s %10s %10s %8s", "-", "-", "-", "-"))
	}
	displayList(batchmode, "io", layout, io)
	// Print top-N diskIO usage
	// disk stats
	diskIOByUsage := stats.dstat.ByUsage()
	var diskio []string
	// TODO(syamp): remove magic number
	for i := 0; i < 5; i++ {
		diskName := "-"
		diskIO := 0.0
		if len(diskIOByUsage) > i {
			d := diskIOByUsage[i]
			diskName = d.Name
			diskIO = d.Usage()
		}
		diskio = append(diskio, fmt.Sprintf("%6s %5s", diskName, fmt.Sprintf("%3.1f%% ", diskIO)))
	}
	displayList(batchmode, "diskio", layout, diskio)
	// Print top-N File system  usage
	// disk stats
	fsByUsage := stats.fsstat.ByUsage()
	var fs []string
	// TODO(syamp): remove magic number
	for i := 0; i < 5; i++ {
		fsName := "-"
		fsUsage := 0.0
		fsInodes := 0.0
		if len(fsByUsage) > i {
			f := fsByUsage[i]
			fsName = f.Name
			fsUsage = f.Usage()
			fsInodes = f.FileUsage()
		}
		fs = append(fs, fmt.Sprintf("%20s %6s i:%6s", truncate(fsName, 20),
			fmt.Sprintf("%3.1f%%", fsUsage),
			fmt.Sprintf("%3.1f%%", fsInodes)))
	}
	displayList(batchmode, "filesystem", layout, fs)
	// Detect potential problems for disk/fs
	for _, d := range diskIOByUsage {
		if d.Usage() > 75.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("Disk IO usage on (%v): %3.1f%%", d.Name, d.Usage()))
		}
	}
	for _, fs := range fsByUsage {
		if fs.Usage() > 90.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("FS block usage on (%v): %3.1f%%", fs.Name, fs.Usage()))
		}
		if fs.FileUsage() > 90.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("FS inode usage on (%v): %3.1f%%", fs.Name, fs.FileUsage()))
		}
	}
	// Interface usage statistics
	var interfaces []string
	interfaceByUsage := stats.ifstat.ByUsage()
	for i := 0; i < 5; i++ {
		name := "-"
		var rx misc.BitSize
		var tx misc.BitSize
		if len(interfaceByUsage) > i {
			iface := interfaceByUsage[i]
			name = truncate(iface.Name, 10)
			rx = misc.BitSize(iface.RXBandwidth())
			tx = misc.BitSize(iface.TXBandwidth())
		}
		interfaces = append(interfaces, fmt.Sprintf("%10s r:%8s t:%8s", name, rx, tx))
	}
	for _, iface := range interfaceByUsage {
		if iface.TXBandwidthUsage() > 75.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("TX bandwidth usage on (%v): %3.1f%%",
					iface.Name, iface.TXBandwidthUsage()))
		}
		if iface.RXBandwidthUsage() > 75.0 {
			stats.osind.Problems = append(stats.osind.Problems,
				fmt.Sprintf("RX bandwidth usage on (%v): %3.1f%%",
					iface.Name, iface.RXBandwidthUsage()))
		}
	}
	displayList(batchmode, "interface", layout, interfaces)
	// CPU stats by cgroup
	// TODO(syamp): should be sorted by quota usage
	var cgcpu, keys []string
	for name := range stats.cgCPU.Cgroups {
		keys = append(keys, name)
	}
	sort.Strings(keys)
	for _, name := range keys {
		v, ok := stats.cgCPU.Cgroups[name]
		if ok {
			name, _ = filepath.Rel(stats.cgCPU.Mountpoint, name)
			cpuUsagePct := (v.Usage() / stats.osind.CPUStat.Total()) * 100
			cpuQuotaPct := (v.Usage() / v.Quota()) * 100
			cpuThrottle := v.Throttle() * 100
			cgcpu = append(cgcpu, fmt.Sprintf("%20s %5s %6s %5s",
				truncate(name, 34),
				fmt.Sprintf("%3.1f%%", cpuUsagePct),
				fmt.Sprintf("q:%3.1f", v.Quota()),
				fmt.Sprintf("qpct:%3.1f%%", cpuQuotaPct)))
			if cpuThrottle > 0.1 {
				stats.osind.Problems =
					append(stats.osind.Problems, fmt.Sprintf(
						"CPU throttling on cgroup(%s): %3.1f%%",
						name, cpuThrottle))
			}
		}
	}
	displayList(batchmode, "cpu(cgroup)", layout, cgcpu)

	// Memory stats by cgroup
	// TODO(syamp): should be sorted by usage
	var cgmem []string
	keys = keys[:0]
	for name := range stats.cgMem.Cgroups {
		keys = append(keys, name)
	}
	sort.Strings(keys)
	for _, name := range keys {
		v, ok := stats.cgMem.Cgroups[name]
		if ok {
			name, _ = filepath.Rel(stats.cgMem.Mountpoint, name)
			memUsagePct := (v.Usage() / stats.osind.MemStat.Total()) * 100
			memQuota := v.SoftLimit()
			if memQuota > stats.osind.MemStat.Total() {
				memQuota = stats.osind.MemStat.Total()
			}
			memQuotaPct := (v.Usage() / v.SoftLimit()) * 100
			cgmem = append(cgmem, fmt.Sprintf("%20s %5s %10s %5s",
				truncate(name, 34),
				fmt.Sprintf("%3.1f%%", memUsagePct),
				fmt.Sprintf("q:%8s", misc.ByteSize(memQuota)),
				fmt.Sprintf("qpct:%3.1f%%", memQuotaPct)))
			if memQuotaPct > 75 {
				stats.osind.Problems =
					append(stats.osind.Problems, fmt.Sprintf(
						"Memory close to quota on cgroup(%s): %3.1f%%",
						name, memQuotaPct))
			}
		}
	}
	displayList(batchmode, "memory(cgroup)", layout, cgmem)
	entropy := fmt.Sprintf("%10.0f", stats.entropystat.Available.Get())
	displayList(batchmode, "entropy", layout, []string{entropy})
}
<|endoftext|>


<commit_before>package cmd

import (
	"encoding/csv"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/howeyc/ledger"
	"github.com/howeyc/ledger/internal/decimal"
	"github.com/jbrukh/bayesian"
	"github.com/spf13/cobra"
)

var csvDateFormat string
var destAccSearch string
var negateAmount bool
var allowMatching bool
var fieldDelimiter string
var scaleFactor float64

// importCmd represents the import command
var importCmd = &cobra.Command{
	Use:   "import <account-substring> <csv-file>",
	Args:  cobra.ExactArgs(2),
	Short: "Import transactions from csv to ledger format",
	Run: func(cmd *cobra.Command, args []string) {
		var accountSubstring, csvFileName string
		accountSubstring = args[0]
		csvFileName = args[1]

		decScale := decimal.NewFromFloat(scaleFactor)

		csvFileReader, err := os.Open(csvFileName)
		if err != nil {
			fmt.Println("CSV: ", err)
			return
		}
		defer csvFileReader.Close()

		generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath)
		if parseError != nil {
			fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error())
			return
		}

		var matchingAccount string
		matchingAccounts := ledger.GetBalances(generalLedger, []string{accountSubstring})
		if len(matchingAccounts) < 1 {
			fmt.Println("Unable to find matching account.")
			return
		}
		matchingAccount = matchingAccounts[len(matchingAccounts)-1].Name

		allAccounts := ledger.GetBalances(generalLedger, []string{})

		csvReader := csv.NewReader(csvFileReader)
		csvReader.Comma, _ = utf8.DecodeRuneInString(fieldDelimiter)
		csvRecords, cerr := csvReader.ReadAll()
		if cerr != nil {
			fmt.Println("CSV parse error:", cerr.Error())
			return
		}

		classes := make([]bayesian.Class, len(allAccounts))
		for i, bal := range allAccounts {
			classes[i] = bayesian.Class(bal.Name)
		}
		classifier := bayesian.NewClassifier(classes...)
		for _, tran := range generalLedger {
			payeeWords := strings.Fields(tran.Payee)
			for _, accChange := range tran.AccountChanges {
				if strings.Contains(accChange.Name, destAccSearch) {
					classifier.Learn(payeeWords, bayesian.Class(accChange.Name))
				}
			}
		}

		// Find columns from header
		var dateColumn, payeeColumn, amountColumn, commentColumn int
		dateColumn, payeeColumn, amountColumn, commentColumn = -1, -1, -1, -1
		for fieldIndex, fieldName := range csvRecords[0] {
			fieldName = strings.ToLower(fieldName)
			if strings.Contains(fieldName, "date") {
				dateColumn = fieldIndex
			} else if strings.Contains(fieldName, "description") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "payee") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "amount") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "expense") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "note") {
				commentColumn = fieldIndex
			} else if strings.Contains(fieldName, "comment") {
				commentColumn = fieldIndex
			}
		}

		if dateColumn < 0 || payeeColumn < 0 || amountColumn < 0 {
			fmt.Println("Unable to find columns required from header field names.")
			return
		}

		expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero}
		csvAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero}
		for _, record := range csvRecords[1:] {
			inputPayeeWords := strings.Fields(record[payeeColumn])
			csvDate, _ := time.Parse(csvDateFormat, record[dateColumn])
			if allowMatching || !existingTransaction(generalLedger, csvDate, inputPayeeWords[0]) {
				// Classify into expense account
				_, likely, _ := classifier.LogScores(inputPayeeWords)
				if likely >= 0 {
					expenseAccount.Name = string(classifier.Classes[likely])
				}

				// Parse error, set to zero
				if dec, derr := decimal.NewFromString(record[amountColumn]); derr != nil {
					expenseAccount.Balance = decimal.Zero
				} else {
					expenseAccount.Balance = dec
				}

				// Negate amount if required
				if negateAmount {
					expenseAccount.Balance = expenseAccount.Balance.Neg()
				}

				// Apply scale
				expenseAccount.Balance = expenseAccount.Balance.Mul(decScale)

				// Csv amount is the negative of the expense amount
				csvAccount.Balance = expenseAccount.Balance.Neg()

				// Create valid transaction for print in ledger format
				trans := &ledger.Transaction{Date: csvDate, Payee: record[payeeColumn]}
				trans.AccountChanges = []ledger.Account{csvAccount, expenseAccount}

				// Comment
				if commentColumn >= 0 && record[commentColumn] != "" {
					trans.Comments = []string{";" + record[commentColumn]}
				}
				PrintTransaction(trans, 80)
			}
		}

	},
}

func init() {
	rootCmd.AddCommand(importCmd)

	importCmd.Flags().BoolVar(&negateAmount, "neg", false, "Negate amount column value.")
	importCmd.Flags().BoolVar(&allowMatching, "allow-matching", false, "Have output include imported transactions that\nmatch existing ledger transactions.")
	importCmd.Flags().Float64Var(&scaleFactor, "scale", 1.0, "Scale factor to multiply against every imported amount.")
	importCmd.Flags().StringVar(&destAccSearch, "set-search", "Expense", "Search string used to find set of accounts for classification.")
	importCmd.Flags().StringVar(&csvDateFormat, "date-format", "01/02/2006", "Date format.")
	importCmd.Flags().StringVar(&fieldDelimiter, "delimiter", ",", "Field delimiter.")
}

func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool {
	for _, trans := range generalLedger {
		if trans.Date == transDate && strings.HasPrefix(trans.Payee, payee) {
			return true
		}
	}
	return false
}
<commit_msg>Fix import account matching<commit_after>package cmd

import (
	"encoding/csv"
	"fmt"
	"os"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/howeyc/ledger"
	"github.com/howeyc/ledger/internal/decimal"
	"github.com/jbrukh/bayesian"
	"github.com/spf13/cobra"
)

var csvDateFormat string
var destAccSearch string
var negateAmount bool
var allowMatching bool
var fieldDelimiter string
var scaleFactor float64

// importCmd represents the import command
var importCmd = &cobra.Command{
	Use:   "import <account-substring> <csv-file>",
	Args:  cobra.ExactArgs(2),
	Short: "Import transactions from csv to ledger format",
	Run: func(cmd *cobra.Command, args []string) {
		var accountSubstring, csvFileName string
		accountSubstring = args[0]
		csvFileName = args[1]

		decScale := decimal.NewFromFloat(scaleFactor)

		csvFileReader, err := os.Open(csvFileName)
		if err != nil {
			fmt.Println("CSV: ", err)
			return
		}
		defer csvFileReader.Close()

		generalLedger, parseError := ledger.ParseLedgerFile(ledgerFilePath)
		if parseError != nil {
			fmt.Printf("%s:%s\n", ledgerFilePath, parseError.Error())
			return
		}

		var matchingAccount string
		matchingAccounts := ledger.GetBalances(generalLedger, []string{accountSubstring})
		if len(matchingAccounts) < 1 {
			fmt.Println("Unable to find matching account.")
			return
		}
		matchingAccount = matchingAccounts[len(matchingAccounts)-1].Name

		allAccounts := ledger.GetBalances(generalLedger, []string{})

		csvReader := csv.NewReader(csvFileReader)
		csvReader.Comma, _ = utf8.DecodeRuneInString(fieldDelimiter)
		csvRecords, cerr := csvReader.ReadAll()
		if cerr != nil {
			fmt.Println("CSV parse error:", cerr.Error())
			return
		}

		classes := make([]bayesian.Class, len(allAccounts))
		for i, bal := range allAccounts {
			classes[i] = bayesian.Class(bal.Name)
		}
		classifier := bayesian.NewClassifier(classes...)
		for _, tran := range generalLedger {
			payeeWords := strings.Fields(tran.Payee)
			for _, accChange := range tran.AccountChanges {
				if strings.Contains(accChange.Name, destAccSearch) {
					classifier.Learn(payeeWords, bayesian.Class(accChange.Name))
				}
			}
		}

		// Find columns from header
		var dateColumn, payeeColumn, amountColumn, commentColumn int
		dateColumn, payeeColumn, amountColumn, commentColumn = -1, -1, -1, -1
		for fieldIndex, fieldName := range csvRecords[0] {
			fieldName = strings.ToLower(fieldName)
			if strings.Contains(fieldName, "date") {
				dateColumn = fieldIndex
			} else if strings.Contains(fieldName, "description") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "payee") {
				payeeColumn = fieldIndex
			} else if strings.Contains(fieldName, "amount") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "expense") {
				amountColumn = fieldIndex
			} else if strings.Contains(fieldName, "note") {
				commentColumn = fieldIndex
			} else if strings.Contains(fieldName, "comment") {
				commentColumn = fieldIndex
			}
		}

		if dateColumn < 0 || payeeColumn < 0 || amountColumn < 0 {
			fmt.Println("Unable to find columns required from header field names.")
			return
		}

		expenseAccount := ledger.Account{Name: "unknown:unknown", Balance: decimal.Zero}
		csvAccount := ledger.Account{Name: matchingAccount, Balance: decimal.Zero}
		for _, record := range csvRecords[1:] {
			inputPayeeWords := strings.Fields(record[payeeColumn])
			csvDate, _ := time.Parse(csvDateFormat, record[dateColumn])
			if allowMatching || !existingTransaction(generalLedger, csvDate, inputPayeeWords[0]) {
				// Classify into expense account
				scores, likely, _ := classifier.LogScores(inputPayeeWords)
				if likely >= 0 {
					matchScore := 0.0
					matchIdx := -1
					for j, score := range scores {
						if j == 0 {
							matchScore = score
						}
						if string(classifier.Classes[j]) == csvAccount.Name {
							continue
						}
						if score > matchScore {
							matchScore = score
							matchIdx = j
						}
					}
					if matchIdx >= 0 {
						expenseAccount.Name = string(classifier.Classes[matchIdx])
					} else {
						expenseAccount.Name = string(classifier.Classes[likely])
					}
				}

				// Parse error, set to zero
				if dec, derr := decimal.NewFromString(record[amountColumn]); derr != nil {
					expenseAccount.Balance = decimal.Zero
				} else {
					expenseAccount.Balance = dec
				}

				// Negate amount if required
				if negateAmount {
					expenseAccount.Balance = expenseAccount.Balance.Neg()
				}

				// Apply scale
				expenseAccount.Balance = expenseAccount.Balance.Mul(decScale)

				// Csv amount is the negative of the expense amount
				csvAccount.Balance = expenseAccount.Balance.Neg()

				// Create valid transaction for print in ledger format
				trans := &ledger.Transaction{Date: csvDate, Payee: record[payeeColumn]}
				trans.AccountChanges = []ledger.Account{csvAccount, expenseAccount}

				// Comment
				if commentColumn >= 0 && record[commentColumn] != "" {
					trans.Comments = []string{";" + record[commentColumn]}
				}
				PrintTransaction(trans, 80)
			}
		}

	},
}

func init() {
	rootCmd.AddCommand(importCmd)

	importCmd.Flags().BoolVar(&negateAmount, "neg", false, "Negate amount column value.")
	importCmd.Flags().BoolVar(&allowMatching, "allow-matching", false, "Have output include imported transactions that\nmatch existing ledger transactions.")
	importCmd.Flags().Float64Var(&scaleFactor, "scale", 1.0, "Scale factor to multiply against every imported amount.")
	importCmd.Flags().StringVar(&destAccSearch, "set-search", "Expense", "Search string used to find set of accounts for classification.")
	importCmd.Flags().StringVar(&csvDateFormat, "date-format", "01/02/2006", "Date format.")
	importCmd.Flags().StringVar(&fieldDelimiter, "delimiter", ",", "Field delimiter.")
}

func existingTransaction(generalLedger []*ledger.Transaction, transDate time.Time, payee string) bool {
	for _, trans := range generalLedger {
		if trans.Date == transDate && strings.HasPrefix(trans.Payee, payee) {
			return true
		}
	}
	return false
}
<|endoftext|>