<commit_before>// Copyright 2021 The gopass Authors. All rights reserved.
// Use of this source code is governed by the MIT license,
// that can be found in the LICENSE file.

// Release is the first part of the gopass release automation. It's supposed
// to be run by a member of the gopass team. It will ensure that the repository
// is in a clean state and make it trivial to trigger a new release.
// You can run it without any parameters and as long as you pay close attention
// to the output it will be a breeze.
package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/blang/semver/v4"
)

var sleep = time.Second
var issueRE = regexp.MustCompile(`#(\d+)\b`)
var verTmpl = `package main

import (
	"strings"

	"github.com/blang/semver/v4"
)

func getVersion() semver.Version {
	sv, err := semver.Parse(strings.TrimPrefix(version, "v"))
	if err == nil {
		if commit != "" {
			sv.Build = []string{commit}
		}
		return sv
	}
	return semver.Version{
		Major: {{ .Major }},
		Minor: {{ .Minor }},
		Patch: {{ .Patch }},
		Pre: []semver.PRVersion{
			{VersionStr: "git"},
		},
		Build: []string{"HEAD"},
	}
}
`

const logo = `
   __     _    _ _      _ _   ___   ___
 /'_ '\ /'_'\ ( '_'\  /'_' )/',__)/',__)
( (_) |( (_) )| (_) )( (_| |\__, \\__, \
'\__  |'\___/'| ,__/''\__,_)(____/(____/
( )_) |       | |
 \___/'       (_)
`

func main() {
	fmt.Print(logo)
	fmt.Println()
	fmt.Println("🌟 Preparing a new gopass release.")
	fmt.Println("☝  Checking pre-conditions ...")
	// - check that workdir is clean
	if !isGitClean() {
		panic("❌ git is dirty")
	}
	fmt.Println("✅ git is clean")
	// - check out master
	if err := gitCoMaster(); err != nil {
		panic(err)
	}
	fmt.Println("✅ Switched to master branch")
	// - pull from origin
	if err := gitPom(); err != nil {
		panic(err)
	}
	fmt.Println("✅ Fetched changes for master")
	// - check that workdir is clean
	if !isGitClean() {
		panic("git is dirty")
	}
	fmt.Println("✅ git is still clean")

	prevVer, nextVer := getVersions()

	fmt.Println()
	fmt.Printf("✅ New version will be: %s\n", nextVer.String())
	fmt.Println()
	fmt.Println("❓ Do you want to continue? (press any key to continue or Ctrl+C to abort)")
	fmt.Scanln()

	// - update VERSION
	if err := writeVersion(nextVer); err != nil {
		panic(err)
	}
	fmt.Println("✅ Wrote VERSION")
	time.Sleep(sleep)
	// - update version.go
	if err := writeVersionGo(nextVer); err != nil {
		panic(err)
	}
	fmt.Println("✅ Wrote version.go")
	time.Sleep(sleep)
	// - update CHANGELOG.md
	if err := writeChangelog(prevVer, nextVer); err != nil {
		panic(err)
	}
	fmt.Println("✅ Updated CHANGELOG.md")
	time.Sleep(sleep)
	// - update shell completions
	if err := updateCompletion(); err != nil {
		panic(err)
	}
	fmt.Println("✅ Updated shell completions")
	time.Sleep(sleep)
	// - update man page
	if err := updateManpage(); err != nil {
		panic(err)
	}
	fmt.Println("✅ Updated man page")
	time.Sleep(sleep)

	// - create PR
	//   git checkout -b release/vX.Y.Z
	if err := gitCoRel(nextVer); err != nil {
		panic(err)
	}
	fmt.Printf("✅ Created branch release/v%s\n", nextVer.String())
	time.Sleep(sleep)

	// commit changes
	if err := gitCommit(nextVer); err != nil {
		panic(err)
	}
	fmt.Printf("✅ Committed changes to release/v%s\n", nextVer.String())
	time.Sleep(sleep)

	fmt.Println("🏁 Preparation finished")
	time.Sleep(sleep)

	fmt.Printf("⚠ Prepared release of gopass %s.\n", nextVer.String())
	time.Sleep(sleep)

	fmt.Printf("⚠ Run 'git push <remote> release/v%s' to push this branch and open a PR against gopasspw/gopass master.\n", nextVer.String())
	time.Sleep(sleep)

	fmt.Printf("⚠ Get the PR merged and run 'git tag -s v%s && git push origin v%s' to kick off the release process.\n", nextVer.String(), nextVer.String())
	time.Sleep(sleep)
	fmt.Println()

	fmt.Println("💎🙌 Done 🚀🚀🚀🚀🚀🚀")
}

func getVersions() (semver.Version, semver.Version) {
	nextVerFlag := ""
	if len(os.Args) > 1 {
		nextVerFlag = strings.TrimSpace(strings.TrimPrefix(os.Args[1], "v"))
	}
	prevVerFlag := ""
	if len(os.Args) > 2 {
		prevVerFlag = strings.TrimSpace(strings.TrimPrefix(os.Args[2], "v"))
	}

	// obtain the last tagged version from git
	gitVer, err := gitVersion()
	if err != nil {
		panic(err)
	}

	// read the version file to get the last committed version
	vfVer, err := versionFile()
	if err != nil {
		panic(err)
	}

	prevVer := gitVer
	if prevVerFlag != "" {
		prevVer = semver.MustParse(prevVerFlag)
	}

	if gitVer.NE(vfVer) {
		fmt.Printf("git version: %q != VERSION: %q\n", gitVer.String(), vfVer.String())
		if prevVerFlag == "" && len(vfVer.Pre) < 1 {
			usage()
			panic("version mismatch")
		}
	}

	nextVer := prevVer
	if nextVerFlag != "" {
		nextVer = semver.MustParse(nextVerFlag)
		if nextVer.LTE(prevVer) {
			usage()
			panic("next version must be greather than the previous version")
		}
	} else {
		nextVer.IncrementPatch()
		if len(vfVer.Pre) > 0 {
			nextVer = vfVer
			nextVer.Pre = nil
		}
	}

	fmt.Printf(`☝ Version overview
  Git (latest tag):  %q
  VERSION:           %q
  Next version flag: %q
  Prev version flag: %q

Will use
  Previous: %q
  Next:     %q
`,
		gitVer,
		vfVer,
		prevVerFlag,
		nextVerFlag,
		prevVer,
		nextVer)

	return prevVer, nextVer
}

func gitCoMaster() error {
	cmd := exec.Command("git", "checkout", "master")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func gitPom() error {
	cmd := exec.Command("git", "pull", "origin", "master")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func gitCoRel(v semver.Version) error {
	cmd := exec.Command("git", "checkout", "-b", "release/v"+v.String())
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func gitCommit(v semver.Version) error {
	cmd := exec.Command("git", "add", "CHANGELOG.md", "VERSION", "version.go", "gopass.1", "*.completion")
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	cmd = exec.Command("git", "commit", "-s", "-m", "Tag v"+v.String(), "-m", "RELEASE_NOTES=n/a")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func writeChangelog(prev, next semver.Version) error {
	cl, err := changelogEntries(prev)
	if err != nil {
		panic(err)
	}

	// prepend the new changelog entries by first writing the
	// new content in a new file ...
	fh, err := os.Create("CHANGELOG.new")
	if err != nil {
		return err
	}
	defer fh.Close()

	fmt.Fprintf(fh, "## %s / %s\n\n", next.String(), time.Now().UTC().Format("2006-01-02"))
	for _, e := range cl {
		fmt.Fprint(fh, "* ")
		fmt.Fprintln(fh, e)
	}
	fmt.Fprintln(fh)

	ofh, err := os.Open("CHANGELOG.md")
	if err != nil {
		return err
	}
	defer ofh.Close()

	// then appending any existing content from the old file and ...
	if _, err := io.Copy(fh, ofh); err != nil {
		return err
	}

	// renaming the new file to the old file
	return os.Rename("CHANGELOG.new", "CHANGELOG.md")
}

func updateCompletion() error {
	cmd := exec.Command("make", "completion")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func updateManpage() error {
	cmd := exec.Command("make", "man")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func writeVersion(v semver.Version) error {
	return os.WriteFile("VERSION", []byte(v.String()+"\n"), 0644)
}

type tplPayload struct {
	Major uint64
	Minor uint64
	Patch uint64
}

func writeVersionGo(v semver.Version) error {
	tmpl, err := template.New("version").Parse(verTmpl)
	if err != nil {
		return err
	}
	fh, err := os.Create("version.go")
	if err != nil {
		return err
	}
	defer fh.Close()
	return tmpl.Execute(fh, tplPayload{
		Major: v.Major,
		Minor: v.Minor,
		Patch: v.Patch,
	})
}

func isGitClean() bool {
	buf, err := exec.Command("git", "diff", "--stat").CombinedOutput()
	if err != nil {
		panic(err)
	}
	return strings.TrimSpace(string(buf)) == ""
}

func versionFile() (semver.Version, error) {
	buf, err := os.ReadFile("VERSION")
	if err != nil {
		return semver.Version{}, err
	}
	return semver.Parse(strings.TrimSpace(string(buf)))
}

func gitVersion() (semver.Version, error) {
	buf, err := exec.Command("git", "tag", "--sort=version:refname").CombinedOutput()
	if err != nil {
		return semver.Version{}, err
	}
	lines := strings.Split(strings.TrimSpace(string(buf)), "\n")
	if len(lines) < 1 {
		return semver.Version{}, fmt.Errorf("no output")
	}
	return semver.Parse(strings.TrimPrefix(lines[len(lines)-1], "v"))
}

func changelogEntries(since semver.Version) ([]string, error) {
	gitSep := "@@@GIT-SEP@@@"
	gitDelim := "@@@GIT-DELIM@@@"
	// full hash - subject - body
	// note: we don't use the hash at the moment
	prettyFormat := gitSep + "%H" + gitDelim + "%s" + gitDelim + "%b" + gitSep
	args := []string{
		"log",
		"v" + since.String() + "..HEAD",
		"--pretty=" + prettyFormat,
	}
	buf, err := exec.Command("git", args...).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("failed to run git %+v with error %w: %s", args, err, string(buf))
	}

	notes := make([]string, 0, 10)
	commits := strings.Split(string(buf), gitSep)
	for _, commit := range commits {
		commit := strings.TrimSpace(commit)
		if commit == "" {
			continue
		}
		p := strings.Split(commit, gitDelim)
		if len(p) < 3 {
			// invalid entry, shouldn't happen
			continue
		}

		issues := []string{}
		if m := issueRE.FindStringSubmatch(strings.TrimSpace(p[1])); len(m) > 1 {
			issues = append(issues, m[1])
		}

		for _, line := range strings.Split(p[2], "\n") {
			line := strings.TrimSpace(line)

			if m := issueRE.FindStringSubmatch(line); len(m) > 1 {
				issues = append(issues, m[1])
			}

			if !strings.HasPrefix(line, "RELEASE_NOTES=") {
				continue
			}
			_, val, found := strings.Cut(line, "=")
			if !found {
				continue
			}
			if strings.ToLower(val) == "n/a" {
				continue
			}
			if len(issues) > 0 {
				val += " (#" + strings.Join(issues, ", #") + ")"
			}
			notes = append(notes, val)
		}
	}

	sort.Strings(notes)
	return notes, nil
}

func usage() {
	fmt.Printf("Usage: %s [next version] [prev version]\n", "go run helpers/release/main.go")
}
<commit_msg>Add patch release workaround to the helper<commit_after>// Copyright 2021 The gopass Authors. All rights reserved.
// Use of this source code is governed by the MIT license,
// that can be found in the LICENSE file.

// Release is the first part of the gopass release automation. It's supposed
// to be run by a member of the gopass team. It will ensure that the repository
// is in a clean state and make it trivial to trigger a new release.
// You can run it without any parameters and as long as you pay close attention
// to the output it will be a breeze.
package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/blang/semver/v4"
)

var sleep = time.Second
var issueRE = regexp.MustCompile(`#(\d+)\b`)
var verTmpl = `package main

import (
	"strings"

	"github.com/blang/semver/v4"
)

func getVersion() semver.Version {
	sv, err := semver.Parse(strings.TrimPrefix(version, "v"))
	if err == nil {
		if commit != "" {
			sv.Build = []string{commit}
		}
		return sv
	}
	return semver.Version{
		Major: {{ .Major }},
		Minor: {{ .Minor }},
		Patch: {{ .Patch }},
		Pre: []semver.PRVersion{
			{VersionStr: "git"},
		},
		Build: []string{"HEAD"},
	}
}
`

const logo = `
   __     _    _ _      _ _   ___   ___
 /'_ '\ /'_'\ ( '_'\  /'_' )/',__)/',__)
( (_) |( (_) )| (_) )( (_| |\__, \\__, \
'\__  |'\___/'| ,__/''\__,_)(____/(____/
( )_) |       | |
 \___/'       (_)

 `

func main() {
	fmt.Println(logo)
	fmt.Println()
	fmt.Println("🌟 Preparing a new gopass release.")
	fmt.Println("☝  Checking pre-conditions ...")
	// - check that workdir is clean
	if !isGitClean() {
		panic("❌ git is dirty")
	}
	fmt.Println("✅ git is clean")

	if sv := os.Getenv("PATCH_RELEASE"); sv == "" {
		// - check out master
		if err := gitCoMaster(); err != nil {
			panic(err)
		}
		fmt.Println("✅ Switched to master branch")
		// - pull from origin
		if err := gitPom(); err != nil {
			panic(err)
		}
		fmt.Println("✅ Fetched changes for master")
	}
	// - check that workdir is clean
	if !isGitClean() {
		panic("git is dirty")
	}
	fmt.Println("✅ git is still clean")

	prevVer, nextVer := getVersions()

	fmt.Println()
	fmt.Printf("✅ New version will be: %s\n", nextVer.String())
	fmt.Println()
	fmt.Println("❓ Do you want to continue? (press any key to continue or Ctrl+C to abort)")
	fmt.Scanln()

	// - update VERSION
	if err := writeVersion(nextVer); err != nil {
		panic(err)
	}
	fmt.Println("✅ Wrote VERSION")
	time.Sleep(sleep)
	// - update version.go
	if err := writeVersionGo(nextVer); err != nil {
		panic(err)
	}
	fmt.Println("✅ Wrote version.go")
	time.Sleep(sleep)
	// - update CHANGELOG.md
	if err := writeChangelog(prevVer, nextVer); err != nil {
		panic(err)
	}
	fmt.Println("✅ Updated CHANGELOG.md")
	time.Sleep(sleep)
	// - update shell completions
	if err := updateCompletion(); err != nil {
		panic(err)
	}
	fmt.Println("✅ Updated shell completions")
	time.Sleep(sleep)
	// - update man page
	if err := updateManpage(); err != nil {
		panic(err)
	}
	fmt.Println("✅ Updated man page")
	time.Sleep(sleep)

	// - create PR
	//   git checkout -b release/vX.Y.Z
	if err := gitCoRel(nextVer); err != nil {
		panic(err)
	}
	fmt.Printf("✅ Created branch release/v%s\n", nextVer.String())
	time.Sleep(sleep)

	// commit changes
	if err := gitCommit(nextVer); err != nil {
		panic(err)
	}
	fmt.Printf("✅ Committed changes to release/v%s\n", nextVer.String())
	time.Sleep(sleep)

	fmt.Println("🏁 Preparation finished")
	time.Sleep(sleep)

	fmt.Printf("⚠ Prepared release of gopass %s.\n", nextVer.String())
	time.Sleep(sleep)

	fmt.Printf("⚠ Run 'git push <remote> release/v%s' to push this branch and open a PR against gopasspw/gopass master.\n", nextVer.String())
	time.Sleep(sleep)

	fmt.Printf("⚠ Get the PR merged and run 'git tag -s v%s && git push origin v%s' to kick off the release process.\n", nextVer.String(), nextVer.String())
	time.Sleep(sleep)
	fmt.Println()

	fmt.Println("💎🙌 Done 🚀🚀🚀🚀🚀🚀")
}

func getVersions() (semver.Version, semver.Version) {
	nextVerFlag := ""
	if len(os.Args) > 1 {
		nextVerFlag = strings.TrimSpace(strings.TrimPrefix(os.Args[1], "v"))
	}
	prevVerFlag := ""
	if len(os.Args) > 2 {
		prevVerFlag = strings.TrimSpace(strings.TrimPrefix(os.Args[2], "v"))
	}

	// obtain the last tagged version from git
	gitVer, err := gitVersion()
	if err != nil {
		panic(err)
	}

	// read the version file to get the last committed version
	vfVer, err := versionFile()
	if err != nil {
		panic(err)
	}

	prevVer := gitVer
	if prevVerFlag != "" {
		prevVer = semver.MustParse(prevVerFlag)
	}

	if gitVer.NE(vfVer) {
		fmt.Printf("git version: %q != VERSION: %q\n", gitVer.String(), vfVer.String())
		if prevVerFlag == "" && len(vfVer.Pre) < 1 {
			usage()
			panic("version mismatch")
		}
	}

	nextVer := prevVer
	if nextVerFlag != "" {
		nextVer = semver.MustParse(nextVerFlag)
		if nextVer.LTE(prevVer) {
			usage()
			panic("next version must be greather than the previous version")
		}
	} else {
		nextVer.IncrementPatch()
		if len(vfVer.Pre) > 0 {
			nextVer = vfVer
			nextVer.Pre = nil
		}
	}

	fmt.Printf(`☝ Version overview
  Git (latest tag):  %q
  VERSION:           %q
  Next version flag: %q
  Prev version flag: %q

Will use
  Previous: %q
  Next:     %q
`,
		gitVer,
		vfVer,
		prevVerFlag,
		nextVerFlag,
		prevVer,
		nextVer)

	return prevVer, nextVer
}

func gitCoMaster() error {
	cmd := exec.Command("git", "checkout", "master")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func gitPom() error {
	cmd := exec.Command("git", "pull", "origin", "master")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func gitCoRel(v semver.Version) error {
	cmd := exec.Command("git", "checkout", "-b", "release/v"+v.String())
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func gitCommit(v semver.Version) error {
	cmd := exec.Command("git", "add", "CHANGELOG.md", "VERSION", "version.go", "gopass.1", "*.completion")
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}
	cmd = exec.Command("git", "commit", "-s", "-m", "Tag v"+v.String(), "-m", "RELEASE_NOTES=n/a")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func writeChangelog(prev, next semver.Version) error {
	cl, err := changelogEntries(prev)
	if err != nil {
		panic(err)
	}

	// prepend the new changelog entries by first writing the
	// new content in a new file ...
	fh, err := os.Create("CHANGELOG.new")
	if err != nil {
		return err
	}
	defer fh.Close()

	fmt.Fprintf(fh, "## %s / %s\n\n", next.String(), time.Now().UTC().Format("2006-01-02"))
	for _, e := range cl {
		fmt.Fprint(fh, "* ")
		fmt.Fprintln(fh, e)
	}
	fmt.Fprintln(fh)

	ofh, err := os.Open("CHANGELOG.md")
	if err != nil {
		return err
	}
	defer ofh.Close()

	// then appending any existing content from the old file and ...
	if _, err := io.Copy(fh, ofh); err != nil {
		return err
	}

	// renaming the new file to the old file
	return os.Rename("CHANGELOG.new", "CHANGELOG.md")
}

func updateCompletion() error {
	cmd := exec.Command("make", "completion")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func updateManpage() error {
	cmd := exec.Command("make", "man")
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func writeVersion(v semver.Version) error {
	return os.WriteFile("VERSION", []byte(v.String()+"\n"), 0644)
}

type tplPayload struct {
	Major uint64
	Minor uint64
	Patch uint64
}

func writeVersionGo(v semver.Version) error {
	tmpl, err := template.New("version").Parse(verTmpl)
	if err != nil {
		return err
	}
	fh, err := os.Create("version.go")
	if err != nil {
		return err
	}
	defer fh.Close()
	return tmpl.Execute(fh, tplPayload{
		Major: v.Major,
		Minor: v.Minor,
		Patch: v.Patch,
	})
}

func isGitClean() bool {
	buf, err := exec.Command("git", "diff", "--stat").CombinedOutput()
	if err != nil {
		panic(err)
	}
	return strings.TrimSpace(string(buf)) == ""
}

func versionFile() (semver.Version, error) {
	buf, err := os.ReadFile("VERSION")
	if err != nil {
		return semver.Version{}, err
	}
	return semver.Parse(strings.TrimSpace(string(buf)))
}

func gitVersion() (semver.Version, error) {
	buf, err := exec.Command("git", "tag", "--sort=version:refname").CombinedOutput()
	if err != nil {
		return semver.Version{}, err
	}
	lines := strings.Split(strings.TrimSpace(string(buf)), "\n")
	if len(lines) < 1 {
		return semver.Version{}, fmt.Errorf("no output")
	}
	return semver.Parse(strings.TrimPrefix(lines[len(lines)-1], "v"))
}

func changelogEntries(since semver.Version) ([]string, error) {
	gitSep := "@@@GIT-SEP@@@"
	gitDelim := "@@@GIT-DELIM@@@"
	// full hash - subject - body
	// note: we don't use the hash at the moment
	prettyFormat := gitSep + "%H" + gitDelim + "%s" + gitDelim + "%b" + gitSep
	args := []string{
		"log",
		"v" + since.String() + "..HEAD",
		"--pretty=" + prettyFormat,
	}
	buf, err := exec.Command("git", args...).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("failed to run git %+v with error %w: %s", args, err, string(buf))
	}

	notes := make([]string, 0, 10)
	commits := strings.Split(string(buf), gitSep)
	for _, commit := range commits {
		commit := strings.TrimSpace(commit)
		if commit == "" {
			continue
		}
		p := strings.Split(commit, gitDelim)
		if len(p) < 3 {
			// invalid entry, shouldn't happen
			continue
		}

		issues := []string{}
		if m := issueRE.FindStringSubmatch(strings.TrimSpace(p[1])); len(m) > 1 {
			issues = append(issues, m[1])
		}

		for _, line := range strings.Split(p[2], "\n") {
			line := strings.TrimSpace(line)

			if m := issueRE.FindStringSubmatch(line); len(m) > 1 {
				issues = append(issues, m[1])
			}

			if !strings.HasPrefix(line, "RELEASE_NOTES=") {
				continue
			}
			p := strings.Split(line, "=")
			if len(p) < 2 {
				continue
			}
			val := p[1]
			if strings.ToLower(val) == "n/a" {
				continue
			}
			if len(issues) > 0 {
				val += " (#" + strings.Join(issues, ", #") + ")"
			}
			notes = append(notes, val)
		}
	}

	sort.Strings(notes)
	return notes, nil
}

func usage() {
	fmt.Printf("Usage: %s [next version] [prev version]\n", "go run helpers/release/main.go")
}
<|endoftext|>


<commit_before>package openrtb2

import "encoding/json"

// 3.2.14 Object: App
//
// This object should be included if the ad supported content is a non-browser application (typically in mobile) as opposed to a website.
// A bid request must not contain both an App and a Site object.
// At a minimum, it is useful to provide an App ID or bundle, but this is not strictly required.
type App struct {

	// Attribute:
	//   id
	// Type:
	//   string; recommended
	// Description:
	//   Exchange-specific app ID.
	ID string `json:"id,omitempty"`

	// Attribute:
	//   name
	// Type:
	//   string
	// Description:
	//   App name (may be aliased at the publisher’s request).
	Name string `json:"name,omitempty"`

	// Attribute:
	//   bundle
	// Type:
	//   string
	// Description:
	//   The store ID of the app in an app store. See OTT/CTV Store
	//   Assigned App Identification Guidelines for more details about
	//   expected strings for CTV app stores. For mobile apps in
	//   Google Play Store, these should be bundle or package names
	//   (e.g. com.foo.mygame). For apps in Apple App Store, these
	//   should be a numeric ID.
	Bundle string `json:"bundle,omitempty"`

	// Attribute:
	//   domain
	// Type:
	//   string
	// Description:
	//   Domain of the app (e.g., “mygame.foo.com”).
	Domain string `json:"domain,omitempty"`

	// Attribute:
	//   storeurl
	// Type:
	//   string
	// Description:
	//   App store URL for an installed app; for IQG 2.1 compliance.
	StoreURL string `json:"storeurl,omitempty"`

	// Attribute:
	//   cattax
	// Type:
	//   integer; default 1
	// Description:
	//   The taxonomy in use. Refer to the AdCOM list List: Category
	//   Taxonomies for values.
	CatTax int64 `json:"cattax,omitempty"`

	// Attribute:
	//   cat
	// Type:
	//   string array
	// Description:
	//   Array of IAB content categories of the app. The taxonomy to be
	//   used is defined by the cattax field. If no cattax field is supplied
	//   IAB Content Category Taxonomy 1.0 is assumed.
	Cat []string `json:"cat,omitempty"`

	// Attribute:
	//   sectioncat
	// Type:
	//   string array
	// Description:
	//   Array of IAB content categories that describe the current
	//   section of the app.
	//   The taxonomy to be used is defined by the cattax field.
	SectionCat []string `json:"sectioncat,omitempty"`

	// Attribute:
	//   pagecat
	// Type:
	//   string array
	// Description:
	//   Array of IAB content categories that describe the current page
	//   or view of the app.
	//   The taxonomy to be used is defined by the cattax field.
	PageCat []string `json:"pagecat,omitempty"`

	// Attribute:
	//   ver
	// Type:
	//   string
	// Description:
	//   Application version.
	Ver string `json:"ver,omitempty"`

	// Attribute:
	//   privacypolicy
	// Type:
	//   integer
	// Description:
	//   Indicates if the app has a privacy policy, where 0 = no, 1 = yes.
	PrivacyPolicy int8 `json:"privacypolicy,omitempty"`

	// Attribute:
	//   paid
	// Type:
	//   integer
	// Description:
	//   0 = app is free, 1 = the app is a paid version.
	Paid int8 `json:"paid,omitempty"`

	// Attribute:
	//   publisher
	// Type:
	//   object
	// Description:
	//   Details about the Publisher (Section 3.2.15) of the app.
	Publisher *Publisher `json:"publisher,omitempty"`

	// Attribute:
	//   content
	// Type:
	//   object
	// Description:
	//   Details about the Content (Section 3.2.16) within the app
	Content *Content `json:"content,omitempty"`

	// Attribute:
	//   keywords
	// Type:
	//   string
	// Description:
	//   Comma separated list of keywords about the app. Only one of
	//   ‘keywords’ or ‘kwarray’ may be present.
	Keywords string `json:"keywords,omitempty"`

	// Attribute:
	//   kwarray
	// Type:
	//   string
	// Description:
	//   Array of keywords about the site. Only one of ‘keywords’ or
	//   ‘kwarray’ may be present.
	KwArray []string `json:"kwarray,omitempty"`

	// Attribute:
	//   ext
	// Type:
	//   object
	// Description:
	//   Placeholder for exchange-specific extensions to OpenRTB.
	Ext json.RawMessage `json:"ext,omitempty"`
}
<commit_msg>openrtb2: impl TODOs for App<commit_after>package openrtb2

import (
	"encoding/json"

	"github.com/mxmCherry/openrtb/v16/adcom1"
)

// 3.2.14 Object: App
//
// This object should be included if the ad supported content is a non-browser application (typically in mobile) as opposed to a website.
// A bid request must not contain both an App and a Site object.
// At a minimum, it is useful to provide an App ID or bundle, but this is not strictly required.
type App struct {

	// Attribute:
	//   id
	// Type:
	//   string; recommended
	// Description:
	//   Exchange-specific app ID.
	ID string `json:"id,omitempty"`

	// Attribute:
	//   name
	// Type:
	//   string
	// Description:
	//   App name (may be aliased at the publisher’s request).
	Name string `json:"name,omitempty"`

	// Attribute:
	//   bundle
	// Type:
	//   string
	// Description:
	//   The store ID of the app in an app store. See OTT/CTV Store
	//   Assigned App Identification Guidelines for more details about
	//   expected strings for CTV app stores. For mobile apps in
	//   Google Play Store, these should be bundle or package names
	//   (e.g. com.foo.mygame). For apps in Apple App Store, these
	//   should be a numeric ID.
	Bundle string `json:"bundle,omitempty"`

	// Attribute:
	//   domain
	// Type:
	//   string
	// Description:
	//   Domain of the app (e.g., “mygame.foo.com”).
	Domain string `json:"domain,omitempty"`

	// Attribute:
	//   storeurl
	// Type:
	//   string
	// Description:
	//   App store URL for an installed app; for IQG 2.1 compliance.
	StoreURL string `json:"storeurl,omitempty"`

	// Attribute:
	//   cattax
	// Type:
	//   integer; default 1
	// Description:
	//   The taxonomy in use. Refer to the AdCOM list List: Category
	//   Taxonomies for values.
	CatTax adcom1.CategoryTaxonomy `json:"cattax,omitempty"`

	// Attribute:
	//   cat
	// Type:
	//   string array
	// Description:
	//   Array of IAB content categories of the app. The taxonomy to be
	//   used is defined by the cattax field. If no cattax field is supplied
	//   IAB Content Category Taxonomy 1.0 is assumed.
	Cat []string `json:"cat,omitempty"`

	// Attribute:
	//   sectioncat
	// Type:
	//   string array
	// Description:
	//   Array of IAB content categories that describe the current
	//   section of the app.
	//   The taxonomy to be used is defined by the cattax field.
	SectionCat []string `json:"sectioncat,omitempty"`

	// Attribute:
	//   pagecat
	// Type:
	//   string array
	// Description:
	//   Array of IAB content categories that describe the current page
	//   or view of the app.
	//   The taxonomy to be used is defined by the cattax field.
	PageCat []string `json:"pagecat,omitempty"`

	// Attribute:
	//   ver
	// Type:
	//   string
	// Description:
	//   Application version.
	Ver string `json:"ver,omitempty"`

	// Attribute:
	//   privacypolicy
	// Type:
	//   integer
	// Description:
	//   Indicates if the app has a privacy policy, where 0 = no, 1 = yes.
	PrivacyPolicy int8 `json:"privacypolicy,omitempty"`

	// Attribute:
	//   paid
	// Type:
	//   integer
	// Description:
	//   0 = app is free, 1 = the app is a paid version.
	Paid int8 `json:"paid,omitempty"`

	// Attribute:
	//   publisher
	// Type:
	//   object
	// Description:
	//   Details about the Publisher (Section 3.2.15) of the app.
	Publisher *Publisher `json:"publisher,omitempty"`

	// Attribute:
	//   content
	// Type:
	//   object
	// Description:
	//   Details about the Content (Section 3.2.16) within the app
	Content *Content `json:"content,omitempty"`

	// Attribute:
	//   keywords
	// Type:
	//   string
	// Description:
	//   Comma separated list of keywords about the app. Only one of
	//   ‘keywords’ or ‘kwarray’ may be present.
	Keywords string `json:"keywords,omitempty"`

	// Attribute:
	//   kwarray
	// Type:
	//   string
	// Description:
	//   Array of keywords about the site. Only one of ‘keywords’ or
	//   ‘kwarray’ may be present.
	KwArray []string `json:"kwarray,omitempty"`

	// Attribute:
	//   ext
	// Type:
	//   object
	// Description:
	//   Placeholder for exchange-specific extensions to OpenRTB.
	Ext json.RawMessage `json:"ext,omitempty"`
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"github.com/ernestio/definition-mapper/libmapper/providers/azure/components"
	"github.com/ernestio/definition-mapper/libmapper/providers/azure/definition"
	"github.com/ernestio/ernestprovider/providers/azure/networkinterface"
	graph "gopkg.in/r3labs/graph.v2"
)

// MapNetworkInterfaces ...
func MapNetworkInterfaces(d *definition.Definition) (interfaces []*components.NetworkInterface) {
	for _, rg := range d.ResourceGroups {
		for _, ni := range rg.NetworkInterfaces {
			cv := components.NetworkInterface{}
			cv.Name = ni.Name
			cv.NetworkSecurityGroup = ni.SecurityGroup
			cv.DNSServers = ni.DNSServers
			cv.InternalDNSNameLabel = ni.InternalDNSNameLabel
			cv.ResourceGroupName = rg.Name
			cv.Tags = mapTags(ni.Name, d.Name)

			for _, ip := range ni.IPConfigurations {
				nIP := networkinterface.IPConfiguration{
					Name:                       ip.Name,
					Subnet:                     ip.Subnet,
					PrivateIPAddress:           ip.PrivateIPAddress,
					PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
					PublicIPAddress:            ip.PublicIPAddressID,
				}
				cv.IPConfigurations = append(cv.IPConfigurations, nIP)
			}

			if ni.ID != "" {
				cv.SetAction("none")
			}

			cv.SetDefaultVariables()

			interfaces = append(interfaces, &cv)
		}
	}

	return
}

// MapDefinitionNetworkInterfaces : ...
func MapDefinitionNetworkInterfaces(g *graph.Graph, rg *definition.ResourceGroup) (nis []definition.NetworkInterface) {
	for _, c := range g.GetComponents().ByType("network_interface") {
		ni := c.(*components.NetworkInterface)

		if ni.ResourceGroupName != rg.Name {
			continue
		}

		nNi := definition.NetworkInterface{
			ID:                   ni.GetProviderID(),
			Name:                 ni.Name,
			SecurityGroup:        ni.NetworkSecurityGroup,
			DNSServers:           ni.DNSServers,
			InternalDNSNameLabel: ni.InternalDNSNameLabel,
		}

		for _, ip := range ni.IPConfigurations {
			nIP := definition.IPConfiguration{
				Name:                       ip.Name,
				Subnet:                     ip.Subnet,
				PrivateIPAddress:           ip.PrivateIPAddress,
				PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
				PublicIPAddressID:          ip.PublicIPAddress,
			}
			nNi.IPConfigurations = append(nNi.IPConfigurations, nIP)
		}

		nis = append(nis, nNi)
	}

	return
}
<commit_msg>Setting some defaults to network interfaces<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package mapper

import (
	"github.com/ernestio/definition-mapper/libmapper/providers/azure/components"
	"github.com/ernestio/definition-mapper/libmapper/providers/azure/definition"
	"github.com/ernestio/ernestprovider/providers/azure/networkinterface"
	graph "gopkg.in/r3labs/graph.v2"
)

// MapNetworkInterfaces ...
func MapNetworkInterfaces(d *definition.Definition) (interfaces []*components.NetworkInterface) {
	for _, rg := range d.ResourceGroups {
		for _, ni := range rg.NetworkInterfaces {
			cv := components.NetworkInterface{}
			cv.Name = ni.Name
			cv.NetworkSecurityGroup = ni.SecurityGroup
			cv.DNSServers = ni.DNSServers
			cv.InternalDNSNameLabel = ni.InternalDNSNameLabel
			cv.ResourceGroupName = rg.Name
			cv.Location = rg.Location
			cv.Tags = mapTags(ni.Name, d.Name)

			for _, ip := range ni.IPConfigurations {
				nIP := networkinterface.IPConfiguration{
					Name:                       ip.Name,
					Subnet:                     ip.Subnet,
					PrivateIPAddress:           ip.PrivateIPAddress,
					PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
					PublicIPAddress:            ip.PublicIPAddressID,
				}
				if nIP.PrivateIPAddressAllocation == "" {
					nIP.PrivateIPAddressAllocation = "static"
				}
				cv.IPConfigurations = append(cv.IPConfigurations, nIP)
			}

			if ni.ID != "" {
				cv.SetAction("none")
			}

			cv.SetDefaultVariables()

			interfaces = append(interfaces, &cv)
		}
	}

	return
}

// MapDefinitionNetworkInterfaces : ...
func MapDefinitionNetworkInterfaces(g *graph.Graph, rg *definition.ResourceGroup) (nis []definition.NetworkInterface) {
	for _, c := range g.GetComponents().ByType("network_interface") {
		ni := c.(*components.NetworkInterface)

		if ni.ResourceGroupName != rg.Name {
			continue
		}

		nNi := definition.NetworkInterface{
			ID:                   ni.GetProviderID(),
			Name:                 ni.Name,
			SecurityGroup:        ni.NetworkSecurityGroup,
			DNSServers:           ni.DNSServers,
			InternalDNSNameLabel: ni.InternalDNSNameLabel,
		}

		for _, ip := range ni.IPConfigurations {
			nIP := definition.IPConfiguration{
				Name:                       ip.Name,
				Subnet:                     ip.Subnet,
				PrivateIPAddress:           ip.PrivateIPAddress,
				PrivateIPAddressAllocation: ip.PrivateIPAddressAllocation,
				PublicIPAddressID:          ip.PublicIPAddress,
			}
			nNi.IPConfigurations = append(nNi.IPConfigurations, nIP)
		}

		nis = append(nis, nNi)
	}

	return
}
<|endoftext|>


<commit_before>package zd

import (
	"fmt"
)

// OrganizationSearchResponse struct
type OrganizationSearchResponse struct {
	Organizations Organization `json:"results,omitempty"`
	NextPage      *string      `json:"next_page,omitempty"`
	PreviousPage  *string      `json:"previous_page,omitempty"`
	Count         *int         `json:"count,omitempty"`
}

// OrganizationWrapper struct
type OrganizationWrapper struct {
	Organization *Organization `   json:"organization"`
}

// OrganizationResponse struct
type OrganizationResponse struct {
	Organizations Organization `json:"organizations,omitempty"`
	NextPage      *string      `json:"next_page,omitempty"`
	PreviousPage  *string      `json:"previous_page,omitempty"`
	Count         *int         `json:"count,omitempty"`
}

// Organization struct
type Organization struct {
	URL                *string           `json:"url,omitempty"`
	ID                 *int              `json:"id,omitempty"`
	Name               *string           `json:"name,omitempty"`
	SharedTickets      *bool             `json:"shared_tickets,omitempty"`
	SharedComments     *bool             `json:"shared_comments,omitempty"`
	ExternalID         *string           `json:"external_id,omitempty"`
	CreatedAt          *string           `json:"created_at,omitempty"`
	UpdatedAt          *string           `json:"updated_at,omitempty"`
	DomainNames        []string          `json:"domain_names,omitempty"`
	Details            *string           `json:"details,omitempty"`
	Notes              *string           `json:"notes,omitempty"`
	GroupID            *string           `json:"group_id,omitempty"`
	Tags               []string          `json:"tags,omitempty"`
	OrganizationFields map[string]string `json:"organization_fields,omitempty"`
}

// OrganizationService struct
type OrganizationService struct {
	client *Client
}

// GetOrganizationByID finds an organization in Zendesk by ID
func (s *OrganizationService) GetOrganizationByID(organizationID string) (*Organization, *Response, error) {
	org := OrganizationWrapper{}

	url := fmt.Sprintf("organizations/%s.json", organizationID)

	req, err := s.client.NewRequest("GET", url, nil)
	if err != nil {
		return nil, nil, err
	}

	resp, err := s.client.Do(req, &org)
	if err != nil {
		return nil, nil, err
	}

	return org.Organization, resp, err
}

// UpdateOrganization updates and organization by id
func (s *OrganizationService) UpdateOrganization(org *Organization) (*Organization, error) {
	organization := &Organization{}

	url := fmt.Sprintf("organizations/%v.json", org.ID)
	or := &OrganizationWrapper{Organization: org}

	req, err := s.client.NewRequest("PUT", url, or)
	if err != nil {
		return organization, err
	}

	result := OrganizationWrapper{}
	_, err = s.client.Do(req, &result)
	if err != nil {
		return organization, err
	}

	organization = result.Organization
	return organization, err
}

//CreateOrganization creates a new organization
func (s *OrganizationService) CreateOrganization(org *Organization) (*Organization, error) {
	organization := &Organization{}

	or := &OrganizationWrapper{Organization: org}
	url := fmt.Sprintf("organizations.json")

	req, err := s.client.NewRequest("POST", url, or)
	if err != nil {
		return organization, err
	}

	result := OrganizationWrapper{}
	_, err = s.client.Do(req, &result)
	if err != nil {
		return organization, err
	}

	organization = result.Organization
	return organization, nil
}

// SearchOrganizationByName searches the organization by name
func (s *OrganizationService) SearchOrganizationByName(orgName string) (*OrganizationSearchResponse, error) {
	org := &OrganizationSearchResponse{}
	url := fmt.Sprintf("search?query=type:organization+name:%s", orgName)

	req, err := s.client.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	_, err = s.client.Do(req, &org)
	if err != nil {
		return nil, err
	}

	return org, nil
}
<commit_msg>allow multiple org returns<commit_after>package zd

import (
	"fmt"
)

// OrganizationSearchResponse struct
type OrganizationSearchResponse struct {
	Organizations []Organization `json:"results,omitempty"`
	NextPage      *string        `json:"next_page,omitempty"`
	PreviousPage  *string        `json:"previous_page,omitempty"`
	Count         *int           `json:"count,omitempty"`
}

// OrganizationWrapper struct
type OrganizationWrapper struct {
	Organization *Organization `   json:"organization"`
}

// OrganizationResponse struct
type OrganizationResponse struct {
	Organizations Organization `json:"organizations,omitempty"`
	NextPage      *string      `json:"next_page,omitempty"`
	PreviousPage  *string      `json:"previous_page,omitempty"`
	Count         *int         `json:"count,omitempty"`
}

// Organization struct
type Organization struct {
	URL                *string           `json:"url,omitempty"`
	ID                 *int              `json:"id,omitempty"`
	Name               *string           `json:"name,omitempty"`
	SharedTickets      *bool             `json:"shared_tickets,omitempty"`
	SharedComments     *bool             `json:"shared_comments,omitempty"`
	ExternalID         *string           `json:"external_id,omitempty"`
	CreatedAt          *string           `json:"created_at,omitempty"`
	UpdatedAt          *string           `json:"updated_at,omitempty"`
	DomainNames        []string          `json:"domain_names,omitempty"`
	Details            *string           `json:"details,omitempty"`
	Notes              *string           `json:"notes,omitempty"`
	GroupID            *string           `json:"group_id,omitempty"`
	Tags               []string          `json:"tags,omitempty"`
	OrganizationFields map[string]string `json:"organization_fields,omitempty"`
}

// OrganizationService struct
type OrganizationService struct {
	client *Client
}

// GetOrganizationByID finds an organization in Zendesk by ID
func (s *OrganizationService) GetOrganizationByID(organizationID string) (*Organization, *Response, error) {
	org := OrganizationWrapper{}

	url := fmt.Sprintf("organizations/%s.json", organizationID)

	req, err := s.client.NewRequest("GET", url, nil)
	if err != nil {
		return nil, nil, err
	}

	resp, err := s.client.Do(req, &org)
	if err != nil {
		return nil, nil, err
	}

	return org.Organization, resp, err
}

// UpdateOrganization updates and organization by id
func (s *OrganizationService) UpdateOrganization(org *Organization) (*Organization, error) {
	organization := &Organization{}

	url := fmt.Sprintf("organizations/%v.json", org.ID)
	or := &OrganizationWrapper{Organization: org}

	req, err := s.client.NewRequest("PUT", url, or)
	if err != nil {
		return organization, err
	}

	result := OrganizationWrapper{}
	_, err = s.client.Do(req, &result)
	if err != nil {
		return organization, err
	}

	organization = result.Organization
	return organization, err
}

//CreateOrganization creates a new organization
func (s *OrganizationService) CreateOrganization(org *Organization) (*Organization, error) {
	organization := &Organization{}

	or := &OrganizationWrapper{Organization: org}
	url := fmt.Sprintf("organizations.json")

	req, err := s.client.NewRequest("POST", url, or)
	if err != nil {
		return organization, err
	}

	result := OrganizationWrapper{}
	_, err = s.client.Do(req, &result)
	if err != nil {
		return organization, err
	}

	organization = result.Organization
	return organization, nil
}

// SearchOrganizationByName searches the organization by name
func (s *OrganizationService) SearchOrganizationByName(orgName string) (*OrganizationSearchResponse, error) {
	org := &OrganizationSearchResponse{}
	url := fmt.Sprintf("search?query=type:organization+name:%s", orgName)

	req, err := s.client.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	_, err = s.client.Do(req, &org)
	if err != nil {
		return nil, err
	}

	return org, nil
}
<|endoftext|>


<commit_before>package removenonorgmembers

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import (
	"context"
	"encoding/json"
	"log"
	"strings"

	pb "github.com/googlecloudplatform/threat-automation/compiled/sha/protos"
	"github.com/googlecloudplatform/threat-automation/entities"
	"github.com/googlecloudplatform/threat-automation/providers/sha"
	"github.com/pkg/errors"
	"google.golang.org/api/cloudresourcemanager/v1"
)

// Required contains the required values needed for this function.
type Required struct {
	OrganizationName string
}

// ReadFinding will attempt to deserialize all supported findings for this function.
func ReadFinding(b []byte) (*Required, error) {
	var finding pb.IamScanner
	r := &Required{}
	if err := json.Unmarshal(b, &finding); err != nil {
		return nil, errors.Wrap(entities.ErrUnmarshal, err.Error())
	}
	switch finding.GetFinding().GetCategory() {
	case "NON_ORG_IAM_MEMBER":
		r.OrganizationName = sha.OrganizationName(finding.GetFinding().GetParent())
	}
	if r.OrganizationName == "" {
		return nil, entities.ErrValueNotFound
	}
	return r, nil
}

// Execute removes non-organization members.
func Execute(ctx context.Context, required *Required, ent *entities.Entity) error {
	conf := ent.Configuration
	if conf.RemoveNonOrgMembers.Enabled {
		allowedDomains := conf.RemoveNonOrgMembers.AllowDomains
		organization, err := ent.Resource.Organization(ctx, required.OrganizationName)
		if err != nil {
			return errors.Wrapf(err, "failed to get organization: %s", required.OrganizationName)
		}
		policy, err := ent.Resource.PolicyOrganization(ctx, organization.Name)
		if err != nil {
			return errors.Wrap(err, "failed to retrieve organization policies")
		}
		membersToRemove := filterNonOrgMembers(organization.DisplayName, policy.Bindings, allowedDomains)
		if _, err = ent.Resource.RemoveMembersOrganization(ctx, organization.Name, membersToRemove, policy); err != nil {
			return errors.Wrap(err, "failed to remove organization policies")
		}
		log.Printf("removed members: %s", membersToRemove)
		return nil
	}
	log.Println("remove non-org members execution disabled: check settings.json.")
	return nil
}

func filterNonOrgMembers(organizationDisplayName string, bindings []*cloudresourcemanager.Binding, allowedDomains []string) (nonOrgMembers []string) {
	for _, b := range bindings {
		for _, m := range b.Members {
			if notFromOrg(m, "user:", organizationDisplayName) && notWhitelisted(m, allowedDomains) {
				nonOrgMembers = append(nonOrgMembers, m)
			}
		}
	}
	return nonOrgMembers
}

func notWhitelisted(member string, domains []string) bool {
	for _, d := range domains {
		if strings.Contains(member, d) {
			return false
		}
	}
	return true
}

func notFromOrg(member, prefix, content string) bool {
	return strings.HasPrefix(member, prefix) && !strings.Contains(member, content)
}
<commit_msg>[#22] fix validation function semantics<commit_after>package removenonorgmembers

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import (
	"context"
	"encoding/json"
	"log"
	"strings"

	pb "github.com/googlecloudplatform/threat-automation/compiled/sha/protos"
	"github.com/googlecloudplatform/threat-automation/entities"
	"github.com/googlecloudplatform/threat-automation/providers/sha"
	"github.com/pkg/errors"
	"google.golang.org/api/cloudresourcemanager/v1"
)

// Required contains the required values needed for this function.
type Required struct {
	OrganizationName string
}

// ReadFinding will attempt to deserialize all supported findings for this function.
func ReadFinding(b []byte) (*Required, error) {
	var finding pb.IamScanner
	r := &Required{}
	if err := json.Unmarshal(b, &finding); err != nil {
		return nil, errors.Wrap(entities.ErrUnmarshal, err.Error())
	}
	switch finding.GetFinding().GetCategory() {
	case "NON_ORG_IAM_MEMBER":
		r.OrganizationName = sha.OrganizationName(finding.GetFinding().GetParent())
	}
	if r.OrganizationName == "" {
		return nil, entities.ErrValueNotFound
	}
	return r, nil
}

// Execute removes non-organization members.
func Execute(ctx context.Context, required *Required, ent *entities.Entity) error {
	conf := ent.Configuration
	if conf.RemoveNonOrgMembers.Enabled {
		allowedDomains := conf.RemoveNonOrgMembers.AllowDomains
		organization, err := ent.Resource.Organization(ctx, required.OrganizationName)
		if err != nil {
			return errors.Wrapf(err, "failed to get organization: %s", required.OrganizationName)
		}
		policy, err := ent.Resource.PolicyOrganization(ctx, organization.Name)
		if err != nil {
			return errors.Wrap(err, "failed to retrieve organization policies")
		}
		membersToRemove := filterNonOrgMembers(organization.DisplayName, policy.Bindings, allowedDomains)
		if _, err = ent.Resource.RemoveMembersOrganization(ctx, organization.Name, membersToRemove, policy); err != nil {
			return errors.Wrap(err, "failed to remove organization policies")
		}
		log.Printf("removed members: %s", membersToRemove)
		return nil
	}
	log.Println("remove non-org members execution disabled: check settings.json.")
	return nil
}

func filterNonOrgMembers(organizationDisplayName string, bindings []*cloudresourcemanager.Binding, allowedDomains []string) (nonOrgMembers []string) {
	for _, b := range bindings {
		for _, m := range b.Members {
			if strings.HasPrefix(m, "user:") && !inOrg(m, "user:", organizationDisplayName) && !allowed(m, allowedDomains) {
				nonOrgMembers = append(nonOrgMembers, m)
			}
		}
	}
	return nonOrgMembers
}

func allowed(member string, domains []string) bool {
	for _, d := range domains {
		if strings.Contains(member, d) {
			return true
		}
	}
	return false
}

func inOrg(member, prefix, content string) bool {
	return strings.Contains(member, content)
}
<|endoftext|>


<commit_before>// Copyright 2015 - 2017 Ka-Hing Cheung
// Copyright 2015 - 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"

	"github.com/codegangsta/cli"
)

var flagCategories map[string]string

// Set up custom help text for goofys; in particular the usage section.
func filterCategory(flags []cli.Flag, category string) (ret []cli.Flag) {
	for _, f := range flags {
		if flagCategories[f.GetName()] == category {
			ret = append(ret, f)
		}
	}
	return
}

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range category .Flags ""}}{{.}}
   {{end}}
TUNING OPTIONS:
   {{range category .Flags "tuning"}}{{.}}
   {{end}}
AWS S3 OPTIONS:
   {{range category .Flags "aws"}}{{.}}
   {{end}}
MISC OPTIONS:
   {{range category .Flags "misc"}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

var VersionHash string

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.15-" + VersionHash,
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.StringFlag{
				Name: "cache",
				Usage: "Directory to use for data cache. " +
					"Requires catfs and `-o allow_other'. " +
					"Can also pass in other catfs options " +
					"(ex: --cache \"--free:10%:$HOME/cache\") (default: off)",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-east-1",
				Usage: "The region to connect to. Usually this is auto-detected." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3) in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
			cli.StringFlag{
				Name:  "acl",
				Usage: "The canned ACL to apply to the object. Possible values: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.BoolFlag{
				Name:  "cheap",
				Usage: "Reduce S3 operation costs at the expense of some performance (default: off)",
			},

			cli.BoolFlag{
				Name:  "no-implicit-dir",
				Usage: "Assume all directory objects (\"dir/\") exist (default: off)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	var funcMap = template.FuncMap{
		"category": filterCategory,
		"join":     strings.Join,
	}

	flagCategories = map[string]string{}

	for _, f := range []string{"region", "sse", "sse-kms", "storage-class", "acl"} {
		flagCategories[f] = "aws"
	}

	for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl"} {
		flagCategories[f] = "tuning"
	}

	for _, f := range []string{"help, h", "debug_fuse", "debug_s3", "version, v", "f"} {
		flagCategories[f] = "misc"
	}

	cli.HelpPrinter = func(w io.Writer, templ string, data interface{}) {
		w = tabwriter.NewWriter(w, 1, 8, 2, ' ', 0)
		var tmplGet = template.Must(template.New("help").Funcs(funcMap).Parse(templ))
		tmplGet.Execute(w, app)
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions      map[string]string
	MountPoint        string
	MountPointArg     string
	MountPointCreated string

	Cache    []string
	DirMode  os.FileMode
	FileMode os.FileMode
	Uid      uint32
	Gid      uint32

	// S3
	Endpoint       string
	Region         string
	RegionSet      bool
	StorageClass   string
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string
	ACL            string

	// Tuning
	Cheap        bool
	ExplicitDir  bool
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

func (flags *FlagStorage) Cleanup() {
	if flags.MountPointCreated != flags.MountPointArg {
		err := os.Remove(flags.MountPointCreated)
		if err != nil {
			log.Errorf("rmdir %v = %v", flags.MountPointCreated, err)
		}
	}
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (ret *FlagStorage) {
	flags := &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		Cheap:        c.Bool("cheap"),
		ExplicitDir:  c.Bool("no-implicit-dir"),
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		RegionSet:      c.IsSet("region"),
		StorageClass:   c.String("storage-class"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),
		ACL:            c.String("acl"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}

	flags.MountPointArg = c.Args()[1]
	flags.MountPoint = flags.MountPointArg
	var err error

	defer func() {
		if err != nil {
			flags.Cleanup()
		}
	}()

	if c.IsSet("cache") {
		cache := c.String("cache")
		cacheArgs := strings.Split(c.String("cache"), ":")
		cacheDir := cacheArgs[len(cacheArgs)-1]

		fi, err := os.Stat(cacheDir)
		if err != nil || !fi.IsDir() {
			io.WriteString(cli.ErrWriter,
				fmt.Sprintf("Invalid value \"%v\" for --cache: not a directory\n\n",
					cacheDir))
			return nil
		}

		if _, ok := flags.MountOptions["allow_other"]; !ok {
			flags.MountPointCreated, err = ioutil.TempDir("", ".goofys-mnt")
			if err != nil {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Unable to create temp dir: %v", err))
				return nil
			}
			flags.MountPoint = flags.MountPointCreated
		}

		cacheArgs = append(cacheArgs, "")
		cacheArgs[len(cacheArgs)-1] = cacheDir
		cacheArgs[len(cacheArgs)-2] = flags.MountPoint

		cacheArgs = append(cacheArgs, "", "", "")
		copy(cacheArgs[3:], cacheArgs[0:])
		cacheArgs[0] = "--test"
		cacheArgs[1] = "-o"
		cacheArgs[2] = "nonempty"
		cacheArgs = append(cacheArgs, flags.MountPointArg)

		catfs := exec.Command("catfs", cacheArgs...)
		_, err = catfs.Output()
		if err != nil {
			if ee, ok := err.(*exec.Error); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("--cache requires catfs (%v) but %v\n\n",
						"http://github.com/kahing/catfs",
						ee.Error()))
			} else if ee, ok := err.(*exec.ExitError); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Invalid value \"%v\" for --cache: %v\n\n",
						cache, string(ee.Stderr)))
			}
			return nil
		}

		flags.Cache = cacheArgs[1:]
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	return flags
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<commit_msg>only pass -o nonempty if we are allow_other<commit_after>// Copyright 2015 - 2017 Ka-Hing Cheung
// Copyright 2015 - 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"

	"github.com/codegangsta/cli"
)

var flagCategories map[string]string

// Set up custom help text for goofys; in particular the usage section.
func filterCategory(flags []cli.Flag, category string) (ret []cli.Flag) {
	for _, f := range flags {
		if flagCategories[f.GetName()] == category {
			ret = append(ret, f)
		}
	}
	return
}

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range category .Flags ""}}{{.}}
   {{end}}
TUNING OPTIONS:
   {{range category .Flags "tuning"}}{{.}}
   {{end}}
AWS S3 OPTIONS:
   {{range category .Flags "aws"}}{{.}}
   {{end}}
MISC OPTIONS:
   {{range category .Flags "misc"}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

var VersionHash string

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.15-" + VersionHash,
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.StringFlag{
				Name: "cache",
				Usage: "Directory to use for data cache. " +
					"Requires catfs and `-o allow_other'. " +
					"Can also pass in other catfs options " +
					"(ex: --cache \"--free:10%:$HOME/cache\") (default: off)",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-east-1",
				Usage: "The region to connect to. Usually this is auto-detected." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3) in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
			cli.StringFlag{
				Name:  "acl",
				Usage: "The canned ACL to apply to the object. Possible values: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.BoolFlag{
				Name:  "cheap",
				Usage: "Reduce S3 operation costs at the expense of some performance (default: off)",
			},

			cli.BoolFlag{
				Name:  "no-implicit-dir",
				Usage: "Assume all directory objects (\"dir/\") exist (default: off)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	var funcMap = template.FuncMap{
		"category": filterCategory,
		"join":     strings.Join,
	}

	flagCategories = map[string]string{}

	for _, f := range []string{"region", "sse", "sse-kms", "storage-class", "acl"} {
		flagCategories[f] = "aws"
	}

	for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl"} {
		flagCategories[f] = "tuning"
	}

	for _, f := range []string{"help, h", "debug_fuse", "debug_s3", "version, v", "f"} {
		flagCategories[f] = "misc"
	}

	cli.HelpPrinter = func(w io.Writer, templ string, data interface{}) {
		w = tabwriter.NewWriter(w, 1, 8, 2, ' ', 0)
		var tmplGet = template.Must(template.New("help").Funcs(funcMap).Parse(templ))
		tmplGet.Execute(w, app)
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions      map[string]string
	MountPoint        string
	MountPointArg     string
	MountPointCreated string

	Cache    []string
	DirMode  os.FileMode
	FileMode os.FileMode
	Uid      uint32
	Gid      uint32

	// S3
	Endpoint       string
	Region         string
	RegionSet      bool
	StorageClass   string
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string
	ACL            string

	// Tuning
	Cheap        bool
	ExplicitDir  bool
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

func (flags *FlagStorage) Cleanup() {
	if flags.MountPointCreated != flags.MountPointArg {
		err := os.Remove(flags.MountPointCreated)
		if err != nil {
			log.Errorf("rmdir %v = %v", flags.MountPointCreated, err)
		}
	}
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (ret *FlagStorage) {
	flags := &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		Cheap:        c.Bool("cheap"),
		ExplicitDir:  c.Bool("no-implicit-dir"),
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		RegionSet:      c.IsSet("region"),
		StorageClass:   c.String("storage-class"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),
		ACL:            c.String("acl"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}

	flags.MountPointArg = c.Args()[1]
	flags.MountPoint = flags.MountPointArg
	var err error

	defer func() {
		if err != nil {
			flags.Cleanup()
		}
	}()

	if c.IsSet("cache") {
		cache := c.String("cache")
		cacheArgs := strings.Split(c.String("cache"), ":")
		cacheDir := cacheArgs[len(cacheArgs)-1]

		fi, err := os.Stat(cacheDir)
		if err != nil || !fi.IsDir() {
			io.WriteString(cli.ErrWriter,
				fmt.Sprintf("Invalid value \"%v\" for --cache: not a directory\n\n",
					cacheDir))
			return nil
		}

		if _, ok := flags.MountOptions["allow_other"]; !ok {
			flags.MountPointCreated, err = ioutil.TempDir("", ".goofys-mnt")
			if err != nil {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Unable to create temp dir: %v", err))
				return nil
			}
			flags.MountPoint = flags.MountPointCreated
		}

		cacheArgs = append(cacheArgs, "")
		cacheArgs[len(cacheArgs)-1] = cacheDir
		cacheArgs[len(cacheArgs)-2] = flags.MountPoint
		cacheArgs = append(cacheArgs, flags.MountPointArg)

		cacheArgs = append(cacheArgs, "")
		copy(cacheArgs[1:], cacheArgs[0:])
		cacheArgs[0] = "--test"

		if flags.MountPointArg == flags.MountPoint {
			cacheArgs = append(cacheArgs, "", "")
			copy(cacheArgs[3:], cacheArgs[1:])
			cacheArgs[1] = "-o"
			cacheArgs[2] = "nonempty"
		}

		catfs := exec.Command("catfs", cacheArgs...)
		_, err = catfs.Output()
		if err != nil {
			if ee, ok := err.(*exec.Error); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("--cache requires catfs (%v) but %v\n\n",
						"http://github.com/kahing/catfs",
						ee.Error()))
			} else if ee, ok := err.(*exec.ExitError); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Invalid value \"%v\" for --cache: %v\n\n",
						cache, string(ee.Stderr)))
			}
			return nil
		}

		flags.Cache = cacheArgs[1:]
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	return flags
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 - 2017 Ka-Hing Cheung
// Copyright 2015 - 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"

	"github.com/codegangsta/cli"
)

var flagCategories map[string]string

// Set up custom help text for goofys; in particular the usage section.
func filterCategory(flags []cli.Flag, category string) (ret []cli.Flag) {
	for _, f := range flags {
		if flagCategories[f.GetName()] == category {
			ret = append(ret, f)
		}
	}
	return
}

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range category .Flags ""}}{{.}}
   {{end}}
TUNING OPTIONS:
   {{range category .Flags "tuning"}}{{.}}
   {{end}}
AWS S3 OPTIONS:
   {{range category .Flags "aws"}}{{.}}
   {{end}}
MISC OPTIONS:
   {{range category .Flags "misc"}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

var VersionHash string

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.16-" + VersionHash,
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.StringFlag{
				Name: "cache",
				Usage: "Directory to use for data cache. " +
					"Requires catfs and `-o allow_other'. " +
					"Can also pass in other catfs options " +
					"(ex: --cache \"--free:10%:$HOME/cache\") (default: off)",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-east-1",
				Usage: "The region to connect to. Usually this is auto-detected." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3) in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
			cli.StringFlag{
				Name:  "acl",
				Usage: "The canned ACL to apply to the object. Possible values: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.BoolFlag{
				Name:  "cheap",
				Usage: "Reduce S3 operation costs at the expense of some performance (default: off)",
			},

			cli.BoolFlag{
				Name:  "no-implicit-dir",
				Usage: "Assume all directory objects (\"dir/\") exist (default: off)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	var funcMap = template.FuncMap{
		"category": filterCategory,
		"join":     strings.Join,
	}

	flagCategories = map[string]string{}

	for _, f := range []string{"region", "sse", "sse-kms", "storage-class", "acl"} {
		flagCategories[f] = "aws"
	}

	for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl"} {
		flagCategories[f] = "tuning"
	}

	for _, f := range []string{"help, h", "debug_fuse", "debug_s3", "version, v", "f"} {
		flagCategories[f] = "misc"
	}

	cli.HelpPrinter = func(w io.Writer, templ string, data interface{}) {
		w = tabwriter.NewWriter(w, 1, 8, 2, ' ', 0)
		var tmplGet = template.Must(template.New("help").Funcs(funcMap).Parse(templ))
		tmplGet.Execute(w, app)
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions      map[string]string
	MountPoint        string
	MountPointArg     string
	MountPointCreated string

	Cache    []string
	DirMode  os.FileMode
	FileMode os.FileMode
	Uid      uint32
	Gid      uint32

	// S3
	Endpoint       string
	Region         string
	RegionSet      bool
	StorageClass   string
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string
	ACL            string

	// Tuning
	Cheap        bool
	ExplicitDir  bool
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

func (flags *FlagStorage) Cleanup() {
	if flags.MountPointCreated != "" && flags.MountPointCreated != flags.MountPointArg {
		err := os.Remove(flags.MountPointCreated)
		if err != nil {
			log.Errorf("rmdir %v = %v", flags.MountPointCreated, err)
		}
	}
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (ret *FlagStorage) {
	flags := &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		Cheap:        c.Bool("cheap"),
		ExplicitDir:  c.Bool("no-implicit-dir"),
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		RegionSet:      c.IsSet("region"),
		StorageClass:   c.String("storage-class"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),
		ACL:            c.String("acl"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}

	flags.MountPointArg = c.Args()[1]
	flags.MountPoint = flags.MountPointArg
	var err error

	defer func() {
		if err != nil {
			flags.Cleanup()
		}
	}()

	if c.IsSet("cache") {
		cache := c.String("cache")
		cacheArgs := strings.Split(c.String("cache"), ":")
		cacheDir := cacheArgs[len(cacheArgs)-1]
		cacheArgs = cacheArgs[:len(cacheArgs)-1]

		fi, err := os.Stat(cacheDir)
		if err != nil || !fi.IsDir() {
			io.WriteString(cli.ErrWriter,
				fmt.Sprintf("Invalid value \"%v\" for --cache: not a directory\n\n",
					cacheDir))
			return nil
		}

		if _, ok := flags.MountOptions["allow_other"]; !ok {
			flags.MountPointCreated, err = ioutil.TempDir("", ".goofys-mnt")
			if err != nil {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Unable to create temp dir: %v", err))
				return nil
			}
			flags.MountPoint = flags.MountPointCreated
		}

		cacheArgs = append([]string{"--test"}, cacheArgs...)

		if flags.MountPointArg == flags.MountPoint {
			cacheArgs = append(cacheArgs, "-ononempty")
		}

		cacheArgs = append(cacheArgs, "--")
		cacheArgs = append(cacheArgs, flags.MountPoint)
		cacheArgs = append(cacheArgs, cacheDir)
		cacheArgs = append(cacheArgs, flags.MountPointArg)

		fuseLog.Debugf("catfs %v", cacheArgs)
		catfs := exec.Command("catfs", cacheArgs...)
		_, err = catfs.Output()
		if err != nil {
			if ee, ok := err.(*exec.Error); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("--cache requires catfs (%v) but %v\n\n",
						"http://github.com/kahing/catfs",
						ee.Error()))
			} else if ee, ok := err.(*exec.ExitError); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Invalid value \"%v\" for --cache: %v\n\n",
						cache, string(ee.Stderr)))
			}
			return nil
		}

		flags.Cache = cacheArgs[1:]
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	return flags
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<commit_msg>v0.0.17<commit_after>// Copyright 2015 - 2017 Ka-Hing Cheung
// Copyright 2015 - 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"

	"github.com/codegangsta/cli"
)

var flagCategories map[string]string

// Set up custom help text for goofys; in particular the usage section.
func filterCategory(flags []cli.Flag, category string) (ret []cli.Flag) {
	for _, f := range flags {
		if flagCategories[f.GetName()] == category {
			ret = append(ret, f)
		}
	}
	return
}

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options]{{end}} bucket[:prefix] mountpoint
   {{if .Version}}
VERSION:
   {{.Version}}
   {{end}}{{if len .Authors}}
AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}
   {{end}}{{if .Commands}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range category .Flags ""}}{{.}}
   {{end}}
TUNING OPTIONS:
   {{range category .Flags "tuning"}}{{.}}
   {{end}}
AWS S3 OPTIONS:
   {{range category .Flags "aws"}}{{.}}
   {{end}}
MISC OPTIONS:
   {{range category .Flags "misc"}}{{.}}
   {{end}}{{end}}{{if .Copyright }}
COPYRIGHT:
   {{.Copyright}}
   {{end}}
`
}

var VersionHash string

func NewApp() (app *cli.App) {
	uid, gid := MyUserAndGroup()

	app = &cli.App{
		Name:     "goofys",
		Version:  "0.0.17-" + VersionHash,
		Usage:    "Mount an S3 bucket locally",
		HideHelp: true,
		Writer:   os.Stderr,
		Flags: []cli.Flag{

			cli.BoolFlag{
				Name:  "help, h",
				Usage: "Print this help text and exit successfully.",
			},

			/////////////////////////
			// File system
			/////////////////////////

			cli.StringSliceFlag{
				Name:  "o",
				Usage: "Additional system-specific mount options. Be careful!",
			},

			cli.StringFlag{
				Name: "cache",
				Usage: "Directory to use for data cache. " +
					"Requires catfs and `-o allow_other'. " +
					"Can also pass in other catfs options " +
					"(ex: --cache \"--free:10%:$HOME/cache\") (default: off)",
			},

			cli.IntFlag{
				Name:  "dir-mode",
				Value: 0755,
				Usage: "Permission bits for directories. (default: 0755)",
			},

			cli.IntFlag{
				Name:  "file-mode",
				Value: 0644,
				Usage: "Permission bits for files. (default: 0644)",
			},

			cli.IntFlag{
				Name:  "uid",
				Value: uid,
				Usage: "UID owner of all inodes.",
			},

			cli.IntFlag{
				Name:  "gid",
				Value: gid,
				Usage: "GID owner of all inodes.",
			},

			/////////////////////////
			// S3
			/////////////////////////

			cli.StringFlag{
				Name:  "endpoint",
				Value: "",
				Usage: "The non-AWS endpoint to connect to." +
					" Possible values: http://127.0.0.1:8081/",
			},

			cli.StringFlag{
				Name:  "region",
				Value: "us-east-1",
				Usage: "The region to connect to. Usually this is auto-detected." +
					" Possible values: us-east-1, us-west-1, us-west-2, eu-west-1, " +
					"eu-central-1, ap-southeast-1, ap-southeast-2, ap-northeast-1, " +
					"sa-east-1, cn-north-1",
			},

			cli.StringFlag{
				Name:  "storage-class",
				Value: "STANDARD",
				Usage: "The type of storage to use when writing objects." +
					" Possible values: REDUCED_REDUNDANCY, STANDARD, STANDARD_IA.",
			},

			cli.StringFlag{
				Name:  "profile",
				Usage: "Use a named profile from $HOME/.aws/credentials instead of \"default\"",
			},

			cli.BoolFlag{
				Name:  "use-content-type",
				Usage: "Set Content-Type according to file extension and /etc/mime.types (default: off)",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
			/// See http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
			cli.BoolFlag{
				Name:  "sse",
				Usage: "Enable basic server-side encryption at rest (SSE-S3) in S3 for all writes (default: off)",
			},

			cli.StringFlag{
				Name:  "sse-kms",
				Usage: "Enable KMS encryption (SSE-KMS) for all writes using this particular KMS `key-id`. Leave blank to Use the account's CMK - customer master key (default: off)",
				Value: "",
			},

			/// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
			cli.StringFlag{
				Name:  "acl",
				Usage: "The canned ACL to apply to the object. Possible values: private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control (default: off)",
				Value: "",
			},

			/////////////////////////
			// Tuning
			/////////////////////////

			cli.BoolFlag{
				Name:  "cheap",
				Usage: "Reduce S3 operation costs at the expense of some performance (default: off)",
			},

			cli.BoolFlag{
				Name:  "no-implicit-dir",
				Usage: "Assume all directory objects (\"dir/\") exist (default: off)",
			},

			cli.DurationFlag{
				Name:  "stat-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache StatObject results and inode attributes.",
			},

			cli.DurationFlag{
				Name:  "type-cache-ttl",
				Value: time.Minute,
				Usage: "How long to cache name -> file/dir mappings in directory " +
					"inodes.",
			},

			/////////////////////////
			// Debugging
			/////////////////////////

			cli.BoolFlag{
				Name:  "debug_fuse",
				Usage: "Enable fuse-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "debug_s3",
				Usage: "Enable S3-related debugging output.",
			},

			cli.BoolFlag{
				Name:  "f",
				Usage: "Run goofys in foreground.",
			},
		},
	}

	var funcMap = template.FuncMap{
		"category": filterCategory,
		"join":     strings.Join,
	}

	flagCategories = map[string]string{}

	for _, f := range []string{"region", "sse", "sse-kms", "storage-class", "acl"} {
		flagCategories[f] = "aws"
	}

	for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl"} {
		flagCategories[f] = "tuning"
	}

	for _, f := range []string{"help, h", "debug_fuse", "debug_s3", "version, v", "f"} {
		flagCategories[f] = "misc"
	}

	cli.HelpPrinter = func(w io.Writer, templ string, data interface{}) {
		w = tabwriter.NewWriter(w, 1, 8, 2, ' ', 0)
		var tmplGet = template.Must(template.New("help").Funcs(funcMap).Parse(templ))
		tmplGet.Execute(w, app)
	}

	return
}

type FlagStorage struct {
	// File system
	MountOptions      map[string]string
	MountPoint        string
	MountPointArg     string
	MountPointCreated string

	Cache    []string
	DirMode  os.FileMode
	FileMode os.FileMode
	Uid      uint32
	Gid      uint32

	// S3
	Endpoint       string
	Region         string
	RegionSet      bool
	StorageClass   string
	Profile        string
	UseContentType bool
	UseSSE         bool
	UseKMS         bool
	KMSKeyID       string
	ACL            string

	// Tuning
	Cheap        bool
	ExplicitDir  bool
	StatCacheTTL time.Duration
	TypeCacheTTL time.Duration

	// Debugging
	DebugFuse  bool
	DebugS3    bool
	Foreground bool
}

func parseOptions(m map[string]string, s string) {
	// NOTE(jacobsa): The man pages don't define how escaping works, and as far
	// as I can tell there is no way to properly escape or quote a comma in the
	// options list for an fstab entry. So put our fingers in our ears and hope
	// that nobody needs a comma.
	for _, p := range strings.Split(s, ",") {
		var name string
		var value string

		// Split on the first equals sign.
		if equalsIndex := strings.IndexByte(p, '='); equalsIndex != -1 {
			name = p[:equalsIndex]
			value = p[equalsIndex+1:]
		} else {
			name = p
		}

		m[name] = value
	}

	return
}

func (flags *FlagStorage) Cleanup() {
	if flags.MountPointCreated != "" && flags.MountPointCreated != flags.MountPointArg {
		err := os.Remove(flags.MountPointCreated)
		if err != nil {
			log.Errorf("rmdir %v = %v", flags.MountPointCreated, err)
		}
	}
}

// Add the flags accepted by run to the supplied flag set, returning the
// variables into which the flags will parse.
func PopulateFlags(c *cli.Context) (ret *FlagStorage) {
	flags := &FlagStorage{
		// File system
		MountOptions: make(map[string]string),
		DirMode:      os.FileMode(c.Int("dir-mode")),
		FileMode:     os.FileMode(c.Int("file-mode")),
		Uid:          uint32(c.Int("uid")),
		Gid:          uint32(c.Int("gid")),

		// Tuning,
		Cheap:        c.Bool("cheap"),
		ExplicitDir:  c.Bool("no-implicit-dir"),
		StatCacheTTL: c.Duration("stat-cache-ttl"),
		TypeCacheTTL: c.Duration("type-cache-ttl"),

		// S3
		Endpoint:       c.String("endpoint"),
		Region:         c.String("region"),
		RegionSet:      c.IsSet("region"),
		StorageClass:   c.String("storage-class"),
		Profile:        c.String("profile"),
		UseContentType: c.Bool("use-content-type"),
		UseSSE:         c.Bool("sse"),
		UseKMS:         c.IsSet("sse-kms"),
		KMSKeyID:       c.String("sse-kms"),
		ACL:            c.String("acl"),

		// Debugging,
		DebugFuse:  c.Bool("debug_fuse"),
		DebugS3:    c.Bool("debug_s3"),
		Foreground: c.Bool("f"),
	}

	// Handle the repeated "-o" flag.
	for _, o := range c.StringSlice("o") {
		parseOptions(flags.MountOptions, o)
	}

	flags.MountPointArg = c.Args()[1]
	flags.MountPoint = flags.MountPointArg
	var err error

	defer func() {
		if err != nil {
			flags.Cleanup()
		}
	}()

	if c.IsSet("cache") {
		cache := c.String("cache")
		cacheArgs := strings.Split(c.String("cache"), ":")
		cacheDir := cacheArgs[len(cacheArgs)-1]
		cacheArgs = cacheArgs[:len(cacheArgs)-1]

		fi, err := os.Stat(cacheDir)
		if err != nil || !fi.IsDir() {
			io.WriteString(cli.ErrWriter,
				fmt.Sprintf("Invalid value \"%v\" for --cache: not a directory\n\n",
					cacheDir))
			return nil
		}

		if _, ok := flags.MountOptions["allow_other"]; !ok {
			flags.MountPointCreated, err = ioutil.TempDir("", ".goofys-mnt")
			if err != nil {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Unable to create temp dir: %v", err))
				return nil
			}
			flags.MountPoint = flags.MountPointCreated
		}

		cacheArgs = append([]string{"--test"}, cacheArgs...)

		if flags.MountPointArg == flags.MountPoint {
			cacheArgs = append(cacheArgs, "-ononempty")
		}

		cacheArgs = append(cacheArgs, "--")
		cacheArgs = append(cacheArgs, flags.MountPoint)
		cacheArgs = append(cacheArgs, cacheDir)
		cacheArgs = append(cacheArgs, flags.MountPointArg)

		fuseLog.Debugf("catfs %v", cacheArgs)
		catfs := exec.Command("catfs", cacheArgs...)
		_, err = catfs.Output()
		if err != nil {
			if ee, ok := err.(*exec.Error); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("--cache requires catfs (%v) but %v\n\n",
						"http://github.com/kahing/catfs",
						ee.Error()))
			} else if ee, ok := err.(*exec.ExitError); ok {
				io.WriteString(cli.ErrWriter,
					fmt.Sprintf("Invalid value \"%v\" for --cache: %v\n\n",
						cache, string(ee.Stderr)))
			}
			return nil
		}

		flags.Cache = cacheArgs[1:]
	}

	// KMS implies SSE
	if flags.UseKMS {
		flags.UseSSE = true
	}

	return flags
}

func MassageMountFlags(args []string) (ret []string) {
	if len(args) == 5 && args[3] == "-o" {
		// looks like it's coming from fstab!
		mountOptions := ""
		ret = append(ret, args[0])

		for _, p := range strings.Split(args[4], ",") {
			if strings.HasPrefix(p, "-") {
				ret = append(ret, p)
			} else {
				mountOptions += p
				mountOptions += ","
			}
		}

		if len(mountOptions) != 0 {
			// remove trailing ,
			mountOptions = mountOptions[:len(mountOptions)-1]
			ret = append(ret, "-o")
			ret = append(ret, mountOptions)
		}

		ret = append(ret, args[1])
		ret = append(ret, args[2])
	} else {
		return args
	}

	return
}
<|endoftext|>