<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strconv"
	"strings"
)

type configuration struct {
	Name           string `json: "name"`
	Url            string `json: "url"`
	CallbackSecret string `json: "callbacksecret"`
	BasicPrice     string `json: "baseprice"`
	MinimumPrice   string `json: "minprice"`
}

type transactionResult struct {
	Success bool `json:"success"`
	Button  struct {
		Code string `json:"code"`
	} `json:"button"`
}

type callbackResult struct {
	Order struct {
		Filename string `json:"custom"`
	} `json:"order"`
}

// Get an appropriate name for the file.
func newFileName(fname string) string {
	result := strings.Replace(strings.Replace(fname, "/", "-", -1), " ", "-", -1)
	if _, err := os.Stat("f/" + result); os.IsNotExist(err) {
		if _, err := os.Stat("tmp/" + result); os.IsNotExist(err) {
			// Don't do anything.
		} else {
			result = getname("p" + result)
		}
	} else {
		result = getname("p" + result)
	}
	return result
}

// Create a coinbase button.
func createButton(n string, p float64) string {
	coinbaserequest := "{ \"button\": {" +
		"\"name\": \"One-Time Hosting Purchase\"," +
		"\"type\": \"buy_now\"," +
		"\"price_string\": \"" + strconv.FormatFloat(p, 'f', 8, 64) + "\"," +
		"\"price_currency_iso\": \"BTC\"," +
		"\"custom\": \"" + n + "\"," +
		"\"callback_url\": \"whatever\"," +
		"\"description\": \"Indefinite storage of the provided file. Your file will be available at: http://btcdl.bearbin.net/f/" + n + " when the transaction processes.\"," +
		"\"type\": \"buy_now\"," +
		"\"style\": \"custom_large\"" +
		"} }"
	apikey := "InsertKeyHere"
	fmt.Println(coinbaserequest)
	request_body := bytes.NewBuffer([]byte(coinbaserequest))

	client := &http.Client{}
	req, err := http.NewRequest("POST", "https://coinbase.com/api/v1/buttons?api_key="+apikey, request_body)
	if err != nil {
		fmt.Println(err)
	}

	req.Header.Add("content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
	}

	response_body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()
	res := transactionResult{}
	fmt.Println(string(response_body))
	err = json.Unmarshal(response_body, &res)
	return res.Button.Code

}

// hello world, the web server
func upload(w http.ResponseWriter, req *http.Request) {

	// Get the form file.
	file, header, err := req.FormFile("file")
	if err != nil {
		fmt.Println(err)
		return
	}

	// Get the name for the file.
	filename := header.Filename
	filename = newFileName(filename)
	fmt.Println(filename)

	data, err := ioutil.ReadAll(file)
	if err != nil {
		log.Print(err)
		return
	}
	err = ioutil.WriteFile("tmp/"+filename, data, 0777)
	if err != nil {
		log.Print(err)
		return
	}

	// Get file size.
	supfil, _ := os.Stat("tmp/" + filename)
	filesize := float64(supfil.Size())
	price := math.Floor(math.Floor(filesize/1024)*4.8828125) / 100000000
	if price < 0.000025 {
		price = 0.000025
	}
	fmt.Println(strconv.FormatFloat(price, 'f', 8, 64))

	// Redirect the user.
	http.Redirect(w, req, "https://coinbase.com/checkouts/"+createButton(filename, price), 302)

}

func coinbaseCallback(w http.ResponseWriter, req *http.Request) {
	fmt.Println("LELELELE")
	body, _ := ioutil.ReadAll(req.Body)
	res := callbackResult{}
	fmt.Println(body)
	json.Unmarshal([]byte(body), &res)
	fmt.Println(res.Order.Filename)
	os.Rename("tmp/"+res.Order.Filename, "f/"+res.Order.Filename)
}

func MainPage(w http.ResponseWriter, req *http.Request) {
	t, _ := template.ParseFiles("index.html")
	t.Execute(w, "")
}

func main() {
	// Main page
	http.HandleFunc("/", MainPage)
	// Upload page
	http.HandleFunc("/upload", upload)
	// Coinbase callback
	http.HandleFunc("/wheatver", coinbaseCallback)
	// Static files
	http.Handle("/f/", http.FileServer(http.Dir("")))

	// Try and serve port 80.
	err := http.ListenAndServe(":80", nil)
	if err != nil {
		// Failed for some reason, try port 8080
		log.Print("Failed to bind to port 80, trying 8080.")
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			// Failed.
			log.Fatal("ListenAndServe: ", err)
		}
	}
}
<commit_msg>Changed some stuff.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os"
	"strconv"
	"strings"
)

// Define the types

type configuration struct {
	Name           string `json: "name"`
	Url            string `json: "url"`
	CallbackSecret string `json: "callbacksecret"`
	BasePrice      int    `json: "baseprice"`
	MinimumPrice   int    `json: "minprice"`
	ApiKey         string `json: "coinbasekey"`
}

type transactionResult struct {
	Success bool `json:"success"`
	Button  struct {
		Code string `json:"code"`
	} `json:"button"`
}

type callbackResult struct {
	Order struct {
		Filename string `json:"custom"`
	} `json:"order"`
}

// Create the configuration
var config = configuration{}

// Do stuff

// Get an appropriate name for the file.
func newFileName(fname string) string {
	result := strings.Replace(strings.Replace(fname, "/", "-", -1), " ", "-", -1)
	if _, err := os.Stat("f/" + result); os.IsNotExist(err) {
		if _, err := os.Stat("tmp/" + result); os.IsNotExist(err) {
			// Don't do anything.
		} else {
			result = newFileName("p" + result)
		}
	} else {
		result = newFileName("p" + result)
	}
	return result
}

// Create a coinbase button.
func createButton(n string, p int) string {
	coinbaserequest := "{ \"button\": {" +
		"\"name\": \"One-Time Hosting Purchase\"," +
		"\"type\": \"buy_now\"," +
		"\"price_string\": \"" + strconv.FormatFloat(float64(p)/float64(100000000), 'f', 8, 64) + "\"," +
		"\"price_currency_iso\": \"BTC\"," +
		"\"custom\": \"" + n + "\"," +
		"\"callback_url\": \"whatever\"," +
		"\"description\": \"Indefinite storage of the provided file. Your file will be available at: http://btcdl.bearbin.net/f/" + n + " when the transaction processes.\"," +
		"\"type\": \"buy_now\"," +
		"\"style\": \"custom_large\"" +
		"} }"
	fmt.Println(coinbaserequest)
	request_body := bytes.NewBuffer([]byte(coinbaserequest))

	client := &http.Client{}
	req, err := http.NewRequest("POST", "https://coinbase.com/api/v1/buttons?api_key="+config.ApiKey, request_body)
	if err != nil {
		fmt.Println(err)
	}

	req.Header.Add("content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err)
	}

	response_body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()
	res := transactionResult{}
	fmt.Println(string(response_body))
	err = json.Unmarshal(response_body, &res)
	return res.Button.Code

}

// hello world, the web server
func upload(w http.ResponseWriter, req *http.Request) {

	// Get the form file.
	file, header, err := req.FormFile("file")
	if err != nil {
		fmt.Println(err)
		return
	}

	// Get the name for the file.
	fileName := newFileName(header.Filename)
	log.Print("Uploaded new file: ", fileName)

	data, err := ioutil.ReadAll(file)
	if err != nil {
		log.Print(err)
		return
	}
	err = ioutil.WriteFile("tmp/"+fileName, data, 0777)
	if err != nil {
		log.Print(err)
		return
	}

	// Get file size.
	supfil, _ := os.Stat("tmp/" + fileName)
	fileSize := math.Floor(float64(supfil.Size()) / 1024)
	price := int(math.Floor(float64(config.BasePrice) * (fileSize/1024)))
	if price < config.MinimumPrice {
		price = config.MinimumPrice
	}
	// Redirect the user.
	http.Redirect(w, req, "https://coinbase.com/checkouts/"+createButton(fileName, price), 302)

}

func coinbaseCallback(w http.ResponseWriter, req *http.Request) {
	fmt.Println("LELELELE")
	body, _ := ioutil.ReadAll(req.Body)
	res := callbackResult{}
	fmt.Println(body)
	json.Unmarshal([]byte(body), &res)
	fmt.Println(res.Order.Filename)
	os.Rename("tmp/"+res.Order.Filename, "f/"+res.Order.Filename)
}

func MainPage(w http.ResponseWriter, req *http.Request) {
	t, _ := template.ParseFiles("index.html")
	t.Execute(w, "")
}

func main() {
	// Inititalize the config.
	configFile, err := os.Open("config.json")
	if err != nil {
		log.Fatal("Failed to open config: ", err)
	}
	decoder := json.NewDecoder(configFile)

	err = decoder.Decode(&config)
	if err != nil {
		log.Fatal("Failed to open config: ", err)
	}

	// Main page
	http.HandleFunc("/", MainPage)
	// Upload page
	http.HandleFunc("/upload", upload)
	// Coinbase callback
	http.HandleFunc("/wheatver", coinbaseCallback)
	// Static files
	http.Handle("/f/", http.FileServer(http.Dir("")))

	// Try and serve port 80.
	err = http.ListenAndServe(":80", nil)
	if err != nil {
		// Failed for some reason, try port 8080
		log.Print("Failed to bind to port 80, trying 8080.")
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			// Failed.
			log.Fatal("ListenAndServe: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package instructions

import (
	"encoding/csv"
	"regexp"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

const MountTypeBind = "bind"
const MountTypeCache = "cache"
const MountTypeTmpfs = "tmpfs"
const MountTypeSecret = "secret"
const MountTypeSSH = "ssh"

var allowedMountTypes = map[string]struct{}{
	MountTypeBind:   {},
	MountTypeCache:  {},
	MountTypeTmpfs:  {},
	MountTypeSecret: {},
	MountTypeSSH:    {},
}

const MountSharingShared = "shared"
const MountSharingPrivate = "private"
const MountSharingLocked = "locked"

var allowedSharingTypes = map[string]struct{}{
	MountSharingShared:  {},
	MountSharingPrivate: {},
	MountSharingLocked:  {},
}

type mountsKeyT string

var mountsKey = mountsKeyT("dockerfile/run/mounts")

func init() {
	parseRunPreHooks = append(parseRunPreHooks, runMountPreHook)
	parseRunPostHooks = append(parseRunPostHooks, runMountPostHook)
}

func isValidMountType(s string) bool {
	if s == "secret" {
		if !isSecretMountsSupported() {
			return false
		}
	}
	if s == "ssh" {
		if !isSSHMountsSupported() {
			return false
		}
	}
	_, ok := allowedMountTypes[s]
	return ok
}

func runMountPreHook(cmd *RunCommand, req parseRequest) error {
	st := &mountState{}
	st.flag = req.flags.AddStrings("mount")
	cmd.setExternalValue(mountsKey, st)
	return nil
}

func runMountPostHook(cmd *RunCommand, req parseRequest) error {
	return setMountState(cmd, nil)
}

func setMountState(cmd *RunCommand, expander SingleWordExpander) error {
	st := getMountState(cmd)
	if st == nil {
		return errors.Errorf("no mount state")
	}
	var mounts []*Mount
	for _, str := range st.flag.StringValues {
		m, err := parseMount(str, expander)
		if err != nil {
			return err
		}
		mounts = append(mounts, m)
	}
	st.mounts = mounts
	return nil
}

func getMountState(cmd *RunCommand) *mountState {
	v := cmd.getExternalValue(mountsKey)
	if v == nil {
		return nil
	}
	return v.(*mountState)
}

func GetMounts(cmd *RunCommand) []*Mount {
	return getMountState(cmd).mounts
}

type mountState struct {
	flag   *Flag
	mounts []*Mount
}

type Mount struct {
	Type         string
	From         string
	Source       string
	Target       string
	ReadOnly     bool
	CacheID      string
	CacheSharing string
	Required     bool
	Mode         *uint64
	UID          *uint64
	GID          *uint64
}

func parseMount(value string, expander SingleWordExpander) (*Mount, error) {
	csvReader := csv.NewReader(strings.NewReader(value))
	fields, err := csvReader.Read()
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse csv mounts")
	}

	m := &Mount{Type: MountTypeBind}

	roAuto := true

	for _, field := range fields {
		parts := strings.SplitN(field, "=", 2)
		key := strings.ToLower(parts[0])

		if len(parts) == 1 {
			switch key {
			case "readonly", "ro":
				m.ReadOnly = true
				roAuto = false
				continue
			case "readwrite", "rw":
				m.ReadOnly = false
				roAuto = false
				continue
			case "required":
				if m.Type == "secret" || m.Type == "ssh" {
					m.Required = true
					continue
				} else {
					return nil, errors.Errorf("unexpected key '%s' for mount type '%s'", key, m.Type)
				}
			}
		}

		if len(parts) != 2 {
			return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field)
		}

		value := parts[1]
		// check for potential variable
		if expander != nil {
			processed, err := expander(value)
			if err != nil {
				return nil, err
			}
			value = processed
		} else if key == "from" {
			if matched, err := regexp.MatchString(`\$.`, value); err != nil { //nolint
				return nil, err
			} else if matched {
				return nil, errors.Errorf("'%s' doesn't support variable expansion, define alias stage instead", key)
			}
		} else {
			// if we don't have an expander, defer evaluation to later
			continue
		}
		switch key {
		case "type":
			if !isValidMountType(strings.ToLower(value)) {
				return nil, errors.Errorf("unsupported mount type %q", value)
			}
			m.Type = strings.ToLower(value)
		case "from":
			m.From = value
		case "source", "src":
			m.Source = value
		case "target", "dst", "destination":
			m.Target = value
		case "readonly", "ro":
			m.ReadOnly, err = strconv.ParseBool(value)
			if err != nil {
				return nil, errors.Errorf("invalid value for %s: %s", key, value)
			}
			roAuto = false
		case "readwrite", "rw":
			rw, err := strconv.ParseBool(value)
			if err != nil {
				return nil, errors.Errorf("invalid value for %s: %s", key, value)
			}
			m.ReadOnly = !rw
			roAuto = false
		case "required":
			if m.Type == "secret" || m.Type == "ssh" {
				v, err := strconv.ParseBool(value)
				if err != nil {
					return nil, errors.Errorf("invalid value for %s: %s", key, value)
				}
				m.Required = v
			} else {
				return nil, errors.Errorf("unexpected key '%s' for mount type '%s'", key, m.Type)
			}
		case "id":
			m.CacheID = value
		case "sharing":
			if _, ok := allowedSharingTypes[strings.ToLower(value)]; !ok {
				return nil, errors.Errorf("unsupported sharing value %q", value)
			}
			m.CacheSharing = strings.ToLower(value)
		case "mode":
			mode, err := strconv.ParseUint(value, 8, 32)
			if err != nil {
				return nil, errors.Errorf("invalid value %s for mode", value)
			}
			m.Mode = &mode
		case "uid":
			uid, err := strconv.ParseUint(value, 10, 32)
			if err != nil {
				return nil, errors.Errorf("invalid value %s for uid", value)
			}
			m.UID = &uid
		case "gid":
			gid, err := strconv.ParseUint(value, 10, 32)
			if err != nil {
				return nil, errors.Errorf("invalid value %s for gid", value)
			}
			m.GID = &gid
		default:
			return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field)
		}
	}

	fileInfoAllowed := m.Type == MountTypeSecret || m.Type == MountTypeSSH || m.Type == MountTypeCache

	if m.Mode != nil && !fileInfoAllowed {
		return nil, errors.Errorf("mode not allowed for %q type mounts", m.Type)
	}

	if m.UID != nil && !fileInfoAllowed {
		return nil, errors.Errorf("uid not allowed for %q type mounts", m.Type)
	}

	if m.GID != nil && !fileInfoAllowed {
		return nil, errors.Errorf("gid not allowed for %q type mounts", m.Type)
	}

	if roAuto {
		if m.Type == MountTypeCache || m.Type == MountTypeTmpfs {
			m.ReadOnly = false
		} else {
			m.ReadOnly = true
		}
	}

	if m.CacheSharing != "" && m.Type != MountTypeCache {
		return nil, errors.Errorf("invalid cache sharing set for %v mount", m.Type)
	}

	if m.Type == MountTypeSecret {
		if m.From != "" {
			return nil, errors.Errorf("secret mount should not have a from")
		}
		if m.CacheSharing != "" {
			return nil, errors.Errorf("secret mount should not define sharing")
		}
		if m.Source == "" && m.Target == "" && m.CacheID == "" {
			return nil, errors.Errorf("invalid secret mount. one of source, target required")
		}
		if m.Source != "" && m.CacheID != "" {
			return nil, errors.Errorf("both source and id can't be set")
		}
	}

	return m, nil
}
<commit_msg>dockerfile: provide suggestions for mount options<commit_after>package instructions

import (
	"encoding/csv"
	"regexp"
	"strconv"
	"strings"

	"github.com/moby/buildkit/util/suggest"
	"github.com/pkg/errors"
)

const MountTypeBind = "bind"
const MountTypeCache = "cache"
const MountTypeTmpfs = "tmpfs"
const MountTypeSecret = "secret"
const MountTypeSSH = "ssh"

var allowedMountTypes = map[string]struct{}{
	MountTypeBind:   {},
	MountTypeCache:  {},
	MountTypeTmpfs:  {},
	MountTypeSecret: {},
	MountTypeSSH:    {},
}

const MountSharingShared = "shared"
const MountSharingPrivate = "private"
const MountSharingLocked = "locked"

var allowedSharingTypes = map[string]struct{}{
	MountSharingShared:  {},
	MountSharingPrivate: {},
	MountSharingLocked:  {},
}

type mountsKeyT string

var mountsKey = mountsKeyT("dockerfile/run/mounts")

func init() {
	parseRunPreHooks = append(parseRunPreHooks, runMountPreHook)
	parseRunPostHooks = append(parseRunPostHooks, runMountPostHook)
}

func isValidMountType(s string) bool {
	if s == "secret" {
		if !isSecretMountsSupported() {
			return false
		}
	}
	if s == "ssh" {
		if !isSSHMountsSupported() {
			return false
		}
	}
	_, ok := allowedMountTypes[s]
	return ok
}

func allMountTypes() []string {
	types := make([]string, 0, len(allowedMountTypes)+2)
	for k := range allowedMountTypes {
		types = append(types, k)
	}
	if isSecretMountsSupported() {
		types = append(types, "secret")
	}
	if isSSHMountsSupported() {
		types = append(types, "ssh")
	}
	return types
}

func runMountPreHook(cmd *RunCommand, req parseRequest) error {
	st := &mountState{}
	st.flag = req.flags.AddStrings("mount")
	cmd.setExternalValue(mountsKey, st)
	return nil
}

func runMountPostHook(cmd *RunCommand, req parseRequest) error {
	return setMountState(cmd, nil)
}

func setMountState(cmd *RunCommand, expander SingleWordExpander) error {
	st := getMountState(cmd)
	if st == nil {
		return errors.Errorf("no mount state")
	}
	var mounts []*Mount
	for _, str := range st.flag.StringValues {
		m, err := parseMount(str, expander)
		if err != nil {
			return err
		}
		mounts = append(mounts, m)
	}
	st.mounts = mounts
	return nil
}

func getMountState(cmd *RunCommand) *mountState {
	v := cmd.getExternalValue(mountsKey)
	if v == nil {
		return nil
	}
	return v.(*mountState)
}

func GetMounts(cmd *RunCommand) []*Mount {
	return getMountState(cmd).mounts
}

type mountState struct {
	flag   *Flag
	mounts []*Mount
}

type Mount struct {
	Type         string
	From         string
	Source       string
	Target       string
	ReadOnly     bool
	CacheID      string
	CacheSharing string
	Required     bool
	Mode         *uint64
	UID          *uint64
	GID          *uint64
}

func parseMount(value string, expander SingleWordExpander) (*Mount, error) {
	csvReader := csv.NewReader(strings.NewReader(value))
	fields, err := csvReader.Read()
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse csv mounts")
	}

	m := &Mount{Type: MountTypeBind}

	roAuto := true

	for _, field := range fields {
		parts := strings.SplitN(field, "=", 2)
		key := strings.ToLower(parts[0])

		if len(parts) == 1 {
			switch key {
			case "readonly", "ro":
				m.ReadOnly = true
				roAuto = false
				continue
			case "readwrite", "rw":
				m.ReadOnly = false
				roAuto = false
				continue
			case "required":
				if m.Type == "secret" || m.Type == "ssh" {
					m.Required = true
					continue
				} else {
					return nil, errors.Errorf("unexpected key '%s' for mount type '%s'", key, m.Type)
				}
			}
		}

		if len(parts) != 2 {
			return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field)
		}

		value := parts[1]
		// check for potential variable
		if expander != nil {
			processed, err := expander(value)
			if err != nil {
				return nil, err
			}
			value = processed
		} else if key == "from" {
			if matched, err := regexp.MatchString(`\$.`, value); err != nil { //nolint
				return nil, err
			} else if matched {
				return nil, errors.Errorf("'%s' doesn't support variable expansion, define alias stage instead", key)
			}
		} else {
			// if we don't have an expander, defer evaluation to later
			continue
		}

		switch key {
		case "type":
			if !isValidMountType(strings.ToLower(value)) {
				return nil, suggest.WrapError(errors.Errorf("unsupported mount type %q", value), value, allMountTypes(), true)
			}
			m.Type = strings.ToLower(value)
		case "from":
			m.From = value
		case "source", "src":
			m.Source = value
		case "target", "dst", "destination":
			m.Target = value
		case "readonly", "ro":
			m.ReadOnly, err = strconv.ParseBool(value)
			if err != nil {
				return nil, errors.Errorf("invalid value for %s: %s", key, value)
			}
			roAuto = false
		case "readwrite", "rw":
			rw, err := strconv.ParseBool(value)
			if err != nil {
				return nil, errors.Errorf("invalid value for %s: %s", key, value)
			}
			m.ReadOnly = !rw
			roAuto = false
		case "required":
			if m.Type == "secret" || m.Type == "ssh" {
				v, err := strconv.ParseBool(value)
				if err != nil {
					return nil, errors.Errorf("invalid value for %s: %s", key, value)
				}
				m.Required = v
			} else {
				return nil, errors.Errorf("unexpected key '%s' for mount type '%s'", key, m.Type)
			}
		case "id":
			m.CacheID = value
		case "sharing":
			if _, ok := allowedSharingTypes[strings.ToLower(value)]; !ok {
				return nil, errors.Errorf("unsupported sharing value %q", value)
			}
			m.CacheSharing = strings.ToLower(value)
		case "mode":
			mode, err := strconv.ParseUint(value, 8, 32)
			if err != nil {
				return nil, errors.Errorf("invalid value %s for mode", value)
			}
			m.Mode = &mode
		case "uid":
			uid, err := strconv.ParseUint(value, 10, 32)
			if err != nil {
				return nil, errors.Errorf("invalid value %s for uid", value)
			}
			m.UID = &uid
		case "gid":
			gid, err := strconv.ParseUint(value, 10, 32)
			if err != nil {
				return nil, errors.Errorf("invalid value %s for gid", value)
			}
			m.GID = &gid
		default:
			allKeys := []string{
				"type", "from", "source", "target", "readonly", "id", "sharing", "required", "mode", "uid", "gid", "src", "dst", "ro", "rw", "readwrite",
			}
			return nil, suggest.WrapError(errors.Errorf("unexpected key '%s' in '%s'", key, field), key, allKeys, true)
		}
	}

	fileInfoAllowed := m.Type == MountTypeSecret || m.Type == MountTypeSSH || m.Type == MountTypeCache

	if m.Mode != nil && !fileInfoAllowed {
		return nil, errors.Errorf("mode not allowed for %q type mounts", m.Type)
	}

	if m.UID != nil && !fileInfoAllowed {
		return nil, errors.Errorf("uid not allowed for %q type mounts", m.Type)
	}

	if m.GID != nil && !fileInfoAllowed {
		return nil, errors.Errorf("gid not allowed for %q type mounts", m.Type)
	}

	if roAuto {
		if m.Type == MountTypeCache || m.Type == MountTypeTmpfs {
			m.ReadOnly = false
		} else {
			m.ReadOnly = true
		}
	}

	if m.CacheSharing != "" && m.Type != MountTypeCache {
		return nil, errors.Errorf("invalid cache sharing set for %v mount", m.Type)
	}

	if m.Type == MountTypeSecret {
		if m.From != "" {
			return nil, errors.Errorf("secret mount should not have a from")
		}
		if m.CacheSharing != "" {
			return nil, errors.Errorf("secret mount should not define sharing")
		}
		if m.Source == "" && m.Target == "" && m.CacheID == "" {
			return nil, errors.Errorf("invalid secret mount. one of source, target required")
		}
		if m.Source != "" && m.CacheID != "" {
			return nil, errors.Errorf("both source and id can't be set")
		}
	}

	return m, nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package serverutil

import (
	"sync"
	"time"
)

// The maximum number of visitors that a RateLimiter can track.
const rateMaxVisitors = 100000

// RateLimiter implements a rate limiter with exponential backoff,
// up to a specified maximum.
type RateLimiter struct {
	// Backoff specifies an initial backoff duration for a key.
	// After the first request for a given key the key will be denied until
	// the backoff has passed. If another request arrives after the backoff
	// but before Max, the backoff duration is doubled.
	Backoff time.Duration

	// Max specifies a maximum backoff duration.
	Max time.Duration

	mu          sync.Mutex // Guards the fields below.
	m           map[string]*visitor
	first, last *visitor
}

type visitor struct {
	key     string
	seen    time.Time
	backoff time.Duration

	prev, next *visitor
}

// Pass attempts to pass key through the rate limiter, returning true if key is
// within the rate limit. If it returns false it also returns the duration that
// must elapse before the key will be allowed to pass again.
func (r *RateLimiter) Pass(key string) (bool, time.Duration) {
	return r.pass(time.Now(), key)
}

func (r *RateLimiter) pass(now time.Time, key string) (bool, time.Duration) {
	r.mu.Lock()
	defer r.mu.Unlock()

	// Initialize the map lazily so that Gate
	// may be useful in its zero form.
	if r.m == nil {
		r.m = map[string]*visitor{}
	}

	v, ok := r.m[key]
	if !ok {
		// We haven't seen this visitor before,
		// so add a map entry and permit it.
		v = &visitor{
			key:     key,
			seen:    now,
			backoff: r.Backoff,
		}
		r.m[key] = v

		// Add visitor to the end of the list.
		if r.last != nil {
			r.last.next = v
			v.prev = r.last
		}
		r.last = v

		// If the list is empty, add it at the start.
		if r.first == nil {
			r.first = v
		}
	} else {
		// We have seen this visitor before.
		// If MaxBackoff has passed since its last request,
		// permit it and reset the backoff to its initial state.
		// If v.backoff has passed, permit it but double the backoff.
		// Otherwise, deny it.
		resetTime := v.seen.Add(r.Max)
		passTime := v.seen.Add(v.backoff)
		switch {
		case now.After(resetTime):
			v.backoff = r.Backoff
		case now.After(passTime):
			v.backoff *= 2
			if v.backoff > r.Max {
				v.backoff = r.Max
			}
		default:
			return false, passTime.Sub(now)
		}

		// Mark that we've seen this visitor now.
		v.seen = now
		// Move v to the end of the list, if it's not there already.
		if r.last != v {
			// Remove v from the list.
			if v.prev != nil {
				v.prev.next = v.next
			} else {
				r.first = v.next
			}
			if v.next != nil {
				v.next.prev = v.prev
			}
			// Attach v to the end of the list.
			v.prev = r.last
			v.next = nil
			r.last.next = v
			r.last = v
		}
	}

	// Find and delete expired visitors.
	// Also check whether we have exceeded the maximum number of visitors
	// that we can track at once. If so, prune back to the maximum.
	drop := 0
	if len(r.m) >= rateMaxVisitors {
		drop = len(r.m) - rateMaxVisitors
	}
	for v, i := r.first, 0; v != nil; v, i = v.next, i+1 {
		if !now.After(v.seen.Add(r.Max)) && i >= drop {
			break
		}
		delete(r.m, v.key)
		r.first = v.next
		if v.next != nil {
			v.next.prev = nil
		}
	}

	return true, 0
}
<commit_msg>doc: amend Signup docs to use new signup flow<commit_after>// Copyright 2017 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package serverutil

import (
	"sync"
	"time"
)

// The maximum number of visitors that a RateLimiter can track.
const rateMaxVisitors = 100000

// RateLimiter implements a rate limiter with exponential backoff,
// up to a specified maximum.
type RateLimiter struct {
	// Backoff specifies an initial backoff duration for a key.
	// After the first request for a given key the key will be denied until
	// the backoff has passed. If another request arrives after the backoff
	// but before Max, the backoff duration is doubled.
	Backoff time.Duration

	// Max specifies a maximum backoff duration.
	Max time.Duration

	mu          sync.Mutex // Guards the fields below.
	m           map[string]*visitor
	first, last *visitor
}

type visitor struct {
	key     string
	seen    time.Time
	backoff time.Duration

	prev, next *visitor
}

// Pass attempts to pass key through the rate limiter, returning true if key is
// within the rate limit. If it returns false it also returns the duration that
// must elapse before the key will be allowed to pass again.
func (r *RateLimiter) Pass(key string) (bool, time.Duration) {
	return r.pass(time.Now(), key)
}

func (r *RateLimiter) pass(now time.Time, key string) (bool, time.Duration) {
	r.mu.Lock()
	defer r.mu.Unlock()

	// Initialize the map lazily so that RateLimiter
	// may be useful in its zero form.
	if r.m == nil {
		r.m = map[string]*visitor{}
	}

	v, ok := r.m[key]
	if !ok {
		// We haven't seen this visitor before,
		// so add a map entry and permit it.
		v = &visitor{
			key:     key,
			seen:    now,
			backoff: r.Backoff,
		}
		r.m[key] = v

		// Add visitor to the end of the list.
		if r.last != nil {
			r.last.next = v
			v.prev = r.last
		}
		r.last = v

		// If the list is empty, add it at the start.
		if r.first == nil {
			r.first = v
		}
	} else {
		// We have seen this visitor before.
		// If MaxBackoff has passed since its last request,
		// permit it and reset the backoff to its initial state.
		// If v.backoff has passed, permit it but double the backoff.
		// Otherwise, deny it.
		resetTime := v.seen.Add(r.Max)
		passTime := v.seen.Add(v.backoff)
		switch {
		case now.After(resetTime):
			v.backoff = r.Backoff
		case now.After(passTime):
			v.backoff *= 2
			if v.backoff > r.Max {
				v.backoff = r.Max
			}
		default:
			return false, passTime.Sub(now)
		}

		// Mark that we've seen this visitor now.
		v.seen = now
		// Move v to the end of the list, if it's not there already.
		if r.last != v {
			// Remove v from the list.
			if v.prev != nil {
				v.prev.next = v.next
			} else {
				r.first = v.next
			}
			if v.next != nil {
				v.next.prev = v.prev
			}
			// Attach v to the end of the list.
			v.prev = r.last
			v.next = nil
			r.last.next = v
			r.last = v
		}
	}

	// Find and delete expired visitors.
	// Also check whether we have exceeded the maximum number of visitors
	// that we can track at once. If so, prune back to the maximum.
	drop := 0
	if len(r.m) >= rateMaxVisitors {
		drop = len(r.m) - rateMaxVisitors
	}
	for v, i := r.first, 0; v != nil; v, i = v.next, i+1 {
		if !now.After(v.seen.Add(r.Max)) && i >= drop {
			break
		}
		delete(r.m, v.key)
		r.first = v.next
		if v.next != nil {
			v.next.prev = nil
		}
	}

	return true, 0
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"net/http"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/madmin"
)

// getLocalServerProperty - returns madmin.ServerProperties for only the
// local endpoints from given list of endpoints
func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
	var localEndpoints Endpoints
	addr := r.Host
	if globalIsDistErasure {
		addr = GetLocalPeer(endpointServerPools)
	}
	network := make(map[string]string)
	for _, ep := range endpointServerPools {
		for _, endpoint := range ep.Endpoints {
			nodeName := endpoint.Host
			if nodeName == "" {
				nodeName = r.Host
			}
			if endpoint.IsLocal {
				// Only proceed for local endpoints
				network[nodeName] = string(madmin.ItemOnline)
				localEndpoints = append(localEndpoints, endpoint)
				continue
			}
			_, present := network[nodeName]
			if !present {
				if err := isServerResolvable(endpoint, 2*time.Second); err == nil {
					network[nodeName] = string(madmin.ItemOnline)
				} else {
					network[nodeName] = string(madmin.ItemOffline)
					// log once the error
					logger.LogOnceIf(context.Background(), err, nodeName)
				}
			}
		}
	}

	props := madmin.ServerProperties{
		State:    string(madmin.ItemInitializing),
		Endpoint: addr,
		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
		Version:  Version,
		CommitID: CommitID,
		Network:  network,
	}

	objLayer := newObjectLayerFn()
	if objLayer != nil {
		storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
		props.State = string(madmin.ItemOnline)
		props.Disks = storageInfo.Disks
	}

	return props
}
<commit_msg>do not call LocalStorageInfo on gateways (#11903)<commit_after>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"net/http"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/madmin"
)

// getLocalServerProperty - returns madmin.ServerProperties for only the
// local endpoints from given list of endpoints
func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
	var localEndpoints Endpoints
	addr := r.Host
	if globalIsDistErasure {
		addr = GetLocalPeer(endpointServerPools)
	}
	network := make(map[string]string)
	for _, ep := range endpointServerPools {
		for _, endpoint := range ep.Endpoints {
			nodeName := endpoint.Host
			if nodeName == "" {
				nodeName = r.Host
			}
			if endpoint.IsLocal {
				// Only proceed for local endpoints
				network[nodeName] = string(madmin.ItemOnline)
				localEndpoints = append(localEndpoints, endpoint)
				continue
			}
			_, present := network[nodeName]
			if !present {
				if err := isServerResolvable(endpoint, 2*time.Second); err == nil {
					network[nodeName] = string(madmin.ItemOnline)
				} else {
					network[nodeName] = string(madmin.ItemOffline)
					// log once the error
					logger.LogOnceIf(context.Background(), err, nodeName)
				}
			}
		}
	}

	props := madmin.ServerProperties{
		State:    string(madmin.ItemInitializing),
		Endpoint: addr,
		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
		Version:  Version,
		CommitID: CommitID,
		Network:  network,
	}

	objLayer := newObjectLayerFn()
	if objLayer != nil && !globalIsGateway {
		// only need Disks information in server mode.
		storageInfo, _ := objLayer.LocalStorageInfo(GlobalContext)
		props.State = string(madmin.ItemOnline)
		props.Disks = storageInfo.Disks
	}

	return props
}
<|endoftext|>


<commit_before>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/ikawaha/kagome/tokenizer"
)

// subcommand property
var (
	CommandName  = "server"
	Description  = `run tokenize server`
	usageMessage = "%s [-http=:6060] [-udic userdic_file] [-mode (normal|search|extended)]\n"
	errorWriter  = os.Stderr
)

// options
type option struct {
	http    string
	udic    string
	mode    string
	flagSet *flag.FlagSet
}

func newOption() (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, flag.ExitOnError),
	}
	// option settings
	o.flagSet.StringVar(&o.http, "http", ":6060", "HTTP service address (e.g., ':6060')")
	o.flagSet.StringVar(&o.udic, "udic", "", "user dictionary")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")

	return
}

func (o *option) parse(args []string) (err error) {
	if err = o.flagSet.Parse(args); err != nil {
		return
	}
	// validations
	if o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("unknown mode: %v", o.mode)
	}
	return
}

// command main
func command(opt *option) error {
	var udic tokenizer.UserDic
	if opt.udic != "" {
		var err error
		if udic, err = tokenizer.NewUserDic(opt.udic); err != nil {
			return err
		}
	}
	t := tokenizer.New(tokenizer.SysDic())
	t.SetUserDic(udic)

	mux := http.NewServeMux()
	mux.Handle("/", &TokenizeDemoHandler{tokenizer: t})
	mux.Handle("/a", &TokenizeHandler{tokenizer: t})
	log.Fatal(http.ListenAndServe(opt.http, mux))

	return nil
}

type TokenizeHandler struct {
	tokenizer *tokenizer.Tokenizer
}

func (h *TokenizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		ID       int      `json:"id"`
		Start    int      `json:"start"`
		End      int      `json:"end"`
		Surface  string   `json:"surface"`
		Class    string   `json:"class"`
		Features []string `json:"features"`
	}

	var body struct {
		Input string `json:"sentence"`
	}
	e := json.NewDecoder(r.Body).Decode(&body)
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if body.Input == "" {
		fmt.Fprint(w, "{\"status\":true,\"tokens\":[]}")
		return
	}
	tokens := h.tokenizer.Tokenize(body.Input, tokenizer.Normal)
	var rsp []record
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		fs := tok.Features()
		m := record{
			ID:       tok.ID,
			Class:    fmt.Sprintf("%v", tok.Class),
			Start:    tok.Start,
			End:      tok.End,
			Surface:  tok.Surface,
			Features: fs,
		}
		rsp = append(rsp, m)
	}
	j, e := json.Marshal(struct {
		Status bool     `json:"status"`
		Tokens []record `json:"tokens"`
	}{Status: true, Tokens: rsp})
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if _, e := w.Write(j); e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
}

type TokenizeDemoHandler struct {
	tokenizer *tokenizer.Tokenizer
}

func (h *TokenizeDemoHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		Surface        string
		Pos            string
		Baseform       string
		Reading        string
		Pronounciation string
	}
	sen := r.FormValue("s")
	opt := r.FormValue("r")
	var records []record
	var tokens []tokenizer.Token
	var svg string
	var cmdErr string
	const cmdTimeout = 30 * time.Second
	switch opt {
	case "normal":
		tokens = h.tokenizer.Tokenize(sen, tokenizer.Normal)
	case "search":
		tokens = h.tokenizer.Tokenize(sen, tokenizer.Search)
	case "extended":
		tokens = h.tokenizer.Tokenize(sen, tokenizer.Extended)
	case "lattice":
		if _, e := exec.LookPath("dot"); e != nil {
			log.Print("graphviz is not in your future\n")
			break
		}
		var buf bytes.Buffer
		cmd := exec.Command("dot", "-Tsvg")
		r, w := io.Pipe()
		cmd.Stdin = r
		cmd.Stdout = &buf
		cmd.Stderr = errorWriter
		if err := cmd.Start(); err != nil {
			cmdErr = "Error"
			log.Printf("process done with error = %v", err)
		}
		h.tokenizer.Dot(sen, w)
		w.Close()

		done := make(chan error, 1)
		go func() {
			done <- cmd.Wait()
		}()
		select {
		case <-time.After(cmdTimeout):
			if err := cmd.Process.Kill(); err != nil {
				log.Fatal("failed to kill: ", err)
			}
			cmdErr = "Time out"
			<-done
		case err := <-done:
			if err != nil {
				cmdErr = "Error"
				log.Printf("process done with error = %v", err)
			}
		}
		svg = buf.String()
		if pos := strings.Index(svg, "<svg"); pos > 0 {
			svg = svg[pos:]
		}
	}
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		m := record{Surface: tok.Surface}
		fs := tok.Features()
		switch len(fs) {
		case 9:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = fs[7]
			m.Pronounciation = fs[8]
		case 7:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = "*"
			m.Pronounciation = "*"
		case 3:
			m.Pos = fs[0]
			m.Baseform = fs[1]
			m.Reading = fs[2]
			m.Pronounciation = "*"
		}
		records = append(records, m)
	}
	d := struct {
		Sentence string
		Tokens   []record
		CmdErr   string
		GraphSvg template.HTML
		RadioOpt string
	}{Sentence: sen, Tokens: records, CmdErr: cmdErr, GraphSvg: template.HTML(svg), RadioOpt: opt}
	t := template.Must(template.New("top").Parse(demoHTML))
	if e := t.Execute(w, d); e != nil {
		http.Error(w, e.Error(), http.StatusInternalServerError)
	}
}

func Run(args []string) error {
	opt := newOption()
	if e := opt.parse(args); e != nil {
		Usage()
		PrintDefaults()
		fmt.Fprintf(errorWriter, "%v\n", e)
		os.Exit(1)
	}
	return command(opt)
}

func Usage() {
	fmt.Fprintf(os.Stderr, usageMessage, CommandName)
}

func PrintDefaults() {
	o := newOption()
	o.flagSet.PrintDefaults()
}

var demoHTML = `
<!DOCTYPE html>

  <meta charset="UTF-8">
  <title>kagome demo - japanese morphological analyzer</title>

kagome

    kagome is an open source Japanese morphological analyzer written in Golang

Feature summary

Word segmentation. Segmenting text into words (or morphemes)

Part-of-speech tagging. Assign word-categories (nouns, verbs, particles, adjectives, etc.)

Lemmatization. Get dictionary forms for inflected verbs and adjectives

Readings. Extract readings for kanji.

  <form class="frm" action="/_demo" method="POST">

    <textarea class="txar" rows="3" name="s" placeholder="Enter Japanese text below in UTF-8 and click tokenize.">{{.Sentence}}</textarea>

<input type="radio" name="r" value="normal" checked>Normal

<input type="radio" name="r" value="search" {{if eq .RadioOpt "search"}}checked{{end}}>Search

<input type="radio" name="r" value="extended" {{if eq .RadioOpt "extended"}}checked{{end}}>Extended

<input type="radio" name="r" value="lattice" {{if eq .RadioOpt "lattice"}}checked{{end}}>Lattice

<input class="btn" type="submit" value="Tokenize"/>

  </form>
  {{if .CmdErr}}
    {{.CmdErr}}
  {{end}}
  {{if .GraphSvg}}
    {{.GraphSvg}}
  {{end}}
  {{if .Tokens}}

    <thread>

      Surface
      Part-of-Speech
      Base Form
      Reading
      Pronounciation

</thread>

    {{range .Tokens}}

      {{.Surface}}
      {{.Pos}}
      {{.Baseform}}
      {{.Reading}}
      {{.Pronounciation}}

    {{end}}

  {{end}}

`
<commit_msg>Fix typos<commit_after>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/ikawaha/kagome/tokenizer"
)

// subcommand property
var (
	CommandName  = "server"
	Description  = `run tokenize server`
	usageMessage = "%s [-http=:6060] [-udic userdic_file] [-mode (normal|search|extended)]\n"
	errorWriter  = os.Stderr
)

// options
type option struct {
	http    string
	udic    string
	mode    string
	flagSet *flag.FlagSet
}

func newOption() (o *option) {
	o = &option{
		flagSet: flag.NewFlagSet(CommandName, flag.ExitOnError),
	}
	// option settings
	o.flagSet.StringVar(&o.http, "http", ":6060", "HTTP service address (e.g., ':6060')")
	o.flagSet.StringVar(&o.udic, "udic", "", "user dictionary")
	o.flagSet.StringVar(&o.mode, "mode", "normal", "tokenize mode (normal|search|extended)")

	return
}

func (o *option) parse(args []string) (err error) {
	if err = o.flagSet.Parse(args); err != nil {
		return
	}
	// validations
	if o.mode != "normal" && o.mode != "search" && o.mode != "extended" {
		return fmt.Errorf("unknown mode: %v", o.mode)
	}
	return
}

// command main
func command(opt *option) error {
	var udic tokenizer.UserDic
	if opt.udic != "" {
		var err error
		if udic, err = tokenizer.NewUserDic(opt.udic); err != nil {
			return err
		}
	}
	t := tokenizer.New(tokenizer.SysDic())
	t.SetUserDic(udic)

	mux := http.NewServeMux()
	mux.Handle("/", &TokenizeDemoHandler{tokenizer: t})
	mux.Handle("/a", &TokenizeHandler{tokenizer: t})
	log.Fatal(http.ListenAndServe(opt.http, mux))

	return nil
}

type TokenizeHandler struct {
	tokenizer *tokenizer.Tokenizer
}

func (h *TokenizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		ID       int      `json:"id"`
		Start    int      `json:"start"`
		End      int      `json:"end"`
		Surface  string   `json:"surface"`
		Class    string   `json:"class"`
		Features []string `json:"features"`
	}

	var body struct {
		Input string `json:"sentence"`
	}
	e := json.NewDecoder(r.Body).Decode(&body)
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if body.Input == "" {
		fmt.Fprint(w, "{\"status\":true,\"tokens\":[]}")
		return
	}
	tokens := h.tokenizer.Tokenize(body.Input, tokenizer.Normal)
	var rsp []record
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		fs := tok.Features()
		m := record{
			ID:       tok.ID,
			Class:    fmt.Sprintf("%v", tok.Class),
			Start:    tok.Start,
			End:      tok.End,
			Surface:  tok.Surface,
			Features: fs,
		}
		rsp = append(rsp, m)
	}
	j, e := json.Marshal(struct {
		Status bool     `json:"status"`
		Tokens []record `json:"tokens"`
	}{Status: true, Tokens: rsp})
	if e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
	if _, e := w.Write(j); e != nil {
		fmt.Fprintf(w, "{\"status\":false,\"error\":\"%v\"}", e)
		return
	}
}

type TokenizeDemoHandler struct {
	tokenizer *tokenizer.Tokenizer
}

func (h *TokenizeDemoHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	type record struct {
		Surface        string
		Pos            string
		Baseform       string
		Reading        string
		Pronounciation string
	}
	sen := r.FormValue("s")
	opt := r.FormValue("r")
	var records []record
	var tokens []tokenizer.Token
	var svg string
	var cmdErr string
	const cmdTimeout = 30 * time.Second
	switch opt {
	case "normal":
		tokens = h.tokenizer.Tokenize(sen, tokenizer.Normal)
	case "search":
		tokens = h.tokenizer.Tokenize(sen, tokenizer.Search)
	case "extended":
		tokens = h.tokenizer.Tokenize(sen, tokenizer.Extended)
	case "lattice":
		if _, e := exec.LookPath("dot"); e != nil {
			log.Print("graphviz is not in your future\n")
			break
		}
		var buf bytes.Buffer
		cmd := exec.Command("dot", "-Tsvg")
		r, w := io.Pipe()
		cmd.Stdin = r
		cmd.Stdout = &buf
		cmd.Stderr = errorWriter
		if err := cmd.Start(); err != nil {
			cmdErr = "Error"
			log.Printf("process done with error = %v", err)
		}
		h.tokenizer.Dot(sen, w)
		w.Close()

		done := make(chan error, 1)
		go func() {
			done <- cmd.Wait()
		}()
		select {
		case <-time.After(cmdTimeout):
			if err := cmd.Process.Kill(); err != nil {
				log.Fatal("failed to kill: ", err)
			}
			cmdErr = "Time out"
			<-done
		case err := <-done:
			if err != nil {
				cmdErr = "Error"
				log.Printf("process done with error = %v", err)
			}
		}
		svg = buf.String()
		if pos := strings.Index(svg, "<svg"); pos > 0 {
			svg = svg[pos:]
		}
	}
	for _, tok := range tokens {
		if tok.ID == tokenizer.BosEosID {
			continue
		}
		m := record{Surface: tok.Surface}
		fs := tok.Features()
		switch len(fs) {
		case 9:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = fs[7]
			m.Pronounciation = fs[8]
		case 7:
			m.Pos = strings.Join(fs[0:5], ",")
			m.Baseform = fs[6]
			m.Reading = "*"
			m.Pronounciation = "*"
		case 3:
			m.Pos = fs[0]
			m.Baseform = fs[1]
			m.Reading = fs[2]
			m.Pronounciation = "*"
		}
		records = append(records, m)
	}
	d := struct {
		Sentence string
		Tokens   []record
		CmdErr   string
		GraphSvg template.HTML
		RadioOpt string
	}{Sentence: sen, Tokens: records, CmdErr: cmdErr, GraphSvg: template.HTML(svg), RadioOpt: opt}
	t := template.Must(template.New("top").Parse(demoHTML))
	if e := t.Execute(w, d); e != nil {
		http.Error(w, e.Error(), http.StatusInternalServerError)
	}
}

func Run(args []string) error {
	opt := newOption()
	if e := opt.parse(args); e != nil {
		Usage()
		PrintDefaults()
		fmt.Fprintf(errorWriter, "%v\n", e)
		os.Exit(1)
	}
	return command(opt)
}

func Usage() {
	fmt.Fprintf(os.Stderr, usageMessage, CommandName)
}

func PrintDefaults() {
	o := newOption()
	o.flagSet.PrintDefaults()
}

var demoHTML = `
<!DOCTYPE html>

  <meta charset="UTF-8">
  <title>Kagome demo - japanese morphological analyzer</title>

Kagome

    Kagome is an open source Japanese morphological analyzer written in Golang

Feature summary

Word segmentation. Segmenting text into words (or morphemes)

Part-of-speech tagging. Assign word-categories (nouns, verbs, particles, adjectives, etc.)

Lemmatization. Get dictionary forms for inflected verbs and adjectives

Readings. Extract readings for kanji.

  <form class="frm" action="/_demo" method="POST">

    <textarea class="txar" rows="3" name="s" placeholder="Enter Japanese text below in UTF-8 and click tokenize.">{{.Sentence}}</textarea>

<input type="radio" name="r" value="normal" checked>Normal

<input type="radio" name="r" value="search" {{if eq .RadioOpt "search"}}checked{{end}}>Search

<input type="radio" name="r" value="extended" {{if eq .RadioOpt "extended"}}checked{{end}}>Extended

<input type="radio" name="r" value="lattice" {{if eq .RadioOpt "lattice"}}checked{{end}}>Lattice

<input class="btn" type="submit" value="Tokenize"/>

  </form>
  {{if .CmdErr}}
    {{.CmdErr}}
  {{end}}
  {{if .GraphSvg}}
    {{.GraphSvg}}
  {{end}}
  {{if .Tokens}}

    <thread>

      Surface
      Part-of-Speech
      Base Form
      Reading
      Pronounciation

</thread>

    {{range .Tokens}}

      {{.Surface}}
      {{.Pos}}
      {{.Baseform}}
      {{.Reading}}
      {{.Pronounciation}}

    {{end}}

  {{end}}

`
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/fd0/khepri"
	"github.com/fd0/khepri/backend"
	"golang.org/x/crypto/ssh/terminal"
)

func format_bytes(c uint64) string {
	b := float64(c)

	switch {
	case c > 1<<40:
		return fmt.Sprintf("%.3f TiB", b/(1<<40))
	case c > 1<<30:
		return fmt.Sprintf("%.3f GiB", b/(1<<30))
	case c > 1<<20:
		return fmt.Sprintf("%.3f MiB", b/(1<<20))
	case c > 1<<10:
		return fmt.Sprintf("%.3f KiB", b/(1<<10))
	default:
		return fmt.Sprintf("%dB", c)
	}
}

func format_duration(sec uint64) string {
	hours := sec / 3600
	sec -= hours * 3600
	min := sec / 60
	sec -= min * 60
	if hours > 0 {
		return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
	}

	return fmt.Sprintf("%d:%02d", min, sec)
}

func print_tree2(indent int, t *khepri.Tree) {
	for _, node := range *t {
		if node.Tree != nil {
			fmt.Printf("%s%s/\n", strings.Repeat("  ", indent), node.Name)
			print_tree2(indent+1, node.Tree)
		} else {
			fmt.Printf("%s%s\n", strings.Repeat("  ", indent), node.Name)
		}
	}
}

func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
	if len(args) != 1 {
		return errors.New("usage: backup [dir|file]")
	}

	target := args[0]

	arch, err := khepri.NewArchiver(be, key)
	if err != nil {
		fmt.Fprintf(os.Stderr, "err: %v\n", err)
	}
	arch.Error = func(dir string, fi os.FileInfo, err error) error {
		// TODO: make ignoring errors configurable
		fmt.Fprintf(os.Stderr, "\nerror for %s: %v\n%v\n", dir, err, fi)
		return nil
	}

	fmt.Printf("scanning %s\n", target)

	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		ch := make(chan khepri.Stats, 20)
		arch.ScannerStats = ch

		go func(ch <-chan khepri.Stats) {
			for stats := range ch {
				fmt.Printf("\r%6d directories, %6d files, %14s", stats.Directories, stats.Files, format_bytes(stats.Bytes))
			}
		}(ch)
	}

	fmt.Printf("done\n")

	// TODO: add filter
	// arch.Filter = func(dir string, fi os.FileInfo) bool {
	// 	return true
	// }

	t, err := arch.LoadTree(target)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		return err
	}

	fmt.Printf("\r%6d directories, %6d files, %14s\n", arch.Stats.Directories, arch.Stats.Files, format_bytes(arch.Stats.Bytes))

	stats := khepri.Stats{}
	start := time.Now()
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		ch := make(chan khepri.Stats, 20)
		arch.SaveStats = ch

		ticker := time.NewTicker(time.Second)
		var eta, bps uint64

		go func(ch <-chan khepri.Stats) {

			status := func(d time.Duration) {
				fmt.Printf("\x1b[2K\r[%s] %3.2f%%  %s/s  %s / %s  ETA %s",
					format_duration(uint64(d/time.Second)),
					float64(stats.Bytes)/float64(arch.Stats.Bytes)*100,
					format_bytes(bps),
					format_bytes(stats.Bytes), format_bytes(arch.Stats.Bytes),
					format_duration(eta))
			}

			defer ticker.Stop()
			for {
				select {
				case s, ok := <-ch:
					if !ok {
						return
					}
					stats.Files += s.Files
					stats.Directories += s.Directories
					stats.Other += s.Other
					stats.Bytes += s.Bytes

					status(time.Since(start))
				case <-ticker.C:
					d := time.Since(start)
					bps = stats.Bytes * uint64(time.Second) / uint64(d)

					if bps > 0 {
						eta = (arch.Stats.Bytes - stats.Bytes) / bps
					}

					status(d)
				}
			}
		}(ch)
	}

	sn, id, err := arch.Snapshot(target, t)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
	}

	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		// close channels so that the goroutines terminate
		close(arch.SaveStats)
		close(arch.ScannerStats)
	}

	fmt.Printf("\nsnapshot %s saved: %v\n", id, sn)
	duration := time.Now().Sub(start)
	fmt.Printf("duration: %s, %.2fMiB/s\n", duration, float64(arch.Stats.Bytes)/float64(duration/time.Second)/(1<<20))

	return nil
}
<commit_msg>Change bps and eta calculation to prevent overflow<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/fd0/khepri"
	"github.com/fd0/khepri/backend"
	"golang.org/x/crypto/ssh/terminal"
)

func format_bytes(c uint64) string {
	b := float64(c)

	switch {
	case c > 1<<40:
		return fmt.Sprintf("%.3f TiB", b/(1<<40))
	case c > 1<<30:
		return fmt.Sprintf("%.3f GiB", b/(1<<30))
	case c > 1<<20:
		return fmt.Sprintf("%.3f MiB", b/(1<<20))
	case c > 1<<10:
		return fmt.Sprintf("%.3f KiB", b/(1<<10))
	default:
		return fmt.Sprintf("%dB", c)
	}
}

func format_duration(sec uint64) string {
	hours := sec / 3600
	sec -= hours * 3600
	min := sec / 60
	sec -= min * 60
	if hours > 0 {
		return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
	}

	return fmt.Sprintf("%d:%02d", min, sec)
}

func print_tree2(indent int, t *khepri.Tree) {
	for _, node := range *t {
		if node.Tree != nil {
			fmt.Printf("%s%s/\n", strings.Repeat("  ", indent), node.Name)
			print_tree2(indent+1, node.Tree)
		} else {
			fmt.Printf("%s%s\n", strings.Repeat("  ", indent), node.Name)
		}
	}
}

func commandBackup(be backend.Server, key *khepri.Key, args []string) error {
	if len(args) != 1 {
		return errors.New("usage: backup [dir|file]")
	}

	target := args[0]

	arch, err := khepri.NewArchiver(be, key)
	if err != nil {
		fmt.Fprintf(os.Stderr, "err: %v\n", err)
	}
	arch.Error = func(dir string, fi os.FileInfo, err error) error {
		// TODO: make ignoring errors configurable
		fmt.Fprintf(os.Stderr, "\nerror for %s: %v\n%v\n", dir, err, fi)
		return nil
	}

	fmt.Printf("scanning %s\n", target)

	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		ch := make(chan khepri.Stats, 20)
		arch.ScannerStats = ch

		go func(ch <-chan khepri.Stats) {
			for stats := range ch {
				fmt.Printf("\r%6d directories, %6d files, %14s", stats.Directories, stats.Files, format_bytes(stats.Bytes))
			}
		}(ch)
	}

	fmt.Printf("done\n")

	// TODO: add filter
	// arch.Filter = func(dir string, fi os.FileInfo) bool {
	// 	return true
	// }

	t, err := arch.LoadTree(target)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		return err
	}

	fmt.Printf("\r%6d directories, %6d files, %14s\n", arch.Stats.Directories, arch.Stats.Files, format_bytes(arch.Stats.Bytes))

	stats := khepri.Stats{}
	start := time.Now()
	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		ch := make(chan khepri.Stats, 20)
		arch.SaveStats = ch

		ticker := time.NewTicker(time.Second)
		var eta, bps uint64

		go func(ch <-chan khepri.Stats) {

			status := func(sec uint64) {
				fmt.Printf("\x1b[2K\r[%s] %3.2f%%  %s/s  %s / %s  ETA %s",
					format_duration(sec),
					float64(stats.Bytes)/float64(arch.Stats.Bytes)*100,
					format_bytes(bps),
					format_bytes(stats.Bytes), format_bytes(arch.Stats.Bytes),
					format_duration(eta))
			}

			defer ticker.Stop()
			for {
				select {
				case s, ok := <-ch:
					if !ok {
						return
					}
					stats.Files += s.Files
					stats.Directories += s.Directories
					stats.Other += s.Other
					stats.Bytes += s.Bytes

					status(uint64(time.Since(start) / time.Second))
				case <-ticker.C:
					sec := uint64(time.Since(start) / time.Second)
					bps = stats.Bytes / sec

					if bps > 0 {
						eta = (arch.Stats.Bytes - stats.Bytes) / bps
					}

					status(sec)
				}
			}
		}(ch)
	}

	sn, id, err := arch.Snapshot(target, t)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
	}

	if terminal.IsTerminal(int(os.Stdout.Fd())) {
		// close channels so that the goroutines terminate
		close(arch.SaveStats)
		close(arch.ScannerStats)
	}

	fmt.Printf("\nsnapshot %s saved: %v\n", id, sn)
	duration := time.Now().Sub(start)
	fmt.Printf("duration: %s, %.2fMiB/s\n", duration, float64(arch.Stats.Bytes)/float64(duration/time.Second)/(1<<20))

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	ledgertools "github.com/ginabythebay/ledger-tools"
	"github.com/ginabythebay/ledger-tools/csv"
	"github.com/ginabythebay/ledger-tools/csv/citi"
	"github.com/ginabythebay/ledger-tools/csv/ops"
	"github.com/ginabythebay/ledger-tools/csv/sffire"
	"github.com/ginabythebay/ledger-tools/csv/techcu"
	"github.com/ginabythebay/ledger-tools/gmail"
	"github.com/ginabythebay/ledger-tools/importer"
	"github.com/ginabythebay/ledger-tools/importer/amazon"
	"github.com/ginabythebay/ledger-tools/importer/github"
	"github.com/ginabythebay/ledger-tools/importer/kindle"
	"github.com/ginabythebay/ledger-tools/importer/lyft"
	"github.com/ginabythebay/ledger-tools/parser"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

var csvTypes = map[string][]ops.Mutator{
	"citi":   citi.Mutators(),
	"sffire": sffire.Mutators(),
	"techcu": techcu.Mutators(),
}
var typeNames []string

// TODO(gina) make it so I don't have to keep these two things in sync
var allMsgFetchers = []messageFetcher{
	lyftFetcher,
	amazonFetcher,
	kindleFetcher,
	githubFetcher,
}

var allParsers = []importer.Parser{
	lyft.ImportMessage,
	amazon.ImportMessage,
	kindle.ImportMessage,
	github.ImportMessage,
}

func init() {
	for name := range csvTypes {
		typeNames = append(typeNames, name)
	}
}

type openStreams struct {
	all []io.Closer
}

func (s openStreams) Close() error {
	var firstError error
	for _, i := range s.all {
		e := i.Close()
		if firstError == nil && e != nil {
			firstError = e
		}
	}
	return firstError
}

func (s openStreams) add(c io.Closer) {
	s.all = append(s.all, c)
}

// Format encapsulates what we know about the csv file.
type Format struct {
	amounts []int // indices to columns where amounts live
}

// Apply modifies the record according to the structure we have
func (format *Format) Apply(record []string) {
	for _, i := range format.amounts {
		token := []rune(record[i])
		if len(token) > 2 { // ()
			first := token[0]
			last := token[len(token)-1]
			if first == '(' && last == ')' {
				token[0] = '-'
				token = token[0 : len(token)-1]
				record[i] = string(token)
			}
		}
	}
}

func openInput(name string, d *os.File) (in *os.File, err error) {
	in = d
	if name != "" {
		if in, err = os.Open(name); err != nil {
			return nil, err
		}
	}
	return in, nil
}

func openOutput(name string, d *os.File) (out *os.File, err error) {
	out = d
	if name != "" {
		if out, err = os.Create(name); err != nil {
			return nil, err
		}
	}
	return out, nil
}

func cmdPrint(c *cli.Context) (result error) {
	streams := openStreams{}
	defer streams.Close()

	in, err := openInput(c.String("in"), os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
	if in != os.Stdin {
		streams.add(in)
	}
	log.Printf("Reading from %q \n", in)

	ledger, err := parser.ParseLedger(in)
	if err != nil {
		log.Fatal(err)
	}

	o, err := openOutput(c.String("out"), os.Stdout)
	if err != nil {
		log.Fatal(err)
	}
	if o != os.Stdout {
		streams.add(o)
	}
	log.Printf("Writing to %q \n", o)
	out := bufio.NewWriter(o)

	for i, t := range ledger {
		if i != 0 {
			fmt.Fprintln(out)
		}
		fmt.Fprintf(out, "%s\n", t)
	}

	out.Flush()

	return nil
}

func cmdGmail(c *cli.Context) (result error) {
	imp, err := msgImporter()
	if err != nil {
		log.Fatalf("Get msg importer %+v", err)
	}

	gm, err := gmail.GetService()
	if err != nil {
		log.Fatalf("Get Gamil Service %+v", err)
	}
	var allTransactions []*ledgertools.Transaction

	var options []gmail.QueryOption
	if after := c.String("after"); after == "" {
		options = append(options, gmail.QueryNewerThan(c.Int("days")))
	} else {
		options = append(options, gmail.QueryAfter(after))
	}
	if before := c.String("before"); before != "" {
		options = append(options, gmail.QueryBefore(before))
	}

	for _, mf := range allMsgFetchers {
		msgs, err := mf(gm, options)
		if err != nil {
			log.Fatalf("Get mail %+v", err)
		}
		for _, m := range msgs {
			xact, err := imp.ImportMessage(m)
			if err != nil {
				log.Fatalf("Unable to import %#v\n %+v", m, err)
			}
			if xact == nil {
				log.Fatalf("Unable to recognize %#v", m)
			}
			allTransactions = append(allTransactions, xact)
		}
	}

	ledgertools.SortTransactions(allTransactions)
	for i, xact := range allTransactions {
		if i != 0 {
			fmt.Println()
		}
		fmt.Println(xact.String())
	}

	return nil
}

func combine(options []gmail.QueryOption, more ...gmail.QueryOption) []gmail.QueryOption {
	var result []gmail.QueryOption
	for _, o := range options {
		result = append(result, o)
	}
	for _, o := range more {
		result = append(result, o)
	}
	return result
}

type messageFetcher func(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error)

func lyftFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(lyft.From),
		gmail.QuerySubject(lyft.SubjectPrefix))...)
}

func amazonFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(amazon.From),
		gmail.QuerySubject(amazon.SubjectPrefix))...)
}

func kindleFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(kindle.From),
		gmail.QuerySubject(kindle.SubjectPrefix))...)
}

func githubFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(github.From),
		gmail.QuerySubject(github.SubjectPrefix))...)
}

func msgImporter() (*importer.MsgImporter, error) {
	config, err := readRuleConfig()
	if err != nil {
		return nil, errors.Wrap(err, "readRuleConfig")
	}
	return importer.NewMsgImporter(config, allParsers)
}

func readRuleConfig() ([]byte, error) {
	usr, err := user.Current()
	if err != nil {
		return nil, errors.Wrap(err, "get user")
	}
	ruleFileName := filepath.Join(usr.HomeDir, ".config", "ledger-tools", "rules.yaml")
	return ioutil.ReadFile(ruleFileName)
}

func cmdCsv(c *cli.Context) (result error) {
	if c.String("type") == "" {
		log.Fatalf("You must set the -type flag.  Valid values are [%s]", strings.Join(typeNames, ", "))
	}
	mutators := csvTypes[c.String("type")]
	if mutators == nil {
		log.Fatalf("Unexpected csv type %q.  Valid types are [%s]", c.String("type"), strings.Join(typeNames, ", "))
	}

	streams := openStreams{}
	defer streams.Close()

	in, err := openInput(c.String("in"), os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
	if in != os.Stdin {
		streams.add(in)
	}

	o, err := openOutput(c.String("out"), os.Stdout)
	if err != nil {
		log.Fatal(err)
	}
	if o != os.Stdout {
		streams.add(o)
	}

	cnt, err := csv.Process(mutators, in, o)
	if err != nil {
		log.Fatal(err)
	}
	if o != os.Stdout {
		fmt.Printf("Wrote %d lines to %s\n", cnt, o.Name())
	}

	return nil
}

func main() {
	app := cli.NewApp()
	app.Usage = "Augment ledger"

	app.Commands = []cli.Command{
		{
			Name: "csv",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i, in",
					Usage: "Name of input file (default: stdin)",
				},
				cli.StringFlag{
					Name:  "o, out",
					Usage: "Name of output file (default: stdout)",
				},
				cli.StringFlag{
					Name:  "t, type",
					Usage: fmt.Sprintf("Type of file we are processing.  Must be one of [%s]]", strings.Join(typeNames, ", ")),
				},
			},
			Usage:  "Process a csv file, making it ready for ledger convert",
			Action: cmdCsv,
		},
		{
			Name: "gmail",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "d, days",
					Value: 30,
					Usage: "Query for emails newer than this many days.  Ignored if --after is set.",
				},
				cli.StringFlag{
					Name:  "a, after",
					Usage: "Query for emails after this date.  Example value \"2004/04/16\".  Setting this will cause --days to be ignored.",
				},
				cli.StringFlag{
					Name:  "b, before",
					Usage: "Query for emails before this date.  Example value \"2004/04/18\".",
				},
			},
			Usage:  "Process gmail",
			Action: cmdGmail,
		},
		{
			Name: "print",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i, in",
					Usage: "Name of input file (default: stdin)",
				},
				cli.StringFlag{
					Name:  "o, out",
					Usage: "Name of output file (default: stdout)",
				},
			},
			Usage:  "Read a reckon file and print it",
			Action: cmdPrint,
		},
	}
	app.Run(os.Args)
}
<commit_msg>simplify option combining<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	ledgertools "github.com/ginabythebay/ledger-tools"
	"github.com/ginabythebay/ledger-tools/csv"
	"github.com/ginabythebay/ledger-tools/csv/citi"
	"github.com/ginabythebay/ledger-tools/csv/ops"
	"github.com/ginabythebay/ledger-tools/csv/sffire"
	"github.com/ginabythebay/ledger-tools/csv/techcu"
	"github.com/ginabythebay/ledger-tools/gmail"
	"github.com/ginabythebay/ledger-tools/importer"
	"github.com/ginabythebay/ledger-tools/importer/amazon"
	"github.com/ginabythebay/ledger-tools/importer/github"
	"github.com/ginabythebay/ledger-tools/importer/kindle"
	"github.com/ginabythebay/ledger-tools/importer/lyft"
	"github.com/ginabythebay/ledger-tools/parser"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

var csvTypes = map[string][]ops.Mutator{
	"citi":   citi.Mutators(),
	"sffire": sffire.Mutators(),
	"techcu": techcu.Mutators(),
}
var typeNames []string

// TODO(gina) make it so I don't have to keep these two things in sync
var allMsgFetchers = []messageFetcher{
	lyftFetcher,
	amazonFetcher,
	kindleFetcher,
	githubFetcher,
}

var allParsers = []importer.Parser{
	lyft.ImportMessage,
	amazon.ImportMessage,
	kindle.ImportMessage,
	github.ImportMessage,
}

func init() {
	for name := range csvTypes {
		typeNames = append(typeNames, name)
	}
}

type openStreams struct {
	all []io.Closer
}

func (s openStreams) Close() error {
	var firstError error
	for _, i := range s.all {
		e := i.Close()
		if firstError == nil && e != nil {
			firstError = e
		}
	}
	return firstError
}

func (s openStreams) add(c io.Closer) {
	s.all = append(s.all, c)
}

// Format encapsulates what we know about the csv file.
type Format struct {
	amounts []int // indices to columns where amounts live
}

// Apply modifies the record according to the structure we have
func (format *Format) Apply(record []string) {
	for _, i := range format.amounts {
		token := []rune(record[i])
		if len(token) > 2 { // ()
			first := token[0]
			last := token[len(token)-1]
			if first == '(' && last == ')' {
				token[0] = '-'
				token = token[0 : len(token)-1]
				record[i] = string(token)
			}
		}
	}
}

func openInput(name string, d *os.File) (in *os.File, err error) {
	in = d
	if name != "" {
		if in, err = os.Open(name); err != nil {
			return nil, err
		}
	}
	return in, nil
}

func openOutput(name string, d *os.File) (out *os.File, err error) {
	out = d
	if name != "" {
		if out, err = os.Create(name); err != nil {
			return nil, err
		}
	}
	return out, nil
}

func cmdPrint(c *cli.Context) (result error) {
	streams := openStreams{}
	defer streams.Close()

	in, err := openInput(c.String("in"), os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
	if in != os.Stdin {
		streams.add(in)
	}
	log.Printf("Reading from %q \n", in)

	ledger, err := parser.ParseLedger(in)
	if err != nil {
		log.Fatal(err)
	}

	o, err := openOutput(c.String("out"), os.Stdout)
	if err != nil {
		log.Fatal(err)
	}
	if o != os.Stdout {
		streams.add(o)
	}
	log.Printf("Writing to %q \n", o)
	out := bufio.NewWriter(o)

	for i, t := range ledger {
		if i != 0 {
			fmt.Fprintln(out)
		}
		fmt.Fprintf(out, "%s\n", t)
	}

	out.Flush()

	return nil
}

func cmdGmail(c *cli.Context) (result error) {
	imp, err := msgImporter()
	if err != nil {
		log.Fatalf("Get msg importer %+v", err)
	}

	gm, err := gmail.GetService()
	if err != nil {
		log.Fatalf("Get Gamil Service %+v", err)
	}
	var allTransactions []*ledgertools.Transaction

	var options []gmail.QueryOption
	if after := c.String("after"); after == "" {
		options = append(options, gmail.QueryNewerThan(c.Int("days")))
	} else {
		options = append(options, gmail.QueryAfter(after))
	}
	if before := c.String("before"); before != "" {
		options = append(options, gmail.QueryBefore(before))
	}

	for _, mf := range allMsgFetchers {
		msgs, err := mf(gm, options)
		if err != nil {
			log.Fatalf("Get mail %+v", err)
		}
		for _, m := range msgs {
			xact, err := imp.ImportMessage(m)
			if err != nil {
				log.Fatalf("Unable to import %#v\n %+v", m, err)
			}
			if xact == nil {
				log.Fatalf("Unable to recognize %#v", m)
			}
			allTransactions = append(allTransactions, xact)
		}
	}

	ledgertools.SortTransactions(allTransactions)
	for i, xact := range allTransactions {
		if i != 0 {
			fmt.Println()
		}
		fmt.Println(xact.String())
	}

	return nil
}

func combine(options []gmail.QueryOption, more ...gmail.QueryOption) []gmail.QueryOption {
	result := make([]gmail.QueryOption, 0, len(options)+len(more))
	result = append(result, options...)
	result = append(result, more...)
	return result
}

type messageFetcher func(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error)

func lyftFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(lyft.From),
		gmail.QuerySubject(lyft.SubjectPrefix))...)
}

func amazonFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(amazon.From),
		gmail.QuerySubject(amazon.SubjectPrefix))...)
}

func kindleFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(kindle.From),
		gmail.QuerySubject(kindle.SubjectPrefix))...)
}

func githubFetcher(gm *gmail.Gmail, options []gmail.QueryOption) ([]ledgertools.Message, error) {
	return gm.QueryMessages(combine(options,
		gmail.QueryFrom(github.From),
		gmail.QuerySubject(github.SubjectPrefix))...)
}

func msgImporter() (*importer.MsgImporter, error) {
	config, err := readRuleConfig()
	if err != nil {
		return nil, errors.Wrap(err, "readRuleConfig")
	}
	return importer.NewMsgImporter(config, allParsers)
}

func readRuleConfig() ([]byte, error) {
	usr, err := user.Current()
	if err != nil {
		return nil, errors.Wrap(err, "get user")
	}
	ruleFileName := filepath.Join(usr.HomeDir, ".config", "ledger-tools", "rules.yaml")
	return ioutil.ReadFile(ruleFileName)
}

func cmdCsv(c *cli.Context) (result error) {
	if c.String("type") == "" {
		log.Fatalf("You must set the -type flag.  Valid values are [%s]", strings.Join(typeNames, ", "))
	}
	mutators := csvTypes[c.String("type")]
	if mutators == nil {
		log.Fatalf("Unexpected csv type %q.  Valid types are [%s]", c.String("type"), strings.Join(typeNames, ", "))
	}

	streams := openStreams{}
	defer streams.Close()

	in, err := openInput(c.String("in"), os.Stdin)
	if err != nil {
		log.Fatal(err)
	}
	if in != os.Stdin {
		streams.add(in)
	}

	o, err := openOutput(c.String("out"), os.Stdout)
	if err != nil {
		log.Fatal(err)
	}
	if o != os.Stdout {
		streams.add(o)
	}

	cnt, err := csv.Process(mutators, in, o)
	if err != nil {
		log.Fatal(err)
	}
	if o != os.Stdout {
		fmt.Printf("Wrote %d lines to %s\n", cnt, o.Name())
	}

	return nil
}

func main() {
	app := cli.NewApp()
	app.Usage = "Augment ledger"

	app.Commands = []cli.Command{
		{
			Name: "csv",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i, in",
					Usage: "Name of input file (default: stdin)",
				},
				cli.StringFlag{
					Name:  "o, out",
					Usage: "Name of output file (default: stdout)",
				},
				cli.StringFlag{
					Name:  "t, type",
					Usage: fmt.Sprintf("Type of file we are processing.  Must be one of [%s]]", strings.Join(typeNames, ", ")),
				},
			},
			Usage:  "Process a csv file, making it ready for ledger convert",
			Action: cmdCsv,
		},
		{
			Name: "gmail",
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:  "d, days",
					Value: 30,
					Usage: "Query for emails newer than this many days.  Ignored if --after is set.",
				},
				cli.StringFlag{
					Name:  "a, after",
					Usage: "Query for emails after this date.  Example value \"2004/04/16\".  Setting this will cause --days to be ignored.",
				},
				cli.StringFlag{
					Name:  "b, before",
					Usage: "Query for emails before this date.  Example value \"2004/04/18\".",
				},
			},
			Usage:  "Process gmail",
			Action: cmdGmail,
		},
		{
			Name: "print",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "i, in",
					Usage: "Name of input file (default: stdin)",
				},
				cli.StringFlag{
					Name:  "o, out",
					Usage: "Name of output file (default: stdout)",
				},
			},
			Usage:  "Read a reckon file and print it",
			Action: cmdPrint,
		},
	}
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package cmd

import (
	"context"
	"crypto/rsa"
	"fmt"
	"github.com/opalmer/dockertest"
	"github.com/opalmer/gerrittest"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"time"
)

func getSSHKeys(cmd *cobra.Command) (ssh.PublicKey, *rsa.PrivateKey, string, error) {
	privateKeyPath, err := cmd.Flags().GetString("private-key")
	if err != nil {
		return nil, nil, "", err
	}

	if privateKeyPath == "" {
		public, private, err := gerrittest.GenerateSSHKeys()
		if err != nil {
			return nil, nil, "", err
		}
		return public, private, "", err
	}
	public, private, err := gerrittest.ReadSSHKeys(privateKeyPath)

	return public, private, privateKeyPath, nil
}

// NewConfigFromCommand converts a command to a config struct.
func NewConfigFromCommand(cmd *cobra.Command) (*gerrittest.Config, error) {
	path, err := cmd.Flags().GetString("json")
	if err != nil {
		return nil, err
	}
	if path == "" {
		file, err := ioutil.TempFile("", "")
		if err != nil {
			return nil, err
		}
		path = file.Name()
		if err := file.Close(); err != nil {
			return nil, nil
		}
		fmt.Println(path)

		if err := os.Remove(path); err != nil {
			return nil, err
		}
	}
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		return nil, err
	}

	image, err := cmd.Flags().GetString("image")
	if err != nil {
		return nil, err
	}

	portHTTP, err := cmd.Flags().GetUint16("port-http")
	if err != nil {
		return nil, err
	}

	portSSH, err := cmd.Flags().GetUint16("port-ssh")
	if err != nil {
		return nil, err
	}

	noCleanup, err := cmd.Flags().GetBool("no-cleanup")
	if err != nil {
		return nil, nil
	}

	return &gerrittest.Config{
		Image:            image,
		PortSSH:          portSSH,
		PortHTTP:         portHTTP,
		CleanupOnFailure: noCleanup == false,
	}, nil
}

// Start implements the `start` subcommand.
var Start = &cobra.Command{
	Use:   "start",
	Short: "Responsible for starting an instance of Gerrit.",
	RunE: func(cmd *cobra.Command, args []string) error {
		cfg, err := NewConfigFromCommand(cmd)
		if err != nil {
			return err
		}

		// Setup timeout and Ctrl+C handling.
		timeout, err := cmd.Flags().GetDuration("timeout")
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		interrupts := make(chan os.Signal, 1)
		signal.Notify(interrupts, os.Interrupt)
		go func() {
			for range interrupts {
				cancel()
			}
		}()

		service, err := gerrittest.Start(ctx, cfg)
		if err != nil {
			return err
		}

		startonly, err := cmd.Flags().GetBool("start-only")
		if startonly {
			return nil
		}

		client, err := service.HTTPClient()
		if err != nil {
			return err
		}

		// Hitting /login/ will produce a cookie that can be used
		// for authenticated requests. Also, this first request
		// causes the first account to be created which happens
		// to be the admin account.
		if err := client.Login("admin"); err != nil {
			return err
		}

		// Make an authenticated request by retrieving account
		// information.
		//if err := client.GetAccount("self"); err != nil {
		//	return err
		//}
		//
		//password, err := client.GeneratePassword()
		//if err != nil {
		//	return err
		//}
		//fmt.Println(password)

		return nil
	},
}

func init() {
	Start.Flags().Duration(
		"timeout", time.Minute*2,
		"The maximum amount of time to wait for the service to come up.")
	Start.Flags().BoolP(
		"no-cleanup", "n", false,
		"If provided then do not cleanup the container on failure. "+
			"Useful when debugging changes to the docker image.")
	Start.Flags().String(
		"json", "",
		"The location to write information about the service to. Any "+
			"existing content will be overwritten.")
	Start.Flags().String(
		"image", "opalmer/gerrittest:2.14.2",
		"The Docker image to spin up Gerrit.")
	Start.Flags().Uint16(
		"port-http", dockertest.RandomPort,
		"The local port to map to Gerrit's REST API. Random by default.")
	Start.Flags().Uint16(
		"port-ssh", dockertest.RandomPort,
		"The local port to map to Gerrit's REST API. Random by default.")
	Start.Flags().StringP(
		"private-key", "i", "",
		"If provided then use this private key instead of generating one.")
	Start.Flags().Bool(
		"start-only", false,
		"If provided just start the container, don't setup anything else.")
}
<commit_msg>readding setup code<commit_after>package cmd

import (
	"context"
	"crypto/rsa"
	"fmt"
	"github.com/opalmer/dockertest"
	"github.com/opalmer/gerrittest"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh"
	"io/ioutil"
	"os"
	"os/signal"
	"path/filepath"
	"time"
)

func getSSHKeys(cmd *cobra.Command) (ssh.PublicKey, *rsa.PrivateKey, string, error) {
	privateKeyPath, err := cmd.Flags().GetString("private-key")
	if err != nil {
		return nil, nil, "", err
	}

	if privateKeyPath == "" {
		public, private, err := gerrittest.GenerateSSHKeys()
		if err != nil {
			return nil, nil, "", err
		}
		return public, private, "", err
	}
	public, private, err := gerrittest.ReadSSHKeys(privateKeyPath)

	return public, private, privateKeyPath, nil
}

// NewConfigFromCommand converts a command to a config struct.
func NewConfigFromCommand(cmd *cobra.Command) (*gerrittest.Config, error) {
	path, err := cmd.Flags().GetString("json")
	if err != nil {
		return nil, err
	}
	if path == "" {
		file, err := ioutil.TempFile("", "")
		if err != nil {
			return nil, err
		}
		path = file.Name()
		if err := file.Close(); err != nil {
			return nil, nil
		}
		fmt.Println(path)

		if err := os.Remove(path); err != nil {
			return nil, err
		}
	}
	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
		return nil, err
	}

	image, err := cmd.Flags().GetString("image")
	if err != nil {
		return nil, err
	}

	portHTTP, err := cmd.Flags().GetUint16("port-http")
	if err != nil {
		return nil, err
	}

	portSSH, err := cmd.Flags().GetUint16("port-ssh")
	if err != nil {
		return nil, err
	}

	noCleanup, err := cmd.Flags().GetBool("no-cleanup")
	if err != nil {
		return nil, nil
	}

	return &gerrittest.Config{
		Image:            image,
		PortSSH:          portSSH,
		PortHTTP:         portHTTP,
		CleanupOnFailure: noCleanup == false,
	}, nil
}

// Start implements the `start` subcommand.
var Start = &cobra.Command{
	Use:   "start",
	Short: "Responsible for starting an instance of Gerrit.",
	RunE: func(cmd *cobra.Command, args []string) error {
		cfg, err := NewConfigFromCommand(cmd)
		if err != nil {
			return err
		}

		// Setup timeout and Ctrl+C handling.
		timeout, err := cmd.Flags().GetDuration("timeout")
		ctx, cancel := context.WithTimeout(context.Background(), timeout)
		interrupts := make(chan os.Signal, 1)
		signal.Notify(interrupts, os.Interrupt)
		go func() {
			for range interrupts {
				cancel()
			}
		}()

		service, err := gerrittest.Start(ctx, cfg)
		if err != nil {
			return err
		}

		startonly, err := cmd.Flags().GetBool("start-only")
		if startonly {
			return nil
		}

		client, err := service.HTTPClient()
		if err != nil {
			return err
		}

		// Hitting /login/ will produce a cookie that can be used
		// for authenticated requests. Also, this first request
		// causes the first account to be created which happens
		// to be the admin account.
		if err := client.Login(); err != nil {
			return err
		}

		// Make an authenticated request by retrieving account
		// information.
		if err := client.GetAccount("self"); err != nil {
			return err
		}

		password, err := client.GeneratePassword()
		if err != nil {
			return err
		}
		fmt.Println(password)

		return nil
	},
}

func init() {
	Start.Flags().Duration(
		"timeout", time.Minute*2,
		"The maximum amount of time to wait for the service to come up.")
	Start.Flags().BoolP(
		"no-cleanup", "n", false,
		"If provided then do not cleanup the container on failure. "+
			"Useful when debugging changes to the docker image.")
	Start.Flags().String(
		"json", "",
		"The location to write information about the service to. Any "+
			"existing content will be overwritten.")
	Start.Flags().String(
		"image", "opalmer/gerrittest:2.14.2",
		"The Docker image to spin up Gerrit.")
	Start.Flags().Uint16(
		"port-http", dockertest.RandomPort,
		"The local port to map to Gerrit's REST API. Random by default.")
	Start.Flags().Uint16(
		"port-ssh", dockertest.RandomPort,
		"The local port to map to Gerrit's REST API. Random by default.")
	Start.Flags().StringP(
		"private-key", "i", "",
		"If provided then use this private key instead of generating one.")
	Start.Flags().Bool(
		"start-only", false,
		"If provided just start the container, don't setup anything else.")
}
<|endoftext|>


<commit_before>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Read the system log.
//
// Synopsis:
//     dmesg [-clear|-read-clear]
//
// Options:
//     -clear: clear the log
//     -read-clear: clear the log after printing
package main

import (
	"flag"
	"log"
	"os"

	"golang.org/x/sys/unix"
)

const (
	_SYSLOG_ACTION_READ_ALL   = 3
	_SYSLOG_ACTION_READ_CLEAR = 4
	_SYSLOG_ACTION_CLEAR      = 5
)

var (
	clear     bool
	readClear bool
)

func init() {
	flag.BoolVar(&clear, "clear", false, "Clear the log")
	flag.BoolVar(&readClear, "read-clear", false, "Clear the log after printing")
	flag.BoolVar(&readClear, "c", false, "Clear the log after printing")
}

func main() {
	flag.Parse()
	if clear && readClear {
		log.Fatalf("cannot specify both -clear and -read-clear")
	}

	level := _SYSLOG_ACTION_READ_ALL
	if clear {
		level = _SYSLOG_ACTION_CLEAR
	}
	if readClear {
		level = _SYSLOG_ACTION_READ_CLEAR
	}

	b := make([]byte, 256*1024)
	amt, err := unix.Klogctl(level, b)
	if err != nil {
		log.Fatalf("syslog failed: %v", err)
	}

	os.Stdout.Write(b[:amt])
}
<commit_msg>cmds/core/dmesg: use SYSLOG_ACTION_* consts from x/sys/unix<commit_after>// Copyright 2012-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Read the system log.
//
// Synopsis:
//     dmesg [-clear|-read-clear]
//
// Options:
//     -clear: clear the log
//     -read-clear: clear the log after printing
package main

import (
	"flag"
	"log"
	"os"

	"golang.org/x/sys/unix"
)

var (
	clear     bool
	readClear bool
)

func init() {
	flag.BoolVar(&clear, "clear", false, "Clear the log")
	flag.BoolVar(&readClear, "read-clear", false, "Clear the log after printing")
	flag.BoolVar(&readClear, "c", false, "Clear the log after printing")
}

func main() {
	flag.Parse()
	if clear && readClear {
		log.Fatalf("cannot specify both -clear and -read-clear")
	}

	level := unix.SYSLOG_ACTION_READ_ALL
	if clear {
		level = unix.SYSLOG_ACTION_CLEAR
	}
	if readClear {
		level = unix.SYSLOG_ACTION_READ_CLEAR
	}

	b := make([]byte, 256*1024)
	amt, err := unix.Klogctl(level, b)
	if err != nil {
		log.Fatalf("syslog failed: %v", err)
	}

	os.Stdout.Write(b[:amt])
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"errors"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
)

// CmdSimpleFSRemove is the 'fs rm' command.
type CmdSimpleFSRemove struct {
	libkb.Contextified
	paths []keybase1.Path
}

// NewCmdSimpleFSRemove creates a new cli.Command.
func NewCmdSimpleFSRemove(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "rm",
		ArgumentHelp: "<path> [path...]",
		Usage:        "remove one or more directory elements",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSimpleFSRemove{Contextified: libkb.NewContextified(g)}, "rm", c)
			cl.SetNoStandalone()
		},
	}
}

// Run runs the command in client/server mode.
func (c *CmdSimpleFSRemove) Run() error {
	cli, err := GetSimpleFSClient(c.G())
	if err != nil {
		return err
	}

	ctx := context.TODO()

	paths, err := doSimpleFSGlob(ctx, c.G(), cli, c.paths)
	if err != nil {
		return err
	}

	for _, path := range paths {
		opid, err2 := cli.SimpleFSMakeOpid(ctx)
		if err2 != nil {
			return err2
		}
		defer cli.SimpleFSClose(ctx, opid)
		c.G().Log.Debug("SimpleFSRemove %s", path.Kbfs())
		err = cli.SimpleFSRemove(ctx, keybase1.SimpleFSRemoveArg{
			OpID: opid,
			Path: path,
		})
		if err != nil {
			break
		}
		err = cli.SimpleFSWait(ctx, opid)
		if err != nil {
			break
		}
	}
	return err
}

// ParseArgv gets the required path argument for this command.
func (c *CmdSimpleFSRemove) ParseArgv(ctx *cli.Context) error {
	nargs := len(ctx.Args())
	var err error

	if nargs < 1 {
		return errors.New("rm requires at least one KBFS path argument")
	}

	for _, src := range ctx.Args() {
		argPath, err := makeSimpleFSPath(src)
		if err != nil {
			return err
		}
		pathType, err := argPath.PathType()
		if err != nil {
			return err
		}
		if pathType != keybase1.PathType_KBFS {
			return errors.New("rm requires KBFS path arguments")
		}
		c.paths = append(c.paths, argPath)
	}
	return err
}

// GetUsage says what this command needs to operate.
func (c *CmdSimpleFSRemove) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		KbKeyring: true,
		API:       true,
	}
}
<commit_msg>client: add -r option for simplefs remove<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"errors"

	"golang.org/x/net/context"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol/keybase1"
)

// CmdSimpleFSRemove is the 'fs rm' command.
type CmdSimpleFSRemove struct {
	libkb.Contextified
	paths   []keybase1.Path
	recurse bool
}

// NewCmdSimpleFSRemove creates a new cli.Command.
func NewCmdSimpleFSRemove(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "rm",
		ArgumentHelp: "<path> [path...]",
		Usage:        "remove one or more directory elements",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSimpleFSRemove{Contextified: libkb.NewContextified(g)}, "rm", c)
			cl.SetNoStandalone()
		},
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "r, recursive",
				Usage: "Recursively delete everything in a directory",
			},
		},
	}
}

// Run runs the command in client/server mode.
func (c *CmdSimpleFSRemove) Run() error {
	cli, err := GetSimpleFSClient(c.G())
	if err != nil {
		return err
	}

	ctx := context.TODO()

	paths, err := doSimpleFSGlob(ctx, c.G(), cli, c.paths)
	if err != nil {
		return err
	}

	for _, path := range paths {
		opid, err2 := cli.SimpleFSMakeOpid(ctx)
		if err2 != nil {
			return err2
		}
		defer cli.SimpleFSClose(ctx, opid)
		c.G().Log.Debug("SimpleFSRemove %s", path.Kbfs())
		err = cli.SimpleFSRemove(ctx, keybase1.SimpleFSRemoveArg{
			OpID:      opid,
			Path:      path,
			Recursive: c.recurse,
		})
		if err != nil {
			break
		}
		err = cli.SimpleFSWait(ctx, opid)
		if err != nil {
			break
		}
	}
	return err
}

// ParseArgv gets the required path argument for this command.
func (c *CmdSimpleFSRemove) ParseArgv(ctx *cli.Context) error {
	c.recurse = ctx.Bool("recursive")

	nargs := len(ctx.Args())
	var err error

	if nargs < 1 {
		return errors.New("rm requires at least one KBFS path argument")
	}

	for _, src := range ctx.Args() {
		argPath, err := makeSimpleFSPath(src)
		if err != nil {
			return err
		}
		pathType, err := argPath.PathType()
		if err != nil {
			return err
		}
		if pathType != keybase1.PathType_KBFS {
			return errors.New("rm requires KBFS path arguments")
		}
		c.paths = append(c.paths, argPath)
	}
	return err
}

// GetUsage says what this command needs to operate.
func (c *CmdSimpleFSRemove) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:    true,
		KbKeyring: true,
		API:       true,
	}
}
<|endoftext|>