<commit_before>package main

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/projectatomic/buildah/imagebuildah"
	buildahcli "github.com/projectatomic/buildah/pkg/cli"
	"github.com/projectatomic/buildah/pkg/parse"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var (
	budDescription = "Builds an OCI image using instructions in one or more Dockerfiles."
	budCommand     = cli.Command{
		Name:           "build-using-dockerfile",
		Aliases:        []string{"bud"},
		Usage:          "Build an image using instructions in a Dockerfile",
		Description:    budDescription,
		Flags:          append(buildahcli.BudFlags, buildahcli.FromAndBudFlags...),
		Action:         budCmd,
		ArgsUsage:      "CONTEXT-DIRECTORY | URL",
		SkipArgReorder: true,
	}
)

func budCmd(c *cli.Context) error {
	output := ""
	tags := []string{}
	if c.IsSet("tag") || c.IsSet("t") {
		tags = c.StringSlice("tag")
		if len(tags) > 0 {
			output = tags[0]
			tags = tags[1:]
		}
	}
	pullPolicy := imagebuildah.PullNever
	if c.BoolT("pull") {
		pullPolicy = imagebuildah.PullIfMissing
	}
	if c.Bool("pull-always") {
		pullPolicy = imagebuildah.PullAlways
	}

	args := make(map[string]string)
	if c.IsSet("build-arg") {
		for _, arg := range c.StringSlice("build-arg") {
			av := strings.SplitN(arg, "=", 2)
			if len(av) > 1 {
				args[av[0]] = av[1]
			} else {
				delete(args, av[0])
			}
		}
	}

	dockerfiles := c.StringSlice("file")
	format := "oci"
	if c.IsSet("format") {
		format = strings.ToLower(c.String("format"))
	}
	if strings.HasPrefix(format, "oci") {
		format = imagebuildah.OCIv1ImageFormat
	} else if strings.HasPrefix(format, "docker") {
		format = imagebuildah.Dockerv2ImageFormat
	} else {
		return errors.Errorf("unrecognized image type %q", format)
	}
	contextDir := ""
	cliArgs := c.Args()
	if len(cliArgs) > 0 {
		// The context directory could be a URL.  Try to handle that.
		tempDir, subDir, err := imagebuildah.TempDirForURL("", "buildah", cliArgs[0])
		if err != nil {
			return errors.Wrapf(err, "error prepping temporary context directory")
		}
		if tempDir != "" {
			// We had to download it to a temporary directory.
			// Delete it later.
			defer func() {
				if err = os.RemoveAll(tempDir); err != nil {
					logrus.Errorf("error removing temporary directory %q: %v", contextDir, err)
				}
			}()
			contextDir = filepath.Join(tempDir, subDir)
		} else {
			// Nope, it was local.  Use it as is.
			absDir, err := filepath.Abs(cliArgs[0])
			if err != nil {
				return errors.Wrapf(err, "error determining path to directory %q", cliArgs[0])
			}
			contextDir = absDir
		}
		cliArgs = cliArgs.Tail()
	} else {
		// No context directory or URL was specified.  Try to use the
		// home of the first locally-available Dockerfile.
		for i := range dockerfiles {
			if strings.HasPrefix(dockerfiles[i], "http://") ||
				strings.HasPrefix(dockerfiles[i], "https://") ||
				strings.HasPrefix(dockerfiles[i], "git://") ||
				strings.HasPrefix(dockerfiles[i], "github.com/") {
				continue
			}
			absFile, err := filepath.Abs(dockerfiles[i])
			if err != nil {
				return errors.Wrapf(err, "error determining path to file %q", dockerfiles[i])
			}
			contextDir = filepath.Dir(absFile)
			dockerfiles[i], err = filepath.Rel(contextDir, absFile)
			if err != nil {
				return errors.Wrapf(err, "error determining path to file %q", dockerfiles[i])
			}
			break
		}
	}
	if contextDir == "" {
		return errors.Errorf("no context directory specified, and no dockerfile specified")
	}
	if len(dockerfiles) == 0 {
		dockerfiles = append(dockerfiles, filepath.Join(contextDir, "Dockerfile"))
	}
	if err := parse.ValidateFlags(c, buildahcli.BudFlags); err != nil {
		return err
	}

	store, err := getStore(c)
	if err != nil {
		return err
	}

	systemContext, err := parse.SystemContextFromOptions(c)
	if err != nil {
		return errors.Wrapf(err, "error building system context")
	}

	runtimeFlags := []string{}
	for _, arg := range c.StringSlice("runtime-flag") {
		runtimeFlags = append(runtimeFlags, "--"+arg)
	}

	commonOpts, err := parse.ParseCommonBuildOptions(c)
	if err != nil {
		return err
	}

	if c.IsSet("compress") {
		logrus.Debugf("--compress option specified but is ignored")
	}

	if c.IsSet("rm") {
		logrus.Debugf("build caching not enabled so --rm flag has no effect")
	}

	if c.IsSet("squash") {
		logrus.Debugf("build caching not enabled so --squash flag has no effect")
	}

	options := imagebuildah.BuildOptions{
		ContextDirectory:      contextDir,
		PullPolicy:            pullPolicy,
		Compression:           imagebuildah.Gzip,
		Quiet:                 c.Bool("quiet"),
		SignaturePolicyPath:   c.String("signature-policy"),
		Args:                  args,
		Output:                output,
		AdditionalTags:        tags,
		Runtime:               c.String("runtime"),
		RuntimeArgs:           runtimeFlags,
		OutputFormat:          format,
		SystemContext:         systemContext,
		CommonBuildOpts:       commonOpts,
		DefaultMountsFilePath: c.GlobalString("default-mounts-file"),
		IIDFile:               c.String("iidfile"),
	}

	if !c.Bool("quiet") {
		options.ReportWriter = os.Stderr
	}

	return imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...)
}
<commit_msg>Adding noop for --force-rm to match --rm<commit_after>package main

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/projectatomic/buildah/imagebuildah"
	buildahcli "github.com/projectatomic/buildah/pkg/cli"
	"github.com/projectatomic/buildah/pkg/parse"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var (
	budDescription = "Builds an OCI image using instructions in one or more Dockerfiles."
	budCommand     = cli.Command{
		Name:           "build-using-dockerfile",
		Aliases:        []string{"bud"},
		Usage:          "Build an image using instructions in a Dockerfile",
		Description:    budDescription,
		Flags:          append(buildahcli.BudFlags, buildahcli.FromAndBudFlags...),
		Action:         budCmd,
		ArgsUsage:      "CONTEXT-DIRECTORY | URL",
		SkipArgReorder: true,
	}
)

func budCmd(c *cli.Context) error {
	output := ""
	tags := []string{}
	if c.IsSet("tag") || c.IsSet("t") {
		tags = c.StringSlice("tag")
		if len(tags) > 0 {
			output = tags[0]
			tags = tags[1:]
		}
	}
	pullPolicy := imagebuildah.PullNever
	if c.BoolT("pull") {
		pullPolicy = imagebuildah.PullIfMissing
	}
	if c.Bool("pull-always") {
		pullPolicy = imagebuildah.PullAlways
	}

	args := make(map[string]string)
	if c.IsSet("build-arg") {
		for _, arg := range c.StringSlice("build-arg") {
			av := strings.SplitN(arg, "=", 2)
			if len(av) > 1 {
				args[av[0]] = av[1]
			} else {
				delete(args, av[0])
			}
		}
	}

	dockerfiles := c.StringSlice("file")
	format := "oci"
	if c.IsSet("format") {
		format = strings.ToLower(c.String("format"))
	}
	if strings.HasPrefix(format, "oci") {
		format = imagebuildah.OCIv1ImageFormat
	} else if strings.HasPrefix(format, "docker") {
		format = imagebuildah.Dockerv2ImageFormat
	} else {
		return errors.Errorf("unrecognized image type %q", format)
	}
	contextDir := ""
	cliArgs := c.Args()
	if len(cliArgs) > 0 {
		// The context directory could be a URL.  Try to handle that.
		tempDir, subDir, err := imagebuildah.TempDirForURL("", "buildah", cliArgs[0])
		if err != nil {
			return errors.Wrapf(err, "error prepping temporary context directory")
		}
		if tempDir != "" {
			// We had to download it to a temporary directory.
			// Delete it later.
			defer func() {
				if err = os.RemoveAll(tempDir); err != nil {
					logrus.Errorf("error removing temporary directory %q: %v", contextDir, err)
				}
			}()
			contextDir = filepath.Join(tempDir, subDir)
		} else {
			// Nope, it was local.  Use it as is.
			absDir, err := filepath.Abs(cliArgs[0])
			if err != nil {
				return errors.Wrapf(err, "error determining path to directory %q", cliArgs[0])
			}
			contextDir = absDir
		}
		cliArgs = cliArgs.Tail()
	} else {
		// No context directory or URL was specified.  Try to use the
		// home of the first locally-available Dockerfile.
		for i := range dockerfiles {
			if strings.HasPrefix(dockerfiles[i], "http://") ||
				strings.HasPrefix(dockerfiles[i], "https://") ||
				strings.HasPrefix(dockerfiles[i], "git://") ||
				strings.HasPrefix(dockerfiles[i], "github.com/") {
				continue
			}
			absFile, err := filepath.Abs(dockerfiles[i])
			if err != nil {
				return errors.Wrapf(err, "error determining path to file %q", dockerfiles[i])
			}
			contextDir = filepath.Dir(absFile)
			dockerfiles[i], err = filepath.Rel(contextDir, absFile)
			if err != nil {
				return errors.Wrapf(err, "error determining path to file %q", dockerfiles[i])
			}
			break
		}
	}
	if contextDir == "" {
		return errors.Errorf("no context directory specified, and no dockerfile specified")
	}
	if len(dockerfiles) == 0 {
		dockerfiles = append(dockerfiles, filepath.Join(contextDir, "Dockerfile"))
	}
	if err := parse.ValidateFlags(c, buildahcli.BudFlags); err != nil {
		return err
	}

	store, err := getStore(c)
	if err != nil {
		return err
	}

	systemContext, err := parse.SystemContextFromOptions(c)
	if err != nil {
		return errors.Wrapf(err, "error building system context")
	}

	runtimeFlags := []string{}
	for _, arg := range c.StringSlice("runtime-flag") {
		runtimeFlags = append(runtimeFlags, "--"+arg)
	}

	commonOpts, err := parse.ParseCommonBuildOptions(c)
	if err != nil {
		return err
	}

	if c.IsSet("compress") {
		logrus.Debugf("--compress option specified but is ignored")
	}

	if c.IsSet("force-rm") {
		logrus.Debugf("build caching not enabled so --force-rm flag has no effect")
	}

	if c.IsSet("rm") {
		logrus.Debugf("build caching not enabled so --rm flag has no effect")
	}

	if c.IsSet("squash") {
		logrus.Debugf("build caching not enabled so --squash flag has no effect")
	}

	options := imagebuildah.BuildOptions{
		ContextDirectory:      contextDir,
		PullPolicy:            pullPolicy,
		Compression:           imagebuildah.Gzip,
		Quiet:                 c.Bool("quiet"),
		SignaturePolicyPath:   c.String("signature-policy"),
		Args:                  args,
		Output:                output,
		AdditionalTags:        tags,
		Runtime:               c.String("runtime"),
		RuntimeArgs:           runtimeFlags,
		OutputFormat:          format,
		SystemContext:         systemContext,
		CommonBuildOpts:       commonOpts,
		DefaultMountsFilePath: c.GlobalString("default-mounts-file"),
		IIDFile:               c.String("iidfile"),
	}

	if !c.Bool("quiet") {
		options.ReportWriter = os.Stderr
	}

	return imagebuildah.BuildDockerfiles(getContext(), store, options, dockerfiles...)
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/domainr/whois"
	"github.com/domainr/whoistest"
)

func main() {
	test := flag.Bool("t", false, "load from whois test data instead of the network")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <domain>\n\nAvailable arguments:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	flag.Parse()
	query := flag.Arg(0)

	if query == "" {
		flag.Usage()
	}

	req, err := whois.NewRequest(query)
	FatalIf(err)

	var res *whois.Response
	if *test {
		fns, err := whoistest.ResponseFiles()
		FatalIf(err)

		// FIXME: UNIX-specific
		sfx := "/" + query + ".mime"
		fmt.Fprintf(os.Stderr, "Looking for test file ...%s\n", sfx)
		// FIXME: slow
		for _, fn := range fns {
			if strings.HasSuffix(fn, sfx) {
				res, err = whois.ReadMIMEFile(fn)
				FatalIf(err)
				break
			}
		}
	} else {
		res, err = whois.DefaultClient.Fetch(req)
		FatalIf(err)
	}

	fmt.Println(res.String())
}

func FatalIf(err error) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "Error: %s\n", err)
	os.Exit(-1)
}
<commit_msg>cmd/client: use FetchContext API<commit_after>// +build ignore

package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/domainr/whois"
	"github.com/domainr/whoistest"
)

func main() {
	test := flag.Bool("t", false, "load from whois test data instead of the network")
	timeout := flag.Duration("timeout", 5*time.Second, "timeout")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <domain>\n\nAvailable arguments:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}

	flag.Parse()
	query := flag.Arg(0)

	if query == "" {
		flag.Usage()
	}

	c := whois.NewClient(0)
	ctx, cancel := context.WithTimeout(context.Background(), *timeout)
	defer cancel()
	req, err := whois.NewRequest(query)
	FatalIf(err)

	var res *whois.Response
	if *test {
		fns, err := whoistest.ResponseFiles()
		FatalIf(err)

		// FIXME: UNIX-specific
		sfx := "/" + query + ".mime"
		fmt.Fprintf(os.Stderr, "Looking for test file ...%s\n", sfx)
		// FIXME: slow
		for _, fn := range fns {
			if strings.HasSuffix(fn, sfx) {
				res, err = whois.ReadMIMEFile(fn)
				FatalIf(err)
				break
			}
		}
	} else {
		res, err = c.FetchContext(ctx, req)
		FatalIf(err)
	}

	fmt.Println(res.String())
}

func FatalIf(err error) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "Error: %s\n", err)
	os.Exit(-1)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
)

const Version = "0.1.4"

var EmptyLocations = []string{"NOT_AVAILABLE"}

type Result struct {
	Status    string
	URL       string
	Took      float64
	Locations []string
	Epoch     int64
}

func (r Result) String() string {
	return fmt.Sprintf("%s\t%0.4f\t%d\t%s\t%s\t", r.Status, r.Took, r.Epoch, r.URL, strings.Join(r.Locations, "|"))
}

type URLValue struct {
	Format string `json:"format"`
	Value  string `json:"value"`
}

type Value struct {
	Index     int             `json:"index"`
	Type      string          `json:"type"`
	Data      json.RawMessage `json:"data"`
	TTL       int             `json:"ttl"`
	Timestamp string          `json:"timestamp"`
}

type APIResponse struct {
	Code   int     `json:"responseCode"`
	Handle string  `json:"handle"`
	Values []Value `json:"values"`
}

func worker(queue chan *url.URL, out chan Result, wg *sync.WaitGroup) {
	defer wg.Done()
	for u := range queue {
		r := retrieve(u)
		out <- r
	}
}

func sink(out chan Result, done chan bool) {
	for r := range out {
		fmt.Println(r)
	}
	done <- true
}

// retrieve will try to GET and parse a DOI API response and will always
// return a Result, which will contain status (either HTTP or internal error designations)
func retrieve(target *url.URL) Result {

	rt := http.DefaultTransport
	var req *http.Request

	err := backoff.Retry(func() (e error) {
		req, e = http.NewRequest("GET", target.String(), nil)
		return
	}, backoff.NewExponentialBackOff())

	if err != nil {
		return Result{Status: "E_REQ", URL: target.String(), Took: 0,
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	var resp *http.Response

	start := time.Now()
	err = backoff.Retry(func() (e error) {
		resp, e = rt.RoundTrip(req)
		if e != nil {
			log.Printf("retrying %s", req.URL.String())
		}
		return e
	}, backoff.NewExponentialBackOff())
	elapsed := time.Since(start)

	if err != nil {
		return Result{Status: "E_REQ", URL: target.String(), Took: elapsed.Seconds(),
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return Result{Status: "E_READ", URL: target.String(), Took: elapsed.Seconds(),
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	resp.Body.Close()

	var ar APIResponse
	err = json.Unmarshal(body, &ar)
	if err != nil {
		return Result{Status: "E_JSON", URL: target.String(), Took: elapsed.Seconds(),
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	if resp.StatusCode != 200 {
		return Result{Status: resp.Status, URL: target.String(), Took: elapsed.Seconds(), Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	result := Result{Status: resp.Status, URL: target.String(), Took: elapsed.Seconds(), Epoch: time.Now().Unix()}

	for _, value := range ar.Values {
		if value.Type == "URL" {
			var v URLValue
			err := json.Unmarshal(value.Data, &v)
			if err != nil {
				return Result{Status: "E_JSON", URL: target.String(), Took: elapsed.Seconds(), Epoch: time.Now().Unix()}
			}
			result.Locations = append(result.Locations, v.Value)
		}
	}
	if len(result.Locations) == 0 {
		result.Locations = EmptyLocations
	}
	return result
}

func main() {

	prefix := flag.String("prefix", "http://doi.org/api/handles", "string to prepend to line")
	numWorkers := flag.Int("w", runtime.NumCPU(), "number of workers")
	version := flag.Bool("v", false, "prints current program version")

	flag.Parse()

	if *version {
		fmt.Println(Version)
		os.Exit(0)
	}

	runtime.GOMAXPROCS(*numWorkers)

	reader := bufio.NewReader(os.Stdin)

	queue := make(chan *url.URL)
	out := make(chan Result)
	done := make(chan bool)

	go sink(out, done)

	var wg sync.WaitGroup

	for i := 0; i < *numWorkers; i++ {
		wg.Add(1)
		go worker(queue, out, &wg)
	}

	for {
		line, err := reader.ReadString('\n')

		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		target := strings.TrimSpace(line)

		if target == "" {
			continue
		}

		if !strings.HasPrefix(target, *prefix) {
			target = fmt.Sprintf("%s/%s", *prefix, target)
		}

		parsed, err := url.Parse(target)
		if err != nil {
			continue
		}

		queue <- parsed
	}

	close(queue)
	wg.Wait()
	close(out)
	<-done
}
<commit_msg>add some docs<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/cenkalti/backoff"
)

// Version of application
const Version = "0.1.4"

// EmptyLocations when we do not get any result or fail.
var EmptyLocations = []string{"NOT_AVAILABLE"}

// Result is the output of this program.
type Result struct {
	Status    string
	URL       string
	Took      float64
	Locations []string
	Epoch     int64
}

// String formats the result as tab-separated values.
func (r Result) String() string {
	return fmt.Sprintf("%s\t%0.4f\t%d\t%s\t%s\t", r.Status, r.Took, r.Epoch, r.URL, strings.Join(r.Locations, "|"))
}

type URLValue struct {
	Format string `json:"format"`
	Value  string `json:"value"`
}

// Value is part of the response.
type Value struct {
	Index     int             `json:"index"`
	Type      string          `json:"type"`
	Data      json.RawMessage `json:"data"`
	TTL       int             `json:"ttl"`
	Timestamp string          `json:"timestamp"`
}

// APIResponse is the response from DOI.
type APIResponse struct {
	Code   int     `json:"responseCode"`
	Handle string  `json:"handle"`
	Values []Value `json:"values"`
}

func worker(queue chan *url.URL, out chan Result, wg *sync.WaitGroup) {
	defer wg.Done()
	for u := range queue {
		r := retrieve(u)
		out <- r
	}
}

func sink(out chan Result, done chan bool) {
	for r := range out {
		fmt.Println(r)
	}
	done <- true
}

// retrieve will try to GET and parse a DOI API response and will always
// return a Result, which will contain status (either HTTP or internal error designations)
func retrieve(target *url.URL) Result {

	rt := http.DefaultTransport
	var req *http.Request

	err := backoff.Retry(func() (e error) {
		req, e = http.NewRequest("GET", target.String(), nil)
		return
	}, backoff.NewExponentialBackOff())

	if err != nil {
		return Result{Status: "E_REQ", URL: target.String(), Took: 0,
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	var resp *http.Response

	start := time.Now()
	err = backoff.Retry(func() (e error) {
		resp, e = rt.RoundTrip(req)
		if e != nil {
			log.Printf("retrying %s", req.URL.String())
		}
		return e
	}, backoff.NewExponentialBackOff())
	elapsed := time.Since(start)

	if err != nil {
		return Result{Status: "E_REQ", URL: target.String(), Took: elapsed.Seconds(),
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	body, err := ioutil.ReadAll(resp.Body)

	if err != nil {
		return Result{Status: "E_READ", URL: target.String(), Took: elapsed.Seconds(),
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	resp.Body.Close()

	var ar APIResponse
	err = json.Unmarshal(body, &ar)
	if err != nil {
		return Result{Status: "E_JSON", URL: target.String(), Took: elapsed.Seconds(),
			Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	if resp.StatusCode != 200 {
		return Result{Status: resp.Status, URL: target.String(), Took: elapsed.Seconds(), Epoch: time.Now().Unix(), Locations: EmptyLocations}
	}

	result := Result{Status: resp.Status, URL: target.String(), Took: elapsed.Seconds(), Epoch: time.Now().Unix()}

	for _, value := range ar.Values {
		if value.Type == "URL" {
			var v URLValue
			err := json.Unmarshal(value.Data, &v)
			if err != nil {
				return Result{Status: "E_JSON", URL: target.String(), Took: elapsed.Seconds(), Epoch: time.Now().Unix()}
			}
			result.Locations = append(result.Locations, v.Value)
		}
	}
	if len(result.Locations) == 0 {
		result.Locations = EmptyLocations
	}
	return result
}

func main() {

	prefix := flag.String("prefix", "http://doi.org/api/handles", "string to prepend to line")
	numWorkers := flag.Int("w", runtime.NumCPU(), "number of workers")
	version := flag.Bool("v", false, "prints current program version")

	flag.Parse()

	if *version {
		fmt.Println(Version)
		os.Exit(0)
	}

	runtime.GOMAXPROCS(*numWorkers)

	reader := bufio.NewReader(os.Stdin)

	queue := make(chan *url.URL)
	out := make(chan Result)
	done := make(chan bool)

	go sink(out, done)

	var wg sync.WaitGroup

	for i := 0; i < *numWorkers; i++ {
		wg.Add(1)
		go worker(queue, out, &wg)
	}

	for {
		line, err := reader.ReadString('\n')

		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		target := strings.TrimSpace(line)

		if target == "" {
			continue
		}

		if !strings.HasPrefix(target, *prefix) {
			target = fmt.Sprintf("%s/%s", *prefix, target)
		}

		parsed, err := url.Parse(target)
		if err != nil {
			continue
		}

		queue <- parsed
	}

	close(queue)
	wg.Wait()
	close(out)
	<-done
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	cmds "github.com/jbenet/go-ipfs/commands"
	"github.com/jbenet/go-ipfs/core"
	commands "github.com/jbenet/go-ipfs/core/commands"
	corehttp "github.com/jbenet/go-ipfs/core/corehttp"
	"github.com/jbenet/go-ipfs/core/corerouting"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
	util "github.com/jbenet/go-ipfs/util"
	"github.com/jbenet/go-ipfs/util/debugerror"
)

const (
	initOptionKwd = "init"
	gcrKwd        = "gcr"
	mountKwd      = "mount"
	writableKwd   = "writable"
	ipfsMountKwd  = "mount-ipfs"
	ipnsMountKwd  = "mount-ipns"
	// apiAddrKwd    = "address-api"
	// swarmAddrKwd  = "address-swarm"
)

var daemonCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Run a network-connected IPFS node",
		ShortDescription: `
'ipfs daemon' runs a persistent IPFS daemon that can serve commands
over the network. Most applications that use IPFS will do so by
communicating with a daemon over the HTTP API. While the daemon is
running, calls to 'ipfs' commands will be sent over the network to
the daemon.
`,
	},

	Options: []cmds.Option{
		cmds.BoolOption(initOptionKwd, "Initialize IPFS with default settings if not already initialized"),
		cmds.BoolOption(gcrKwd, "Enables Grandcentral Routing"),
		cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem"),
		cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"),
		cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount)"),
		cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount)"),

		// TODO: add way to override addresses. tricky part: updating the config if also --init.
		// cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"),
		// cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"),
	},
	Subcommands: map[string]*cmds.Command{},
	Run:         daemonFunc,
}

func daemonFunc(req cmds.Request, res cmds.Response) {
	// let the user know we're going.
	fmt.Printf("Initializing daemon...\n")

	// first, whether user has provided the initialization flag. we may be
	// running in an uninitialized state.
	initialize, _, err := req.Option(initOptionKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	if initialize {

		// now, FileExists is our best method of detecting whether IPFS is
		// configured. Consider moving this into a config helper method
		// `IsInitialized` where the quality of the signal can be improved over
		// time, and many call-sites can benefit.
		if !util.FileExists(req.Context().ConfigRoot) {
			err := initWithDefaults(os.Stdout, req.Context().ConfigRoot)
			if err != nil {
				res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
				return
			}
		}
	}

	// To ensure that IPFS has been initialized, fetch the config. Do this
	// _before_ acquiring the daemon lock so the user gets an appropriate error
	// message.
	// NB: It's safe to read the config without the daemon lock, but not safe
	// to write.
	ctx := req.Context()
	cfg, err := ctx.GetConfig()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	// acquire the repo lock _before_ constructing a node. we need to make
	// sure we are permitted to access the resources (datastore, etc.)
	repo := fsrepo.At(req.Context().ConfigRoot)
	if err := repo.Open(); err != nil {
		res.SetError(debugerror.Errorf("Couldn't obtain lock. Is another daemon already running?"), cmds.ErrNormal)
		return
	}

	// Start assembling corebuilder
	nb := core.NewNodeBuilder().Online()
	nb.SetRepo(repo)

	useGCR, _, err := req.Option(gcrKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if useGCR {
		servers, err := repo.Config().GCR.ServerIPFSAddrs()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			repo.Close() // because ownership hasn't been transferred to the node
			return
		}
		var infos []peer.PeerInfo
		for _, addr := range servers {
			infos = append(infos, peer.PeerInfo{
				ID:    addr.ID(),
				Addrs: []ma.Multiaddr{addr.Transport()},
			})
		}
		nb.SetRouting(corerouting.SupernodeClient(infos...))
	}

	node, err := nb.Build(ctx.Context)
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	defer node.Close()
	req.Context().ConstructNode = func() (*core.IpfsNode, error) {
		return node, nil
	}

	// verify api address is valid multiaddr
	apiMaddr, err := ma.NewMultiaddr(cfg.Addresses.API)
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	var gatewayMaddr ma.Multiaddr
	if len(cfg.Addresses.Gateway) > 0 {
		// ignore error for gateway address
		// if there is an error (invalid address), then don't run the gateway
		gatewayMaddr, _ = ma.NewMultiaddr(cfg.Addresses.Gateway)
		if gatewayMaddr == nil {
			log.Errorf("Invalid gateway address: %s", cfg.Addresses.Gateway)
		}
	}

	// mount if the user provided the --mount flag
	mount, _, err := req.Option(mountKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if mount {
		fsdir, found, err := req.Option(ipfsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			fsdir = cfg.Mounts.IPFS
		}

		nsdir, found, err := req.Option(ipnsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			nsdir = cfg.Mounts.IPNS
		}

		err = commands.Mount(node, fsdir, nsdir)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		fmt.Printf("IPFS mounted at: %s\n", fsdir)
		fmt.Printf("IPNS mounted at: %s\n", nsdir)
	}

	var rootRedirect corehttp.ServeOption
	if len(cfg.Gateway.RootRedirect) > 0 {
		rootRedirect = corehttp.RedirectOption("", cfg.Gateway.RootRedirect)
	}

	writable, writableOptionFound, err := req.Option(writableKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if !writableOptionFound {
		writable = cfg.Gateway.Writable
	}

	if gatewayMaddr != nil {
		go func() {
			var opts = []corehttp.ServeOption{
				corehttp.IPNSHostnameOption(),
				corehttp.GatewayOption(writable),
			}
			if rootRedirect != nil {
				opts = append(opts, rootRedirect)
			}
			fmt.Printf("Gateway server listening on %s\n", gatewayMaddr)
			if writable {
				fmt.Printf("Gateway server is writable\n")
			}
			err := corehttp.ListenAndServe(node, gatewayMaddr.String(), opts...)
			if err != nil {
				log.Error(err)
			}
		}()
	}

	gateway := corehttp.NewGateway(corehttp.GatewayConfig{
		Writable: true,
		BlockList: &corehttp.BlockList{
			Decider: func(s string) bool {
				// for now, only allow paths in the WebUI path
				for _, webuipath := range corehttp.WebUIPaths {
					if strings.HasPrefix(s, webuipath) {
						return true
					}
				}
				return false
			},
		},
	})
	var opts = []corehttp.ServeOption{
		corehttp.CommandsOption(*req.Context()),
		corehttp.WebUIOption,
		gateway.ServeOption(),
	}
	if rootRedirect != nil {
		opts = append(opts, rootRedirect)
	}
	fmt.Printf("API server listening on %s\n", apiMaddr)
	if err := corehttp.ListenAndServe(node, apiMaddr.String(), opts...); err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
}
<commit_msg>feat(daemon) learns --routing=supernode<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	cmds "github.com/jbenet/go-ipfs/commands"
	"github.com/jbenet/go-ipfs/core"
	commands "github.com/jbenet/go-ipfs/core/commands"
	corehttp "github.com/jbenet/go-ipfs/core/corehttp"
	"github.com/jbenet/go-ipfs/core/corerouting"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
	util "github.com/jbenet/go-ipfs/util"
	"github.com/jbenet/go-ipfs/util/debugerror"
)

const (
	initOptionKwd             = "init"
	routingOptionKwd          = "routing"
	routingOptionSupernodeKwd = "supernode"
	mountKwd                  = "mount"
	writableKwd               = "writable"
	ipfsMountKwd              = "mount-ipfs"
	ipnsMountKwd              = "mount-ipns"
	// apiAddrKwd    = "address-api"
	// swarmAddrKwd  = "address-swarm"
)

var daemonCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Run a network-connected IPFS node",
		ShortDescription: `
'ipfs daemon' runs a persistent IPFS daemon that can serve commands
over the network. Most applications that use IPFS will do so by
communicating with a daemon over the HTTP API. While the daemon is
running, calls to 'ipfs' commands will be sent over the network to
the daemon.
`,
	},

	Options: []cmds.Option{
		cmds.BoolOption(initOptionKwd, "Initialize IPFS with default settings if not already initialized"),
		cmds.StringOption(routingOptionKwd, "Overrides the routing option (dht, supernode)"),
		cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem"),
		cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"),
		cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount)"),
		cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount)"),

		// TODO: add way to override addresses. tricky part: updating the config if also --init.
		// cmds.StringOption(apiAddrKwd, "Address for the daemon rpc API (overrides config)"),
		// cmds.StringOption(swarmAddrKwd, "Address for the swarm socket (overrides config)"),
	},
	Subcommands: map[string]*cmds.Command{},
	Run:         daemonFunc,
}

func daemonFunc(req cmds.Request, res cmds.Response) {
	// let the user know we're going.
	fmt.Printf("Initializing daemon...\n")

	// first, whether user has provided the initialization flag. we may be
	// running in an uninitialized state.
	initialize, _, err := req.Option(initOptionKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	if initialize {

		// now, FileExists is our best method of detecting whether IPFS is
		// configured. Consider moving this into a config helper method
		// `IsInitialized` where the quality of the signal can be improved over
		// time, and many call-sites can benefit.
		if !util.FileExists(req.Context().ConfigRoot) {
			err := initWithDefaults(os.Stdout, req.Context().ConfigRoot)
			if err != nil {
				res.SetError(debugerror.Wrap(err), cmds.ErrNormal)
				return
			}
		}
	}

	// To ensure that IPFS has been initialized, fetch the config. Do this
	// _before_ acquiring the daemon lock so the user gets an appropriate error
	// message.
	// NB: It's safe to read the config without the daemon lock, but not safe
	// to write.
	ctx := req.Context()
	cfg, err := ctx.GetConfig()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	// acquire the repo lock _before_ constructing a node. we need to make
	// sure we are permitted to access the resources (datastore, etc.)
	repo := fsrepo.At(req.Context().ConfigRoot)
	if err := repo.Open(); err != nil {
		res.SetError(debugerror.Errorf("Couldn't obtain lock. Is another daemon already running?"), cmds.ErrNormal)
		return
	}

	// Start assembling corebuilder
	nb := core.NewNodeBuilder().Online()
	nb.SetRepo(repo)

	routingOption, _, err := req.Option(routingOptionKwd).String()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if routingOption == routingOptionSupernodeKwd {
		servers, err := repo.Config().GCR.ServerIPFSAddrs()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			repo.Close() // because ownership hasn't been transferred to the node
			return
		}
		var infos []peer.PeerInfo
		for _, addr := range servers {
			infos = append(infos, peer.PeerInfo{
				ID:    addr.ID(),
				Addrs: []ma.Multiaddr{addr.Transport()},
			})
		}
		nb.SetRouting(corerouting.SupernodeClient(infos...))
	}

	node, err := nb.Build(ctx.Context)
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	defer node.Close()
	req.Context().ConstructNode = func() (*core.IpfsNode, error) {
		return node, nil
	}

	// verify api address is valid multiaddr
	apiMaddr, err := ma.NewMultiaddr(cfg.Addresses.API)
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}

	var gatewayMaddr ma.Multiaddr
	if len(cfg.Addresses.Gateway) > 0 {
		// ignore error for gateway address
		// if there is an error (invalid address), then don't run the gateway
		gatewayMaddr, _ = ma.NewMultiaddr(cfg.Addresses.Gateway)
		if gatewayMaddr == nil {
			log.Errorf("Invalid gateway address: %s", cfg.Addresses.Gateway)
		}
	}

	// mount if the user provided the --mount flag
	mount, _, err := req.Option(mountKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if mount {
		fsdir, found, err := req.Option(ipfsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			fsdir = cfg.Mounts.IPFS
		}

		nsdir, found, err := req.Option(ipnsMountKwd).String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if !found {
			nsdir = cfg.Mounts.IPNS
		}

		err = commands.Mount(node, fsdir, nsdir)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		fmt.Printf("IPFS mounted at: %s\n", fsdir)
		fmt.Printf("IPNS mounted at: %s\n", nsdir)
	}

	var rootRedirect corehttp.ServeOption
	if len(cfg.Gateway.RootRedirect) > 0 {
		rootRedirect = corehttp.RedirectOption("", cfg.Gateway.RootRedirect)
	}

	writable, writableOptionFound, err := req.Option(writableKwd).Bool()
	if err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
	if !writableOptionFound {
		writable = cfg.Gateway.Writable
	}

	if gatewayMaddr != nil {
		go func() {
			var opts = []corehttp.ServeOption{
				corehttp.IPNSHostnameOption(),
				corehttp.GatewayOption(writable),
			}
			if rootRedirect != nil {
				opts = append(opts, rootRedirect)
			}
			fmt.Printf("Gateway server listening on %s\n", gatewayMaddr)
			if writable {
				fmt.Printf("Gateway server is writable\n")
			}
			err := corehttp.ListenAndServe(node, gatewayMaddr.String(), opts...)
			if err != nil {
				log.Error(err)
			}
		}()
	}

	gateway := corehttp.NewGateway(corehttp.GatewayConfig{
		Writable: true,
		BlockList: &corehttp.BlockList{
			Decider: func(s string) bool {
				// for now, only allow paths in the WebUI path
				for _, webuipath := range corehttp.WebUIPaths {
					if strings.HasPrefix(s, webuipath) {
						return true
					}
				}
				return false
			},
		},
	})
	var opts = []corehttp.ServeOption{
		corehttp.CommandsOption(*req.Context()),
		corehttp.WebUIOption,
		gateway.ServeOption(),
	}
	if rootRedirect != nil {
		opts = append(opts, rootRedirect)
	}
	fmt.Printf("API server listening on %s\n", apiMaddr)
	if err := corehttp.ListenAndServe(node, apiMaddr.String(), opts...); err != nil {
		res.SetError(err, cmds.ErrNormal)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/fatih/color"
	"github.com/pkg/errors"

	"github.com/ahmetb/kubectx/internal/cmdutil"
	"github.com/ahmetb/kubectx/internal/kubeconfig"
	"github.com/ahmetb/kubectx/internal/printer"
)

type ListOp struct{}

func (op ListOp) Run(stdout, stderr io.Writer) error {
	kc := new(kubeconfig.Kubeconfig).WithLoader(cmdutil.DefaultLoader)
	defer kc.Close()
	if err := kc.Parse(); err != nil {
		return errors.Wrap(err, "kubeconfig error")
	}

	ctx := kc.GetCurrentContext()
	if ctx == "" {
		return errors.New("current-context is not set")
	}
	curNs, err := kc.NamespaceOfContext(ctx)
	if err != nil {
		return errors.Wrap(err, "cannot read current namespace")
	}

	ns, err := queryNamespaces()
	if err != nil {
		return errors.Wrap(err, "could not list namespaces (is the cluster accessible?)")
	}

	for _, c := range ns {
		s := c
		if c == curNs {
			s = printer.ActiveItemColor.Sprint(c)
		}
		fmt.Fprintf(stdout, "%s\n", s)
	}
	return nil
}

func findKubectl() (string, error) {
	if v := os.Getenv("KUBECTL"); v != "" {
		return v, nil
	}
	v, err := exec.LookPath("kubectl")
	return v, errors.Wrap(err, "kubectl not found, needed for kubens")
}

func queryNamespaces() ([]string, error) {
	kubectl ,err := findKubectl()
	if err != nil {
		return nil ,err
	}

	// TODO add a log message to user if kubectl is taking >1s

	var b bytes.Buffer
	cmd := exec.Command(kubectl, "get", "namespaces", `-o=jsonpath={range .items[*].metadata.name}{@}{"\n"}{end}`)
	cmd.Env = os.Environ()
	cmd.Stdout, cmd.Stderr = &b, &b
	if err := cmd.Run(); err != nil {
		return nil, errors.Wrapf(err, "failed to query namespaces: %v", b.String())
	}
	return strings.Split(strings.TrimSpace(b.String()), "\n"), nil
}
<commit_msg>fix compile error<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"

	"github.com/pkg/errors"

	"github.com/ahmetb/kubectx/internal/cmdutil"
	"github.com/ahmetb/kubectx/internal/kubeconfig"
	"github.com/ahmetb/kubectx/internal/printer"
)

type ListOp struct{}

func (op ListOp) Run(stdout, stderr io.Writer) error {
	kc := new(kubeconfig.Kubeconfig).WithLoader(cmdutil.DefaultLoader)
	defer kc.Close()
	if err := kc.Parse(); err != nil {
		return errors.Wrap(err, "kubeconfig error")
	}

	ctx := kc.GetCurrentContext()
	if ctx == "" {
		return errors.New("current-context is not set")
	}
	curNs, err := kc.NamespaceOfContext(ctx)
	if err != nil {
		return errors.Wrap(err, "cannot read current namespace")
	}

	ns, err := queryNamespaces()
	if err != nil {
		return errors.Wrap(err, "could not list namespaces (is the cluster accessible?)")
	}

	for _, c := range ns {
		s := c
		if c == curNs {
			s = printer.ActiveItemColor.Sprint(c)
		}
		fmt.Fprintf(stdout, "%s\n", s)
	}
	return nil
}

func findKubectl() (string, error) {
	if v := os.Getenv("KUBECTL"); v != "" {
		return v, nil
	}
	v, err := exec.LookPath("kubectl")
	return v, errors.Wrap(err, "kubectl not found, needed for kubens")
}

func queryNamespaces() ([]string, error) {
	kubectl, err := findKubectl()
	if err != nil {
		return nil, err
	}

	// TODO add a log message to user if kubectl is taking >1s

	var b bytes.Buffer
	cmd := exec.Command(kubectl, "get", "namespaces", `-o=jsonpath={range .items[*].metadata.name}{@}{"\n"}{end}`)
	cmd.Env = os.Environ()
	cmd.Stdout, cmd.Stderr = &b, &b
	if err := cmd.Run(); err != nil {
		return nil, errors.Wrapf(err, "failed to query namespaces: %v", b.String())
	}
	return strings.Split(strings.TrimSpace(b.String()), "\n"), nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"syscall"
	"time"

	"github.com/go-errors/errors"
	isatty "github.com/mattn/go-isatty"
	lua "github.com/yuin/gopher-lua"
	"github.com/zyedidia/micro/v2/internal/action"
	"github.com/zyedidia/micro/v2/internal/buffer"
	"github.com/zyedidia/micro/v2/internal/clipboard"
	"github.com/zyedidia/micro/v2/internal/config"
	ulua "github.com/zyedidia/micro/v2/internal/lua"
	"github.com/zyedidia/micro/v2/internal/screen"
	"github.com/zyedidia/micro/v2/internal/shell"
	"github.com/zyedidia/micro/v2/internal/util"
	"github.com/zyedidia/tcell"
)

var (
	// Event channel
	autosave chan bool

	// Command line flags
	flagVersion   = flag.Bool("version", false, "Show the version number and information")
	flagConfigDir = flag.String("config-dir", "", "Specify a custom location for the configuration directory")
	flagOptions   = flag.Bool("options", false, "Show all option help")
	flagDebug     = flag.Bool("debug", false, "Enable debug mode (prints debug info to ./log.txt)")
	flagPlugin    = flag.String("plugin", "", "Plugin command")
	flagClean     = flag.Bool("clean", false, "Clean configuration directory")
	optionFlags   map[string]*string
)

func InitFlags() {
	flag.Usage = func() {
		fmt.Println("Usage: micro [OPTIONS] [FILE]...")
		fmt.Println("-clean")
		fmt.Println("    \tCleans the configuration directory")
		fmt.Println("-config-dir dir")
		fmt.Println("    \tSpecify a custom location for the configuration directory")
		fmt.Println("[FILE]:LINE:COL (if the `parsecursor` option is enabled)")
		fmt.Println("+LINE:COL")
		fmt.Println("    \tSpecify a line and column to start the cursor at when opening a buffer")
		fmt.Println("-options")
		fmt.Println("    \tShow all option help")
		fmt.Println("-debug")
		fmt.Println("    \tEnable debug mode (enables logging to ./log.txt)")
		fmt.Println("-version")
		fmt.Println("    \tShow the version number and information")

		fmt.Print("\nMicro's plugin's can be managed at the command line with the following commands.\n")
		fmt.Println("-plugin install [PLUGIN]...")
		fmt.Println("    \tInstall plugin(s)")
		fmt.Println("-plugin remove [PLUGIN]...")
		fmt.Println("    \tRemove plugin(s)")
		fmt.Println("-plugin update [PLUGIN]...")
		fmt.Println("    \tUpdate plugin(s) (if no argument is given, updates all plugins)")
		fmt.Println("-plugin search [PLUGIN]...")
		fmt.Println("    \tSearch for a plugin")
		fmt.Println("-plugin list")
		fmt.Println("    \tList installed plugins")
		fmt.Println("-plugin available")
		fmt.Println("    \tList available plugins")

		fmt.Print("\nMicro's options can also be set via command line arguments for quick\nadjustments. For real configuration, please use the settings.json\nfile (see 'help options').\n\n")
		fmt.Println("-option value")
		fmt.Println("    \tSet `option` to `value` for this session")
		fmt.Println("    \tFor example: `micro -syntax off file.c`")
		fmt.Println("\nUse `micro -options` to see the full list of configuration options")
	}

	optionFlags = make(map[string]*string)

	for k, v := range config.DefaultAllSettings() {
		optionFlags[k] = flag.String(k, "", fmt.Sprintf("The %s option. Default value: '%v'.", k, v))
	}

	flag.Parse()

	if *flagVersion {
		// If -version was passed
		fmt.Println("Version:", util.Version)
		fmt.Println("Commit hash:", util.CommitHash)
		fmt.Println("Compiled on", util.CompileDate)
		os.Exit(0)
	}

	if *flagOptions {
		// If -options was passed
		var keys []string
		m := config.DefaultAllSettings()
		for k := range m {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			v := m[k]
			fmt.Printf("-%s value\n", k)
			fmt.Printf("    \tDefault value: '%v'\n", v)
		}
		os.Exit(0)
	}

	if util.Debug == "OFF" && *flagDebug {
		util.Debug = "ON"
	}
}

// DoPluginFlags parses and executes any flags that require LoadAllPlugins (-plugin and -clean)
func DoPluginFlags() {
	if *flagClean || *flagPlugin != "" {
		config.LoadAllPlugins()

		if *flagPlugin != "" {
			args := flag.Args()

			config.PluginCommand(os.Stdout, *flagPlugin, args)
		} else if *flagClean {
			CleanConfig()
		}

		os.Exit(0)
	}
}

// LoadInput determines which files should be loaded into buffers
// based on the input stored in flag.Args()
func LoadInput(args []string) []*buffer.Buffer {
	// There are a number of ways micro should start given its input

	// 1. If it is given a files in flag.Args(), it should open those

	// 2. If there is no input file and the input is not a terminal, that means
	// something is being piped in and the stdin should be opened in an
	// empty buffer

	// 3. If there is no input file and the input is a terminal, an empty buffer
	// should be opened

	var filename string
	var input []byte
	var err error
	buffers := make([]*buffer.Buffer, 0, len(args))

	btype := buffer.BTDefault
	if !isatty.IsTerminal(os.Stdout.Fd()) {
		btype = buffer.BTStdout
	}

	files := make([]string, 0, len(args))
	flagStartPos := buffer.Loc{-1, -1}
	flagr := regexp.MustCompile(`^\+(\d+)(?::(\d+))?$`)
	for _, a := range args {
		match := flagr.FindStringSubmatch(a)
		if len(match) == 3 && match[2] != "" {
			line, err := strconv.Atoi(match[1])
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			col, err := strconv.Atoi(match[2])
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			flagStartPos = buffer.Loc{col - 1, line - 1}
		} else if len(match) == 3 && match[2] == "" {
			line, err := strconv.Atoi(match[1])
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			flagStartPos = buffer.Loc{0, line - 1}
		} else {
			files = append(files, a)
		}
	}

	if len(files) > 0 {
		// Option 1
		// We go through each file and load it
		for i := 0; i < len(files); i++ {
			buf, err := buffer.NewBufferFromFileAtLoc(files[i], btype, flagStartPos)
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			// If the file didn't exist, input will be empty, and we'll open an empty buffer
			buffers = append(buffers, buf)
		}
	} else if !isatty.IsTerminal(os.Stdin.Fd()) {
		// Option 2
		// The input is not a terminal, so something is being piped in
		// and we should read from stdin
		input, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			screen.TermMessage("Error reading from stdin: ", err)
			input = []byte{}
		}
		buffers = append(buffers, buffer.NewBufferFromStringAtLoc(string(input), filename, btype, flagStartPos))
	} else {
		// Option 3, just open an empty buffer
		buffers = append(buffers, buffer.NewBufferFromStringAtLoc(string(input), filename, btype, flagStartPos))
	}

	return buffers
}

func main() {
	defer func() {
		if util.Stdout.Len() > 0 {
			fmt.Fprint(os.Stdout, util.Stdout.String())
		}
		os.Exit(0)
	}()

	// runtime.SetCPUProfileRate(400)
	// f, _ := os.Create("micro.prof")
	// pprof.StartCPUProfile(f)
	// defer pprof.StopCPUProfile()

	var err error

	InitFlags()

	InitLog()

	err = config.InitConfigDir(*flagConfigDir)
	if err != nil {
		screen.TermMessage(err)
	}

	config.InitRuntimeFiles()
	err = config.ReadSettings()
	if err != nil {
		screen.TermMessage(err)
	}
	err = config.InitGlobalSettings()
	if err != nil {
		screen.TermMessage(err)
	}

	// flag options
	for k, v := range optionFlags {
		if *v != "" {
			nativeValue, err := config.GetNativeValue(k, config.DefaultAllSettings()[k], *v)
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			config.GlobalSettings[k] = nativeValue
		}
	}

	DoPluginFlags()

	err = screen.Init()
	if err != nil {
		fmt.Println(err)
		fmt.Println("Fatal: Micro could not initialize a Screen.")
		os.Exit(1)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Kill, syscall.SIGTERM)

	go func() {
		<-c
		if screen.Screen != nil {
			screen.Screen.Fini()
		}
		os.Exit(0)
	}()

	m := clipboard.SetMethod(config.GetGlobalOption("clipboard").(string))
	clipErr := clipboard.Initialize(m)

	defer func() {
		if err := recover(); err != nil {
			if screen.Screen != nil {
				screen.Screen.Fini()
			}
			fmt.Println("Micro encountered an error:", err)
			// backup all open buffers
			for _, b := range buffer.OpenBuffers {
				b.Backup()
			}
			// Print the stack trace too
			fmt.Print(errors.Wrap(err, 2).ErrorStack())
			os.Exit(1)
		}
	}()

	err = config.LoadAllPlugins()
	if err != nil {
		screen.TermMessage(err)
	}

	action.InitBindings()
	action.InitCommands()

	err = config.InitColorscheme()
	if err != nil {
		screen.TermMessage(err)
	}

	err = config.RunPluginFn("preinit")
	if err != nil {
		screen.TermMessage(err)
	}

	args := flag.Args()
	b := LoadInput(args)

	if len(b) == 0 {
		// No buffers to open
		screen.Screen.Fini()
		runtime.Goexit()
	}

	action.InitTabs(b)
	action.InitGlobals()

	err = config.RunPluginFn("init")
	if err != nil {
		screen.TermMessage(err)
	}

	err = config.RunPluginFn("postinit")
	if err != nil {
		screen.TermMessage(err)
	}

	if clipErr != nil {
		action.InfoBar.Error(clipErr, " or change 'clipboard' option")
	}

	screen.Events = make(chan tcell.Event)

	// Here is the event loop which runs in a separate thread
	go func() {
		for {
			screen.Lock()
			e := screen.Screen.PollEvent()
			screen.Unlock()
			if e != nil {
				screen.Events <- e
			}
		}
	}()

	// clear the drawchan so we don't redraw excessively
	// if someone requested a redraw before we started displaying
	for len(screen.DrawChan()) > 0 {
		<-screen.DrawChan()
	}

	// wait for initial resize event
	select {
	case event := <-screen.Events:
		action.Tabs.HandleEvent(event)
	case <-time.After(10 * time.Millisecond):
		// time out after 10ms
	}

	// Since this loop is very slow (waits for user input every time) it's
	// okay to be inefficient and run it via a function every time
	// We do this so we can recover from panics without crashing the editor
	for {
		DoEvent()
	}
}

// DoEvent runs the main action loop of the editor
func DoEvent() {
	var event tcell.Event

	// recover from errors without crashing the editor
	defer func() {
		if err := recover(); err != nil {
			if e, ok := err.(*lua.ApiError); ok {
				screen.TermMessage("Lua API error:", e)
			} else {
				screen.TermMessage("Micro encountered an error:", errors.Wrap(err, 2).ErrorStack(), "\nIf you can reproduce this error, please report it at https://github.com/zyedidia/micro/issues")
			}
		}
	}()
	// Display everything
	screen.Screen.Fill(' ', config.DefStyle)
	screen.Screen.HideCursor()
	action.Tabs.Display()
	for _, ep := range action.MainTab().Panes {
		ep.Display()
	}
	action.MainTab().Display()
	action.InfoBar.Display()
	screen.Screen.Show()

	// Check for new events
	select {
	case f := <-shell.Jobs:
		// If a new job has finished while running in the background we should execute the callback
		ulua.Lock.Lock()
		f.Function(f.Output, f.Args)
		ulua.Lock.Unlock()
	case <-config.Autosave:
		ulua.Lock.Lock()
		for _, b := range buffer.OpenBuffers {
			b.Save()
		}
		ulua.Lock.Unlock()
	case <-shell.CloseTerms:
	case event = <-screen.Events:
	case <-screen.DrawChan():
		for len(screen.DrawChan()) > 0 {
			<-screen.DrawChan()
		}
	}

	ulua.Lock.Lock()
	if action.InfoBar.HasPrompt {
		action.InfoBar.HandleEvent(event)
	} else {
		action.Tabs.HandleEvent(event)
	}
	ulua.Lock.Unlock()
}
<commit_msg>Properly close unmodified buffers on sigterm<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"syscall"
	"time"

	"github.com/go-errors/errors"
	isatty "github.com/mattn/go-isatty"
	lua "github.com/yuin/gopher-lua"
	"github.com/zyedidia/micro/v2/internal/action"
	"github.com/zyedidia/micro/v2/internal/buffer"
	"github.com/zyedidia/micro/v2/internal/clipboard"
	"github.com/zyedidia/micro/v2/internal/config"
	ulua "github.com/zyedidia/micro/v2/internal/lua"
	"github.com/zyedidia/micro/v2/internal/screen"
	"github.com/zyedidia/micro/v2/internal/shell"
	"github.com/zyedidia/micro/v2/internal/util"
	"github.com/zyedidia/tcell"
)

var (
	// Event channel
	autosave chan bool

	// Command line flags
	flagVersion   = flag.Bool("version", false, "Show the version number and information")
	flagConfigDir = flag.String("config-dir", "", "Specify a custom location for the configuration directory")
	flagOptions   = flag.Bool("options", false, "Show all option help")
	flagDebug     = flag.Bool("debug", false, "Enable debug mode (prints debug info to ./log.txt)")
	flagPlugin    = flag.String("plugin", "", "Plugin command")
	flagClean     = flag.Bool("clean", false, "Clean configuration directory")
	optionFlags   map[string]*string
)

func InitFlags() {
	flag.Usage = func() {
		fmt.Println("Usage: micro [OPTIONS] [FILE]...")
		fmt.Println("-clean")
		fmt.Println("    \tCleans the configuration directory")
		fmt.Println("-config-dir dir")
		fmt.Println("    \tSpecify a custom location for the configuration directory")
		fmt.Println("[FILE]:LINE:COL (if the `parsecursor` option is enabled)")
		fmt.Println("+LINE:COL")
		fmt.Println("    \tSpecify a line and column to start the cursor at when opening a buffer")
		fmt.Println("-options")
		fmt.Println("    \tShow all option help")
		fmt.Println("-debug")
		fmt.Println("    \tEnable debug mode (enables logging to ./log.txt)")
		fmt.Println("-version")
		fmt.Println("    \tShow the version number and information")

		fmt.Print("\nMicro's plugin's can be managed at the command line with the following commands.\n")
		fmt.Println("-plugin install [PLUGIN]...")
		fmt.Println("    \tInstall plugin(s)")
		fmt.Println("-plugin remove [PLUGIN]...")
		fmt.Println("    \tRemove plugin(s)")
		fmt.Println("-plugin update [PLUGIN]...")
		fmt.Println("    \tUpdate plugin(s) (if no argument is given, updates all plugins)")
		fmt.Println("-plugin search [PLUGIN]...")
		fmt.Println("    \tSearch for a plugin")
		fmt.Println("-plugin list")
		fmt.Println("    \tList installed plugins")
		fmt.Println("-plugin available")
		fmt.Println("    \tList available plugins")

		fmt.Print("\nMicro's options can also be set via command line arguments for quick\nadjustments. For real configuration, please use the settings.json\nfile (see 'help options').\n\n")
		fmt.Println("-option value")
		fmt.Println("    \tSet `option` to `value` for this session")
		fmt.Println("    \tFor example: `micro -syntax off file.c`")
		fmt.Println("\nUse `micro -options` to see the full list of configuration options")
	}

	optionFlags = make(map[string]*string)

	for k, v := range config.DefaultAllSettings() {
		optionFlags[k] = flag.String(k, "", fmt.Sprintf("The %s option. Default value: '%v'.", k, v))
	}

	flag.Parse()

	if *flagVersion {
		// If -version was passed
		fmt.Println("Version:", util.Version)
		fmt.Println("Commit hash:", util.CommitHash)
		fmt.Println("Compiled on", util.CompileDate)
		os.Exit(0)
	}

	if *flagOptions {
		// If -options was passed
		var keys []string
		m := config.DefaultAllSettings()
		for k := range m {
			keys = append(keys, k)
		}
		sort.Strings(keys)
		for _, k := range keys {
			v := m[k]
			fmt.Printf("-%s value\n", k)
			fmt.Printf("    \tDefault value: '%v'\n", v)
		}
		os.Exit(0)
	}

	if util.Debug == "OFF" && *flagDebug {
		util.Debug = "ON"
	}
}

// DoPluginFlags parses and executes any flags that require LoadAllPlugins (-plugin and -clean)
func DoPluginFlags() {
	if *flagClean || *flagPlugin != "" {
		config.LoadAllPlugins()

		if *flagPlugin != "" {
			args := flag.Args()

			config.PluginCommand(os.Stdout, *flagPlugin, args)
		} else if *flagClean {
			CleanConfig()
		}

		os.Exit(0)
	}
}

// LoadInput determines which files should be loaded into buffers
// based on the input stored in flag.Args()
func LoadInput(args []string) []*buffer.Buffer {
	// There are a number of ways micro should start given its input

	// 1. If it is given a files in flag.Args(), it should open those

	// 2. If there is no input file and the input is not a terminal, that means
	// something is being piped in and the stdin should be opened in an
	// empty buffer

	// 3. If there is no input file and the input is a terminal, an empty buffer
	// should be opened

	var filename string
	var input []byte
	var err error
	buffers := make([]*buffer.Buffer, 0, len(args))

	btype := buffer.BTDefault
	if !isatty.IsTerminal(os.Stdout.Fd()) {
		btype = buffer.BTStdout
	}

	files := make([]string, 0, len(args))
	flagStartPos := buffer.Loc{-1, -1}
	flagr := regexp.MustCompile(`^\+(\d+)(?::(\d+))?$`)
	for _, a := range args {
		match := flagr.FindStringSubmatch(a)
		if len(match) == 3 && match[2] != "" {
			line, err := strconv.Atoi(match[1])
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			col, err := strconv.Atoi(match[2])
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			flagStartPos = buffer.Loc{col - 1, line - 1}
		} else if len(match) == 3 && match[2] == "" {
			line, err := strconv.Atoi(match[1])
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			flagStartPos = buffer.Loc{0, line - 1}
		} else {
			files = append(files, a)
		}
	}

	if len(files) > 0 {
		// Option 1
		// We go through each file and load it
		for i := 0; i < len(files); i++ {
			buf, err := buffer.NewBufferFromFileAtLoc(files[i], btype, flagStartPos)
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			// If the file didn't exist, input will be empty, and we'll open an empty buffer
			buffers = append(buffers, buf)
		}
	} else if !isatty.IsTerminal(os.Stdin.Fd()) {
		// Option 2
		// The input is not a terminal, so something is being piped in
		// and we should read from stdin
		input, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			screen.TermMessage("Error reading from stdin: ", err)
			input = []byte{}
		}
		buffers = append(buffers, buffer.NewBufferFromStringAtLoc(string(input), filename, btype, flagStartPos))
	} else {
		// Option 3, just open an empty buffer
		buffers = append(buffers, buffer.NewBufferFromStringAtLoc(string(input), filename, btype, flagStartPos))
	}

	return buffers
}

func main() {
	defer func() {
		if util.Stdout.Len() > 0 {
			fmt.Fprint(os.Stdout, util.Stdout.String())
		}
		os.Exit(0)
	}()

	// runtime.SetCPUProfileRate(400)
	// f, _ := os.Create("micro.prof")
	// pprof.StartCPUProfile(f)
	// defer pprof.StopCPUProfile()

	var err error

	InitFlags()

	InitLog()

	err = config.InitConfigDir(*flagConfigDir)
	if err != nil {
		screen.TermMessage(err)
	}

	config.InitRuntimeFiles()
	err = config.ReadSettings()
	if err != nil {
		screen.TermMessage(err)
	}
	err = config.InitGlobalSettings()
	if err != nil {
		screen.TermMessage(err)
	}

	// flag options
	for k, v := range optionFlags {
		if *v != "" {
			nativeValue, err := config.GetNativeValue(k, config.DefaultAllSettings()[k], *v)
			if err != nil {
				screen.TermMessage(err)
				continue
			}
			config.GlobalSettings[k] = nativeValue
		}
	}

	DoPluginFlags()

	err = screen.Init()
	if err != nil {
		fmt.Println(err)
		fmt.Println("Fatal: Micro could not initialize a Screen.")
		os.Exit(1)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Kill, syscall.SIGTERM)

	go func() {
		<-c

		for _, b := range buffer.OpenBuffers {
			if !b.Modified() {
				b.Fini()
			}
		}

		if screen.Screen != nil {
			screen.Screen.Fini()
		}
		os.Exit(0)
	}()

	m := clipboard.SetMethod(config.GetGlobalOption("clipboard").(string))
	clipErr := clipboard.Initialize(m)

	defer func() {
		if err := recover(); err != nil {
			if screen.Screen != nil {
				screen.Screen.Fini()
			}
			fmt.Println("Micro encountered an error:", err)
			// backup all open buffers
			for _, b := range buffer.OpenBuffers {
				b.Backup()
			}
			// Print the stack trace too
			fmt.Print(errors.Wrap(err, 2).ErrorStack())
			os.Exit(1)
		}
	}()

	err = config.LoadAllPlugins()
	if err != nil {
		screen.TermMessage(err)
	}

	action.InitBindings()
	action.InitCommands()

	err = config.InitColorscheme()
	if err != nil {
		screen.TermMessage(err)
	}

	err = config.RunPluginFn("preinit")
	if err != nil {
		screen.TermMessage(err)
	}

	args := flag.Args()
	b := LoadInput(args)

	if len(b) == 0 {
		// No buffers to open
		screen.Screen.Fini()
		runtime.Goexit()
	}

	action.InitTabs(b)
	action.InitGlobals()

	err = config.RunPluginFn("init")
	if err != nil {
		screen.TermMessage(err)
	}

	err = config.RunPluginFn("postinit")
	if err != nil {
		screen.TermMessage(err)
	}

	if clipErr != nil {
		action.InfoBar.Error(clipErr, " or change 'clipboard' option")
	}

	screen.Events = make(chan tcell.Event)

	// Here is the event loop which runs in a separate thread
	go func() {
		for {
			screen.Lock()
			e := screen.Screen.PollEvent()
			screen.Unlock()
			if e != nil {
				screen.Events <- e
			}
		}
	}()

	// clear the drawchan so we don't redraw excessively
	// if someone requested a redraw before we started displaying
	for len(screen.DrawChan()) > 0 {
		<-screen.DrawChan()
	}

	// wait for initial resize event
	select {
	case event := <-screen.Events:
		action.Tabs.HandleEvent(event)
	case <-time.After(10 * time.Millisecond):
		// time out after 10ms
	}

	// Since this loop is very slow (waits for user input every time) it's
	// okay to be inefficient and run it via a function every time
	// We do this so we can recover from panics without crashing the editor
	for {
		DoEvent()
	}
}

// DoEvent runs the main action loop of the editor
func DoEvent() {
	var event tcell.Event

	// recover from errors without crashing the editor
	defer func() {
		if err := recover(); err != nil {
			if e, ok := err.(*lua.ApiError); ok {
				screen.TermMessage("Lua API error:", e)
			} else {
				screen.TermMessage("Micro encountered an error:", errors.Wrap(err, 2).ErrorStack(), "\nIf you can reproduce this error, please report it at https://github.com/zyedidia/micro/issues")
			}
		}
	}()
	// Display everything
	screen.Screen.Fill(' ', config.DefStyle)
	screen.Screen.HideCursor()
	action.Tabs.Display()
	for _, ep := range action.MainTab().Panes {
		ep.Display()
	}
	action.MainTab().Display()
	action.InfoBar.Display()
	screen.Screen.Show()

	// Check for new events
	select {
	case f := <-shell.Jobs:
		// If a new job has finished while running in the background we should execute the callback
		ulua.Lock.Lock()
		f.Function(f.Output, f.Args)
		ulua.Lock.Unlock()
	case <-config.Autosave:
		ulua.Lock.Lock()
		for _, b := range buffer.OpenBuffers {
			b.Save()
		}
		ulua.Lock.Unlock()
	case <-shell.CloseTerms:
	case event = <-screen.Events:
	case <-screen.DrawChan():
		for len(screen.DrawChan()) > 0 {
			<-screen.DrawChan()
		}
	}

	ulua.Lock.Lock()
	if action.InfoBar.HasPrompt {
		action.InfoBar.HandleEvent(event)
	} else {
		action.Tabs.HandleEvent(event)
	}
	ulua.Lock.Unlock()
}
<|endoftext|>


<commit_before>package cmd

import (
	"net/url"

	"github.com/markbates/pop"
	"github.com/netlify/gotrue/conf"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var migrateCmd = cobra.Command{
	Use:  "migrate",
	Long: "Migrate database strucutures. This will create new tables and add missing columns and indexes.",
	Run:  migrate,
}

func migrate(cmd *cobra.Command, args []string) {
	globalConfig, err := conf.LoadGlobal(configFile)
	if err != nil {
		logrus.Fatalf("Failed to load configuration: %+v", err)
	}
	if globalConfig.DB.Driver == "" && globalConfig.DB.URL != "" {
		u, err := url.Parse(globalConfig.DB.URL)
		if err != nil {
			logrus.Fatalf("%+v", errors.Wrap(err, "parsing db connection url"))
		}
		globalConfig.DB.Driver = u.Scheme
	}
	pop.Debug = true

	deets := &pop.ConnectionDetails{
		Dialect: globalConfig.DB.Driver,
		URL:     globalConfig.DB.URL,
	}
	if globalConfig.DB.Namespace != "" {
		deets.Options = map[string]string{
			"Namespace": globalConfig.DB.Namespace + "_",
		}
	}

	db, err := pop.NewConnection(deets)
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "opening db connection"))
	}
	defer db.Close()

	if err := db.Open(); err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "checking database connection"))
	}

	logrus.Infof("Reading migrations from %s", globalConfig.DB.MigrationsPath)
	mig, err := pop.NewFileMigrator(globalConfig.DB.MigrationsPath, db)
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "creating db migrator"))
	}
	logrus.Infof("%d up / %d down migrations found", len(mig.Migrations["up"]), len(mig.Migrations["down"]))
	// turn off schema dump
	mig.SchemaPath = ""

	err = mig.Up()
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "running db migrations"))
	}
}
<commit_msg>Log full migration status instead<commit_after>package cmd

import (
	"net/url"

	"github.com/markbates/pop"
	"github.com/netlify/gotrue/conf"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var migrateCmd = cobra.Command{
	Use:  "migrate",
	Long: "Migrate database strucutures. This will create new tables and add missing columns and indexes.",
	Run:  migrate,
}

func migrate(cmd *cobra.Command, args []string) {
	globalConfig, err := conf.LoadGlobal(configFile)
	if err != nil {
		logrus.Fatalf("Failed to load configuration: %+v", err)
	}
	if globalConfig.DB.Driver == "" && globalConfig.DB.URL != "" {
		u, err := url.Parse(globalConfig.DB.URL)
		if err != nil {
			logrus.Fatalf("%+v", errors.Wrap(err, "parsing db connection url"))
		}
		globalConfig.DB.Driver = u.Scheme
	}
	pop.Debug = true

	deets := &pop.ConnectionDetails{
		Dialect: globalConfig.DB.Driver,
		URL:     globalConfig.DB.URL,
	}
	if globalConfig.DB.Namespace != "" {
		deets.Options = map[string]string{
			"Namespace": globalConfig.DB.Namespace + "_",
		}
	}

	db, err := pop.NewConnection(deets)
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "opening db connection"))
	}
	defer db.Close()

	if err := db.Open(); err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "checking database connection"))
	}

	logrus.Infof("Reading migrations from %s", globalConfig.DB.MigrationsPath)
	mig, err := pop.NewFileMigrator(globalConfig.DB.MigrationsPath, db)
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "creating db migrator"))
	}
	logrus.Infof("before status")
	err = mig.Status()
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "migration status"))
	}
	// turn off schema dump
	mig.SchemaPath = ""

	err = mig.Up()
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "running db migrations"))
	}

	logrus.Infof("after status")
	err = mig.Status()
	if err != nil {
		logrus.Fatalf("%+v", errors.Wrap(err, "migration status"))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

var clientVersion string
var clientVersionLock sync.Mutex

type Client struct {
	Ticket string
	host   string
	client *http.Client
}

type ListItem struct {
	Name        string
	Description string
	Hosts       []string
	Version     float64
	Features    uint32
}

type upgradeManifest struct {
	URL string
}

type Package struct {
	Package     string
	Description string
}

type EpochTime time.Time

func (t *EpochTime) UnmarshalJSON(bs []byte) error {
	n, err := strconv.Atoi(string(bs))
	if err != nil {
		return err
	}

	*t = EpochTime(time.Unix(int64(n), 0))
	return nil
}

type ParsedTicket struct {
	User     string
	IPs      []string
	Validity EpochTime
}

var obfuscatedRe = regexp.MustCompile(`\$mole\$[0-9a-zA-Z+/-]+`)

func certFingerprint(conn *tls.Conn) []byte {
	cert := conn.ConnectionState().PeerCertificates[0].Raw
	sha := sha1.New()
	_, _ = sha.Write(cert)
	return sha.Sum(nil)
}

func NewClient(host, fingerprint string) *Client {
	if host == "" {
		fatalln(msgNoHost)
	}

	clientVersionLock.Lock()
	if !strings.HasPrefix(clientVersion, "4.") {
		if buildVersion != "" {
			clientVersion = strings.Replace(buildVersion, "v", "", 1)
		} else {
			// Built from go get, so no tag info
			clientVersion = "4.0-unknown-dev"
		}
	}
	clientVersionLock.Unlock()

	transport := &http.Transport{
		Dial: func(n, a string) (net.Conn, error) {
			t0 := time.Now()
			tlsCfg := &tls.Config{InsecureSkipVerify: true}
			conn, err := tls.Dial(n, host, tlsCfg)
			if err != nil {
				return nil, err
			}

			fp := hexBytes(certFingerprint(conn))
			if fingerprint != "" && fp != fingerprint {
				return nil, fmt.Errorf("server fingerprint mismatch (%s != %s)", fp, fingerprint)
			}
			debugf("tls connect %.01f ms", time.Since(t0).Seconds()*1000)
			return conn, err
		},
	}
	client := &http.Client{Transport: transport}
	return &Client{host: host, client: client}
}

func (c *Client) request(method, path string, content io.Reader) (*http.Response, error) {
	url := "http://" + c.host + path
	debugln(method, url)

	req, err := http.NewRequest(method, url, content)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "mole/"+clientVersion)
	req.Header.Set("X-Mole-Version", clientVersion)
	req.Header.Set("X-Mole-Ticket", c.Ticket)

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode == 530 {
		defer resp.Body.Close()
		return nil, fmt.Errorf(msg530)
	}

	if resp.StatusCode != 200 {
		defer resp.Body.Close()
		data, _ := ioutil.ReadAll(resp.Body)
		if len(data) > 0 {
			return nil, fmt.Errorf("%s: %s", resp.Status, data)
		} else {
			return nil, fmt.Errorf(resp.Status)
		}
	}

	debugln(resp.Status, resp.Header.Get("Content-type"), resp.ContentLength)

	if ch := resp.Header.Get("X-Mole-Canonical-Hostname"); ch != "" && ch != moleIni.Get("server", "host") {
		moleIni.Set("server", "host", ch)
		saveMoleIni()
		okf(msgUpdatedHost, ch)
	}

	return resp, nil
}

func (c *Client) ServerVersion() string {
	url := "http://" + c.host + "/ping"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return ""
	}

	req.Header.Set("User-Agent", "mole/"+clientVersion)
	req.Header.Set("X-Mole-Version", clientVersion)
	req.Header.Set("X-Mole-Ticket", c.Ticket)

	resp, err := c.client.Do(req)
	if err != nil {
		return ""
	}
	resp.Body.Close()

	return resp.Header.Get("X-Mole-Version")
}

func (c *Client) List() ([]ListItem, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/store", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var items []ListItem
	err = json.NewDecoder(resp.Body).Decode(&items)
	if err != nil {
		return nil, err
	}

	sort.Sort(listItems(items))

	debugf("list %.01f ms", time.Since(t0).Seconds()*1000)
	return items, nil
}

func (c *Client) Get(tunnel string) (string, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/store/"+tunnel+".ini", nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	res := string(data)

	debugf("get %.01f ms", time.Since(t0).Seconds()*1000)
	return res, nil
}

func (c *Client) Put(tunnel string, data io.Reader) error {
	t0 := time.Now()

	resp, err := c.request("PUT", "/store/"+tunnel+".ini", data)
	if err != nil {
		return err
	}
	resp.Body.Close()

	debugf("put %.01f ms", time.Since(t0).Seconds()*1000)
	return nil
}

func (c *Client) Delete(tunnel string) error {
	t0 := time.Now()

	resp, err := c.request("DELETE", "/store/"+tunnel+".ini", nil)
	if err != nil {
		return err
	}
	resp.Body.Close()

	debugf("delete %.01f ms", time.Since(t0).Seconds()*1000)
	return nil
}

func (c *Client) Deobfuscate(tunnel string) (string, error) {
	t0 := time.Now()

	var err error
	var keylist []string
	var keymap map[string]string

	matches := obfuscatedRe.FindAllString(tunnel, -1)
	for _, o := range matches {
		keylist = append(keylist, o[6:])
	}

	var buf bytes.Buffer
	json.NewEncoder(&buf).Encode(keylist)
	resp, err := c.request("POST", "/keys", &buf)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&keymap)
	fatalErr(err)

	for k, v := range keymap {
		tunnel = strings.Replace(tunnel, "$mole$"+k, strconv.Quote(v), -1)
	}

	debugf("deobfuscate %.01f ms", time.Since(t0).Seconds()*1000)
	return tunnel, nil
}

func (c *Client) GetTicket(username, password string) (string, error) {
	t0 := time.Now()

	resp, err := c.request("POST", "/ticket/"+username, bytes.NewBufferString(password))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	res := string(data)

	debugf("getticket %.01f ms", time.Since(t0).Seconds()*1000)
	return res, nil
}

func (c *Client) ParseTicket() (ParsedTicket, error) {
	t0 := time.Now()
	var res ParsedTicket

	resp, err := c.request("GET", "/ticket/", nil)
	if err != nil {
		return res, err
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return res, err
	}

	debugf("parseticket %.01f ms", time.Since(t0).Seconds()*1000)
	return res, nil
}

func (c *Client) UpgradesURL() (string, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/extra/upgrades.json", nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var manifest upgradeManifest
	err = json.NewDecoder(resp.Body).Decode(&manifest)
	if err != nil {
		return "", err
	}

	debugf("upgradeurl %.01f ms", time.Since(t0).Seconds()*1000)
	return manifest.URL, nil
}

func (c *Client) Packages() (map[string][]Package, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/extra/packages.json", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var packageMap map[string][]Package
	err = json.NewDecoder(resp.Body).Decode(&packageMap)
	if err != nil {
		return nil, err
	}

	debugf("packages %.01f ms", time.Since(t0).Seconds()*1000)
	return packageMap, nil
}

func (c *Client) Package(file string) (io.ReadCloser, error) {
	resp, err := c.request("GET", "/extra/"+file, nil)
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}

type listItems []ListItem

func (l listItems) Len() int {
	return len(l)
}

func (l listItems) Less(i, j int) bool {
	return l[i].Name < l[j].Name
}

func (l listItems) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
<commit_msg>BUG: Don't deobfuscate tunnels that don't have any obfuscated keys<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

var clientVersion string
var clientVersionLock sync.Mutex

type Client struct {
	Ticket string
	host   string
	client *http.Client
}

type ListItem struct {
	Name        string
	Description string
	Hosts       []string
	Version     float64
	Features    uint32
}

type upgradeManifest struct {
	URL string
}

type Package struct {
	Package     string
	Description string
}

type EpochTime time.Time

func (t *EpochTime) UnmarshalJSON(bs []byte) error {
	n, err := strconv.Atoi(string(bs))
	if err != nil {
		return err
	}

	*t = EpochTime(time.Unix(int64(n), 0))
	return nil
}

type ParsedTicket struct {
	User     string
	IPs      []string
	Validity EpochTime
}

var obfuscatedRe = regexp.MustCompile(`\$mole\$[0-9a-zA-Z+/-]+`)

func certFingerprint(conn *tls.Conn) []byte {
	cert := conn.ConnectionState().PeerCertificates[0].Raw
	sha := sha1.New()
	_, _ = sha.Write(cert)
	return sha.Sum(nil)
}

func NewClient(host, fingerprint string) *Client {
	if host == "" {
		fatalln(msgNoHost)
	}

	clientVersionLock.Lock()
	if !strings.HasPrefix(clientVersion, "4.") {
		if buildVersion != "" {
			clientVersion = strings.Replace(buildVersion, "v", "", 1)
		} else {
			// Built from go get, so no tag info
			clientVersion = "4.0-unknown-dev"
		}
	}
	clientVersionLock.Unlock()

	transport := &http.Transport{
		Dial: func(n, a string) (net.Conn, error) {
			t0 := time.Now()
			tlsCfg := &tls.Config{InsecureSkipVerify: true}
			conn, err := tls.Dial(n, host, tlsCfg)
			if err != nil {
				return nil, err
			}

			fp := hexBytes(certFingerprint(conn))
			if fingerprint != "" && fp != fingerprint {
				return nil, fmt.Errorf("server fingerprint mismatch (%s != %s)", fp, fingerprint)
			}
			debugf("tls connect %.01f ms", time.Since(t0).Seconds()*1000)
			return conn, err
		},
	}
	client := &http.Client{Transport: transport}
	return &Client{host: host, client: client}
}

func (c *Client) request(method, path string, content io.Reader) (*http.Response, error) {
	url := "http://" + c.host + path
	debugln(method, url)

	req, err := http.NewRequest(method, url, content)
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", "mole/"+clientVersion)
	req.Header.Set("X-Mole-Version", clientVersion)
	req.Header.Set("X-Mole-Ticket", c.Ticket)

	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode == 530 {
		defer resp.Body.Close()
		return nil, fmt.Errorf(msg530)
	}

	if resp.StatusCode != 200 {
		defer resp.Body.Close()
		data, _ := ioutil.ReadAll(resp.Body)
		if len(data) > 0 {
			return nil, fmt.Errorf("%s: %s", resp.Status, data)
		} else {
			return nil, fmt.Errorf(resp.Status)
		}
	}

	debugln(resp.Status, resp.Header.Get("Content-type"), resp.ContentLength)

	if ch := resp.Header.Get("X-Mole-Canonical-Hostname"); ch != "" && ch != moleIni.Get("server", "host") {
		moleIni.Set("server", "host", ch)
		saveMoleIni()
		okf(msgUpdatedHost, ch)
	}

	return resp, nil
}

func (c *Client) ServerVersion() string {
	url := "http://" + c.host + "/ping"
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return ""
	}

	req.Header.Set("User-Agent", "mole/"+clientVersion)
	req.Header.Set("X-Mole-Version", clientVersion)
	req.Header.Set("X-Mole-Ticket", c.Ticket)

	resp, err := c.client.Do(req)
	if err != nil {
		return ""
	}
	resp.Body.Close()

	return resp.Header.Get("X-Mole-Version")
}

func (c *Client) List() ([]ListItem, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/store", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var items []ListItem
	err = json.NewDecoder(resp.Body).Decode(&items)
	if err != nil {
		return nil, err
	}

	sort.Sort(listItems(items))

	debugf("list %.01f ms", time.Since(t0).Seconds()*1000)
	return items, nil
}

func (c *Client) Get(tunnel string) (string, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/store/"+tunnel+".ini", nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	res := string(data)

	debugf("get %.01f ms", time.Since(t0).Seconds()*1000)
	return res, nil
}

func (c *Client) Put(tunnel string, data io.Reader) error {
	t0 := time.Now()

	resp, err := c.request("PUT", "/store/"+tunnel+".ini", data)
	if err != nil {
		return err
	}
	resp.Body.Close()

	debugf("put %.01f ms", time.Since(t0).Seconds()*1000)
	return nil
}

func (c *Client) Delete(tunnel string) error {
	t0 := time.Now()

	resp, err := c.request("DELETE", "/store/"+tunnel+".ini", nil)
	if err != nil {
		return err
	}
	resp.Body.Close()

	debugf("delete %.01f ms", time.Since(t0).Seconds()*1000)
	return nil
}

func (c *Client) Deobfuscate(tunnel string) (string, error) {
	t0 := time.Now()

	var err error
	var keylist []string
	var keymap map[string]string

	matches := obfuscatedRe.FindAllString(tunnel, -1)
	for _, o := range matches {
		keylist = append(keylist, o[6:])
	}
	if len(keylist) == 0 {
		return tunnel, nil
	}

	var buf bytes.Buffer
	json.NewEncoder(&buf).Encode(keylist)
	resp, err := c.request("POST", "/keys", &buf)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&keymap)
	fatalErr(err)

	for k, v := range keymap {
		tunnel = strings.Replace(tunnel, "$mole$"+k, strconv.Quote(v), -1)
	}

	debugf("deobfuscate %.01f ms", time.Since(t0).Seconds()*1000)
	return tunnel, nil
}

func (c *Client) GetTicket(username, password string) (string, error) {
	t0 := time.Now()

	resp, err := c.request("POST", "/ticket/"+username, bytes.NewBufferString(password))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	res := string(data)

	debugf("getticket %.01f ms", time.Since(t0).Seconds()*1000)
	return res, nil
}

func (c *Client) ParseTicket() (ParsedTicket, error) {
	t0 := time.Now()
	var res ParsedTicket

	resp, err := c.request("GET", "/ticket/", nil)
	if err != nil {
		return res, err
	}
	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return res, err
	}

	debugf("parseticket %.01f ms", time.Since(t0).Seconds()*1000)
	return res, nil
}

func (c *Client) UpgradesURL() (string, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/extra/upgrades.json", nil)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var manifest upgradeManifest
	err = json.NewDecoder(resp.Body).Decode(&manifest)
	if err != nil {
		return "", err
	}

	debugf("upgradeurl %.01f ms", time.Since(t0).Seconds()*1000)
	return manifest.URL, nil
}

func (c *Client) Packages() (map[string][]Package, error) {
	t0 := time.Now()

	resp, err := c.request("GET", "/extra/packages.json", nil)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var packageMap map[string][]Package
	err = json.NewDecoder(resp.Body).Decode(&packageMap)
	if err != nil {
		return nil, err
	}

	debugf("packages %.01f ms", time.Since(t0).Seconds()*1000)
	return packageMap, nil
}

func (c *Client) Package(file string) (io.ReadCloser, error) {
	resp, err := c.request("GET", "/extra/"+file, nil)
	if err != nil {
		return nil, err
	}

	return resp.Body, nil
}

type listItems []ListItem

func (l listItems) Len() int {
	return len(l)
}

func (l listItems) Less(i, j int) bool {
	return l[i].Name < l[j].Name
}

func (l listItems) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"runtime"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/jessevdk/go-flags"
	"github.com/restic/restic"
	"github.com/restic/restic/backend"
)

var version = "compiled manually"

var opts struct {
	Repo string `short:"r" long:"repo"    description:"Repository directory to backup to/restore from"`
}

var parser = flags.NewParser(&opts, flags.Default)

func errx(code int, format string, data ...interface{}) {
	if len(format) > 0 && format[len(format)-1] != '\n' {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, data...)
	os.Exit(code)
}

func readPassword(env string, prompt string) string {

	if env != "" {
		p := os.Getenv(env)

		if p != "" {
			return p
		}
	}

	fmt.Print(prompt)
	pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
	if err != nil {
		errx(2, "unable to read password: %v", err)
	}
	fmt.Println()

	return string(pw)
}

type CmdInit struct{}

func (cmd CmdInit) Execute(args []string) error {
	if opts.Repo == "" {
		return errors.New("Please specify repository location (-r)")
	}

	pw := readPassword("RESTIC_PASSWORD", "enter password for new backend: ")
	pw2 := readPassword("RESTIC_PASSWORD", "enter password again: ")

	if pw != pw2 {
		errx(1, "passwords do not match")
	}

	be, err := create(opts.Repo)
	if err != nil {
		fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", opts.Repo, err)
		os.Exit(1)
	}

	s := restic.NewServer(be)

	_, err = restic.CreateKey(s, pw)
	if err != nil {
		fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err)
		os.Exit(1)
	}

	fmt.Printf("created restic backend at %s\n", opts.Repo)

	return nil
}

// Open the backend specified by URI.
// Valid formats are:
// * /foo/bar -> local repository at /foo/bar
// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar
// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup
func open(u string) (backend.Backend, error) {
	url, err := url.Parse(u)
	if err != nil {
		return nil, err
	}

	if url.Scheme == "" {
		return backend.OpenLocal(url.Path)
	}

	args := []string{url.Host}
	if url.User != nil && url.User.Username() != "" {
		args = append(args, "-l")
		args = append(args, url.User.Username())
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return backend.OpenSFTP(url.Path[1:], "ssh", args...)
}

// Create the backend specified by URI.
func create(u string) (backend.Backend, error) {
	url, err := url.Parse(u)
	if err != nil {
		return nil, err
	}

	if url.Scheme == "" {
		return backend.CreateLocal(url.Path)
	}

	args := []string{url.Host}
	if url.User != nil && url.User.Username() != "" {
		args = append(args, "-l")
		args = append(args, url.User.Username())
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return backend.CreateSFTP(url.Path[1:], "ssh", args...)
}

func OpenRepo() (restic.Server, error) {
	if opts.Repo == "" {
		return restic.Server{}, errors.New("Please specify repository location (-r)")
	}

	be, err := open(opts.Repo)
	if err != nil {
		return restic.Server{}, err
	}

	s := restic.NewServer(be)

	err = s.SearchKey(readPassword("RESTIC_PASSWORD", "Enter Password for Repository: "))
	if err != nil {
		return restic.Server{}, fmt.Errorf("unable to open repo: %v", err)
	}

	return s, nil
}

func init() {
	// set GOMAXPROCS to number of CPUs
	runtime.GOMAXPROCS(runtime.NumCPU())

	_, err := parser.AddCommand("init",
		"create repository",
		"The init command creates a new repository",
		&CmdInit{})
	if err != nil {
		panic(err)
	}
}

func main() {
	// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
	opts.Repo = os.Getenv("RESTIC_REPOSITORY")

	_, err := parser.Parse()
	if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
		os.Exit(0)
	}

	if err != nil {
		os.Exit(1)
	}

	// restic.PoolAlloc()
}
<commit_msg>Write prompt to stderr instead of stdout<commit_after>package main

import (
	"errors"
	"fmt"
	"net/url"
	"os"
	"runtime"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/jessevdk/go-flags"
	"github.com/restic/restic"
	"github.com/restic/restic/backend"
)

var version = "compiled manually"

var opts struct {
	Repo string `short:"r" long:"repo"    description:"Repository directory to backup to/restore from"`
}

var parser = flags.NewParser(&opts, flags.Default)

func errx(code int, format string, data ...interface{}) {
	if len(format) > 0 && format[len(format)-1] != '\n' {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, data...)
	os.Exit(code)
}

func readPassword(env string, prompt string) string {

	if env != "" {
		p := os.Getenv(env)

		if p != "" {
			return p
		}
	}

	fmt.Fprint(os.Stderr, prompt)
	pw, err := terminal.ReadPassword(int(os.Stdin.Fd()))
	if err != nil {
		errx(2, "unable to read password: %v", err)
	}
	fmt.Fprintln(os.Stderr)

	return string(pw)
}

type CmdInit struct{}

func (cmd CmdInit) Execute(args []string) error {
	if opts.Repo == "" {
		return errors.New("Please specify repository location (-r)")
	}

	pw := readPassword("RESTIC_PASSWORD", "enter password for new backend: ")
	pw2 := readPassword("RESTIC_PASSWORD", "enter password again: ")

	if pw != pw2 {
		errx(1, "passwords do not match")
	}

	be, err := create(opts.Repo)
	if err != nil {
		fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", opts.Repo, err)
		os.Exit(1)
	}

	s := restic.NewServer(be)

	_, err = restic.CreateKey(s, pw)
	if err != nil {
		fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err)
		os.Exit(1)
	}

	fmt.Printf("created restic backend at %s\n", opts.Repo)

	return nil
}

// Open the backend specified by URI.
// Valid formats are:
// * /foo/bar -> local repository at /foo/bar
// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar
// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup
func open(u string) (backend.Backend, error) {
	url, err := url.Parse(u)
	if err != nil {
		return nil, err
	}

	if url.Scheme == "" {
		return backend.OpenLocal(url.Path)
	}

	args := []string{url.Host}
	if url.User != nil && url.User.Username() != "" {
		args = append(args, "-l")
		args = append(args, url.User.Username())
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return backend.OpenSFTP(url.Path[1:], "ssh", args...)
}

// Create the backend specified by URI.
func create(u string) (backend.Backend, error) {
	url, err := url.Parse(u)
	if err != nil {
		return nil, err
	}

	if url.Scheme == "" {
		return backend.CreateLocal(url.Path)
	}

	args := []string{url.Host}
	if url.User != nil && url.User.Username() != "" {
		args = append(args, "-l")
		args = append(args, url.User.Username())
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return backend.CreateSFTP(url.Path[1:], "ssh", args...)
}

func OpenRepo() (restic.Server, error) {
	if opts.Repo == "" {
		return restic.Server{}, errors.New("Please specify repository location (-r)")
	}

	be, err := open(opts.Repo)
	if err != nil {
		return restic.Server{}, err
	}

	s := restic.NewServer(be)

	err = s.SearchKey(readPassword("RESTIC_PASSWORD", "Enter Password for Repository: "))
	if err != nil {
		return restic.Server{}, fmt.Errorf("unable to open repo: %v", err)
	}

	return s, nil
}

func init() {
	// set GOMAXPROCS to number of CPUs
	runtime.GOMAXPROCS(runtime.NumCPU())

	_, err := parser.AddCommand("init",
		"create repository",
		"The init command creates a new repository",
		&CmdInit{})
	if err != nil {
		panic(err)
	}
}

func main() {
	// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
	opts.Repo = os.Getenv("RESTIC_REPOSITORY")

	_, err := parser.Parse()
	if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
		os.Exit(0)
	}

	if err != nil {
		os.Exit(1)
	}

	// restic.PoolAlloc()
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"strings"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	"github.com/forseti-security/config-validator/pkg/gcv"
	"github.com/golang/glog"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

var (
	policyPath = flag.String("policyPath", "", "directories, separated by comma, containing policy templates and configs")
	// TODO(corb): Template development will eventually inline library code, but the currently template examples have dependency rego code.
	//  This flag will be deprecated when the template tooling is complete.
	policyLibraryPath  = flag.String("policyLibraryPath", "", "directory containing policy templates and configs")
	port               = flag.Int("port", 10000, "The server port")
	maxMessageRecvSize = flag.Int(
		"maxMessageRecvSize", 128*1024*1024, "The max message receive size for the RPC service")
)

type gcvServer struct {
	validator *gcv.Validator
}

func (s *gcvServer) AddData(ctx context.Context, request *validator.AddDataRequest) (*validator.AddDataResponse, error) {
	err := s.validator.AddData(request)
	if err != nil {
		return &validator.AddDataResponse{}, status.Error(codes.Internal, err.Error())
	}
	return &validator.AddDataResponse{}, nil
}

func (s *gcvServer) Audit(ctx context.Context, request *validator.AuditRequest) (*validator.AuditResponse, error) {
	resp, err := s.validator.Audit(ctx)
	if err != nil {
		return resp, status.Error(codes.Internal, err.Error())
	}
	return resp, nil
}

func (s *gcvServer) Reset(ctx context.Context, request *validator.ResetRequest) (*validator.ResetResponse, error) {
	err := s.validator.Reset(ctx)
	if err != nil {
		return &validator.ResetResponse{}, status.Error(codes.Internal, err.Error())
	}
	return &validator.ResetResponse{}, nil
}

func (s *gcvServer) Review(ctx context.Context, request *validator.ReviewRequest) (*validator.ReviewResponse, error) {
	return s.validator.Review(ctx, request)
}

func newServer(stopChannel chan struct{}, policyPaths []string, policyLibraryPath string) (*gcvServer, error) {
	v, err := gcv.NewValidator(stopChannel, policyPaths, policyLibraryPath)
	if err != nil {
		return nil, err
	}
	return &gcvServer{
		validator: v,
	}, nil
}

func main() {
	flag.Parse()
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		log.Fatalf("failed to listen on port %d: %v", *port, err)
	}

	stopChannel := make(chan struct{})
	defer close(stopChannel)
	grpcServer := grpc.NewServer(
		grpc.MaxRecvMsgSize(*maxMessageRecvSize),
	)
	policyPaths := strings.Split(*policyPath, ",")
	serverImpl, err := newServer(stopChannel, policyPaths, *policyLibraryPath)
	if err != nil {
		log.Fatalf("Failed to load server %v", err)
	}
	validator.RegisterValidatorServer(grpcServer, serverImpl)
	if err := grpcServer.Serve(lis); err != nil {
		glog.Fatalf("RPC server ungracefully stopped: %v", err)
	}
}
<commit_msg>Policy library and path as env vars (#86)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strings"

	"github.com/forseti-security/config-validator/pkg/api/validator"
	"github.com/forseti-security/config-validator/pkg/gcv"
	"github.com/golang/glog"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

var (
	policyPath = flag.String("policyPath", os.Getenv("POLICY_PATH"), "directories, separated by comma, containing policy templates and configs")
	// TODO(corb): Template development will eventually inline library code, but the currently template examples have dependency rego code.
	//  This flag will be deprecated when the template tooling is complete.
	policyLibraryPath  = flag.String("policyLibraryPath", os.Getenv("POLICY_LIBRARY_PATH"), "directory containing policy templates and configs")
	port               = flag.Int("port", 10000, "The server port")
	maxMessageRecvSize = flag.Int(
		"maxMessageRecvSize", 128*1024*1024, "The max message receive size for the RPC service")
)

type gcvServer struct {
	validator *gcv.Validator
}

func (s *gcvServer) AddData(ctx context.Context, request *validator.AddDataRequest) (*validator.AddDataResponse, error) {
	err := s.validator.AddData(request)
	if err != nil {
		return &validator.AddDataResponse{}, status.Error(codes.Internal, err.Error())
	}
	return &validator.AddDataResponse{}, nil
}

func (s *gcvServer) Audit(ctx context.Context, request *validator.AuditRequest) (*validator.AuditResponse, error) {
	resp, err := s.validator.Audit(ctx)
	if err != nil {
		return resp, status.Error(codes.Internal, err.Error())
	}
	return resp, nil
}

func (s *gcvServer) Reset(ctx context.Context, request *validator.ResetRequest) (*validator.ResetResponse, error) {
	err := s.validator.Reset(ctx)
	if err != nil {
		return &validator.ResetResponse{}, status.Error(codes.Internal, err.Error())
	}
	return &validator.ResetResponse{}, nil
}

func (s *gcvServer) Review(ctx context.Context, request *validator.ReviewRequest) (*validator.ReviewResponse, error) {
	return s.validator.Review(ctx, request)
}

func newServer(stopChannel chan struct{}, policyPaths []string, policyLibraryPath string) (*gcvServer, error) {
	v, err := gcv.NewValidator(stopChannel, policyPaths, policyLibraryPath)
	if err != nil {
		return nil, err
	}
	return &gcvServer{
		validator: v,
	}, nil
}

func main() {
	flag.Parse()
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port))
	if err != nil {
		log.Fatalf("failed to listen on port %d: %v", *port, err)
	}

	stopChannel := make(chan struct{})
	defer close(stopChannel)
	grpcServer := grpc.NewServer(
		grpc.MaxRecvMsgSize(*maxMessageRecvSize),
	)
	policyPaths := strings.Split(*policyPath, ",")
	serverImpl, err := newServer(stopChannel, policyPaths, *policyLibraryPath)
	if err != nil {
		log.Fatalf("Failed to load server %v", err)
	}
	validator.RegisterValidatorServer(grpcServer, serverImpl)
	if err := grpcServer.Serve(lis); err != nil {
		glog.Fatalf("RPC server ungracefully stopped: %v", err)
	}
}
<|endoftext|>