<commit_before>// Package cmount implements a FUSE mounting system for rclone remotes.
//
// This uses the cgo based cgofuse library

// +build cmount
// +build cgo
// +build linux darwin freebsd windows

package cmount

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/billziss-gh/cgofuse/fuse"
	"github.com/pkg/errors"
	"github.com/rclone/rclone/cmd/mountlib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/vfs"
)

const (
	// SetCapReaddirPlus informs the host that the hosted file system has the readdir-plus
	// capability [Windows only]. A file system that has the readdir-plus capability can send
	// full stat information during Readdir, thus avoiding extraneous Getattr calls.
	usingReaddirPlus = runtime.GOOS == "windows"
)

func init() {
	name := "cmount"
	if runtime.GOOS == "windows" {
		name = "mount"
	}
	mountlib.NewMountCommand(name, false, mount)
	mountlib.AddRc("cmount", mount)
}

// mountOptions configures the options from the command line flags
func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
	// Options
	options = []string{
		"-o", "fsname=" + device,
		"-o", "subtype=rclone",
		"-o", fmt.Sprintf("max_readahead=%d", opt.MaxReadAhead),
		"-o", fmt.Sprintf("attr_timeout=%g", opt.AttrTimeout.Seconds()),
		// This causes FUSE to supply O_TRUNC with the Open
		// call which is more efficient for cmount.  However
		// it does not work with cgofuse on Windows with
		// WinFSP so cmount must work with or without it.
		"-o", "atomic_o_trunc",
	}
	if opt.DebugFUSE {
		options = append(options, "-o", "debug")
	}

	// OSX options
	if runtime.GOOS == "darwin" {
		if opt.NoAppleDouble {
			options = append(options, "-o", "noappledouble")
		}
		if opt.NoAppleXattr {
			options = append(options, "-o", "noapplexattr")
		}
	}

	// Windows options
	if runtime.GOOS == "windows" {
		// These cause WinFsp to mean the current user
		options = append(options, "-o", "uid=-1")
		options = append(options, "-o", "gid=-1")
		options = append(options, "--FileSystemName=rclone")
	}

	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
		if opt.VolumeName != "" {
			options = append(options, "-o", "volname="+opt.VolumeName)
		}
	}
	if opt.AllowNonEmpty {
		options = append(options, "-o", "nonempty")
	}
	if opt.AllowOther {
		options = append(options, "-o", "allow_other")
	}
	if opt.AllowRoot {
		options = append(options, "-o", "allow_root")
	}
	if opt.DefaultPermissions {
		options = append(options, "-o", "default_permissions")
	}
	if VFS.Opt.ReadOnly {
		options = append(options, "-o", "ro")
	}
	if opt.WritebackCache {
		// FIXME? options = append(options, "-o", WritebackCache())
	}
	if opt.DaemonTimeout != 0 {
		options = append(options, "-o", fmt.Sprintf("daemon_timeout=%d", int(opt.DaemonTimeout.Seconds())))
	}
	for _, option := range opt.ExtraOptions {
		options = append(options, "-o", option)
	}
	for _, option := range opt.ExtraFlags {
		options = append(options, option)
	}
	return options
}

// waitFor runs fn() until it returns true or the timeout expires
func waitFor(fn func() bool) (ok bool) {
	const totalWait = 10 * time.Second
	const individualWait = 10 * time.Millisecond
	for i := 0; i < int(totalWait/individualWait); i++ {
		ok = fn()
		if ok {
			return ok
		}
		time.Sleep(individualWait)
	}
	return false
}

// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error, func() error, error) {
	f := VFS.Fs()
	fs.Debugf(f, "Mounting on %q", mountpoint)

	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
	if runtime.GOOS != "windows" {
		fi, err := os.Stat(mountpoint)
		if err != nil {
			return nil, nil, errors.Wrap(err, "mountpoint")
		}
		if !fi.IsDir() {
			return nil, nil, errors.New("mountpoint is not a directory")
		}
	}

	// Create underlying FS
	fsys := NewFS(VFS)
	host := fuse.NewFileSystemHost(fsys)
	if usingReaddirPlus {
		host.SetCapReaddirPlus(true)
	}
	host.SetCapCaseInsensitive(f.Features().CaseInsensitive)

	// Create options
	options := mountOptions(VFS, f.Name()+":"+f.Root(), mountpoint, opt)
	fs.Debugf(f, "Mounting with options: %q", options)

	// Serve the mount point in the background returning error to errChan
	errChan := make(chan error, 1)
	go func() {
		var err error
		ok := host.Mount(mountpoint, options)
		if !ok {
			err = errors.New("mount failed")
			fs.Errorf(f, "Mount failed")
		}
		errChan <- err
	}()

	// unmount
	unmount := func() error {
		// Shutdown the VFS
		fsys.VFS.Shutdown()
		fs.Debugf(nil, "Calling host.Unmount")
		if host.Unmount() {
			fs.Debugf(nil, "host.Unmount succeeded")
			if runtime.GOOS == "windows" {
				if !waitFor(func() bool {
					_, err := os.Stat(mountpoint)
					return err != nil
				}) {
					fs.Errorf(nil, "mountpoint %q didn't disappear after unmount - continuing anyway", mountpoint)
				}
			}
			return nil
		}
		fs.Debugf(nil, "host.Unmount failed")
		return errors.New("host unmount failed")
	}

	// Wait for the filesystem to become ready, checking the file
	// system didn't blow up before starting
	select {
	case err := <-errChan:
		err = errors.Wrap(err, "mount stopped before calling Init")
		return nil, nil, err
	case <-fsys.ready:
	}

	// Wait for the mount point to be available on Windows
	// On Windows the Init signal comes slightly before the mount is ready
	if runtime.GOOS == "windows" {
		if !waitFor(func() bool {
			_, err := os.Stat(mountpoint)
			return err == nil
		}) {
			fs.Errorf(nil, "mountpoint %q didn't became available on mount - continuing anyway", mountpoint)
		}
	}

	return errChan, unmount, nil
}
<commit_msg>cmount: catch panics in initialization and turn into error messages<commit_after>// Package cmount implements a FUSE mounting system for rclone remotes.
//
// This uses the cgo based cgofuse library

// +build cmount
// +build cgo
// +build linux darwin freebsd windows

package cmount

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/billziss-gh/cgofuse/fuse"
	"github.com/pkg/errors"
	"github.com/rclone/rclone/cmd/mountlib"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/vfs"
)

const (
	// SetCapReaddirPlus informs the host that the hosted file system has the readdir-plus
	// capability [Windows only]. A file system that has the readdir-plus capability can send
	// full stat information during Readdir, thus avoiding extraneous Getattr calls.
	usingReaddirPlus = runtime.GOOS == "windows"
)

func init() {
	name := "cmount"
	if runtime.GOOS == "windows" {
		name = "mount"
	}
	mountlib.NewMountCommand(name, false, mount)
	mountlib.AddRc("cmount", mount)
}

// mountOptions configures the options from the command line flags
func mountOptions(VFS *vfs.VFS, device string, mountpoint string, opt *mountlib.Options) (options []string) {
	// Options
	options = []string{
		"-o", "fsname=" + device,
		"-o", "subtype=rclone",
		"-o", fmt.Sprintf("max_readahead=%d", opt.MaxReadAhead),
		"-o", fmt.Sprintf("attr_timeout=%g", opt.AttrTimeout.Seconds()),
		// This causes FUSE to supply O_TRUNC with the Open
		// call which is more efficient for cmount.  However
		// it does not work with cgofuse on Windows with
		// WinFSP so cmount must work with or without it.
		"-o", "atomic_o_trunc",
	}
	if opt.DebugFUSE {
		options = append(options, "-o", "debug")
	}

	// OSX options
	if runtime.GOOS == "darwin" {
		if opt.NoAppleDouble {
			options = append(options, "-o", "noappledouble")
		}
		if opt.NoAppleXattr {
			options = append(options, "-o", "noapplexattr")
		}
	}

	// Windows options
	if runtime.GOOS == "windows" {
		// These cause WinFsp to mean the current user
		options = append(options, "-o", "uid=-1")
		options = append(options, "-o", "gid=-1")
		options = append(options, "--FileSystemName=rclone")
	}

	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
		if opt.VolumeName != "" {
			options = append(options, "-o", "volname="+opt.VolumeName)
		}
	}
	if opt.AllowNonEmpty {
		options = append(options, "-o", "nonempty")
	}
	if opt.AllowOther {
		options = append(options, "-o", "allow_other")
	}
	if opt.AllowRoot {
		options = append(options, "-o", "allow_root")
	}
	if opt.DefaultPermissions {
		options = append(options, "-o", "default_permissions")
	}
	if VFS.Opt.ReadOnly {
		options = append(options, "-o", "ro")
	}
	if opt.WritebackCache {
		// FIXME? options = append(options, "-o", WritebackCache())
	}
	if opt.DaemonTimeout != 0 {
		options = append(options, "-o", fmt.Sprintf("daemon_timeout=%d", int(opt.DaemonTimeout.Seconds())))
	}
	for _, option := range opt.ExtraOptions {
		options = append(options, "-o", option)
	}
	for _, option := range opt.ExtraFlags {
		options = append(options, option)
	}
	return options
}

// waitFor runs fn() until it returns true or the timeout expires
func waitFor(fn func() bool) (ok bool) {
	const totalWait = 10 * time.Second
	const individualWait = 10 * time.Millisecond
	for i := 0; i < int(totalWait/individualWait); i++ {
		ok = fn()
		if ok {
			return ok
		}
		time.Sleep(individualWait)
	}
	return false
}

// mount the file system
//
// The mount point will be ready when this returns.
//
// returns an error, and an error channel for the serve process to
// report an error when fusermount is called.
func mount(VFS *vfs.VFS, mountpoint string, opt *mountlib.Options) (<-chan error, func() error, error) {
	f := VFS.Fs()
	fs.Debugf(f, "Mounting on %q", mountpoint)

	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
	if runtime.GOOS != "windows" {
		fi, err := os.Stat(mountpoint)
		if err != nil {
			return nil, nil, errors.Wrap(err, "mountpoint")
		}
		if !fi.IsDir() {
			return nil, nil, errors.New("mountpoint is not a directory")
		}
	}

	// Create underlying FS
	fsys := NewFS(VFS)
	host := fuse.NewFileSystemHost(fsys)
	if usingReaddirPlus {
		host.SetCapReaddirPlus(true)
	}
	host.SetCapCaseInsensitive(f.Features().CaseInsensitive)

	// Create options
	options := mountOptions(VFS, f.Name()+":"+f.Root(), mountpoint, opt)
	fs.Debugf(f, "Mounting with options: %q", options)

	// Serve the mount point in the background returning error to errChan
	errChan := make(chan error, 1)
	go func() {
		defer func() {
			if r := recover(); r != nil {
				errChan <- errors.Errorf("mount failed: %v", r)
			}
		}()
		var err error
		ok := host.Mount(mountpoint, options)
		if !ok {
			err = errors.New("mount failed")
			fs.Errorf(f, "Mount failed")
		}
		errChan <- err
	}()

	// unmount
	unmount := func() error {
		// Shutdown the VFS
		fsys.VFS.Shutdown()
		fs.Debugf(nil, "Calling host.Unmount")
		if host.Unmount() {
			fs.Debugf(nil, "host.Unmount succeeded")
			if runtime.GOOS == "windows" {
				if !waitFor(func() bool {
					_, err := os.Stat(mountpoint)
					return err != nil
				}) {
					fs.Errorf(nil, "mountpoint %q didn't disappear after unmount - continuing anyway", mountpoint)
				}
			}
			return nil
		}
		fs.Debugf(nil, "host.Unmount failed")
		return errors.New("host unmount failed")
	}

	// Wait for the filesystem to become ready, checking the file
	// system didn't blow up before starting
	select {
	case err := <-errChan:
		err = errors.Wrap(err, "mount stopped before calling Init")
		return nil, nil, err
	case <-fsys.ready:
	}

	// Wait for the mount point to be available on Windows
	// On Windows the Init signal comes slightly before the mount is ready
	if runtime.GOOS == "windows" {
		if !waitFor(func() bool {
			_, err := os.Stat(mountpoint)
			return err == nil
		}) {
			fs.Errorf(nil, "mountpoint %q didn't became available on mount - continuing anyway", mountpoint)
		}
	}

	return errChan, unmount, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"strconv"

	"chain/core"
	"chain/core/mockhsm"
	"chain/database/sql"
	"chain/env"
	"chain/log"
)

// config vars
var (
	dbURL = env.String("DATABASE_URL", "postgres:///core?sslmode=disable")
)

// We collect log output in this buffer,
// and display it only when there's an error.
var logbuf bytes.Buffer

type command struct {
	f func(*sql.DB, []string)
}

var commands = map[string]*command{
	"config-generator":     {configGenerator},
	"create-block-keypair": {createBlockKeyPair},
	"config":               {configNongenerator},
}

func main() {
	log.SetOutput(&logbuf)
	env.Parse()
	db, err := sql.Open("hapg", *dbURL)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(2)
	}

	if len(os.Args) < 2 {
		help(os.Stdout)
		os.Exit(0)
	}
	cmd := commands[os.Args[1]]
	if cmd == nil {
		fmt.Fprintln(os.Stderr, "unknown command:", os.Args[1])
		help(os.Stderr)
		os.Exit(1)
	}
	cmd.f(db, os.Args[2:])
}

func configGenerator(db *sql.DB, args []string) {
	const usage = "error: corectl config-generator [-s] [<quorum> <<pubkey> <url>...>]"
	var (
		isSigner bool
		quorum   int
		signers  []core.ConfigSigner
		err      error
	)

	if len(args) > 0 && args[0] == "-s" {
		isSigner = true
		args = args[1:]
	}

	if len(args) == 0 {
		if isSigner {
			quorum = 1
		}
	} else if len(args)%2 != 1 {
		fatalln(usage)
	} else {
		quorum, err = strconv.Atoi(args[0])
		if err != nil {
			fatalln(usage)
		}

		for i := 1; i < len(args); i += 2 {
			pubkey, err := hex.DecodeString(args[i])
			if err != nil {
				fatalln(usage)
			}
			url := args[i+1]
			signers = append(signers, core.ConfigSigner{
				Pubkey: pubkey,
				URL:    url,
			})
		}
	}

	config := &core.Config{
		IsGenerator: true,
		IsSigner:    isSigner,
		Quorum:      quorum,
		Signers:     signers,
	}

	ctx := context.Background()
	err = core.Configure(ctx, db, config)
	if err != nil {
		fatalln("error:", err)
	}

	fmt.Println("blockchain id", config.BlockchainID)
}

func createBlockKeyPair(db *sql.DB, args []string) {
	if len(args) != 0 {
		fatalln("error: create-block-keypair takes no args")
	}

	hsm := mockhsm.New(db)
	ctx := context.Background()
	xpub, err := hsm.CreateKey(ctx, "block_key")
	if err != nil {
		fatalln("error:", err)
	}

	fmt.Println("block xpub:", xpub.XPub.String())
}

func configNongenerator(db *sql.DB, args []string) {
	errUsage := "error: corectl config <blockchain-id> <generator-url> [-t <generator-access-token>] [-k <block-pubkey>]"
	if len(args) < 2 {
		fatalln(errUsage)
	}

	var config core.Config
	err := config.BlockchainID.UnmarshalText([]byte(args[0]))
	if err != nil {
		fatalln("error: invalid blockchain ID:", err)
	}
	config.GeneratorURL = args[1]

	for args = args[2:]; len(args) > 0; args = args[2:] {
		if len(args) < 2 {
			fatalln(errUsage)
		}

		switch args[0] {
		case "-t":
			config.GeneratorAccessToken = args[1]
		case "-k":
			config.IsSigner = true
			config.BlockXPub = args[1]
		default:
			fatalln(errUsage)
		}
	}

	ctx := context.Background()
	err = core.Configure(ctx, db, &config)
	if err != nil {
		fatalln("error:", err)
	}
}

func fatalln(v ...interface{}) {
	io.Copy(os.Stderr, &logbuf)
	fmt.Fprintln(os.Stderr, v...)
	os.Exit(2)
}

func help(w io.Writer) {
	fmt.Fprintln(w, "usage: corectl [command] [arguments]")
	fmt.Fprint(w, "\nThe commands are:\n\n")
	for name := range commands {
		fmt.Fprintln(w, "\t", name)
	}
	fmt.Fprintln(w)
}
<commit_msg>cmd/corectl: silently truncate xpubs to bare pubkeys on input<commit_after>package main

import (
	"bytes"
	"context"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"strconv"

	"chain/core"
	"chain/core/mockhsm"
	"chain/crypto/ed25519"
	"chain/database/sql"
	"chain/env"
	"chain/log"
)

// config vars
var (
	dbURL = env.String("DATABASE_URL", "postgres:///core?sslmode=disable")
)

// We collect log output in this buffer,
// and display it only when there's an error.
var logbuf bytes.Buffer

type command struct {
	f func(*sql.DB, []string)
}

var commands = map[string]*command{
	"config-generator":     {configGenerator},
	"create-block-keypair": {createBlockKeyPair},
	"config":               {configNongenerator},
}

func main() {
	log.SetOutput(&logbuf)
	env.Parse()
	db, err := sql.Open("hapg", *dbURL)
	if err != nil {
		fmt.Fprintln(os.Stderr, "error:", err)
		os.Exit(2)
	}

	if len(os.Args) < 2 {
		help(os.Stdout)
		os.Exit(0)
	}
	cmd := commands[os.Args[1]]
	if cmd == nil {
		fmt.Fprintln(os.Stderr, "unknown command:", os.Args[1])
		help(os.Stderr)
		os.Exit(1)
	}
	cmd.f(db, os.Args[2:])
}

func configGenerator(db *sql.DB, args []string) {
	const usage = "error: corectl config-generator [-s] [<quorum> <<pubkey> <url>...>]"
	var (
		isSigner bool
		quorum   int
		signers  []core.ConfigSigner
		err      error
	)

	if len(args) > 0 && args[0] == "-s" {
		isSigner = true
		args = args[1:]
	}

	if len(args) == 0 {
		if isSigner {
			quorum = 1
		}
	} else if len(args)%2 != 1 {
		fatalln(usage)
	} else {
		quorum, err = strconv.Atoi(args[0])
		if err != nil {
			fatalln(usage)
		}

		for i := 1; i < len(args); i += 2 {
			pubkey, err := hex.DecodeString(args[i])
			if err != nil {
				fatalln(usage)
			}
			url := args[i+1]
			signers = append(signers, core.ConfigSigner{
				// Silently truncate the input (which is likely to be an xpub
				// produced by the create-block-keypair subcommand) to
				// bare-pubkey size.
				// TODO(bobg): When the mockhsm can produce bare pubkeys,
				// treat xpubs on input as an error instead.
				Pubkey: pubkey[:ed25519.PublicKeySize],
				URL:    url,
			})
		}
	}

	config := &core.Config{
		IsGenerator: true,
		IsSigner:    isSigner,
		Quorum:      quorum,
		Signers:     signers,
	}

	ctx := context.Background()
	err = core.Configure(ctx, db, config)
	if err != nil {
		fatalln("error:", err)
	}

	fmt.Println("blockchain id", config.BlockchainID)
}

func createBlockKeyPair(db *sql.DB, args []string) {
	if len(args) != 0 {
		fatalln("error: create-block-keypair takes no args")
	}

	hsm := mockhsm.New(db)
	ctx := context.Background()
	xpub, err := hsm.CreateKey(ctx, "block_key")
	if err != nil {
		fatalln("error:", err)
	}

	fmt.Println("block xpub:", xpub.XPub.String())
}

func configNongenerator(db *sql.DB, args []string) {
	errUsage := "error: corectl config <blockchain-id> <generator-url> [-t <generator-access-token>] [-k <block-pubkey>]"
	if len(args) < 2 {
		fatalln(errUsage)
	}

	var config core.Config
	err := config.BlockchainID.UnmarshalText([]byte(args[0]))
	if err != nil {
		fatalln("error: invalid blockchain ID:", err)
	}
	config.GeneratorURL = args[1]

	for args = args[2:]; len(args) > 0; args = args[2:] {
		if len(args) < 2 {
			fatalln(errUsage)
		}

		switch args[0] {
		case "-t":
			config.GeneratorAccessToken = args[1]
		case "-k":
			config.IsSigner = true
			config.BlockXPub = args[1]
		default:
			fatalln(errUsage)
		}
	}

	ctx := context.Background()
	err = core.Configure(ctx, db, &config)
	if err != nil {
		fatalln("error:", err)
	}
}

func fatalln(v ...interface{}) {
	io.Copy(os.Stderr, &logbuf)
	fmt.Fprintln(os.Stderr, v...)
	os.Exit(2)
}

func help(w io.Writer) {
	fmt.Fprintln(w, "usage: corectl [command] [arguments]")
	fmt.Fprint(w, "\nThe commands are:\n\n")
	for name := range commands {
		fmt.Fprintln(w, "\t", name)
	}
	fmt.Fprintln(w)
}
<|endoftext|>


<commit_before>package main

import (
	"dokugen"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
)

//TODO: let people pass in a filename to export to.

type appOptions struct {
	GENERATE            bool
	HELP                bool
	PUZZLE_TO_SOLVE     string
	NUM                 int
	PRINT_STATS         bool
	WALKTHROUGH         bool
	RAW_SYMMETRY        string
	SYMMETRY            sudoku.SymmetryType
	SYMMETRY_PROPORTION float64
}

func main() {

	//TODO: figure out how to test this.

	var options appOptions

	flag.BoolVar(&options.GENERATE, "g", false, "if true, will generate a puzzle.")
	flag.BoolVar(&options.HELP, "h", false, "If provided, will print help and exit.")
	flag.IntVar(&options.NUM, "n", 1, "Number of things to generate")
	flag.BoolVar(&options.PRINT_STATS, "p", false, "If provided, will print stats.")
	flag.StringVar(&options.PUZZLE_TO_SOLVE, "s", "", "If provided, will solve the puzzle at the given filename and print solution.")
	flag.BoolVar(&options.WALKTHROUGH, "w", false, "If provided, will print out a walkthrough to solve the provided puzzle.")
	flag.StringVar(&options.RAW_SYMMETRY, "y", "vertical", "Valid values: 'none', 'both', 'horizontal', 'vertical")
	flag.Float64Var(&options.SYMMETRY_PROPORTION, "r", 1.0, "What proportion of cells should be filled according to symmetry")

	flag.Parse()

	options.RAW_SYMMETRY = strings.ToLower(options.RAW_SYMMETRY)
	switch options.RAW_SYMMETRY {
	case "none":
		options.SYMMETRY = sudoku.SYMMETRY_NONE
	case "both":
		options.SYMMETRY = sudoku.SYMMETRY_BOTH
	case "horizontal":
		options.SYMMETRY = sudoku.SYMMETRY_HORIZONTAL
	case "vertical":
		options.SYMMETRY = sudoku.SYMMETRY_VERTICAL
	default:
		log.Fatal("Unknown symmetry flag: ", options.RAW_SYMMETRY)
	}

	output := os.Stdout

	if options.HELP {
		flag.PrintDefaults()
		return
	}

	var grid *sudoku.Grid

	for i := 0; i < options.NUM; i++ {
		//TODO: allow the type of symmetry to be configured.
		if options.GENERATE {
			grid = sudoku.GenerateGrid(options.SYMMETRY, options.SYMMETRY_PROPORTION)
			fmt.Fprintln(output, grid.DataString())
			fmt.Fprintln(output, "\n")
		} else if options.PUZZLE_TO_SOLVE != "" {
			//TODO: detect if the load failed.
			grid := sudoku.NewGrid()
			grid.LoadFromFile(options.PUZZLE_TO_SOLVE)
		}
		//TODO: use of this option leads to a busy loop somewhere... Is it related to the generate-multiple-and-difficulty hang?
		if options.WALKTHROUGH {
			fmt.Fprintln(output, grid.HumanWalkthrough())
			fmt.Fprintln(output, "\n")
		}
		if options.PRINT_STATS {
			fmt.Fprintln(output, "\n")
			fmt.Fprintln(output, grid.Difficulty())
		}
		if options.PUZZLE_TO_SOLVE != "" {
			grid.Solve()
			fmt.Fprintln(output, grid.DataString())
			//If we're asked to solve, n could only be 1 anyway.
			return
		}
	}

}
<commit_msg>dokugen, when -p is provided, now prints out a SolveDirections.Stats()<commit_after>package main

import (
	"dokugen"
	"flag"
	"fmt"
	"log"
	"os"
	"strings"
)

//TODO: let people pass in a filename to export to.

type appOptions struct {
	GENERATE            bool
	HELP                bool
	PUZZLE_TO_SOLVE     string
	NUM                 int
	PRINT_STATS         bool
	WALKTHROUGH         bool
	RAW_SYMMETRY        string
	SYMMETRY            sudoku.SymmetryType
	SYMMETRY_PROPORTION float64
}

func main() {

	//TODO: figure out how to test this.

	var options appOptions

	flag.BoolVar(&options.GENERATE, "g", false, "if true, will generate a puzzle.")
	flag.BoolVar(&options.HELP, "h", false, "If provided, will print help and exit.")
	flag.IntVar(&options.NUM, "n", 1, "Number of things to generate")
	flag.BoolVar(&options.PRINT_STATS, "p", false, "If provided, will print stats.")
	flag.StringVar(&options.PUZZLE_TO_SOLVE, "s", "", "If provided, will solve the puzzle at the given filename and print solution.")
	flag.BoolVar(&options.WALKTHROUGH, "w", false, "If provided, will print out a walkthrough to solve the provided puzzle.")
	flag.StringVar(&options.RAW_SYMMETRY, "y", "vertical", "Valid values: 'none', 'both', 'horizontal', 'vertical")
	flag.Float64Var(&options.SYMMETRY_PROPORTION, "r", 1.0, "What proportion of cells should be filled according to symmetry")

	flag.Parse()

	options.RAW_SYMMETRY = strings.ToLower(options.RAW_SYMMETRY)
	switch options.RAW_SYMMETRY {
	case "none":
		options.SYMMETRY = sudoku.SYMMETRY_NONE
	case "both":
		options.SYMMETRY = sudoku.SYMMETRY_BOTH
	case "horizontal":
		options.SYMMETRY = sudoku.SYMMETRY_HORIZONTAL
	case "vertical":
		options.SYMMETRY = sudoku.SYMMETRY_VERTICAL
	default:
		log.Fatal("Unknown symmetry flag: ", options.RAW_SYMMETRY)
	}

	output := os.Stdout

	if options.HELP {
		flag.PrintDefaults()
		return
	}

	var grid *sudoku.Grid

	for i := 0; i < options.NUM; i++ {
		//TODO: allow the type of symmetry to be configured.
		if options.GENERATE {
			grid = sudoku.GenerateGrid(options.SYMMETRY, options.SYMMETRY_PROPORTION)
			fmt.Fprintln(output, grid.DataString())
			fmt.Fprintln(output, "\n")
		} else if options.PUZZLE_TO_SOLVE != "" {
			//TODO: detect if the load failed.
			grid := sudoku.NewGrid()
			grid.LoadFromFile(options.PUZZLE_TO_SOLVE)
		}
		//TODO: use of this option leads to a busy loop somewhere... Is it related to the generate-multiple-and-difficulty hang?
		if options.WALKTHROUGH {
			fmt.Fprintln(output, grid.HumanWalkthrough())
			fmt.Fprintln(output, "\n")
		}
		if options.PRINT_STATS {
			fmt.Fprintln(output, "\n")
			fmt.Fprintln(output, grid.Difficulty())
			fmt.Fprintln(output, "\n")
			directions := grid.HumanSolution()
			fmt.Fprintln(output, strings.Join(directions.Stats(), "\n"))
		}
		if options.PUZZLE_TO_SOLVE != "" {
			grid.Solve()
			fmt.Fprintln(output, grid.DataString())
			//If we're asked to solve, n could only be 1 anyway.
			return
		}
	}

}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2021 IBM, Inc.
 *
 */

package watcher

import (
	"fmt"
	"sync"
	"time"

	k8sv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"

	kvv1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/client-go/log"

	"kubevirt.io/kubevirt/pkg/controller"

	"k8s.io/client-go/tools/cache"

	objUtils "kubevirt.io/kubevirt/tools/perfscale-load-generator/object"
)

const (
	informerTimeout = time.Minute
)

type ObjListWatcher struct {
	sync.Mutex

	// runningObjs is a set to avoid recount duplicated events of an object
	runningObjs            map[string]bool
	desiredObjRunningCount int
	updateChannel          chan bool

	virtCli      kubecli.KubevirtClient
	listOpts     metav1.ListOptions
	ResourceKind string

	informer    cache.SharedInformer
	stopChannel chan struct{}
}

func NewObjListWatcher(virtCli kubecli.KubevirtClient, resourceKind string, objCount int, listOpts metav1.ListOptions) *ObjListWatcher {
	w := &ObjListWatcher{
		stopChannel:            make(chan struct{}),
		updateChannel:          make(chan bool),
		runningObjs:            make(map[string]bool),
		virtCli:                virtCli,
		ResourceKind:           resourceKind,
		desiredObjRunningCount: objCount,
		listOpts:               listOpts,
	}

	labelSelector, err := labels.Parse(w.listOpts.LabelSelector)
	if err != nil {
		panic(err)
	}
	objListWatcher := controller.NewListWatchFromClient(
		virtCli.RestClient(),
		w.ResourceKind,
		k8sv1.NamespaceAll,
		fields.Everything(),
		labelSelector)

	w.informer = cache.NewSharedInformer(objListWatcher, nil, 0)
	w.stopChannel = make(chan struct{})
	w.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: func(oldObj, newObj interface{}) {
			w.handleUpdate(newObj)
		},
		DeleteFunc: func(obj interface{}) {
			w.handleDeleted(obj)
		},
	})

	return w
}

func (w *ObjListWatcher) Run() {
	go w.informer.Run(w.stopChannel)
	timeoutCh := make(chan struct{})
	timeoutTimer := time.AfterFunc(informerTimeout, func() {
		close(timeoutCh)
	})
	defer timeoutTimer.Stop()
	if !cache.WaitForCacheSync(timeoutCh, w.informer.HasSynced) {
		panic(fmt.Errorf("watcher timed out waiting for caches to sync"))
	}
	// waiting 1s to prevent the watcher miss events from the test
	time.Sleep(1 * time.Second)
}

func (w *ObjListWatcher) Stop() {
	close(w.stopChannel)
}

func (w *ObjListWatcher) handleUpdate(obj interface{}) {
	switch w.ResourceKind {
	case objUtils.VMIResource:
		vmi, ok := obj.(*kvv1.VirtualMachineInstance)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstance obj: %v", w.ResourceKind)
		}
		if vmi.Status.Phase == kvv1.Running {
			w.addObj(string(vmi.GetUID()))
		}
		if vmi.Status.Phase == kvv1.Succeeded {
			w.deleteObj(string(vmi.GetUID()))
		}

	case objUtils.VMResource:
		vm, ok := obj.(*kvv1.VirtualMachine)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachine obj: %v", w.ResourceKind)
		}
		if vm.Status.Ready {
			w.addObj(string(vm.GetUID()))
		}

	case objUtils.VMIReplicaSetResource:
		replicaSet, ok := obj.(*kvv1.VirtualMachineInstanceReplicaSet)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstanceReplicaSet obj: %v", w.ResourceKind)
		}
		w.addObj(string(replicaSet.GetUID()))

	default:
		log.Log.V(2).Errorf("Watcher does not support object type %s", w.ResourceKind)
		return
	}
}

func (w *ObjListWatcher) handleDeleted(obj interface{}) {
	switch w.ResourceKind {
	case objUtils.VMResource:
		vm, ok := obj.(*kvv1.VirtualMachine)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachine obj: %v", w.ResourceKind)
		}
		w.deleteObj(string(vm.GetUID()))

	case objUtils.VMIReplicaSetResource:
		replicaSet, ok := obj.(*kvv1.VirtualMachineInstanceReplicaSet)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstanceReplicaSet obj: %v", w.ResourceKind)
		}
		w.deleteObj(string(replicaSet.GetUID()))

	default:
		return
	}
}

func (w *ObjListWatcher) WaitRunning(timeout time.Duration) error {
	if w.getRunningCount() == w.desiredObjRunningCount {
		return nil
	}
	for {
		select {
		case <-time.After(timeout):
			return fmt.Errorf("timeout after %s waiting for objects be running", timeout)

		case <-w.updateChannel:
			count := w.getRunningCount()
			if count == w.desiredObjRunningCount {
				return nil
			}
			log.Log.V(6).Infof("Waiting %d %v to be Running", (w.desiredObjRunningCount - count), w.ResourceKind)
		}
	}
}

func (w *ObjListWatcher) WaitDeletion(timeout time.Duration) error {
	runningCount := w.getRunningCount()
	if runningCount == 0 || runningCount == w.desiredObjRunningCount {
		return nil
	}
	for {
		select {
		case <-time.After(timeout):
			return fmt.Errorf("timeout after %s waiting for objects be deleted", timeout)

		case <-w.updateChannel:
			count := w.getRunningCount()
			if count == 0 || count == w.desiredObjRunningCount {
				return nil
			}
			log.Log.V(6).Infof("Still %d Running %v", count, w.ResourceKind)
		}
	}
}

func (w *ObjListWatcher) addObj(id string) {
	w.Lock()
	w.runningObjs[id] = true
	w.Unlock()
	w.updateChannel <- true
}

func (w *ObjListWatcher) deleteObj(id string) {
	w.Lock()
	delete(w.runningObjs, id)
	w.Unlock()
	w.updateChannel <- true
}

func (w *ObjListWatcher) getRunningCount() int {
	var count int
	w.Lock()
	count = len(w.runningObjs)
	w.Unlock()
	return count
}
<commit_msg>Add isGarbageCollected to perfscale-load-generator ObjListWatcher<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2021 IBM, Inc.
 *
 */

package watcher

import (
	"context"
	"fmt"
	"k8s.io/apimachinery/pkg/api/errors"
	"sync"
	"time"

	k8sv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/labels"

	kvv1 "kubevirt.io/api/core/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/client-go/log"

	"kubevirt.io/kubevirt/pkg/controller"

	"k8s.io/client-go/tools/cache"

	objUtils "kubevirt.io/kubevirt/tools/perfscale-load-generator/object"
)

const (
	informerTimeout = time.Minute
)

type ObjListWatcher struct {
	sync.Mutex

	// runningObjs is a set to avoid recount duplicated events of an object
	runningObjs            map[string]bool
	desiredObjRunningCount int
	updateChannel          chan bool

	virtCli      kubecli.KubevirtClient
	listOpts     metav1.ListOptions
	ResourceKind string

	informer    cache.SharedInformer
	stopChannel chan struct{}
}

func NewObjListWatcher(virtCli kubecli.KubevirtClient, resourceKind string, objCount int, listOpts metav1.ListOptions) *ObjListWatcher {
	w := &ObjListWatcher{
		stopChannel:            make(chan struct{}),
		updateChannel:          make(chan bool),
		runningObjs:            make(map[string]bool),
		virtCli:                virtCli,
		ResourceKind:           resourceKind,
		desiredObjRunningCount: objCount,
		listOpts:               listOpts,
	}

	labelSelector, err := labels.Parse(w.listOpts.LabelSelector)
	if err != nil {
		panic(err)
	}
	objListWatcher := controller.NewListWatchFromClient(
		virtCli.RestClient(),
		w.ResourceKind,
		k8sv1.NamespaceAll,
		fields.Everything(),
		labelSelector)

	w.informer = cache.NewSharedInformer(objListWatcher, nil, 0)
	w.stopChannel = make(chan struct{})
	w.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		UpdateFunc: func(oldObj, newObj interface{}) {
			w.handleUpdate(newObj)
		},
		DeleteFunc: func(obj interface{}) {
			w.handleDeleted(obj)
		},
	})

	return w
}

func (w *ObjListWatcher) Run() {
	go w.informer.Run(w.stopChannel)
	timeoutCh := make(chan struct{})
	timeoutTimer := time.AfterFunc(informerTimeout, func() {
		close(timeoutCh)
	})
	defer timeoutTimer.Stop()
	if !cache.WaitForCacheSync(timeoutCh, w.informer.HasSynced) {
		panic(fmt.Errorf("watcher timed out waiting for caches to sync"))
	}
	// waiting 1s to prevent the watcher miss events from the test
	time.Sleep(1 * time.Second)
}

func (w *ObjListWatcher) Stop() {
	close(w.stopChannel)
}

func (w *ObjListWatcher) handleUpdate(obj interface{}) {
	switch w.ResourceKind {
	case objUtils.VMIResource:
		vmi, ok := obj.(*kvv1.VirtualMachineInstance)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstance obj: %v", w.ResourceKind)
		}
		if vmi.Status.Phase == kvv1.Running {
			w.addObj(string(vmi.GetUID()))
		}
		if vmi.Status.Phase == kvv1.Succeeded || vmi.Status.Phase == kvv1.Failed {
			log.Log.V(4).Errorf("VirtualMachineInstance obj: %v Succeeded, waiting for garbage collection", fmt.Sprintf("%s/%s", vmi.Namespace, vmi.Name))
		}

	case objUtils.VMResource:
		vm, ok := obj.(*kvv1.VirtualMachine)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachine obj: %v", w.ResourceKind)
		}
		if vm.Status.Ready {
			w.addObj(string(vm.GetUID()))
		}

	case objUtils.VMIReplicaSetResource:
		replicaSet, ok := obj.(*kvv1.VirtualMachineInstanceReplicaSet)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstanceReplicaSet obj: %v", w.ResourceKind)
		}
		w.addObj(string(replicaSet.GetUID()))

	default:
		log.Log.V(2).Errorf("Watcher does not support object type %s", w.ResourceKind)
		return
	}
}

func (w *ObjListWatcher) handleDeleted(obj interface{}) {
	switch w.ResourceKind {
	case objUtils.VMResource:
		vm, ok := obj.(*kvv1.VirtualMachine)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachine obj: %v", w.ResourceKind)
			return
		}
		isGarbageCollected, err := w.isGarbageCollected(vm)
		if err != nil {
			log.Log.V(2).Errorf("Could not get VirtualMachine: %v, err: %#v", fmt.Sprintf("%s/%s", vm.Namespace, vm.Name), err)
		}
		if !isGarbageCollected {
			log.Log.V(4).Errorf("VirtualMachine obj: %v has not been garbage collected yet", fmt.Sprintf("%s/%s", vm.Namespace, vm.Name))
			return
		}
		w.deleteObj(string(vm.GetUID()))

	case objUtils.VMIReplicaSetResource:
		replicaSet, ok := obj.(*kvv1.VirtualMachineInstanceReplicaSet)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstanceReplicaSet obj: %v", w.ResourceKind)
			return
		}
		isGarbageCollected, err := w.isGarbageCollected(replicaSet)
		if err != nil {
			log.Log.V(2).Errorf("Could not get VirtualMachineInstanceReplicaSet: %v, err: %#v", fmt.Sprintf("%s/%s", replicaSet.Namespace, replicaSet.Name), err)
		}
		if !isGarbageCollected {
			log.Log.V(4).Errorf("VirtualMachineInstanceReplicaSet obj: %v has not been garbage collected yet", fmt.Sprintf("%s/%s", replicaSet.Namespace, replicaSet.Name))
			return
		}
		w.deleteObj(string(replicaSet.GetUID()))
	case objUtils.VMIResource:
		vmi, ok := obj.(*kvv1.VirtualMachineInstance)
		if !ok {
			log.Log.V(2).Errorf("Could not convert VirtualMachineInstance obj: %v", w.ResourceKind)
			return
		}
		isGarbageCollected, err := w.isGarbageCollected(vmi)
		if err != nil {
			log.Log.V(2).Errorf("Could not get VirtualMachineInstance: %v, err: %#v", fmt.Sprintf("%s/%s", vmi.Namespace, vmi.Name), err)
		}
		if !isGarbageCollected {
			log.Log.V(4).Errorf("VirtualMachineInstance obj: %v has not been garbage collected yet", fmt.Sprintf("%s/%s", vmi.Namespace, vmi.Name))
			return
		}
		w.deleteObj(string(vmi.GetUID()))

	default:
		return
	}
}

func (w *ObjListWatcher) WaitRunning(timeout time.Duration) error {
	if w.getRunningCount() == w.desiredObjRunningCount {
		return nil
	}
	for {
		select {
		case <-time.After(timeout):
			return fmt.Errorf("timeout after %s waiting for objects be running", timeout)

		case <-w.updateChannel:
			count := w.getRunningCount()
			if count == w.desiredObjRunningCount {
				return nil
			}
			log.Log.V(6).Infof("Waiting %d %v to be Running", (w.desiredObjRunningCount - count), w.ResourceKind)
		}
	}
}

func (w *ObjListWatcher) WaitDeletion(timeout time.Duration) error {
	runningCount := w.getRunningCount()
	if runningCount == 0 || runningCount == w.desiredObjRunningCount {
		return nil
	}
	for {
		select {
		case <-time.After(timeout):
			return fmt.Errorf("timeout after %s waiting for objects be deleted", timeout)

		case <-w.updateChannel:
			count := w.getRunningCount()
			if count == 0 || count == w.desiredObjRunningCount {
				return nil
			}
			log.Log.V(6).Infof("Still %d %v waiting to be Garbage Collected", count, w.ResourceKind)
		}
	}
}

func (w *ObjListWatcher) addObj(id string) {
	w.Lock()
	w.runningObjs[id] = true
	w.Unlock()
	w.updateChannel <- true
}

func (w *ObjListWatcher) deleteObj(id string) {
	w.Lock()
	delete(w.runningObjs, id)
	w.Unlock()
	w.updateChannel <- true
}

func (w *ObjListWatcher) getRunningCount() int {
	var count int
	w.Lock()
	count = len(w.runningObjs)
	w.Unlock()
	return count
}

func (w *ObjListWatcher) isGarbageCollected(obj metav1.Object) (bool, error) {
	err := w.virtCli.RestClient().Get().
		Namespace(obj.GetNamespace()).
		Name(obj.GetName()).
		Resource(w.ResourceKind).
		Do(context.Background()).Error()
	switch {
	case err != nil && !errors.IsNotFound(err):
		log.Log.V(2).Errorf("Error deleting obj %s %s: %v", w.ResourceKind, obj.GetName(), err)
		return false, err
	case err != nil && errors.IsNotFound(err):
		return true, err
	default:
		return false, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"io"
	"net/http"
	"os"
	"time"

	"github.com/gorilla/mux"
	"github.com/joho/godotenv"
	"github.com/sirupsen/logrus"

	"github.com/skygeario/skygear-server/pkg/core/auth"
	coreConfig "github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	coreMiddleware "github.com/skygeario/skygear-server/pkg/core/middleware"
	"github.com/skygeario/skygear-server/pkg/core/redis"
	"github.com/skygeario/skygear-server/pkg/core/sentry"
	"github.com/skygeario/skygear-server/pkg/gateway"
	gatewayConfig "github.com/skygeario/skygear-server/pkg/gateway/config"
	"github.com/skygeario/skygear-server/pkg/gateway/handler"
	"github.com/skygeario/skygear-server/pkg/gateway/middleware"
	"github.com/skygeario/skygear-server/pkg/gateway/provider"
	"github.com/skygeario/skygear-server/pkg/gateway/store"
	pqStore "github.com/skygeario/skygear-server/pkg/gateway/store/pq"
	standaloneStore "github.com/skygeario/skygear-server/pkg/gateway/store/standalone"
)

var config gatewayConfig.Configuration
var loggerFactory logging.Factory
var logger *logrus.Entry

func init() {
	// logging initialization
	logging.SetModule("gateway")
	loggerFactory = logging.NewFactory(
		logging.NewDefaultLogHook(nil),
		&sentry.LogHook{Hub: sentry.DefaultClient.Hub},
	)
	logger = loggerFactory.NewLogger("gateway")

	if err := godotenv.Load(); err != nil {
		logger.WithError(err).Info(
			"Error in loading .env file, continue without .env")
	}

	if err := config.ReadFromEnv(); err != nil {
		logger.WithError(err).Panic(
			"Fail to load config for starting gateway server")
	}
}

func main() {
	// create gateway store
	var store store.GatewayStore
	var connErr error
	if config.Standalone {
		filename := config.StandaloneTenantConfigurationFile
		reader, err := os.Open(filename)
		if err != nil {
			logger.WithError(err).Panic("Fail to open config file")
		}
		tenantConfig, err := coreConfig.NewTenantConfigurationFromYAML(reader)
		if err != nil {
			logger.WithError(err).Panic("Fail to load config from YAML")
		}
		store = &standaloneStore.Store{
			TenantConfig: *tenantConfig,
		}
	} else {
		store, connErr = pqStore.NewGatewayStore(
			context.Background(),
			config.ConnectionStr,
			loggerFactory,
		)
		if connErr != nil {
			logger.WithError(connErr).Panic("Fail to create db conn")
		}
	}
	defer store.Close()

	gatewayDependency := gateway.DependencyMap{
		UseInsecureCookie: config.UseInsecureCookie,
	}
	dbPool := db.NewPool()
	redisPool, err := redis.NewPool(config.Redis)
	if err != nil {
		logger.Fatalf("fail to create redis pool: %v", err.Error())
	}
	rr := mux.NewRouter()
	rr.HandleFunc("/_healthz", HealthCheckHandler)

	r := rr.PathPrefix("/").Subrouter()
	r.Use(sentry.Middleware(sentry.DefaultClient.Hub))
	r.Use(coreMiddleware.RecoverMiddleware{}.Handle)

	r.Use(coreMiddleware.RequestIDMiddleware{}.Handle)
	r.Use(middleware.FindAppMiddleware{Store: store}.Handle)

	gr := r.PathPrefix("/_{gear}").Subrouter()

	gr.Use(coreMiddleware.WriteTenantConfigMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)
	gr.Use(middleware.TenantAuthzMiddleware{
		Store:         store,
		Configuration: config,
	}.Handle)
	gr.Use(coreMiddleware.CORSMiddleware{}.Handle)

	gr.Handle("/{rest:.*}", handler.NewGearHandler())

	cr := r.PathPrefix("/").Subrouter()

	cr.Use(coreMiddleware.WriteTenantConfigMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)

	cr.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = db.InitRequestDBContext(r, dbPool)
			r = auth.InitRequestAuthContext(r)
			r = r.WithContext(redis.WithRedis(r.Context(), redisPool))
			next.ServeHTTP(w, r)
		})
	})

	cr.Use(middleware.FindDeploymentRouteMiddleware{
		RestPathIdentifier: "rest",
		Store:              store,
	}.Handle)

	// CORS headers should be set right after a proxy backend has been found.
	cr.Use(coreMiddleware.CORSMiddleware{}.Handle)

	cr.Use(coreMiddleware.Injecter{
		MiddlewareFactory: coreMiddleware.AuthnMiddlewareFactory{},
		Dependency:        gatewayDependency,
	}.Handle)

	cr.Use(coreMiddleware.Injecter{
		MiddlewareFactory: middleware.AuthInfoMiddlewareFactory{},
		Dependency:        gatewayDependency,
	}.Handle)

	cr.HandleFunc("/{rest:.*}", handler.NewDeploymentRouteHandler())

	srv := &http.Server{
		Addr:         config.Host,
		ReadTimeout:  time.Second * 60,
		WriteTimeout: time.Second * 60,
		IdleTimeout:  time.Second * 60,
		Handler:      rr, // Pass our instance of gorilla/mux in.
	}

	logger.Info("Start gateway server")
	if err := srv.ListenAndServe(); err != nil {
		logger.WithError(err).Errorf("Fail to start gateway server")
	}
}

// HealthCheckHandler is basic handler for server health check
func HealthCheckHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "application/json")
	io.WriteString(w, "OK")
}
<commit_msg>Return Redis connection to pool properly<commit_after>package main

import (
	"context"
	"io"
	"net/http"
	"os"
	"time"

	"github.com/gorilla/mux"
	"github.com/joho/godotenv"
	"github.com/sirupsen/logrus"

	"github.com/skygeario/skygear-server/pkg/core/auth"
	coreConfig "github.com/skygeario/skygear-server/pkg/core/config"
	"github.com/skygeario/skygear-server/pkg/core/db"
	"github.com/skygeario/skygear-server/pkg/core/logging"
	coreMiddleware "github.com/skygeario/skygear-server/pkg/core/middleware"
	"github.com/skygeario/skygear-server/pkg/core/redis"
	"github.com/skygeario/skygear-server/pkg/core/sentry"
	"github.com/skygeario/skygear-server/pkg/gateway"
	gatewayConfig "github.com/skygeario/skygear-server/pkg/gateway/config"
	"github.com/skygeario/skygear-server/pkg/gateway/handler"
	"github.com/skygeario/skygear-server/pkg/gateway/middleware"
	"github.com/skygeario/skygear-server/pkg/gateway/provider"
	"github.com/skygeario/skygear-server/pkg/gateway/store"
	pqStore "github.com/skygeario/skygear-server/pkg/gateway/store/pq"
	standaloneStore "github.com/skygeario/skygear-server/pkg/gateway/store/standalone"
)

var config gatewayConfig.Configuration
var loggerFactory logging.Factory
var logger *logrus.Entry

func init() {
	// logging initialization
	logging.SetModule("gateway")
	loggerFactory = logging.NewFactory(
		logging.NewDefaultLogHook(nil),
		&sentry.LogHook{Hub: sentry.DefaultClient.Hub},
	)
	logger = loggerFactory.NewLogger("gateway")

	if err := godotenv.Load(); err != nil {
		logger.WithError(err).Info(
			"Error in loading .env file, continue without .env")
	}

	if err := config.ReadFromEnv(); err != nil {
		logger.WithError(err).Panic(
			"Fail to load config for starting gateway server")
	}
}

func main() {
	// create gateway store
	var store store.GatewayStore
	var connErr error
	if config.Standalone {
		filename := config.StandaloneTenantConfigurationFile
		reader, err := os.Open(filename)
		if err != nil {
			logger.WithError(err).Panic("Fail to open config file")
		}
		tenantConfig, err := coreConfig.NewTenantConfigurationFromYAML(reader)
		if err != nil {
			logger.WithError(err).Panic("Fail to load config from YAML")
		}
		store = &standaloneStore.Store{
			TenantConfig: *tenantConfig,
		}
	} else {
		store, connErr = pqStore.NewGatewayStore(
			context.Background(),
			config.ConnectionStr,
			loggerFactory,
		)
		if connErr != nil {
			logger.WithError(connErr).Panic("Fail to create db conn")
		}
	}
	defer store.Close()

	gatewayDependency := gateway.DependencyMap{
		UseInsecureCookie: config.UseInsecureCookie,
	}
	dbPool := db.NewPool()
	redisPool, err := redis.NewPool(config.Redis)
	if err != nil {
		logger.Fatalf("fail to create redis pool: %v", err.Error())
	}
	rr := mux.NewRouter()
	rr.HandleFunc("/_healthz", HealthCheckHandler)

	r := rr.PathPrefix("/").Subrouter()
	r.Use(sentry.Middleware(sentry.DefaultClient.Hub))
	r.Use(coreMiddleware.RecoverMiddleware{}.Handle)

	r.Use(coreMiddleware.RequestIDMiddleware{}.Handle)
	r.Use(middleware.FindAppMiddleware{Store: store}.Handle)

	gr := r.PathPrefix("/_{gear}").Subrouter()

	gr.Use(coreMiddleware.WriteTenantConfigMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)
	gr.Use(middleware.TenantAuthzMiddleware{
		Store:         store,
		Configuration: config,
	}.Handle)
	gr.Use(coreMiddleware.CORSMiddleware{}.Handle)

	gr.Handle("/{rest:.*}", handler.NewGearHandler())

	cr := r.PathPrefix("/").Subrouter()

	cr.Use(coreMiddleware.WriteTenantConfigMiddleware{
		ConfigurationProvider: provider.GatewayTenantConfigurationProvider{
			Store: store,
		},
	}.Handle)

	cr.Use(func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			r = db.InitRequestDBContext(r, dbPool)
			r = auth.InitRequestAuthContext(r)
			r = r.WithContext(redis.WithRedis(r.Context(), redisPool))
			defer redis.CloseConn(r.Context())
			next.ServeHTTP(w, r)
		})
	})

	cr.Use(middleware.FindDeploymentRouteMiddleware{
		RestPathIdentifier: "rest",
		Store:              store,
	}.Handle)

	// CORS headers should be set right after a proxy backend has been found.
	cr.Use(coreMiddleware.CORSMiddleware{}.Handle)

	cr.Use(coreMiddleware.Injecter{
		MiddlewareFactory: coreMiddleware.AuthnMiddlewareFactory{},
		Dependency:        gatewayDependency,
	}.Handle)

	cr.Use(coreMiddleware.Injecter{
		MiddlewareFactory: middleware.AuthInfoMiddlewareFactory{},
		Dependency:        gatewayDependency,
	}.Handle)

	cr.HandleFunc("/{rest:.*}", handler.NewDeploymentRouteHandler())

	srv := &http.Server{
		Addr:         config.Host,
		ReadTimeout:  time.Second * 60,
		WriteTimeout: time.Second * 60,
		IdleTimeout:  time.Second * 60,
		Handler:      rr, // Pass our instance of gorilla/mux in.
	}

	logger.Info("Start gateway server")
	if err := srv.ListenAndServe(); err != nil {
		logger.WithError(err).Errorf("Fail to start gateway server")
	}
}

// HealthCheckHandler is basic handler for server health check
func HealthCheckHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "application/json")
	io.WriteString(w, "OK")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/bgentry/speakeasy"
	"github.com/kopia/kopia/vault"
)

var (
	traceStorage = app.Flag("trace-storage", "Enables tracing of storage operations.").Hidden().Bool()

	vaultPath    = app.Flag("vault", "Specify the vault to use.").PlaceHolder("PATH").Envar("KOPIA_VAULT").Short('v').String()
	password     = app.Flag("password", "Vault password.").Envar("KOPIA_PASSWORD").Short('p').String()
	passwordFile = app.Flag("passwordfile", "Read vault password from a file.").PlaceHolder("FILENAME").Envar("KOPIA_PASSWORD_FILE").ExistingFile()
	key          = app.Flag("key", "Specify vault master key (hexadecimal).").Envar("KOPIA_KEY").Short('k').String()
	keyFile      = app.Flag("keyfile", "Read vault master key from file.").PlaceHolder("FILENAME").Envar("KOPIA_KEY_FILE").ExistingFile()
)

func failOnError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		os.Exit(1)
	}
}

func mustOpenVault() *vault.Vault {
	s, err := openVault()
	failOnError(err)
	return s
}

func getHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}

	return os.Getenv("HOME")
}

func vaultConfigFileName() string {
	return filepath.Join(getHomeDir(), ".kopia/vault.config")
}

func persistVaultConfig(v *vault.Vault) error {
	cfg, err := v.Token()
	if err != nil {
		return err
	}

	fname := vaultConfigFileName()
	log.Printf("saving configuration %v", fname)
	if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil {
		return err
	}

	return ioutil.WriteFile(fname, []byte(cfg), 0600)
}

func getPersistedVaultConfig() string {
	f, err := ioutil.ReadFile(vaultConfigFileName())
	if err != nil {
		return ""
	}

	return string(f)
}

func openVault() (*vault.Vault, error) {
	vc := getPersistedVaultConfig()
	if vc != "" {
		return vault.OpenWithToken(vc)
	}

	if *vaultPath == "" {
		return nil, fmt.Errorf("vault not connected and not specified, use --vault or run 'kopia connect'")
	}

	return openVaultSpecifiedByFlag()
}

func openVaultSpecifiedByFlag() (*vault.Vault, error) {
	if *vaultPath == "" {
		return nil, fmt.Errorf("--vault must be specified")
	}
	storage, err := newStorageFromURL(*vaultPath)
	if err != nil {
		return nil, err
	}

	creds, err := getVaultCredentials(false)
	if err != nil {
		return nil, err
	}

	return vault.Open(storage, creds)
}

var errPasswordTooShort = errors.New("password too short")

func getVaultCredentials(isNew bool) (vault.Credentials, error) {
	if *key != "" {
		k, err := hex.DecodeString(*key)
		if err != nil {
			return nil, fmt.Errorf("invalid key format: %v", err)
		}

		return vault.MasterKey(k)
	}

	if *password != "" {
		return vault.Password(strings.TrimSpace(*password))
	}

	if *keyFile != "" {
		key, err := ioutil.ReadFile(*keyFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read key file: %v", err)
		}

		return vault.MasterKey(key)
	}

	if *passwordFile != "" {
		f, err := ioutil.ReadFile(*passwordFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read password file: %v", err)
		}

		return vault.Password(strings.TrimSpace(string(f)))
	}
	if isNew {
		for {
			p1, err := askPass("Enter password to create new vault: ")
			if err == errPasswordTooShort {
				fmt.Printf("Password too short, must be at least %v characters, you entered %v. Try again.", vault.MinPasswordLength, len(p1))
				fmt.Println()
				continue
			}
			if err != nil {
				return nil, err
			}
			p2, err := askPass("Re-enter password for verification: ")
			if err != nil {
				return nil, err
			}
			if p1 != p2 {
				fmt.Println("Passwords don't match!")
			} else {
				return vault.Password(p1)
			}
		}
	} else {
		p1, err := askPass("Enter password to open vault: ")
		if err != nil {
			return nil, err
		}
		fmt.Println()
		return vault.Password(p1)
	}
}

func askPass(prompt string) (string, error) {
	b, err := speakeasy.Ask(prompt)
	if err != nil {
		return "", err
	}

	p := string(b)

	if len(p) < vault.MinPasswordLength {
		return p, errPasswordTooShort
	}

	return p, nil
}
<commit_msg>cleanup<commit_after>package main

import (
	"encoding/hex"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/bgentry/speakeasy"
	"github.com/kopia/kopia/vault"
)

var (
	traceStorage = app.Flag("trace-storage", "Enables tracing of storage operations.").Hidden().Bool()

	vaultPath    = app.Flag("vault", "Specify the vault to use.").PlaceHolder("PATH").Envar("KOPIA_VAULT").Short('v').String()
	password     = app.Flag("password", "Vault password.").Envar("KOPIA_PASSWORD").Short('p').String()
	passwordFile = app.Flag("passwordfile", "Read vault password from a file.").PlaceHolder("FILENAME").Envar("KOPIA_PASSWORD_FILE").ExistingFile()
	key          = app.Flag("key", "Specify vault master key (hexadecimal).").Envar("KOPIA_KEY").Short('k').String()
	keyFile      = app.Flag("keyfile", "Read vault master key from file.").PlaceHolder("FILENAME").Envar("KOPIA_KEY_FILE").ExistingFile()
)

func failOnError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		os.Exit(1)
	}
}

func mustOpenVault() *vault.Vault {
	s, err := openVault()
	failOnError(err)
	return s
}

func getHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}

	return os.Getenv("HOME")
}

func vaultConfigFileName() string {
	return filepath.Join(getHomeDir(), ".kopia/vault.config")
}

func persistVaultConfig(v *vault.Vault) error {
	cfg, err := v.Token()
	if err != nil {
		return err
	}

	fname := vaultConfigFileName()
	log.Printf("Saving vault configuration to '%v'.", fname)
	if err := os.MkdirAll(filepath.Dir(fname), 0700); err != nil {
		return err
	}

	return ioutil.WriteFile(fname, []byte(cfg), 0600)
}

func getPersistedVaultConfig() string {
	f, err := ioutil.ReadFile(vaultConfigFileName())
	if err != nil {
		return ""
	}

	return string(f)
}

func openVault() (*vault.Vault, error) {
	vc := getPersistedVaultConfig()
	if vc != "" {
		return vault.OpenWithToken(vc)
	}

	if *vaultPath == "" {
		return nil, fmt.Errorf("vault not connected and not specified, use --vault or run 'kopia connect'")
	}

	return openVaultSpecifiedByFlag()
}

func openVaultSpecifiedByFlag() (*vault.Vault, error) {
	if *vaultPath == "" {
		return nil, fmt.Errorf("--vault must be specified")
	}
	storage, err := newStorageFromURL(*vaultPath)
	if err != nil {
		return nil, err
	}

	creds, err := getVaultCredentials(false)
	if err != nil {
		return nil, err
	}

	return vault.Open(storage, creds)
}

var errPasswordTooShort = errors.New("password too short")

func getVaultCredentials(isNew bool) (vault.Credentials, error) {
	if *key != "" {
		k, err := hex.DecodeString(*key)
		if err != nil {
			return nil, fmt.Errorf("invalid key format: %v", err)
		}

		return vault.MasterKey(k)
	}

	if *password != "" {
		return vault.Password(strings.TrimSpace(*password))
	}

	if *keyFile != "" {
		key, err := ioutil.ReadFile(*keyFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read key file: %v", err)
		}

		return vault.MasterKey(key)
	}

	if *passwordFile != "" {
		f, err := ioutil.ReadFile(*passwordFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read password file: %v", err)
		}

		return vault.Password(strings.TrimSpace(string(f)))
	}
	if isNew {
		for {
			p1, err := askPass("Enter password to create new vault: ")
			if err == errPasswordTooShort {
				fmt.Printf("Password too short, must be at least %v characters, you entered %v. Try again.", vault.MinPasswordLength, len(p1))
				fmt.Println()
				continue
			}
			if err != nil {
				return nil, err
			}
			p2, err := askPass("Re-enter password for verification: ")
			if err != nil {
				return nil, err
			}
			if p1 != p2 {
				fmt.Println("Passwords don't match!")
			} else {
				return vault.Password(p1)
			}
		}
	} else {
		p1, err := askPass("Enter password to open vault: ")
		if err != nil {
			return nil, err
		}
		fmt.Println()
		return vault.Password(p1)
	}
}

func askPass(prompt string) (string, error) {
	b, err := speakeasy.Ask(prompt)
	if err != nil {
		return "", err
	}

	p := string(b)

	if len(p) < vault.MinPasswordLength {
		return p, errPasswordTooShort
	}

	return p, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd_test

import (
	"testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/testing/testbase"
)

func Test(t *testing.T) { gc.TestingT(t) }

type Dependencies struct{}

var _ = gc.Suite(&Dependencies{})

func (*Dependencies) TestPackageDependencies(c *gc.C) {
	// This test is to ensure we don't bring in dependencies without thinking.
	// Looking at the "environs/config", it is just for JujuHome.  This should
	// really be moved into "juju/osenv".
	c.Assert(testbase.FindJujuCoreImports(c, "launchpad.net/juju-core/cmd"),
		gc.DeepEquals,
		[]string{"juju/arch", "juju/osenv", "names", "version"})
}
<commit_msg>Fix package dependencies test<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd_test

import (
	"testing"

	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/testing/testbase"
)

func Test(t *testing.T) { gc.TestingT(t) }

type Dependencies struct{}

var _ = gc.Suite(&Dependencies{})

func (*Dependencies) TestPackageDependencies(c *gc.C) {
	// This test is to ensure we don't bring in dependencies without thinking.
	// Looking at the "environs/config", it is just for JujuHome.  This should
	// really be moved into "juju/osenv".
	c.Assert(testbase.FindJujuCoreImports(c, "launchpad.net/juju-core/cmd"),
		gc.DeepEquals,
		[]string{"juju/arch", "juju/osenv", "names", "thirdparty/pbkdf2", "utils", "version"})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/andygrunwald/perseus/config"
	"github.com/andygrunwald/perseus/controller"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// cfgFile contains the path and name of the configuration file.
var cfgFile string

// numOfWorkers reflects the number of workers used for concurrent processes
var numOfWorkers int

// RootCmd represents the base command when called without any subcommands.
var RootCmd = &cobra.Command{
	Use:   "perseus",
	Short: "Local git mirror for your PHP (composer) project dependencies that works together with Satis",
	Long: `perseus is a tool that works together with Satis to create a local git mirror for your PHP (composer) project dependencies.

Every modern PHP project is managed by composer.
To save development time, external packages will be used to focus on your business logic.
Most external packages are downloaded from Packagist, Github or other places every time you hit composer install or update.
To speed up your development workflow, minimize network traffic and being independent from other 3rd party services for building and deploying your apps, a local mirror in your office make sense.

perseus will create a mirror of all your project dependencies and lets you fetch everything from there rather than fetching the whole source from the internet (e.g. Github or Packagist).
Each dependency is entirely mirrored, meaning you'll have all versions, tags, and branches on your local machine or server.`,
}

// addCmd represents the "add" command for the CLI interface.
var addCmd = &cobra.Command{
	Use:   "add",
	Short: "Mirrors one given package and adds it to Satis",
	Long: `Mirrors one given package and adds it to Satis.

If the package is available in the medusa.json configuration file and contains a URL, the URL from the configuration file will be used.
Otherwise perseus will request the URL from packagist.

When "with-deps" is given, dependencies of the package will be mirrored as well.
Dependencies will be determined through API requests to packagist.org.
`,
	Example: `  perseus add "twig/twig"
  perseus add --with-deps "symfony/console"`,
	ValidArgs: []string{"package", "config"},
	RunE:      cmdAddRun,
}

// mirrorCmd represents the "mirror" command for the CLI interface.
var mirrorCmd = &cobra.Command{
	Use:   "mirror",
	Short: "Mirrors all repositories that are specified in the configuration file",
	Long: `The mirror command reads the given configuration file and mirrors the git repository for each package (including dependencies), so they can be used locally.

Both package lists form the configuration file (repositories and require) will be taken into account.
Dependencies will be only resolved from the packages entered in the require section.
Repositories entered in the repositories section will be mirrors as is without resolving the dependencies.
`,
	Example:   "  perseus mirror",
	ValidArgs: []string{"config"},
	RunE:      cmdMirrorRun,
}

// updateCmd represents the "update" command for the CLI interface.
var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Fetch latest updates for each mirrored package",
	Long: `The update command reads the given configuration file file and updates each mirrored package.

This command will not reflect the configured repositories from the configuration.
It will only reflect the packages that were already mirrored in the past.

If you add a new package to the configuration you need either call the "add" command with the package as an argument.
Or you add the new package to the configuration and call the "mirror" command.

The update command is useful to ensure that every branch, tag or change in the configured packages is mirrors downstream.
Otherwise you would stuck with the version from the time you added the package.`,
	Example:   "  perseus update",
	ValidArgs: []string{"config"},
	RunE:      cmdUpdateRun,
}

// init kicks of cobra (our CLI interface) and defines all global flags that can be used across all commands.
func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "medusa.json", "Medusa configuration file")
	RootCmd.PersistentFlags().IntVar(&numOfWorkers, "numOfWorkers", runtime.GOMAXPROCS(0), "Number of worker used for concurrent operations (e.g. resolving a dependency tree or downloads)")

	// Original medusa command
	// 	medusa add [--with-deps] package [config]
	RootCmd.AddCommand(addCmd)
	addCmd.Flags().Bool("with-deps", false, "If set, the package dependencies will be downloaded, too")

	// Original medusa command
	// 	medusa mirror [config]
	RootCmd.AddCommand(mirrorCmd)

	// Original medusa command
	// 	medusa update [config]
	RootCmd.AddCommand(updateCmd)

	// Cobra is only able to define flags, but no arguments
	// If we were able to define arguments we would implement those:
	//
	// 	new InputArgument('package', InputArgument::REQUIRED, 'The name of a composer package', null),
	// 	new InputArgument('config', InputArgument::OPTIONAL, 'A config file', 'medusa.json')
	//
	// See https://github.com/spf13/cobra/issues/378 for details.
}

func main() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	if cfgFile != "" {
		viper.SetConfigFile(cfgFile)
	}

	viper.SetConfigName("medusa")
	viper.AddConfigPath(".")

	// Prefix env vars
	viper.SetEnvPrefix("PERSEUS")
	viper.AutomaticEnv()

	// If a config file is found, read it in.
	// If an error happen, quit.
	if err := viper.ReadInConfig(); err != nil {
		fmt.Println("Configuration file is missing and required.")
		fmt.Println("Please checkout https://github.com/andygrunwald/perseus#configuration for further details.")
		os.Exit(1)
	}
}

// cmdAddRun is the CLI interface for the "add" command
func cmdAddRun(cmd *cobra.Command, args []string) error {
	// Check first argument: package
	if len(args) == 0 {
		return fmt.Errorf("No argument applied. Please apply one argument: package")
	}
	packet := args[0]

	// Initialize logger with structured logging
	l := &logrus.Logger{
		Out: os.Stderr,
		Formatter: &logrus.TextFormatter{
			TimestampFormat: time.RFC3339,
			FullTimestamp:   true,
		},
		Hooks: make(logrus.LevelHooks),
		Level: logrus.InfoLevel,
	}

	// Check if we got minimum 2 arguments.
	// We will only use the second argument here. The rest will be ignored.
	// Second argument is the configuration file, but it is optional.
	// When this is set, we have to overwrite the configuration that viper found before
	if len(args) >= 2 {
		configFileArg := args[1]
		if _, err := os.Stat(configFileArg); os.IsNotExist(err) {
			return fmt.Errorf("Configuration file %s applied as a configuration file, but don't exists", configFileArg)
		}
		viper.SetConfigFile(configFileArg)
	}
	l.WithFields(logrus.Fields{
		"path": viper.ConfigFileUsed(),
	}).Info("Using configuration file")

	// Check "with-deps" flag
	withDepsFlag, err := cmd.Flags().GetBool("with-deps")
	if err != nil {
		return fmt.Errorf("Couldn't determine \"with-deps\" flag: %s\n", err)
	}

	// Create viper based configuration provider for Medusa
	p, err := config.NewViperProvider(viper.GetViper())
	if err != nil {
		return fmt.Errorf("Couldn't create a viper configuration provider: %s\n", err)
	}

	m, err := config.NewMedusa(p)
	if err != nil {
		return fmt.Errorf("Couldn't create medusa configuration object: %s\n", err)
	}

	// Determine number of concurrent workers
	nOfWorkers, err := cmd.Flags().GetInt("numOfWorkers")
	if err != nil {
		return fmt.Errorf("Couldn't determine number of concurrent workers. Please control the 'numOfWorkers' flag. Error message: %s\n", err)
	}

	l.WithFields(logrus.Fields{
		"command": "add",
		"package": packet,
	}).Info("Running command for package")
	// Setup command and run it
	c := &controller.AddController{
		Package:          packet,
		WithDependencies: withDepsFlag,
		Config:           m,
		Log:              logrus.FieldLogger(l),
		NumOfWorker:      nOfWorkers,
	}
	err = c.Run()
	if err != nil {
		return fmt.Errorf("Error during execution of \"add\" command: %s\n", err)
	}

	return nil
}

// cmdMirrorRun is the CLI interface for the "mirror" command
func cmdMirrorRun(cmd *cobra.Command, args []string) error {
	// Initialize logger with structured logging
	l := &logrus.Logger{
		Out: os.Stderr,
		Formatter: &logrus.TextFormatter{
			TimestampFormat: time.RFC3339,
			FullTimestamp:   true,
		},
		Hooks: make(logrus.LevelHooks),
		Level: logrus.InfoLevel,
	}

	// Check if we got minimum 1 argument.
	// We will only use the first argument here. The rest will be ignored.
	// First argument is the configuration file, but it is optional.
	// When this is set, we have to overwrite the configuration that viper found before
	if len(args) >= 1 {
		configFileArg := args[0]
		if _, err := os.Stat(configFileArg); os.IsNotExist(err) {
			return fmt.Errorf("Configuration file %s applied as a configuration file, but don't exists", configFileArg)
		}
		viper.SetConfigFile(configFileArg)
	}
	l.WithFields(logrus.Fields{
		"path": viper.ConfigFileUsed(),
	}).Info("Using configuration file")

	// Create viper based configuration provider for Medusa
	p, err := config.NewViperProvider(viper.GetViper())
	if err != nil {
		return fmt.Errorf("Couldn't create a viper configuration provider: %s\n", err)
	}

	m, err := config.NewMedusa(p)
	if err != nil {
		return fmt.Errorf("Couldn't create medusa configuration object: %s\n", err)
	}

	// Determine number of concurrent workers
	nOfWorkers, err := cmd.Flags().GetInt("numOfWorkers")
	if err != nil {
		return fmt.Errorf("Couldn't determine number of concurrent workers. Please control the 'numOfWorkers' flag. Error message: %s\n", err)
	}

	l.Println("Running \"mirror\" command")
	// Setup command and run it
	c := &controller.MirrorController{
		Config:      m,
		Log:         logrus.FieldLogger(l),
		NumOfWorker: nOfWorkers,
	}
	err = c.Run()
	if err != nil {
		return fmt.Errorf("Error during execution of \"mirror\" command: %s\n", err)
	}

	return nil
}

// cmdUpdateRun is the CLI interface for the "update" command
func cmdUpdateRun(cmd *cobra.Command, args []string) error {
	// Initialize logger with structured logging
	l := &logrus.Logger{
		Out: os.Stderr,
		Formatter: &logrus.TextFormatter{
			TimestampFormat: time.RFC3339,
			FullTimestamp:   true,
		},
		Hooks: make(logrus.LevelHooks),
		Level: logrus.InfoLevel,
	}

	// Check if we got minimum 1 argument.
	// We will only use the first argument here. The rest will be ignored.
	// First argument is the configuration file, but it is optional.
	// When this is set, we have to overwrite the configuration that viper found before
	if len(args) >= 1 {
		configFileArg := args[0]
		if _, err := os.Stat(configFileArg); os.IsNotExist(err) {
			return fmt.Errorf("Configuration file %s applied as a configuration file, but don't exists", configFileArg)
		}
		viper.SetConfigFile(configFileArg)
	}
	l.WithFields(logrus.Fields{
		"path": viper.ConfigFileUsed(),
	}).Info("Using configuration file")

	// Create viper based configuration provider for Medusa
	p, err := config.NewViperProvider(viper.GetViper())
	if err != nil {
		return fmt.Errorf("Couldn't create a viper configuration provider: %s\n", err)
	}

	m, err := config.NewMedusa(p)
	if err != nil {
		return fmt.Errorf("Couldn't create medusa configuration object: %s\n", err)
	}

	// Determine number of concurrent workers
	nOfWorkers, err := cmd.Flags().GetInt("numOfWorkers")
	if err != nil {
		return fmt.Errorf("Couldn't determine number of concurrent workers. Please control the 'numOfWorkers' flag. Error message: %s\n", err)
	}

	l.Println("Running \"update\" command")
	// Setup command and run it
	c := &controller.UpdateController{
		Config:      m,
		Log:         logrus.FieldLogger(l),
		NumOfWorker: nOfWorkers,
	}
	err = c.Run()
	if err != nil {
		return fmt.Errorf("Error during execution of \"update\" command: %s\n", err)
	}

	return nil
}
<commit_msg>Fixed usage of configuration reading<commit_after>package main

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/andygrunwald/perseus/config"
	"github.com/andygrunwald/perseus/controller"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// cfgFile contains the path and name of the configuration file.
var cfgFile string

// numOfWorkers reflects the number of workers used for concurrent processes
var numOfWorkers int

// RootCmd represents the base command when called without any subcommands.
var RootCmd = &cobra.Command{
	Use:   "perseus",
	Short: "Local git mirror for your PHP (composer) project dependencies that works together with Satis",
	Long: `perseus is a tool that works together with Satis to create a local git mirror for your PHP (composer) project dependencies.

Every modern PHP project is managed by composer.
To save development time, external packages will be used to focus on your business logic.
Most external packages are downloaded from Packagist, Github or other places every time you hit composer install or update.
To speed up your development workflow, minimize network traffic and being independent from other 3rd party services for building and deploying your apps, a local mirror in your office make sense.

perseus will create a mirror of all your project dependencies and lets you fetch everything from there rather than fetching the whole source from the internet (e.g. Github or Packagist).
Each dependency is entirely mirrored, meaning you'll have all versions, tags, and branches on your local machine or server.`,
}

// addCmd represents the "add" command for the CLI interface.
var addCmd = &cobra.Command{
	Use:   "add",
	Short: "Mirrors one given package and adds it to Satis",
	Long: `Mirrors one given package and adds it to Satis.

If the package is available in the medusa.json configuration file and contains a URL, the URL from the configuration file will be used.
Otherwise perseus will request the URL from packagist.

When "with-deps" is given, dependencies of the package will be mirrored as well.
Dependencies will be determined through API requests to packagist.org.
`,
	Example: `  perseus add "twig/twig"
  perseus add --with-deps "symfony/console"`,
	ValidArgs: []string{"package", "config"},
	RunE:      cmdAddRun,
}

// mirrorCmd represents the "mirror" command for the CLI interface.
var mirrorCmd = &cobra.Command{
	Use:   "mirror",
	Short: "Mirrors all repositories that are specified in the configuration file",
	Long: `The mirror command reads the given configuration file and mirrors the git repository for each package (including dependencies), so they can be used locally.

Both package lists form the configuration file (repositories and require) will be taken into account.
Dependencies will be only resolved from the packages entered in the require section.
Repositories entered in the repositories section will be mirrors as is without resolving the dependencies.
`,
	Example:   "  perseus mirror",
	ValidArgs: []string{"config"},
	RunE:      cmdMirrorRun,
}

// updateCmd represents the "update" command for the CLI interface.
var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Fetch latest updates for each mirrored package",
	Long: `The update command reads the given configuration file file and updates each mirrored package.

This command will not reflect the configured repositories from the configuration.
It will only reflect the packages that were already mirrored in the past.

If you add a new package to the configuration you need either call the "add" command with the package as an argument.
Or you add the new package to the configuration and call the "mirror" command.

The update command is useful to ensure that every branch, tag or change in the configured packages is mirrors downstream.
Otherwise you would stuck with the version from the time you added the package.`,
	Example:   "  perseus update",
	ValidArgs: []string{"config"},
	RunE:      cmdUpdateRun,
}

// init kicks of cobra (our CLI interface) and defines all global flags that can be used across all commands.
func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "medusa.json", "Medusa configuration file")
	RootCmd.PersistentFlags().IntVar(&numOfWorkers, "numOfWorkers", runtime.GOMAXPROCS(0), "Number of worker used for concurrent operations (e.g. resolving a dependency tree or downloads)")

	// Original medusa command
	// 	medusa add [--with-deps] package [config]
	RootCmd.AddCommand(addCmd)
	addCmd.Flags().Bool("with-deps", false, "If set, the package dependencies will be downloaded, too")

	// Original medusa command
	// 	medusa mirror [config]
	RootCmd.AddCommand(mirrorCmd)

	// Original medusa command
	// 	medusa update [config]
	RootCmd.AddCommand(updateCmd)

	// Cobra is only able to define flags, but no arguments
	// If we were able to define arguments we would implement those:
	//
	// 	new InputArgument('package', InputArgument::REQUIRED, 'The name of a composer package', null),
	// 	new InputArgument('config', InputArgument::OPTIONAL, 'A config file', 'medusa.json')
	//
	// See https://github.com/spf13/cobra/issues/378 for details.
}

func main() {
	if err := RootCmd.Execute(); err != nil {
		os.Exit(-1)
	}
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	viper.SetConfigName("medusa")
	viper.AddConfigPath(".")

	// Check and read command line parameter "--config"
	if cfgFile != "" {
		viper.SetConfigFile(cfgFile)
	}
}

// cmdAddRun is the CLI interface for the "add" command
func cmdAddRun(cmd *cobra.Command, args []string) error {
	// Check first argument: package
	if len(args) == 0 {
		return fmt.Errorf("No argument applied. Please apply one argument: package")
	}
	packet := args[0]

	// Initialize logger with structured logging
	l := &logrus.Logger{
		Out: os.Stderr,
		Formatter: &logrus.TextFormatter{
			TimestampFormat: time.RFC3339,
			FullTimestamp:   true,
		},
		Hooks: make(logrus.LevelHooks),
		Level: logrus.InfoLevel,
	}

	// Check if we got minimum 2 arguments.
	// We will only use the second argument here. The rest will be ignored.
	// Second argument is the configuration file, but it is optional.
	// When this is set, we have to overwrite the configuration that viper found before
	if len(args) >= 2 {
		configFileArg := args[1]
		if _, err := os.Stat(configFileArg); os.IsNotExist(err) {
			return fmt.Errorf("Configuration file %s applied as a configuration file, but doesn't exists", configFileArg)
		}
		viper.SetConfigFile(configFileArg)
	}

	// If a config file is found, read it in.
	// If an error happen, quit.
	if err := viper.ReadInConfig(); err != nil {
		s := fmt.Errorf("Error while reading the configuration file \"%s\": %s\nPlease checkout https://github.com/andygrunwald/perseus#configuration for further details.", viper.ConfigFileUsed(), err)
		return s
	}

	l.WithFields(logrus.Fields{
		"path": viper.ConfigFileUsed(),
	}).Info("Using configuration file")

	// Check "with-deps" flag
	withDepsFlag, err := cmd.Flags().GetBool("with-deps")
	if err != nil {
		return fmt.Errorf("Couldn't determine \"with-deps\" flag: %s\n", err)
	}

	// Create viper based configuration provider for Medusa
	p, err := config.NewViperProvider(viper.GetViper())
	if err != nil {
		return fmt.Errorf("Couldn't create a viper configuration provider: %s\n", err)
	}

	m, err := config.NewMedusa(p)
	if err != nil {
		return fmt.Errorf("Couldn't create medusa configuration object: %s\n", err)
	}

	// Determine number of concurrent workers
	nOfWorkers, err := cmd.Flags().GetInt("numOfWorkers")
	if err != nil {
		return fmt.Errorf("Couldn't determine number of concurrent workers. Please control the 'numOfWorkers' flag. Error message: %s\n", err)
	}

	l.WithFields(logrus.Fields{
		"command": "add",
		"package": packet,
	}).Info("Running command for package")
	// Setup command and run it
	c := &controller.AddController{
		Package:          packet,
		WithDependencies: withDepsFlag,
		Config:           m,
		Log:              logrus.FieldLogger(l),
		NumOfWorker:      nOfWorkers,
	}
	err = c.Run()
	if err != nil {
		return fmt.Errorf("Error during execution of \"add\" command: %s\n", err)
	}

	return nil
}

// cmdMirrorRun is the CLI interface for the "mirror" command
func cmdMirrorRun(cmd *cobra.Command, args []string) error {
	// Initialize logger with structured logging
	l := &logrus.Logger{
		Out: os.Stderr,
		Formatter: &logrus.TextFormatter{
			TimestampFormat: time.RFC3339,
			FullTimestamp:   true,
		},
		Hooks: make(logrus.LevelHooks),
		Level: logrus.InfoLevel,
	}

	// Check if we got minimum 1 argument.
	// We will only use the first argument here. The rest will be ignored.
	// First argument is the configuration file, but it is optional.
	// When this is set, we have to overwrite the configuration that viper found before
	if len(args) >= 1 {
		configFileArg := args[0]
		if _, err := os.Stat(configFileArg); os.IsNotExist(err) {
			return fmt.Errorf("Configuration file %s applied as a configuration file, but doesn't exists", configFileArg)
		}
		viper.SetConfigFile(configFileArg)
	}

	// If a config file is found, read it in.
	// If an error happen, quit.
	if err := viper.ReadInConfig(); err != nil {
		s := fmt.Errorf("Error while reading the configuration file \"%s\": %s\nPlease checkout https://github.com/andygrunwald/perseus#configuration for further details.", viper.ConfigFileUsed(), err)
		return s
	}

	l.WithFields(logrus.Fields{
		"path": viper.ConfigFileUsed(),
	}).Info("Using configuration file")

	// Create viper based configuration provider for Medusa
	p, err := config.NewViperProvider(viper.GetViper())
	if err != nil {
		return fmt.Errorf("Couldn't create a viper configuration provider: %s\n", err)
	}

	m, err := config.NewMedusa(p)
	if err != nil {
		return fmt.Errorf("Couldn't create medusa configuration object: %s\n", err)
	}

	// Determine number of concurrent workers
	nOfWorkers, err := cmd.Flags().GetInt("numOfWorkers")
	if err != nil {
		return fmt.Errorf("Couldn't determine number of concurrent workers. Please control the 'numOfWorkers' flag. Error message: %s\n", err)
	}

	l.Println("Running \"mirror\" command")
	// Setup command and run it
	c := &controller.MirrorController{
		Config:      m,
		Log:         logrus.FieldLogger(l),
		NumOfWorker: nOfWorkers,
	}
	err = c.Run()
	if err != nil {
		return fmt.Errorf("Error during execution of \"mirror\" command: %s\n", err)
	}

	return nil
}

// cmdUpdateRun is the CLI interface for the "update" command
func cmdUpdateRun(cmd *cobra.Command, args []string) error {
	// Initialize logger with structured logging
	l := &logrus.Logger{
		Out: os.Stderr,
		Formatter: &logrus.TextFormatter{
			TimestampFormat: time.RFC3339,
			FullTimestamp:   true,
		},
		Hooks: make(logrus.LevelHooks),
		Level: logrus.InfoLevel,
	}

	// Check if we got minimum 1 argument.
	// We will only use the first argument here. The rest will be ignored.
	// First argument is the configuration file, but it is optional.
	// When this is set, we have to overwrite the configuration that viper found before
	if len(args) >= 1 {
		configFileArg := args[0]
		if _, err := os.Stat(configFileArg); os.IsNotExist(err) {
			return fmt.Errorf("Configuration file %s applied as a configuration file, but doesn't exists", configFileArg)
		}
		viper.SetConfigFile(configFileArg)
	}

	// If a config file is found, read it in.
	// If an error happen, quit.
	if err := viper.ReadInConfig(); err != nil {
		s := fmt.Errorf("Error while reading the configuration file \"%s\": %s\nPlease checkout https://github.com/andygrunwald/perseus#configuration for further details.", viper.ConfigFileUsed(), err)
		return s
	}

	l.WithFields(logrus.Fields{
		"path": viper.ConfigFileUsed(),
	}).Info("Using configuration file")

	// Create viper based configuration provider for Medusa
	p, err := config.NewViperProvider(viper.GetViper())
	if err != nil {
		return fmt.Errorf("Couldn't create a viper configuration provider: %s\n", err)
	}

	m, err := config.NewMedusa(p)
	if err != nil {
		return fmt.Errorf("Couldn't create medusa configuration object: %s\n", err)
	}

	// Determine number of concurrent workers
	nOfWorkers, err := cmd.Flags().GetInt("numOfWorkers")
	if err != nil {
		return fmt.Errorf("Couldn't determine number of concurrent workers. Please control the 'numOfWorkers' flag. Error message: %s\n", err)
	}

	l.Println("Running \"update\" command")
	// Setup command and run it
	c := &controller.UpdateController{
		Config:      m,
		Log:         logrus.FieldLogger(l),
		NumOfWorker: nOfWorkers,
	}
	err = c.Run()
	if err != nil {
		return fmt.Errorf("Error during execution of \"update\" command: %s\n", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/smira/commander"
)

func aptlyPublishDrop(cmd *commander.Command, args []string) error {
	var err error
	if len(args) < 1 || len(args) > 2 {
		cmd.Usage()
		return err
	}

	distribution := args[0]
	prefix := "."

	if len(args) == 2 {
		prefix = args[1]
	}

	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
		context.CollectionFactory(), context.Progress())
	if err != nil {
		return fmt.Errorf("unable to remove: %s", err)
	}

	context.Progress().Printf("\nPublished repositroy has been removed successfully.\n")

	return err
}

func makeCmdPublishDrop() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlyPublishDrop,
		UsageLine: "drop <distribution> [<prefix>]",
		Short:     "remove published repository",
		Long: `
Command removes whatever has been published under specified <prefix> and
<distribution> name.

Example:

    $ aptly publish drop wheezy
`,
	}

	return cmd
}
<commit_msg>Fix misprint. #8<commit_after>package cmd

import (
	"fmt"
	"github.com/smira/commander"
)

func aptlyPublishDrop(cmd *commander.Command, args []string) error {
	var err error
	if len(args) < 1 || len(args) > 2 {
		cmd.Usage()
		return err
	}

	distribution := args[0]
	prefix := "."

	if len(args) == 2 {
		prefix = args[1]
	}

	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
		context.CollectionFactory(), context.Progress())
	if err != nil {
		return fmt.Errorf("unable to remove: %s", err)
	}

	context.Progress().Printf("\nPublished repository has been removed successfully.\n")

	return err
}

func makeCmdPublishDrop() *commander.Command {
	cmd := &commander.Command{
		Run:       aptlyPublishDrop,
		UsageLine: "drop <distribution> [<prefix>]",
		Short:     "remove published repository",
		Long: `
Command removes whatever has been published under specified <prefix> and
<distribution> name.

Example:

    $ aptly publish drop wheezy
`,
	}

	return cmd
}
<|endoftext|>