<commit_before>package cache

import (
	"bytes"
	bin "encoding/binary"
	"errors"
	"github.com/jmhodges/levigo"
	"os"
	"path/filepath"
	"strconv"
)

var levelDbWriteBufferSize, levelDbWriteBlockSize int64
var deltaCacheBunchSize int64

func init() {
	levelDbWriteBufferSize, _ = strconv.ParseInt(
		os.Getenv("GOPOSM_LEVELDB_BUFFERSIZE"), 10, 32)
	levelDbWriteBlockSize, _ = strconv.ParseInt(
		os.Getenv("GOPOSM_LEVELDB_BLOCKSIZE"), 10, 32)

	// bunchSize defines how many coordinates should be stored in a
	// single record. This is the maximum and a bunch will typically contain
	// less coordinates (e.g. when nodes are removes).
	//
	// A higher number improves -read mode (writing the cache) but also
	// increases the overhead during -write mode (reading coords).
	deltaCacheBunchSize, _ = strconv.ParseInt(
		os.Getenv("GOPOSM_DELTACACHE_BUNCHSIZE"), 10, 32)

	if deltaCacheBunchSize == 0 {
		deltaCacheBunchSize = 128
	}
}

var (
	NotFound = errors.New("not found")
)

type OSMCache struct {
	Dir          string
	Coords       *DeltaCoordsCache
	Ways         *WaysCache
	Nodes        *NodesCache
	Relations    *RelationsCache
	InsertedWays *InsertedWaysCache
	opened       bool
}

func (c *OSMCache) Close() {
	if c.Coords != nil {
		c.Coords.Close()
		c.Coords = nil
	}
	if c.Nodes != nil {
		c.Nodes.Close()
		c.Nodes = nil
	}
	if c.Ways != nil {
		c.Ways.Close()
		c.Ways = nil
	}
	if c.Relations != nil {
		c.Relations.Close()
		c.Relations = nil
	}
}

func NewOSMCache(dir string) *OSMCache {
	cache := &OSMCache{Dir: dir}
	return cache
}

func (c *OSMCache) Open() error {
	err := os.MkdirAll(c.Dir, 0755)
	if err != nil {
		return err
	}
	c.Coords, err = NewDeltaCoordsCache(filepath.Join(c.Dir, "coords"))
	if err != nil {
		return err
	}
	c.Nodes, err = NewNodesCache(filepath.Join(c.Dir, "nodes"))
	if err != nil {
		c.Close()
		return err
	}
	c.Ways, err = NewWaysCache(filepath.Join(c.Dir, "ways"))
	if err != nil {
		c.Close()
		return err
	}
	c.Relations, err = NewRelationsCache(filepath.Join(c.Dir, "relations"))
	if err != nil {
		c.Close()
		return err
	}
	c.InsertedWays, err = NewInsertedWaysCache(filepath.Join(c.Dir, "inserted_ways"))
	if err != nil {
		c.Close()
		return err
	}
	c.opened = true
	return nil
}

func (c *OSMCache) Exists() bool {
	if c.opened {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "coords")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "nodes")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "ways")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "relations")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "inserted_ways")); !os.IsNotExist(err) {
		return true
	}
	return false
}

func (c *OSMCache) Remove() error {
	if c.opened {
		c.Close()
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "coords")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "nodes")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "ways")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "relations")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "inserted_ways")); err != nil {
		return err
	}
	return nil
}

type Cache struct {
	db *levigo.DB
	wo *levigo.WriteOptions
	ro *levigo.ReadOptions
}

func (c *Cache) open(path string) error {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(1024 * 1024 * 8))
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(64)
	// save a few bytes by allowing leveldb to use delta enconding
	// for up to n keys (instead of only 16)
	opts.SetBlockRestartInterval(128)
	if levelDbWriteBufferSize != 0 {
		opts.SetWriteBufferSize(int(levelDbWriteBufferSize))
	}
	if levelDbWriteBlockSize != 0 {
		opts.SetBlockSize(int(levelDbWriteBlockSize))
	}
	db, err := levigo.Open(path, opts)
	if err != nil {
		return err
	}
	c.db = db
	c.wo = levigo.NewWriteOptions()
	c.ro = levigo.NewReadOptions()
	return nil
}

func idToKeyBuf(id int64) []byte {
	var b bytes.Buffer
	bin.Write(&b, bin.BigEndian, &id)
	return b.Bytes()
}

func (p *Cache) Close() {
	p.db.Close()
}
<commit_msg>create keybuffer with fixed size<commit_after>package cache

import (
	"bytes"
	bin "encoding/binary"
	"errors"
	"github.com/jmhodges/levigo"
	"os"
	"path/filepath"
	"strconv"
)

var levelDbWriteBufferSize, levelDbWriteBlockSize int64
var deltaCacheBunchSize int64

func init() {
	levelDbWriteBufferSize, _ = strconv.ParseInt(
		os.Getenv("GOPOSM_LEVELDB_BUFFERSIZE"), 10, 32)
	levelDbWriteBlockSize, _ = strconv.ParseInt(
		os.Getenv("GOPOSM_LEVELDB_BLOCKSIZE"), 10, 32)

	// bunchSize defines how many coordinates should be stored in a
	// single record. This is the maximum and a bunch will typically contain
	// less coordinates (e.g. when nodes are removes).
	//
	// A higher number improves -read mode (writing the cache) but also
	// increases the overhead during -write mode (reading coords).
	deltaCacheBunchSize, _ = strconv.ParseInt(
		os.Getenv("GOPOSM_DELTACACHE_BUNCHSIZE"), 10, 32)

	if deltaCacheBunchSize == 0 {
		deltaCacheBunchSize = 128
	}
}

var (
	NotFound = errors.New("not found")
)

type OSMCache struct {
	Dir          string
	Coords       *DeltaCoordsCache
	Ways         *WaysCache
	Nodes        *NodesCache
	Relations    *RelationsCache
	InsertedWays *InsertedWaysCache
	opened       bool
}

func (c *OSMCache) Close() {
	if c.Coords != nil {
		c.Coords.Close()
		c.Coords = nil
	}
	if c.Nodes != nil {
		c.Nodes.Close()
		c.Nodes = nil
	}
	if c.Ways != nil {
		c.Ways.Close()
		c.Ways = nil
	}
	if c.Relations != nil {
		c.Relations.Close()
		c.Relations = nil
	}
}

func NewOSMCache(dir string) *OSMCache {
	cache := &OSMCache{Dir: dir}
	return cache
}

func (c *OSMCache) Open() error {
	err := os.MkdirAll(c.Dir, 0755)
	if err != nil {
		return err
	}
	c.Coords, err = NewDeltaCoordsCache(filepath.Join(c.Dir, "coords"))
	if err != nil {
		return err
	}
	c.Nodes, err = NewNodesCache(filepath.Join(c.Dir, "nodes"))
	if err != nil {
		c.Close()
		return err
	}
	c.Ways, err = NewWaysCache(filepath.Join(c.Dir, "ways"))
	if err != nil {
		c.Close()
		return err
	}
	c.Relations, err = NewRelationsCache(filepath.Join(c.Dir, "relations"))
	if err != nil {
		c.Close()
		return err
	}
	c.InsertedWays, err = NewInsertedWaysCache(filepath.Join(c.Dir, "inserted_ways"))
	if err != nil {
		c.Close()
		return err
	}
	c.opened = true
	return nil
}

func (c *OSMCache) Exists() bool {
	if c.opened {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "coords")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "nodes")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "ways")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "relations")); !os.IsNotExist(err) {
		return true
	}
	if _, err := os.Stat(filepath.Join(c.Dir, "inserted_ways")); !os.IsNotExist(err) {
		return true
	}
	return false
}

func (c *OSMCache) Remove() error {
	if c.opened {
		c.Close()
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "coords")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "nodes")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "ways")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "relations")); err != nil {
		return err
	}
	if err := os.RemoveAll(filepath.Join(c.Dir, "inserted_ways")); err != nil {
		return err
	}
	return nil
}

type Cache struct {
	db *levigo.DB
	wo *levigo.WriteOptions
	ro *levigo.ReadOptions
}

func (c *Cache) open(path string) error {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(1024 * 1024 * 8))
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(64)
	// save a few bytes by allowing leveldb to use delta enconding
	// for up to n keys (instead of only 16)
	opts.SetBlockRestartInterval(128)
	if levelDbWriteBufferSize != 0 {
		opts.SetWriteBufferSize(int(levelDbWriteBufferSize))
	}
	if levelDbWriteBlockSize != 0 {
		opts.SetBlockSize(int(levelDbWriteBlockSize))
	}
	db, err := levigo.Open(path, opts)
	if err != nil {
		return err
	}
	c.db = db
	c.wo = levigo.NewWriteOptions()
	c.ro = levigo.NewReadOptions()
	return nil
}

func idToKeyBuf(id int64) []byte {
	b := make([]byte, 0, 8)
	buf := bytes.NewBuffer(b)
	bin.Write(buf, bin.BigEndian, &id)
	return b
}

func (p *Cache) Close() {
	p.db.Close()
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// root-dump dumps the content of a ROOT file, including the content of
// the Trees (for all entries), if any.
//
// Example:
//
//  $> root-dump ./testdata/small-flat-tree.root
//  >>> file[./testdata/small-flat-tree.root]
//  key[000]: tree;1 "my tree title" (TTree)
//  [000][Int32]: 0
//  [000][Int64]: 0
//  [000][UInt32]: 0
//  [000][UInt64]: 0
//  [000][Float32]: 0
//  [000][Float64]: 0
//  [000][Str]: evt-000
//  [000][ArrayInt32]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayInt64]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayInt32]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayInt64]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayFloat32]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayFloat64]: [0 0 0 0 0 0 0 0 0 0]
//  [000][N]: 0
//  [000][SliceInt32]: []
//  [000][SliceInt64]: []
//  [...]
//
//  $> root-dump -h
//  Usage: root-dump [options] f0.root [f1.root [...]]
//
//  ex:
//   $> root-dump ./testdata/small-flat-tree.root
//   $> root-dump -deep=0 ./testdata/small-flat-tree.root
//
//  options:
//    -deep
//      	enable deep dumping of values (including Trees' entries) (default true)
//
package main // import "go-hep.org/x/hep/rootio/cmd/root-dump"

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"

	"go-hep.org/x/hep/rootio"
)

var (
	deepFlag = flag.Bool("deep", true, "enable deep dumping of values (including Trees' entries)")
)

func main() {
	log.SetPrefix("root-dump: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: root-dump [options] f0.root [f1.root [...]]

ex:
 $> root-dump ./testdata/small-flat-tree.root
 $> root-dump -deep=0 ./testdata/small-flat-tree.root

options:
`,
		)
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		log.Fatalf("need at least one input ROOT file")
	}

	for _, fname := range flag.Args() {
		err := dump(os.Stdout, fname, *deepFlag)
		if err != nil {
			log.Fatalf("error dumping file %q: %v", fname, err)
		}
	}
}

func dump(w io.Writer, fname string, deep bool) error {
	fmt.Fprintf(w, ">>> file[%s]\n", fname)
	f, err := rootio.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()

	for i, key := range f.Keys() {
		obj, err := key.Object()
		if err != nil {
			return err
		}
		fmt.Fprintf(w, "key[%03d]: %s;%d %q (%s)\n", i, key.Name(), key.Cycle(), key.Title(), obj.Class())
		if deep {
			switch obj := obj.(type) {
			case rootio.Tree:
				err = dumpTree(w, obj)
			default:
				err = fmt.Errorf("unhandled type %T", obj)
			}
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func dumpTree(w io.Writer, t rootio.Tree) error {

	var vars []rootio.ScanVar
	for _, b := range t.Branches() {
		for _, leaf := range b.Leaves() {
			if cls := leaf.Class(); cls == "TLeafElement" {
				return fmt.Errorf("trees with TLeafElement(s) not handled (leaf=%q)", leaf.Name())
			}
			ptr := newValue(leaf)
			if leaf.LeafCount() != nil && false {
				continue
			}
			vars = append(vars, rootio.ScanVar{Name: leaf.Name(), Value: ptr})
		}
	}

	sc, err := rootio.NewScannerVars(t, vars...)
	if err != nil {
		return err
	}
	defer sc.Close()

	for sc.Next() {
		err = sc.Scan()
		if err != nil {
			return fmt.Errorf("error scanning entry %d: %v", sc.Entry(), err)
		}
		for _, v := range vars {
			rv := reflect.Indirect(reflect.ValueOf(v.Value))
			fmt.Fprintf(w, "[%03d][%s]: %v\n", sc.Entry(), v.Name, rv.Interface())
		}
	}
	return nil
}

func newValue(leaf rootio.Leaf) interface{} {
	etype := leaf.Type()
	switch {
	case leaf.LeafCount() != nil:
		etype = reflect.SliceOf(etype)
	case leaf.Len() > 1 && leaf.Kind() != reflect.String:
		etype = reflect.ArrayOf(leaf.Len(), etype)
	}
	return reflect.New(etype).Interface()
}
<commit_msg>rootio/cmd/root-dump: handle dirs and subdirs<commit_after>// Copyright 2017 The go-hep Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// root-dump dumps the content of a ROOT file, including the content of
// the Trees (for all entries), if any.
//
// Example:
//
//  $> root-dump ./testdata/small-flat-tree.root
//  >>> file[./testdata/small-flat-tree.root]
//  key[000]: tree;1 "my tree title" (TTree)
//  [000][Int32]: 0
//  [000][Int64]: 0
//  [000][UInt32]: 0
//  [000][UInt64]: 0
//  [000][Float32]: 0
//  [000][Float64]: 0
//  [000][Str]: evt-000
//  [000][ArrayInt32]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayInt64]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayInt32]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayInt64]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayFloat32]: [0 0 0 0 0 0 0 0 0 0]
//  [000][ArrayFloat64]: [0 0 0 0 0 0 0 0 0 0]
//  [000][N]: 0
//  [000][SliceInt32]: []
//  [000][SliceInt64]: []
//  [...]
//
//  $> root-dump -h
//  Usage: root-dump [options] f0.root [f1.root [...]]
//
//  ex:
//   $> root-dump ./testdata/small-flat-tree.root
//   $> root-dump -deep=0 ./testdata/small-flat-tree.root
//
//  options:
//    -deep
//      	enable deep dumping of values (including Trees' entries) (default true)
//
package main // import "go-hep.org/x/hep/rootio/cmd/root-dump"

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"reflect"

	"go-hep.org/x/hep/rootio"
)

var (
	deepFlag = flag.Bool("deep", true, "enable deep dumping of values (including Trees' entries)")
)

func main() {
	log.SetPrefix("root-dump: ")
	log.SetFlags(0)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: root-dump [options] f0.root [f1.root [...]]

ex:
 $> root-dump ./testdata/small-flat-tree.root
 $> root-dump -deep=0 ./testdata/small-flat-tree.root

options:
`,
		)
		flag.PrintDefaults()
	}

	flag.Parse()

	if flag.NArg() == 0 {
		flag.Usage()
		log.Fatalf("need at least one input ROOT file")
	}

	for _, fname := range flag.Args() {
		err := dump(os.Stdout, fname, *deepFlag)
		if err != nil {
			log.Fatalf("error dumping file %q: %v", fname, err)
		}
	}
}

func dump(w io.Writer, fname string, deep bool) error {
	fmt.Fprintf(w, ">>> file[%s]\n", fname)
	f, err := rootio.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()

	return dumpDir(w, f, deep)
}

func dumpDir(w io.Writer, dir rootio.Directory, deep bool) error {
	for i, key := range dir.Keys() {
		obj, err := key.Object()
		if err != nil {
			return err
		}
		fmt.Fprintf(w, "key[%03d]: %s;%d %q (%s)\n", i, key.Name(), key.Cycle(), key.Title(), obj.Class())
		if deep {
			switch obj := obj.(type) {
			case rootio.Tree:
				err = dumpTree(w, obj)
			case rootio.Directory:
				err = dumpDir(w, obj, deep)
			default:
				err = fmt.Errorf("unhandled type %T", obj)
			}
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func dumpTree(w io.Writer, t rootio.Tree) error {

	var vars []rootio.ScanVar
	for _, b := range t.Branches() {
		for _, leaf := range b.Leaves() {
			if cls := leaf.Class(); cls == "TLeafElement" {
				return fmt.Errorf("trees with TLeafElement(s) not handled (leaf=%q)", leaf.Name())
			}
			ptr := newValue(leaf)
			if leaf.LeafCount() != nil && false {
				continue
			}
			vars = append(vars, rootio.ScanVar{Name: leaf.Name(), Value: ptr})
		}
	}

	sc, err := rootio.NewScannerVars(t, vars...)
	if err != nil {
		return err
	}
	defer sc.Close()

	for sc.Next() {
		err = sc.Scan()
		if err != nil {
			return fmt.Errorf("error scanning entry %d: %v", sc.Entry(), err)
		}
		for _, v := range vars {
			rv := reflect.Indirect(reflect.ValueOf(v.Value))
			fmt.Fprintf(w, "[%03d][%s]: %v\n", sc.Entry(), v.Name, rv.Interface())
		}
	}
	return nil
}

func newValue(leaf rootio.Leaf) interface{} {
	etype := leaf.Type()
	switch {
	case leaf.LeafCount() != nil:
		etype = reflect.SliceOf(etype)
	case leaf.Len() > 1 && leaf.Kind() != reflect.String:
		etype = reflect.ArrayOf(leaf.Len(), etype)
	}
	return reflect.New(etype).Interface()
}
<|endoftext|>


<commit_before>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/docopt/docopt-go"

	"github.com/wandoulabs/codis/pkg/models"
	"github.com/wandoulabs/codis/pkg/models/store/zk"
	"github.com/wandoulabs/codis/pkg/topom"
	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/log"
)

const banner = `
  _____  ____    ____/ /  (_)  _____
 / ___/ / __ \  / __  /  / /  / ___/
/ /__  / /_/ / / /_/ /  / /  (__  )
\___/  \____/  \__,_/  /_/  /____/

`

func main() {
	const usage = `
Usage:
	codis-dashboard [--ncpu=N] [--config=CONF] [--log=FILE] [--log-level=LEVEL] (--zookeeper=ADDR|--etcd=ADDR)
	codis-dashboard version

Options:
	--ncpu=N                    set runtime.GOMAXPROCS to N, default is runtime.NumCPU().
	-c CONF, --config=CONF      specify the config file.
	-l FILE, --log=FILE         specify the daliy rotated log file.
	--log-level=LEVEL           specify the log-level, can be INFO,WARN,DEBUG,ERROR, default is INFO.
`

	d, err := docopt.Parse(usage, nil, true, "", false)
	if err != nil {
		log.PanicError(err, "parse arguments failed")
	}

	if v, ok := d["version"].(bool); ok && v {
		fmt.Println("version:", utils.Version)
		fmt.Println("compile:", utils.Compile)
		return
	}

	if s, ok := d["--log"].(string); ok && s != "" {
		w, err := log.NewRollingFile(s, log.DailyRolling)
		if err != nil {
			log.PanicErrorf(err, "open log file %s failed", s)
		} else {
			log.StdLog = log.New(w, "")
		}
	}
	log.SetLevel(log.LEVEL_INFO)

	fmt.Println(banner)

	ncpu := runtime.NumCPU()
	if s, ok := d["--ncpu"].(string); ok && s != "" {
		n, err := strconv.Atoi(s)
		if err != nil {
			log.PanicErrorf(err, "parse --ncpu failed, invalid ncpu = '%s'", s)
		}
		ncpu = n
	}
	runtime.GOMAXPROCS(ncpu)
	log.Infof("set ncpu = %d", ncpu)

	if s, ok := d["--log-level"].(string); ok && s != "" {
		var level = strings.ToUpper(s)
		switch s {
		case "ERROR":
			log.SetLevel(log.LEVEL_ERROR)
		case "DEBUG":
			log.SetLevel(log.LEVEL_DEBUG)
		case "WARN", "WARNING":
			log.SetLevel(log.LEVEL_WARN)
		case "INFO":
			log.SetLevel(log.LEVEL_INFO)
		default:
			log.Panicf("parse --log-level failed, invalid level = '%s'", level)
		}
	}

	config := topom.NewDefaultConfig()
	if s, ok := d["--config"].(string); ok && s != "" {
		if err := config.LoadFromFile(s); err != nil {
			log.PanicErrorf(err, "load config failed, file = '%s'", s)
		}
	}

	var store models.Store
	switch {
	case d["--zookeeper"] != nil:
		store, err = zkstore.NewStore(d["--zookeeper"].(string))
		if err != nil {
			log.PanicErrorf(err, "create zkstore failed")
		}
	case d["--etcd"] != nil:
		log.Panicf("etcd is not supported yet!!")
	}

	if store == nil {
		log.Panicf("nil store for topom")
	}
	defer store.Close()

	s, err := topom.New(store, config)
	if err != nil {
		log.PanicErrorf(err, "create topom with config file failed\n%s\n", config)
	}
	defer s.Close()

	log.Infof("create topom with config\n%s\n", config)

	go func() {
		defer s.Close()
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM)

		sig := <-c
		log.Infof("[%p] dashboard receive signal = '%v'", s, sig)
	}()

	for !s.IsClosed() {
		time.Sleep(time.Second)
	}

	log.Infof("[%p] topom exiting ...", s)
}
<commit_msg>Fix, call StartDaemonRoutines after topom's creation<commit_after>// Copyright 2014 Wandoujia Inc. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package main

import (
	"fmt"
	"os"
	"os/signal"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/docopt/docopt-go"

	"github.com/wandoulabs/codis/pkg/models"
	"github.com/wandoulabs/codis/pkg/models/store/zk"
	"github.com/wandoulabs/codis/pkg/topom"
	"github.com/wandoulabs/codis/pkg/utils"
	"github.com/wandoulabs/codis/pkg/utils/log"
)

const banner = `
  _____  ____    ____/ /  (_)  _____
 / ___/ / __ \  / __  /  / /  / ___/
/ /__  / /_/ / / /_/ /  / /  (__  )
\___/  \____/  \__,_/  /_/  /____/

`

func main() {
	const usage = `
Usage:
	codis-dashboard [--ncpu=N] [--config=CONF] [--log=FILE] [--log-level=LEVEL] (--zookeeper=ADDR|--etcd=ADDR)
	codis-dashboard version

Options:
	--ncpu=N                    set runtime.GOMAXPROCS to N, default is runtime.NumCPU().
	-c CONF, --config=CONF      specify the config file.
	-l FILE, --log=FILE         specify the daliy rotated log file.
	--log-level=LEVEL           specify the log-level, can be INFO,WARN,DEBUG,ERROR, default is INFO.
`

	d, err := docopt.Parse(usage, nil, true, "", false)
	if err != nil {
		log.PanicError(err, "parse arguments failed")
	}

	if v, ok := d["version"].(bool); ok && v {
		fmt.Println("version:", utils.Version)
		fmt.Println("compile:", utils.Compile)
		return
	}

	if s, ok := d["--log"].(string); ok && s != "" {
		w, err := log.NewRollingFile(s, log.DailyRolling)
		if err != nil {
			log.PanicErrorf(err, "open log file %s failed", s)
		} else {
			log.StdLog = log.New(w, "")
		}
	}
	log.SetLevel(log.LEVEL_INFO)

	fmt.Println(banner)

	ncpu := runtime.NumCPU()
	if s, ok := d["--ncpu"].(string); ok && s != "" {
		n, err := strconv.Atoi(s)
		if err != nil {
			log.PanicErrorf(err, "parse --ncpu failed, invalid ncpu = '%s'", s)
		}
		ncpu = n
	}
	runtime.GOMAXPROCS(ncpu)
	log.Infof("set ncpu = %d", ncpu)

	if s, ok := d["--log-level"].(string); ok && s != "" {
		var level = strings.ToUpper(s)
		switch s {
		case "ERROR":
			log.SetLevel(log.LEVEL_ERROR)
		case "DEBUG":
			log.SetLevel(log.LEVEL_DEBUG)
		case "WARN", "WARNING":
			log.SetLevel(log.LEVEL_WARN)
		case "INFO":
			log.SetLevel(log.LEVEL_INFO)
		default:
			log.Panicf("parse --log-level failed, invalid level = '%s'", level)
		}
	}

	config := topom.NewDefaultConfig()
	if s, ok := d["--config"].(string); ok && s != "" {
		if err := config.LoadFromFile(s); err != nil {
			log.PanicErrorf(err, "load config failed, file = '%s'", s)
		}
	}

	var store models.Store
	switch {
	case d["--zookeeper"] != nil:
		store, err = zkstore.NewStore(d["--zookeeper"].(string))
		if err != nil {
			log.PanicErrorf(err, "create zkstore failed")
		}
	case d["--etcd"] != nil:
		log.Panicf("etcd is not supported yet!!")
	}

	if store == nil {
		log.Panicf("nil store for topom")
	}
	defer store.Close()

	s, err := topom.New(store, config)
	if err != nil {
		log.PanicErrorf(err, "create topom with config file failed\n%s\n", config)
	}
	defer s.Close()

	s.StartDaemonRoutines()

	log.Infof("create topom with config\n%s\n", config)

	go func() {
		defer s.Close()
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM)

		sig := <-c
		log.Infof("[%p] dashboard receive signal = '%v'", s, sig)
	}()

	for !s.IsClosed() {
		time.Sleep(time.Second)
	}

	log.Infof("[%p] topom exiting ...", s)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Debian/debiman/internal/manpage"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

//go:generate go run gentmpl.go header footer style manpage manpageerror contents pkgindex index faq
//go:generate go run genbundled.go static/Inconsolata.woff static/Inconsolata.woff2 static/openlogo-50.svg static/opensearch.xml static/Roboto-Bold.woff static/Roboto-Bold.woff2 static/Roboto-Regular.woff static/Roboto-Regular.woff2

type breadcrumb struct {
	Link string
	Text string
}

var commonTmpls = parseCommonTemplates()

func parseCommonTemplates() *template.Template {
	t := template.New("root")
	t = template.Must(t.New("header").Parse(headerContent))
	t = template.Must(t.New("footer").
		Funcs(map[string]interface{}{
			"Now": func() string {
				return time.Now().UTC().Format(iso8601Format)
			}}).
		Parse(footerContent))
	t = template.Must(t.New("style").Parse(styleContent))
	return t
}

type renderingMode int

const (
	regularFiles renderingMode = iota
	symlinks
	packageIndex
)

func walkContents(ctx context.Context, renderChan chan<- renderJob, contents map[string][]os.FileInfo, whitelist map[string]bool, mode renderingMode, gv globalView) error {
	// the invariant is: each file ending in .gz must have a corresponding .html.gz file
	// the .html.gz must have a modtime that is >= the modtime of the .gz file
	for dir, files := range contents {
		if whitelist != nil && !whitelist[filepath.Base(dir)] {
			continue
		}

		fileByName := make(map[string]os.FileInfo, len(files))
		for _, f := range files {
			fileByName[f.Name()] = f
		}

		manpageByName := make(map[string]*manpage.Meta, len(files))

		var indexModTime time.Time
		if fi, ok := fileByName["index.html.gz"]; ok {
			indexModTime = fi.ModTime()
		}
		var indexNeedsUpdate bool

		for _, f := range files {
			full := filepath.Join(dir, f.Name())
			if !strings.HasSuffix(full, ".gz") ||
				strings.HasSuffix(full, ".html.gz") {
				continue
			}

			symlink := f.Mode()&os.ModeSymlink != 0

			if mode == regularFiles && symlink ||
				mode == symlinks && !symlink {
				continue
			}

			if !indexNeedsUpdate && f.ModTime().After(indexModTime) {
				indexNeedsUpdate = true
			}

			m, err := manpage.FromServingPath(*servingDir, full)
			if err != nil {
				// If we run into this case, our code cannot correctly
				// interpret the result of ServingPath().
				log.Printf("BUG: cannot parse manpage from serving path %q: %v", full, err)
				continue
			}

			manpageByName[f.Name()] = m
			if mode == packageIndex {
				continue
			}

			n := strings.TrimSuffix(f.Name(), ".gz") + ".html.gz"
			html, ok := fileByName[n]
			if !ok || html.ModTime().Before(f.ModTime()) || *forceRerender {
				versions := gv.xref[m.Name]
				// Replace m with its corresponding entry in versions
				// so that rendermanpage() can use pointer equality to
				// efficiently skip entries.
				for _, v := range versions {
					if v.ServingPath() == m.ServingPath() {
						m = v
						break
					}
				}
				select {
				case renderChan <- renderJob{
					dest:     filepath.Join(dir, n),
					src:      full,
					meta:     m,
					versions: versions,
					xref:     gv.xref,
					modTime:  f.ModTime(),
					symlink:  symlink,
				}:
				case <-ctx.Done():
					break
				}
			}
		}

		if mode != packageIndex {
			continue
		}

		if !indexNeedsUpdate && !*forceRerender {
			continue
		}

		if err := renderPkgindex(filepath.Join(dir, "index.html.gz"), manpageByName); err != nil {
			return err
		}
	}

	return nil
}

func renderAll(gv globalView) error {
	binsBySuite := make(map[string][]string)

	suitedirs, err := ioutil.ReadDir(*servingDir)
	if err != nil {
		return err
	}
	// To minimize I/O, gather all FileInfos in one pass.
	contents := make(map[string][]os.FileInfo)
	for _, sfi := range suitedirs {
		if !sfi.IsDir() {
			continue
		}
		if !gv.suites[sfi.Name()] {
			continue
		}
		bins, err := ioutil.ReadDir(filepath.Join(*servingDir, sfi.Name()))
		if err != nil {
			return err
		}
		names := make([]string, len(bins))
		for idx, bfi := range bins {
			names[idx] = bfi.Name()
			dir := filepath.Join(*servingDir, sfi.Name(), bfi.Name())
			contents[dir], err = ioutil.ReadDir(dir)
			if err != nil {
				return err
			}
		}
		binsBySuite[sfi.Name()] = names
	}

	eg, ctx := errgroup.WithContext(context.Background())
	renderChan := make(chan renderJob)
	// TODO: flag for parallelism level
	for i := 0; i < 30; i++ {
		eg.Go(func() error {
			for r := range renderChan {
				if err := rendermanpage(r); err != nil {
					// rendermanpage writes an error page if rendering
					// failed, any returned error is severe (e.g. file
					// system full) and should lead to termination.
					return err
				}
			}
			return nil
		})
	}
	var whitelist map[string]bool
	if *onlyRender != "" {
		whitelist = make(map[string]bool)
		log.Printf("Restricting rendering to the following binary packages:")
		for _, e := range strings.Split(strings.TrimSpace(*onlyRender), ",") {
			whitelist[e] = true
			log.Printf("  %q", e)
		}
		log.Printf("(total: %d whitelist entries)", len(whitelist))
	}

	// Render all regular files first
	if err := walkContents(ctx, renderChan, contents, whitelist, regularFiles, gv); err != nil {
		return err
	}

	// then render all symlinks, re-using the rendered fragments
	if err := walkContents(ctx, renderChan, contents, whitelist, symlinks, gv); err != nil {
		return err
	}

	// and finally render the package index files which need to
	// consider both regular files and symlinks.
	if err := walkContents(ctx, renderChan, contents, whitelist, packageIndex, gv); err != nil {
		return err
	}

	close(renderChan)
	if err := eg.Wait(); err != nil {
		return err
	}

	for suite, bins := range binsBySuite {
		if err := renderContents(filepath.Join(*servingDir, fmt.Sprintf("contents-%s.html.gz", suite)), suite, bins); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Skip package index generation for empty directories<commit_after>package main

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/Debian/debiman/internal/manpage"
	"golang.org/x/net/context"
	"golang.org/x/sync/errgroup"
)

//go:generate go run gentmpl.go header footer style manpage manpageerror contents pkgindex index faq
//go:generate go run genbundled.go static/Inconsolata.woff static/Inconsolata.woff2 static/openlogo-50.svg static/opensearch.xml static/Roboto-Bold.woff static/Roboto-Bold.woff2 static/Roboto-Regular.woff static/Roboto-Regular.woff2

type breadcrumb struct {
	Link string
	Text string
}

var commonTmpls = parseCommonTemplates()

func parseCommonTemplates() *template.Template {
	t := template.New("root")
	t = template.Must(t.New("header").Parse(headerContent))
	t = template.Must(t.New("footer").
		Funcs(map[string]interface{}{
			"Now": func() string {
				return time.Now().UTC().Format(iso8601Format)
			}}).
		Parse(footerContent))
	t = template.Must(t.New("style").Parse(styleContent))
	return t
}

type renderingMode int

const (
	regularFiles renderingMode = iota
	symlinks
	packageIndex
)

func walkContents(ctx context.Context, renderChan chan<- renderJob, contents map[string][]os.FileInfo, whitelist map[string]bool, mode renderingMode, gv globalView) error {
	// the invariant is: each file ending in .gz must have a corresponding .html.gz file
	// the .html.gz must have a modtime that is >= the modtime of the .gz file
	for dir, files := range contents {
		if whitelist != nil && !whitelist[filepath.Base(dir)] {
			continue
		}

		fileByName := make(map[string]os.FileInfo, len(files))
		for _, f := range files {
			fileByName[f.Name()] = f
		}

		manpageByName := make(map[string]*manpage.Meta, len(files))

		var indexModTime time.Time
		if fi, ok := fileByName["index.html.gz"]; ok {
			indexModTime = fi.ModTime()
		}
		var indexNeedsUpdate bool

		for _, f := range files {
			full := filepath.Join(dir, f.Name())
			if !strings.HasSuffix(full, ".gz") ||
				strings.HasSuffix(full, ".html.gz") {
				continue
			}

			symlink := f.Mode()&os.ModeSymlink != 0

			if mode == regularFiles && symlink ||
				mode == symlinks && !symlink {
				continue
			}

			if !indexNeedsUpdate && f.ModTime().After(indexModTime) {
				indexNeedsUpdate = true
			}

			m, err := manpage.FromServingPath(*servingDir, full)
			if err != nil {
				// If we run into this case, our code cannot correctly
				// interpret the result of ServingPath().
				log.Printf("BUG: cannot parse manpage from serving path %q: %v", full, err)
				continue
			}

			manpageByName[f.Name()] = m
			if mode == packageIndex {
				continue
			}

			n := strings.TrimSuffix(f.Name(), ".gz") + ".html.gz"
			html, ok := fileByName[n]
			if !ok || html.ModTime().Before(f.ModTime()) || *forceRerender {
				versions := gv.xref[m.Name]
				// Replace m with its corresponding entry in versions
				// so that rendermanpage() can use pointer equality to
				// efficiently skip entries.
				for _, v := range versions {
					if v.ServingPath() == m.ServingPath() {
						m = v
						break
					}
				}
				select {
				case renderChan <- renderJob{
					dest:     filepath.Join(dir, n),
					src:      full,
					meta:     m,
					versions: versions,
					xref:     gv.xref,
					modTime:  f.ModTime(),
					symlink:  symlink,
				}:
				case <-ctx.Done():
					break
				}
			}
		}

		if mode != packageIndex {
			continue
		}

		if !indexNeedsUpdate && !*forceRerender {
			continue
		}

		if len(manpageByName) == 0 {
			log.Printf("WARNING: empty directory %q, not generating package index", dir)
			continue
		}

		if err := renderPkgindex(filepath.Join(dir, "index.html.gz"), manpageByName); err != nil {
			return err
		}
	}

	return nil
}

func renderAll(gv globalView) error {
	binsBySuite := make(map[string][]string)

	suitedirs, err := ioutil.ReadDir(*servingDir)
	if err != nil {
		return err
	}
	// To minimize I/O, gather all FileInfos in one pass.
	contents := make(map[string][]os.FileInfo)
	for _, sfi := range suitedirs {
		if !sfi.IsDir() {
			continue
		}
		if !gv.suites[sfi.Name()] {
			continue
		}
		bins, err := ioutil.ReadDir(filepath.Join(*servingDir, sfi.Name()))
		if err != nil {
			return err
		}
		names := make([]string, len(bins))
		for idx, bfi := range bins {
			names[idx] = bfi.Name()
			dir := filepath.Join(*servingDir, sfi.Name(), bfi.Name())
			contents[dir], err = ioutil.ReadDir(dir)
			if err != nil {
				return err
			}
		}
		binsBySuite[sfi.Name()] = names
	}

	eg, ctx := errgroup.WithContext(context.Background())
	renderChan := make(chan renderJob)
	// TODO: flag for parallelism level
	for i := 0; i < 30; i++ {
		eg.Go(func() error {
			for r := range renderChan {
				if err := rendermanpage(r); err != nil {
					// rendermanpage writes an error page if rendering
					// failed, any returned error is severe (e.g. file
					// system full) and should lead to termination.
					return err
				}
			}
			return nil
		})
	}
	var whitelist map[string]bool
	if *onlyRender != "" {
		whitelist = make(map[string]bool)
		log.Printf("Restricting rendering to the following binary packages:")
		for _, e := range strings.Split(strings.TrimSpace(*onlyRender), ",") {
			whitelist[e] = true
			log.Printf("  %q", e)
		}
		log.Printf("(total: %d whitelist entries)", len(whitelist))
	}

	// Render all regular files first
	if err := walkContents(ctx, renderChan, contents, whitelist, regularFiles, gv); err != nil {
		return err
	}

	// then render all symlinks, re-using the rendered fragments
	if err := walkContents(ctx, renderChan, contents, whitelist, symlinks, gv); err != nil {
		return err
	}

	// and finally render the package index files which need to
	// consider both regular files and symlinks.
	if err := walkContents(ctx, renderChan, contents, whitelist, packageIndex, gv); err != nil {
		return err
	}

	close(renderChan)
	if err := eg.Wait(); err != nil {
		return err
	}

	for suite, bins := range binsBySuite {
		if err := renderContents(filepath.Join(*servingDir, fmt.Sprintf("contents-%s.html.gz", suite)), suite, bins); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strconv"

	"github.com/alienth/fastlyctl/log"
	"github.com/alienth/fastlyctl/util"
	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "fastlyctl"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "config.toml",
			Usage: "Load Fastly configuration from `FILE`",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key. Can be read from 'fastly_key' file in CWD.",
			EnvVar: "FASTLY_KEY",
			Value:  util.GetFastlyKey(),
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "Print more detailed info for debugging.",
		},
		cli.BoolFlag{
			Name:  "assume-yes, y",
			Usage: "Assume 'yes' to all prompts. USE ONLY IF YOU ARE CERTAIN YOUR COMMANDS WON'T BREAK ANYTHING!",
		},
	}

	app.Before = func(c *cli.Context) error {
		if err := util.CheckFastlyKey(c); err != nil {
			return err
		}
		return nil
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "push",
			Aliases:   []string{"p"},
			Usage:     "Push locally defined service configuration options to Fastly.",
			ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>)...",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "all, a",
					Usage: "Push all services listed in config file",
				},
				cli.BoolFlag{
					Name:  "noop, n",
					Usage: "Push new config versions, but do not activate.",
				},
			},
			Before: func(c *cli.Context) error {
				if !util.IsInteractive() {
					return cli.NewExitError(util.ErrNonInteractive.Error(), -1)
				}
				if (!c.Bool("all") && !c.Args().Present()) || (c.Bool("all") && c.Args().Present()) {
					return cli.NewExitError("Error: either specify service names to be pushed, or push all with -a", -1)
				}
				if c.GlobalBool("debug") {
					log.EnableDebug()
				}
				return nil
			},
			Action: syncConfig,
		},
		cli.Command{
			Name:    "version",
			Aliases: []string{"v"},
			Usage:   "Manage service versions.",
			Before: func(c *cli.Context) error {
				// less than 2 here since the subcommand is the first Arg
				if len(c.Args()) < 2 {
					return cli.NewExitError("Please specify service.", -1)
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "list",
					Usage:     "List versions associated with a given service",
					Action:    versionList,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>)",
				},
				cli.Command{
					Name:      "validate",
					Usage:     "Validate a specified VERSION",
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <VERSION>",
					Action:    versionValidate,
					Before: func(c *cli.Context) error {
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to validate.", -1)
						}
						return nil
					},
				},
				cli.Command{
					Name:      "activate",
					Usage:     "Activate a specified VERSION",
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <VERSION>",
					Action:    versionActivate,
					Before: func(c *cli.Context) error {
						if !util.IsInteractive() {
							return cli.NewExitError(util.ErrNonInteractive.Error(), -1)
						}
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to activate.", -1)
						}
						return versionValidate(c)
					},
				},
			},
		},
		cli.Command{
			Name:  "service",
			Usage: "Manage services.",
			Subcommands: cli.Commands{
				cli.Command{
					Name:   "list",
					Usage:  "List services associated with account",
					Action: serviceList,
				},
			},
		},
		cli.Command{
			Name:    "dictionary",
			Aliases: []string{"d"},
			Usage:   "Manage dictionaries.",
			Before: func(c *cli.Context) error {
				// less than 2 here since the subcommand is the first Arg
				if len(c.Args()) < 2 {
					return cli.NewExitError("Please specify service.", -1)
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "list",
					Usage:     "List dictionaries associated with a given service",
					Action:    dictionaryList,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>)",
				},
				cli.Command{
					Name:      "item-add",
					Usage:     "Add an item to a dictionary",
					Action:    dictionaryAddItem,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <DICTIONARY_NAME> <ITEM_KEY> <ITEM_VALUE>",
				},
				cli.Command{
					Name:      "item-rm",
					Usage:     "Remove an item from a dictionary",
					Action:    dictionaryRemoveItem,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <DICTIONARY_NAME> <ITEM_KEY>",
				},
				cli.Command{
					Name:      "item-ls",
					Usage:     "List items in a dictionary",
					Action:    dictionaryListItems,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <DICTIONARY_NAME>",
				},
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Fix the interactive check.<commit_after>package main

import (
	"os"
	"strconv"

	"github.com/alienth/fastlyctl/log"
	"github.com/alienth/fastlyctl/util"
	"github.com/urfave/cli"
)

func main() {
	app := cli.NewApp()
	app.Name = "fastlyctl"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config, c",
			Value: "config.toml",
			Usage: "Load Fastly configuration from `FILE`",
		},
		cli.StringFlag{
			Name:   "fastly-key, K",
			Usage:  "Fastly API Key. Can be read from 'fastly_key' file in CWD.",
			EnvVar: "FASTLY_KEY",
			Value:  util.GetFastlyKey(),
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "Print more detailed info for debugging.",
		},
		cli.BoolFlag{
			Name:  "assume-yes, y",
			Usage: "Assume 'yes' to all prompts. USE ONLY IF YOU ARE CERTAIN YOUR COMMANDS WON'T BREAK ANYTHING!",
		},
	}

	app.Before = func(c *cli.Context) error {
		if err := util.CheckFastlyKey(c); err != nil {
			return err
		}
		return nil
	}

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "push",
			Aliases:   []string{"p"},
			Usage:     "Push locally defined service configuration options to Fastly.",
			ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>)...",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "all, a",
					Usage: "Push all services listed in config file",
				},
				cli.BoolFlag{
					Name:  "noop, n",
					Usage: "Push new config versions, but do not activate.",
				},
			},
			Before: func(c *cli.Context) error {
				if !util.IsInteractive() && !c.GlobalBool("assume-yes") {
					return cli.NewExitError(util.ErrNonInteractive.Error(), -1)
				}
				if (!c.Bool("all") && !c.Args().Present()) || (c.Bool("all") && c.Args().Present()) {
					return cli.NewExitError("Error: either specify service names to be pushed, or push all with -a", -1)
				}
				if c.GlobalBool("debug") {
					log.EnableDebug()
				}
				return nil
			},
			Action: syncConfig,
		},
		cli.Command{
			Name:    "version",
			Aliases: []string{"v"},
			Usage:   "Manage service versions.",
			Before: func(c *cli.Context) error {
				// less than 2 here since the subcommand is the first Arg
				if len(c.Args()) < 2 {
					return cli.NewExitError("Please specify service.", -1)
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "list",
					Usage:     "List versions associated with a given service",
					Action:    versionList,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>)",
				},
				cli.Command{
					Name:      "validate",
					Usage:     "Validate a specified VERSION",
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <VERSION>",
					Action:    versionValidate,
					Before: func(c *cli.Context) error {
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to validate.", -1)
						}
						return nil
					},
				},
				cli.Command{
					Name:      "activate",
					Usage:     "Activate a specified VERSION",
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <VERSION>",
					Action:    versionActivate,
					Before: func(c *cli.Context) error {
						if !util.IsInteractive() && !c.GlobalBool("assume-yes") {
							return cli.NewExitError(util.ErrNonInteractive.Error(), -1)
						}
						if _, err := strconv.Atoi(c.Args().Get(1)); err != nil {
							return cli.NewExitError("Please specify version to activate.", -1)
						}
						return versionValidate(c)
					},
				},
			},
		},
		cli.Command{
			Name:  "service",
			Usage: "Manage services.",
			Subcommands: cli.Commands{
				cli.Command{
					Name:   "list",
					Usage:  "List services associated with account",
					Action: serviceList,
				},
			},
		},
		cli.Command{
			Name:    "dictionary",
			Aliases: []string{"d"},
			Usage:   "Manage dictionaries.",
			Before: func(c *cli.Context) error {
				// less than 2 here since the subcommand is the first Arg
				if len(c.Args()) < 2 {
					return cli.NewExitError("Please specify service.", -1)
				}
				return nil
			},
			Subcommands: cli.Commands{
				cli.Command{
					Name:      "list",
					Usage:     "List dictionaries associated with a given service",
					Action:    dictionaryList,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>)",
				},
				cli.Command{
					Name:      "item-add",
					Usage:     "Add an item to a dictionary",
					Action:    dictionaryAddItem,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <DICTIONARY_NAME> <ITEM_KEY> <ITEM_VALUE>",
				},
				cli.Command{
					Name:      "item-rm",
					Usage:     "Remove an item from a dictionary",
					Action:    dictionaryRemoveItem,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <DICTIONARY_NAME> <ITEM_KEY>",
				},
				cli.Command{
					Name:      "item-ls",
					Usage:     "List items in a dictionary",
					Action:    dictionaryListItems,
					ArgsUsage: "(<SERVICE_NAME> | <SERVICE_ID>) <DICTIONARY_NAME>",
				},
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package internal

import (
	"errors"
	"fmt"

	"github.com/ready-steady/probability"
	"github.com/ready-steady/staircase"
	"github.com/ready-steady/statistics/correlation"

	acorrelation "../../pkg/correlation"
)

var (
	standardGaussian = probability.NewGaussian(0, 1)
)

type model struct {
	taskIndex  []uint
	correlator []float64
	modes      []mode

	nt uint
	nu uint
	nz uint
}

type mode *staircase.Staircase

func newModel(c *ProbabilityConfig, s *system) (*model, error) {
	nt := s.nt

	taskIndex, err := parseNaturalIndex(c.TaskIndex, 0, nt-1)
	if err != nil {
		return nil, err
	}

	correlator, err := computeCorrelator(c, s, taskIndex)
	if err != nil {
		return nil, err
	}

	nu := uint(len(taskIndex))
	nz := uint(len(correlator)) / nu

	modes, err := computeModes(c, nu)
	if err != nil {
		return nil, err
	}

	model := &model{
		taskIndex:  taskIndex,
		correlator: correlator,
		modes:      modes,

		nt: nt,
		nu: nu,
		nz: nz,
	}

	return model, nil
}

func (m *model) String() string {
	return fmt.Sprintf(`{"parameters": %d, "variables": %d}`, m.nu, m.nz)
}

func (m *model) transform(z []float64) []float64 {
	nt, nu, nz := m.nt, m.nu, m.nz

	n := make([]float64, nz)
	u := make([]float64, nu)

	// Independent uniform to independent Gaussian
	for i := range n {
		n[i] = standardGaussian.InvCDF(z[i])
	}

	// Independent Gaussian to dependent Gaussian
	combine(m.correlator, n, u, nu, nz)

	// Dependent Gaussian to dependent uniform
	for i := range u {
		u[i] = standardGaussian.CDF(u[i])
	}

	modes := make([]float64, nt)
	for i, tid := range m.taskIndex {
		modes[tid] = (*staircase.Staircase)(m.modes[i]).Evaluate(u[i])
	}

	return modes
}

func computeCorrelator(c *ProbabilityConfig, s *system, taskIndex []uint) ([]float64, error) {
	if c.CorrLength < 0 {
		return nil, errors.New("the correlation length should be nonnegative")
	}
	if c.VarThreshold <= 0 {
		return nil, errors.New("the variance-reduction threshold should be positive")
	}

	C := acorrelation.Compute(s.application, taskIndex, c.CorrLength)
	correlator, _, err := correlation.Decompose(C, uint(len(taskIndex)), c.VarThreshold)
	if err != nil {
		return nil, err
	}

	return correlator, nil
}

func computeModes(c *ProbabilityConfig, count uint) ([]mode, error) {
	if c.Modes == 0 {
		return nil, errors.New("the number of modes should be positive")
	}
	if c.MinOffset <= -1 || c.MaxOffset <= -1 {
		return nil, errors.New("the offsets should be greater than -1")
	}
	if c.Transition <= 0 || c.Transition > 0.5 {
		return nil, errors.New("the transition parameter should be in (0, 0.5]")
	}

	generator := probability.NewGenerator(newSeed(c.Seed))
	uniform := probability.NewUniform(0, 1)

	result := make([]mode, count)
	for i := range result {
		Σ := 0.0
		values := make([]float64, c.Modes)
		probabilities := make([]float64, c.Modes)
		for j := range values {
			values[j] = uniform.Sample(generator)
			probabilities[j] = uniform.Sample(generator)
			Σ += probabilities[j]
		}
		for j := range values {
			values[j] *= c.MaxOffset - c.MinOffset
			values[j] += c.MinOffset
			probabilities[j] /= Σ
		}

		result[i] = staircase.New(probabilities, values, c.Transition)
	}

	return result, nil
}
<commit_msg>A cosmetic adjustment<commit_after>package internal

import (
	"errors"
	"fmt"

	"github.com/ready-steady/probability"
	"github.com/ready-steady/staircase"
	"github.com/ready-steady/statistics/correlation"

	acorrelation "../../pkg/correlation"
)

var (
	standardGaussian = probability.NewGaussian(0, 1)
)

type model struct {
	taskIndex  []uint
	correlator []float64
	modes      []mode

	nt uint
	nu uint
	nz uint
}

type mode *staircase.Staircase

func newModel(c *ProbabilityConfig, s *system) (*model, error) {
	nt := s.nt

	taskIndex, err := parseNaturalIndex(c.TaskIndex, 0, nt-1)
	if err != nil {
		return nil, err
	}

	correlator, err := computeCorrelator(c, s, taskIndex)
	if err != nil {
		return nil, err
	}

	nu := uint(len(taskIndex))
	nz := uint(len(correlator)) / nu

	modes, err := computeModes(c, nu)
	if err != nil {
		return nil, err
	}

	model := &model{
		taskIndex:  taskIndex,
		correlator: correlator,
		modes:      modes,

		nt: nt,
		nu: nu,
		nz: nz,
	}

	return model, nil
}

func (m *model) String() string {
	return fmt.Sprintf(`{"parameters": %d, "variables": %d}`, m.nu, m.nz)
}

func (m *model) transform(z []float64) []float64 {
	nt, nu, nz := m.nt, m.nu, m.nz

	n := make([]float64, nz)
	u := make([]float64, nu)

	// Independent uniform to independent Gaussian
	for i := range n {
		n[i] = standardGaussian.InvCDF(z[i])
	}

	// Independent Gaussian to dependent Gaussian
	combine(m.correlator, n, u, nu, nz)

	// Dependent Gaussian to dependent uniform
	for i := range u {
		u[i] = standardGaussian.CDF(u[i])
	}

	modes := make([]float64, nt)
	for i, tid := range m.taskIndex {
		modes[tid] = (*staircase.Staircase)(m.modes[i]).Evaluate(u[i])
	}

	return modes
}

func computeCorrelator(c *ProbabilityConfig, s *system, taskIndex []uint) ([]float64, error) {
	if c.CorrLength < 0 {
		return nil, errors.New("the correlation length should be nonnegative")
	}
	if c.VarThreshold <= 0 {
		return nil, errors.New("the variance-reduction threshold should be positive")
	}

	C := acorrelation.Compute(s.application, taskIndex, c.CorrLength)
	correlator, _, err := correlation.Decompose(C, uint(len(taskIndex)), c.VarThreshold)
	if err != nil {
		return nil, err
	}

	return correlator, nil
}

func computeModes(c *ProbabilityConfig, count uint) ([]mode, error) {
	if c.Modes == 0 {
		return nil, errors.New("the number of modes should be positive")
	}
	if c.MinOffset <= -1 || c.MaxOffset <= -1 {
		return nil, errors.New("the offsets should be greater than -1")
	}
	if c.Transition <= 0 || c.Transition > 0.5 {
		return nil, errors.New("the transition parameter should be in (0, 0.5]")
	}

	generator := probability.NewGenerator(newSeed(c.Seed))
	uniform := probability.NewUniform(0, 1)

	result := make([]mode, count)
	for i := range result {
		Σ := 0.0
		values := make([]float64, c.Modes)
		probabilities := make([]float64, c.Modes)
		for j := range values {
			values[j] = uniform.Sample(generator)
			probabilities[j] = uniform.Sample(generator)
			Σ += probabilities[j]
		}
		for j := range values {
			values[j] = c.MinOffset + (c.MaxOffset-c.MinOffset)*values[j]
			probabilities[j] /= Σ
		}

		result[i] = staircase.New(probabilities, values, c.Transition)
	}

	return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	_ "launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/testing"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	stdtesting "testing"
)

func TestPackage(t *stdtesting.T) {
	testing.MgoTestPackage(t)
}

type MainSuite struct{}

var _ = Suite(&MainSuite{})

var flagRunMain = flag.Bool("run-main", false, "Run the application's main function for recursive testing")

// Reentrancy point for testing (something as close as possible to) the juju
// tool itself.
func TestRunMain(t *stdtesting.T) {
	if *flagRunMain {
		Main(flag.Args())
	}
}

func badrun(c *C, exit int, cmd ...string) string {
	args := append([]string{"-test.run", "TestRunMain", "-run-main", "--", "juju"}, cmd...)
	ps := exec.Command(os.Args[0], args...)
	output, err := ps.CombinedOutput()
	if exit != 0 {
		c.Assert(err, ErrorMatches, fmt.Sprintf("exit status %d", exit))
	}
	return string(output)
}

var runMainTests = []struct {
	summary string
	args    []string
	code    int
	out     string
}{
	{
		summary: "missing command",
		code:    2,
		out:     "error: no command specified\n",
	}, {
		summary: "unknown command",
		args:    []string{"discombobulate"},
		code:    2,
		out:     "error: unrecognized command: juju discombobulate\n",
	}, {
		summary: "unknown option before command",
		args:    []string{"--cheese", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "unknown option after command",
		args:    []string{"bootstrap", "--cheese"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "known option, but specified before command",
		args:    []string{"--environment", "blah", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --environment\n",
	},
}

func (s *MainSuite) TestRunMain(c *C) {
	for i, t := range runMainTests {
		c.Logf("test %d: %s", i, t.summary)
		out := badrun(c, t.code, t.args...)
		c.Assert(out, Equals, t.out)
	}
}

var brokenConfig = `
environments:
    one:
        type: dummy
        state-server: false
        authorized-keys: i-am-a-key
        broken: %s
`

// breakJuju forces the dummy environment to return an error
// when environMethod is called.
func breakJuju(c *C, environMethod string) (msg string) {
	yaml := fmt.Sprintf(brokenConfig, environMethod)
	err := ioutil.WriteFile(homePath(".juju", "environments.yaml"), []byte(yaml), 0666)
	c.Assert(err, IsNil)

	return fmt.Sprintf("dummy.%s is broken", environMethod)
}

func (s *MainSuite) TestActualRunJujuArgsBeforeCommand(c *C) {
	defer makeFakeHome(c, "one").restore()

	// Check global args work when specified before command
	msg := breakJuju(c, "Bootstrap")
	logpath := filepath.Join(c.MkDir(), "log")
	out := badrun(c, 1, "--log-file", logpath, "--verbose", "--debug", "bootstrap")
	c.Assert(out, Equals, "error: "+msg+"\n")
	content, err := ioutil.ReadFile(logpath)
	c.Assert(err, IsNil)
	fullmsg := fmt.Sprintf(`(.|\n)*JUJU juju bootstrap command failed: %s\n`, msg)
	c.Assert(string(content), Matches, fullmsg)
}

func (s *MainSuite) TestActualRunJujuArgsAfterCommand(c *C) {
	defer makeFakeHome(c, "one").restore()

	// Check global args work when specified after command
	msg := breakJuju(c, "Bootstrap")
	logpath := filepath.Join(c.MkDir(), "log")
	out := badrun(c, 1, "bootstrap", "--log-file", logpath, "--verbose", "--debug")
	c.Assert(out, Equals, "error: "+msg+"\n")
	content, err := ioutil.ReadFile(logpath)
	c.Assert(err, IsNil)
	fullmsg := fmt.Sprintf(`(.|\n)*JUJU juju bootstrap command failed: %s\n`, msg)
	c.Assert(string(content), Matches, fullmsg)
}

var commandNames = []string{
	"add-relation",
	"add-unit",
	"bootstrap",
	"deploy",
	"destroy-environment",
	"destroy-machine",
	"destroy-relation",
	"destroy-service",
	"destroy-unit",
	"expose",
	"generate-config", // alias for init
	"get",
	"get-constraints",
	"init",
	"remove-relation", // alias for destory-relation
	"remove-unit",     // alias for destroy-unit
	"resolved",
	"scp",
	"set",
	"set-constraints",
	"ssh",
	"stat", // alias for status
	"status",
	"terminate-machine", // alias for destroy-machine
	"unexpose",
	"upgrade-juju",
}

func (s *MainSuite) TestHelp(c *C) {
	// Check that we have correctly registered all the commands
	// by checking the help output.

	out := badrun(c, 0, "-help")
	lines := strings.Split(out, "\n")
	c.Assert(lines[0], Matches, `usage: juju .*`)
	for ; len(lines) > 0; lines = lines[1:] {
		if lines[0] == "commands:" {
			break
		}
	}
	c.Assert(lines, Not(HasLen), 0)

	var names []string
	for lines = lines[1:]; len(lines) > 0; lines = lines[1:] {
		f := strings.Fields(lines[0])
		if len(f) == 0 {
			continue
		}
		c.Assert(f, Not(HasLen), 0)
		names = append(names, f[0])
	}
	sort.Strings(names)
	c.Assert(names, DeepEquals, commandNames)
}

type fakeHome string

func makeFakeHome(c *C, certNames ...string) fakeHome {
	oldHome := os.Getenv("HOME")
	os.Setenv("HOME", c.MkDir())

	err := os.Mkdir(homePath(".juju"), 0777)
	c.Assert(err, IsNil)
	for _, name := range certNames {
		err := ioutil.WriteFile(homePath(".juju", name+"-cert.pem"), []byte(testing.CACert), 0666)
		c.Assert(err, IsNil)

		err = ioutil.WriteFile(homePath(".juju", name+"-private-key.pem"), []byte(testing.CAKey), 0666)
		c.Assert(err, IsNil)
	}

	err = os.Mkdir(homePath(".ssh"), 0777)
	c.Assert(err, IsNil)
	err = ioutil.WriteFile(homePath(".ssh", "id_rsa.pub"), []byte("auth key\n"), 0666)
	c.Assert(err, IsNil)

	return fakeHome(oldHome)
}

func homePath(names ...string) string {
	all := append([]string{os.Getenv("HOME")}, names...)
	return filepath.Join(all...)
}

func (h fakeHome) restore() {
	os.Setenv("HOME", string(h))
}
<commit_msg>It is no longer an error to call the juju command without a command.<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	. "launchpad.net/gocheck"
	_ "launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/testing"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	stdtesting "testing"
)

func TestPackage(t *stdtesting.T) {
	testing.MgoTestPackage(t)
}

type MainSuite struct{}

var _ = Suite(&MainSuite{})

var flagRunMain = flag.Bool("run-main", false, "Run the application's main function for recursive testing")

// Reentrancy point for testing (something as close as possible to) the juju
// tool itself.
func TestRunMain(t *stdtesting.T) {
	if *flagRunMain {
		Main(flag.Args())
	}
}

func badrun(c *C, exit int, cmd ...string) string {
	args := append([]string{"-test.run", "TestRunMain", "-run-main", "--", "juju"}, cmd...)
	ps := exec.Command(os.Args[0], args...)
	output, err := ps.CombinedOutput()
	if exit != 0 {
		c.Assert(err, ErrorMatches, fmt.Sprintf("exit status %d", exit))
	}
	return string(output)
}

var runMainTests = []struct {
	summary string
	args    []string
	code    int
	out     string
}{
	{
		summary: "unknown command",
		args:    []string{"discombobulate"},
		code:    2,
		out:     "error: unrecognized command: juju discombobulate\n",
	}, {
		summary: "unknown option before command",
		args:    []string{"--cheese", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "unknown option after command",
		args:    []string{"bootstrap", "--cheese"},
		code:    2,
		out:     "error: flag provided but not defined: --cheese\n",
	}, {
		summary: "known option, but specified before command",
		args:    []string{"--environment", "blah", "bootstrap"},
		code:    2,
		out:     "error: flag provided but not defined: --environment\n",
	},
}

func (s *MainSuite) TestRunMain(c *C) {
	for i, t := range runMainTests {
		c.Logf("test %d: %s", i, t.summary)
		out := badrun(c, t.code, t.args...)
		c.Assert(out, Equals, t.out)
	}
}

var brokenConfig = `
environments:
    one:
        type: dummy
        state-server: false
        authorized-keys: i-am-a-key
        broken: %s
`

// breakJuju forces the dummy environment to return an error
// when environMethod is called.
func breakJuju(c *C, environMethod string) (msg string) {
	yaml := fmt.Sprintf(brokenConfig, environMethod)
	err := ioutil.WriteFile(homePath(".juju", "environments.yaml"), []byte(yaml), 0666)
	c.Assert(err, IsNil)

	return fmt.Sprintf("dummy.%s is broken", environMethod)
}

func (s *MainSuite) TestActualRunJujuArgsBeforeCommand(c *C) {
	defer makeFakeHome(c, "one").restore()

	// Check global args work when specified before command
	msg := breakJuju(c, "Bootstrap")
	logpath := filepath.Join(c.MkDir(), "log")
	out := badrun(c, 1, "--log-file", logpath, "--verbose", "--debug", "bootstrap")
	c.Assert(out, Equals, "error: "+msg+"\n")
	content, err := ioutil.ReadFile(logpath)
	c.Assert(err, IsNil)
	fullmsg := fmt.Sprintf(`(.|\n)*JUJU juju bootstrap command failed: %s\n`, msg)
	c.Assert(string(content), Matches, fullmsg)
}

func (s *MainSuite) TestActualRunJujuArgsAfterCommand(c *C) {
	defer makeFakeHome(c, "one").restore()

	// Check global args work when specified after command
	msg := breakJuju(c, "Bootstrap")
	logpath := filepath.Join(c.MkDir(), "log")
	out := badrun(c, 1, "bootstrap", "--log-file", logpath, "--verbose", "--debug")
	c.Assert(out, Equals, "error: "+msg+"\n")
	content, err := ioutil.ReadFile(logpath)
	c.Assert(err, IsNil)
	fullmsg := fmt.Sprintf(`(.|\n)*JUJU juju bootstrap command failed: %s\n`, msg)
	c.Assert(string(content), Matches, fullmsg)
}

var commandNames = []string{
	"?", // alias for help
	"add-relation",
	"add-unit",
	"bootstrap",
	"deploy",
	"destroy-environment",
	"destroy-machine",
	"destroy-relation",
	"destroy-service",
	"destroy-unit",
	"expose",
	"generate-config", // alias for init
	"get",
	"get-constraints",
	"help",
	"init",
	"remove-relation", // alias for destory-relation
	"remove-unit",     // alias for destroy-unit
	"resolved",
	"scp",
	"set",
	"set-constraints",
	"ssh",
	"stat", // alias for status
	"status",
	"terminate-machine", // alias for destroy-machine
	"unexpose",
	"upgrade-juju",
}

func (s *MainSuite) TestHelp(c *C) {
	// Check that we have correctly registered all the commands
	// by checking the help output.

	out := badrun(c, 0, "-help")
	lines := strings.Split(out, "\n")
	c.Assert(lines[0], Matches, `usage: juju .*`)
	for ; len(lines) > 0; lines = lines[1:] {
		if lines[0] == "commands:" {
			break
		}
	}
	c.Assert(lines, Not(HasLen), 0)

	var names []string
	for lines = lines[1:]; len(lines) > 0; lines = lines[1:] {
		f := strings.Fields(lines[0])
		if len(f) == 0 {
			continue
		}
		c.Assert(f, Not(HasLen), 0)
		names = append(names, f[0])
	}
	sort.Strings(names)
	c.Assert(names, DeepEquals, commandNames)
}

type fakeHome string

func makeFakeHome(c *C, certNames ...string) fakeHome {
	oldHome := os.Getenv("HOME")
	os.Setenv("HOME", c.MkDir())

	err := os.Mkdir(homePath(".juju"), 0777)
	c.Assert(err, IsNil)
	for _, name := range certNames {
		err := ioutil.WriteFile(homePath(".juju", name+"-cert.pem"), []byte(testing.CACert), 0666)
		c.Assert(err, IsNil)

		err = ioutil.WriteFile(homePath(".juju", name+"-private-key.pem"), []byte(testing.CAKey), 0666)
		c.Assert(err, IsNil)
	}

	err = os.Mkdir(homePath(".ssh"), 0777)
	c.Assert(err, IsNil)
	err = ioutil.WriteFile(homePath(".ssh", "id_rsa.pub"), []byte("auth key\n"), 0666)
	c.Assert(err, IsNil)

	return fakeHome(oldHome)
}

func homePath(names ...string) string {
	all := append([]string{os.Getenv("HOME")}, names...)
	return filepath.Join(all...)
}

func (h fakeHome) restore() {
	os.Setenv("HOME", string(h))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/validation"
	"k8s.io/kops/pkg/kopscodecs"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	"k8s.io/kubernetes/pkg/kubectl/cmd/util/editor"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

type CreateInstanceGroupOptions struct {
	Role    string
	Subnets []string
	// DryRun mode output an ig manifest of Output type.
	DryRun bool
	// Output type during a DryRun
	Output string
}

var (
	create_ig_long = templates.LongDesc(i18n.T(`
		Create an instancegroup configuration.  kops has the concept of "instance groups",
		which are a group of similar virtual machines. On AWS, they map to an
		AutoScalingGroup. An ig work either as a Kubernetes master or a node.`))

	create_ig_example = templates.Examples(i18n.T(`

		# Create an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name

		# Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name --dry-run -oyaml
		`))

	create_ig_short = i18n.T(`Create an instancegroup.`)
)

// NewCmdCreateInstanceGroup create a new cobra command object for creating a instancegroup.
func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateInstanceGroupOptions{
		Role: string(api.InstanceGroupRoleNode),
	}

	cmd := &cobra.Command{
		Use:     "instancegroup",
		Aliases: []string{"instancegroups", "ig"},
		Short:   create_ig_short,
		Long:    create_ig_long,
		Example: create_ig_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunCreateInstanceGroup(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	// TODO: Create Enum helper - or is there one in k8s already?
	var allRoles []string
	for _, r := range api.AllInstanceGroupRoles {
		allRoles = append(allRoles, string(r))
	}

	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")
	cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnets in which to create instance group")
	// DryRun mode that will print YAML or JSON
	cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.")
	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml")

	return cmd
}

func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateInstanceGroupOptions) error {
	if len(args) == 0 {
		return fmt.Errorf("Specify name of instance group to create")
	}
	if len(args) != 1 {
		return fmt.Errorf("Can only create one instance group at a time!")
	}
	groupName := args[0]

	cluster, err := rootCommand.Cluster()
	if err != nil {
		return err
	}

	clientset, err := rootCommand.Clientset()
	if err != nil {
		return err
	}

	channel, err := cloudup.ChannelForCluster(cluster)
	if err != nil {
		return err
	}

	existing, err := clientset.InstanceGroupsFor(cluster).Get(groupName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if existing != nil {
		return fmt.Errorf("instance group %q already exists", groupName)
	}

	// Populate some defaults
	ig := &api.InstanceGroup{}
	ig.ObjectMeta.Name = groupName

	role, ok := api.ParseInstanceGroupRole(options.Role, true)
	if !ok {
		return fmt.Errorf("unknown role %q", options.Role)
	}
	ig.Spec.Role = role

	if len(options.Subnets) == 0 {
		return fmt.Errorf("cannot create instance group without subnets; specify --subnet flag(s)")
	}
	ig.Spec.Subnets = options.Subnets

	ig, err = cloudup.PopulateInstanceGroupSpec(cluster, ig, channel)
	if err != nil {
		return err
	}

	if options.DryRun {

		if options.Output == "" {
			return fmt.Errorf("must set output flag; yaml or json")
		}

		// Cluster name is not populated, and we need it
		ig.ObjectMeta.Labels = make(map[string]string)
		ig.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name

		switch options.Output {
		case OutputYaml:
			if err := fullOutputYAML(out, ig); err != nil {
				return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
			}
			return nil
		case OutputJSON:
			if err := fullOutputJSON(out, ig); err != nil {
				return fmt.Errorf("error writing cluster json to stdout: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("unsupported output type %q", options.Output)
		}
	}

	var (
		edit = editor.NewDefaultEditor(editorEnvs)
	)

	raw, err := kopscodecs.ToVersionedYaml(ig)
	if err != nil {
		return err
	}
	ext := "yaml"

	// launch the editor
	edited, file, err := edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, bytes.NewReader(raw))
	defer func() {
		if file != "" {
			os.Remove(file)
		}
	}()
	if err != nil {
		return fmt.Errorf("error launching editor: %v", err)
	}

	obj, _, err := kopscodecs.ParseVersionedYaml(edited)
	if err != nil {
		return fmt.Errorf("error parsing yaml: %v", err)
	}
	group, ok := obj.(*api.InstanceGroup)
	if !ok {
		return fmt.Errorf("unexpected object type: %T", obj)
	}

	err = validation.ValidateInstanceGroup(group)
	if err != nil {
		return err
	}

	_, err = clientset.InstanceGroupsFor(cluster).Create(group)
	if err != nil {
		return fmt.Errorf("error storing InstanceGroup: %v", err)
	}

	return nil
}
<commit_msg>Add no edit flag so create instancegroup command is usable on scripts<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/validation"
	"k8s.io/kops/pkg/kopscodecs"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	"k8s.io/kubernetes/pkg/kubectl/cmd/util/editor"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

type CreateInstanceGroupOptions struct {
	Role    string
	Subnets []string
	// DryRun mode output an ig manifest of Output type.
	DryRun bool
	// Output type during a DryRun
	Output string
	// Do not launch editor when creating an instance group
	NoEditor bool
}

var (
	create_ig_long = templates.LongDesc(i18n.T(`
		Create an instancegroup configuration.  kops has the concept of "instance groups",
		which are a group of similar virtual machines. On AWS, they map to an
		AutoScalingGroup. An ig work either as a Kubernetes master or a node.`))

	create_ig_example = templates.Examples(i18n.T(`

		# Create an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name

		# Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name --dry-run -oyaml
		`))

	create_ig_short = i18n.T(`Create an instancegroup.`)
)

// NewCmdCreateInstanceGroup create a new cobra command object for creating a instancegroup.
func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateInstanceGroupOptions{
		Role: string(api.InstanceGroupRoleNode),
	}

	cmd := &cobra.Command{
		Use:     "instancegroup",
		Aliases: []string{"instancegroups", "ig"},
		Short:   create_ig_short,
		Long:    create_ig_long,
		Example: create_ig_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunCreateInstanceGroup(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	// TODO: Create Enum helper - or is there one in k8s already?
	var allRoles []string
	for _, r := range api.AllInstanceGroupRoles {
		allRoles = append(allRoles, string(r))
	}

	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")
	cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnets in which to create instance group")
	// DryRun mode that will print YAML or JSON
	cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.")
	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml")
	cmd.Flags().BoolVar(&options.NoEditor, "no-editor", options.NoEditor, "If true, The instance group will be created by default with no option of editing it")

	return cmd
}

func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateInstanceGroupOptions) error {
	if len(args) == 0 {
		return fmt.Errorf("Specify name of instance group to create")
	}
	if len(args) != 1 {
		return fmt.Errorf("Can only create one instance group at a time!")
	}
	groupName := args[0]

	cluster, err := rootCommand.Cluster()
	if err != nil {
		return err
	}

	clientset, err := rootCommand.Clientset()
	if err != nil {
		return err
	}

	channel, err := cloudup.ChannelForCluster(cluster)
	if err != nil {
		return err
	}

	existing, err := clientset.InstanceGroupsFor(cluster).Get(groupName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if existing != nil {
		return fmt.Errorf("instance group %q already exists", groupName)
	}

	// Populate some defaults
	ig := &api.InstanceGroup{}
	ig.ObjectMeta.Name = groupName

	role, ok := api.ParseInstanceGroupRole(options.Role, true)
	if !ok {
		return fmt.Errorf("unknown role %q", options.Role)
	}
	ig.Spec.Role = role

	if len(options.Subnets) == 0 {
		return fmt.Errorf("cannot create instance group without subnets; specify --subnet flag(s)")
	}
	ig.Spec.Subnets = options.Subnets

	ig, err = cloudup.PopulateInstanceGroupSpec(cluster, ig, channel)
	if err != nil {
		return err
	}

	if options.DryRun {

		if options.Output == "" {
			return fmt.Errorf("must set output flag; yaml or json")
		}

		// Cluster name is not populated, and we need it
		ig.ObjectMeta.Labels = make(map[string]string)
		ig.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name

		switch options.Output {
		case OutputYaml:
			if err := fullOutputYAML(out, ig); err != nil {
				return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
			}
			return nil
		case OutputJSON:
			if err := fullOutputJSON(out, ig); err != nil {
				return fmt.Errorf("error writing cluster json to stdout: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("unsupported output type %q", options.Output)
		}
	}

	if !options.NoEditor {
		var (
			edit = editor.NewDefaultEditor(editorEnvs)
		)

		raw, err := kopscodecs.ToVersionedYaml(ig)
		if err != nil {
			return err
		}
		ext := "yaml"

		// launch the editor
		edited, file, err := edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, bytes.NewReader(raw))
		defer func() {
			if file != "" {
				os.Remove(file)
			}
		}()
		if err != nil {
			return fmt.Errorf("error launching editor: %v", err)
		}

		obj, _, err := kopscodecs.ParseVersionedYaml(edited)
		if err != nil {
			return fmt.Errorf("error parsing yaml: %v", err)
		}
		group, ok := obj.(*api.InstanceGroup)
		if !ok {
			return fmt.Errorf("unexpected object type: %T", obj)
		}

		err = validation.ValidateInstanceGroup(group)
		if err != nil {
			return err
		}

		ig = group
	}

	_, err = clientset.InstanceGroupsFor(cluster).Create(ig)
	if err != nil {
		return fmt.Errorf("error storing InstanceGroup: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package mountlib

import (
	"io"
	"log"
	"os"
	"runtime"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/flags"
	"github.com/ncw/rclone/vfs"
	"github.com/ncw/rclone/vfs/vfsflags"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Options set by command line flags
var (
	DebugFUSE                        = false
	AllowNonEmpty                    = false
	AllowRoot                        = false
	AllowOther                       = false
	DefaultPermissions               = false
	WritebackCache                   = false
	MaxReadAhead       fs.SizeSuffix = 128 * 1024
	ExtraOptions       []string
	ExtraFlags         []string
)

// Check is folder is empty
func checkMountEmpty(mountpoint string) error {
	fp, fpErr := os.Open(mountpoint)

	if fpErr != nil {
		return errors.Wrap(fpErr, "Can not open: "+mountpoint)
	}
	defer fs.CheckClose(fp, &fpErr)

	_, fpErr = fp.Readdirnames(1)

	// directory is not empty
	if fpErr != io.EOF {
		var e error
		var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
		if fpErr == nil {
			e = errors.New(errorMsg)
		} else {
			e = errors.Wrap(fpErr, errorMsg)
		}
		return e
	}
	return nil
}

// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
	var commandDefintion = &cobra.Command{
		Use:   commandName + " remote:path /path/to/mountpoint",
		Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`,
		Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.

This is **EXPERIMENTAL** - use with care.

First set up your remote using ` + "`rclone config`" + `.  Check it works with ` + "`rclone ls`" + ` etc.

Start the mount like this

    rclone ` + commandName + ` remote:path/to/files /path/to/local/mount

Or on Windows like this where X: is an unused drive letter

    rclone ` + commandName + ` remote:path/to/files X:

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal,
the mount is automatically stopped.

The umount operation can fail, for example when the mountpoint is busy.
When that happens, it is the user's responsibility to stop the mount manually with

    # Linux
    fusermount -u /path/to/local/mount
    # OS X
    umount /path/to/local/mount

### Installing on Windows

To run rclone ` + commandName + ` on Windows, you will need to
download and install [WinFsp](http://www.secfs.net/winfsp/).

WinFsp is an [open source](https://github.com/billziss-gh/winfsp)
Windows File System Proxy which makes it easy to write user space file
systems for Windows.  It provides a FUSE emulation layer which rclone
uses combination with
[cgofuse](https://github.com/billziss-gh/cgofuse).  Both of these
packages are by Bill Zissimopoulos who was very helpful during the
implementation of rclone ` + commandName + ` for Windows.

#### Windows caveats

Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.

The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system.

### Limitations

This can only write files sequentially, it can only seek when reading.
This means that many applications won't work with their files on an
rclone mount.

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2,
Hubic) won't work from the root - you will need to specify a bucket,
or a path within the bucket.  So ` + "`swift:`" + ` won't work whereas
` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `.
None of these support the concept of directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

### rclone ` + commandName + ` vs rclone sync/copy

File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries.  However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the **EXPERIMENTAL** [file caching](#file-caching)
for solutions to make ` + commandName + ` mount more reliable.

### Filters

Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount.

### systemd

When running rclone ` + commandName + ` as a systemd service, it is possible
to use Type=notify. In this case the service will enter the started state
after the mountpoint has been successfully set up.
Units having the rclone ` + commandName + ` service specified as a requirement
will see all files and folders immediately in this mode.
` + vfs.Help,
		Run: func(command *cobra.Command, args []string) {
			cmd.CheckArgs(2, 2, command, args)
			fdst := cmd.NewFsDst(args)

			// Show stats if the user has specifically requested them
			if cmd.ShowStats() {
				stopStats := cmd.StartStats()
				defer close(stopStats)
			}

			// Skip checkMountEmpty if --allow-non-empty flag is used or if
			// the Operating System is Windows
			if !AllowNonEmpty && runtime.GOOS != "windows" {
				err := checkMountEmpty(args[1])
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			err := Mount(fdst, args[1])
			if err != nil {
				log.Fatalf("Fatal error: %v", err)
			}
		},
	}

	// Register the command
	cmd.Root.AddCommand(commandDefintion)

	// Add flags
	flagSet := commandDefintion.Flags()
	flags.BoolVarP(flagSet, &DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.")
	// mount options
	flags.BoolVarP(flagSet, &AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.")
	flags.BoolVarP(flagSet, &AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.")
	flags.BoolVarP(flagSet, &AllowOther, "allow-other", "", AllowOther, "Allow access to other users.")
	flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.")
	flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.")
	flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.")
	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
	//flags.BoolVarP(flagSet, &foreground, "foreground", "", foreground, "Do not detach.")

	// Add in the generic flags
	vfsflags.AddFlags(flagSet)

	return commandDefintion
}
<commit_msg>mount: update docs showing --vfs-cache-mode to work around limitations<commit_after>package mountlib

import (
	"io"
	"log"
	"os"
	"runtime"

	"github.com/ncw/rclone/cmd"
	"github.com/ncw/rclone/fs"
	"github.com/ncw/rclone/fs/config/flags"
	"github.com/ncw/rclone/vfs"
	"github.com/ncw/rclone/vfs/vfsflags"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// Options set by command line flags
var (
	DebugFUSE                        = false
	AllowNonEmpty                    = false
	AllowRoot                        = false
	AllowOther                       = false
	DefaultPermissions               = false
	WritebackCache                   = false
	MaxReadAhead       fs.SizeSuffix = 128 * 1024
	ExtraOptions       []string
	ExtraFlags         []string
)

// Check is folder is empty
func checkMountEmpty(mountpoint string) error {
	fp, fpErr := os.Open(mountpoint)

	if fpErr != nil {
		return errors.Wrap(fpErr, "Can not open: "+mountpoint)
	}
	defer fs.CheckClose(fp, &fpErr)

	_, fpErr = fp.Readdirnames(1)

	// directory is not empty
	if fpErr != io.EOF {
		var e error
		var errorMsg = "Directory is not empty: " + mountpoint + " If you want to mount it anyway use: --allow-non-empty option"
		if fpErr == nil {
			e = errors.New(errorMsg)
		} else {
			e = errors.Wrap(fpErr, errorMsg)
		}
		return e
	}
	return nil
}

// NewMountCommand makes a mount command with the given name and Mount function
func NewMountCommand(commandName string, Mount func(f fs.Fs, mountpoint string) error) *cobra.Command {
	var commandDefintion = &cobra.Command{
		Use:   commandName + " remote:path /path/to/mountpoint",
		Short: `Mount the remote as a mountpoint. **EXPERIMENTAL**`,
		Long: `
rclone ` + commandName + ` allows Linux, FreeBSD, macOS and Windows to
mount any of Rclone's cloud storage systems as a file system with
FUSE.

This is **EXPERIMENTAL** - use with care.

First set up your remote using ` + "`rclone config`" + `.  Check it works with ` + "`rclone ls`" + ` etc.

Start the mount like this

    rclone ` + commandName + ` remote:path/to/files /path/to/local/mount

Or on Windows like this where X: is an unused drive letter

    rclone ` + commandName + ` remote:path/to/files X:

When the program ends, either via Ctrl+C or receiving a SIGINT or SIGTERM signal,
the mount is automatically stopped.

The umount operation can fail, for example when the mountpoint is busy.
When that happens, it is the user's responsibility to stop the mount manually with

    # Linux
    fusermount -u /path/to/local/mount
    # OS X
    umount /path/to/local/mount

### Installing on Windows

To run rclone ` + commandName + ` on Windows, you will need to
download and install [WinFsp](http://www.secfs.net/winfsp/).

WinFsp is an [open source](https://github.com/billziss-gh/winfsp)
Windows File System Proxy which makes it easy to write user space file
systems for Windows.  It provides a FUSE emulation layer which rclone
uses combination with
[cgofuse](https://github.com/billziss-gh/cgofuse).  Both of these
packages are by Bill Zissimopoulos who was very helpful during the
implementation of rclone ` + commandName + ` for Windows.

#### Windows caveats

Note that drives created as Administrator are not visible by other
accounts (including the account that was elevated as
Administrator). So if you start a Windows drive from an Administrative
Command Prompt and then try to access the same drive from Explorer
(which does not run as Administrator), you will not be able to see the
new drive.

The easiest way around this is to start the drive from a normal
command prompt. It is also possible to start a drive from the SYSTEM
account (using [the WinFsp.Launcher
infrastructure](https://github.com/billziss-gh/winfsp/wiki/WinFsp-Service-Architecture))
which creates drives accessible for everyone on the system.

### Limitations

Without the use of "--vfs-cache-mode" this can only write files
sequentially, it can only seek when reading.  This means that many
applications won't work with their files on an rclone mount without
"--vfs-cache-mode writes" or "--vfs-cache-mode full".  See the [File
Caching](#file-caching) section for more info.

The bucket based remotes (eg Swift, S3, Google Compute Storage, B2,
Hubic) won't work from the root - you will need to specify a bucket,
or a path within the bucket.  So ` + "`swift:`" + ` won't work whereas
` + "`swift:bucket`" + ` will as will ` + "`swift:bucket/path`" + `.
None of these support the concept of directories, so empty
directories will have a tendency to disappear once they fall out of
the directory cache.

Only supported on Linux, FreeBSD, OS X and Windows at the moment.

### rclone ` + commandName + ` vs rclone sync/copy

File systems expect things to be 100% reliable, whereas cloud storage
systems are a long way from 100% reliable. The rclone sync/copy
commands cope with this with lots of retries.  However rclone ` + commandName + `
can't use retries in the same way without making local copies of the
uploads. Look at the **EXPERIMENTAL** [file caching](#file-caching)
for solutions to make ` + commandName + ` mount more reliable.

### Filters

Note that all the rclone filters can be used to select a subset of the
files to be visible in the mount.

### systemd

When running rclone ` + commandName + ` as a systemd service, it is possible
to use Type=notify. In this case the service will enter the started state
after the mountpoint has been successfully set up.
Units having the rclone ` + commandName + ` service specified as a requirement
will see all files and folders immediately in this mode.
` + vfs.Help,
		Run: func(command *cobra.Command, args []string) {
			cmd.CheckArgs(2, 2, command, args)
			fdst := cmd.NewFsDst(args)

			// Show stats if the user has specifically requested them
			if cmd.ShowStats() {
				stopStats := cmd.StartStats()
				defer close(stopStats)
			}

			// Skip checkMountEmpty if --allow-non-empty flag is used or if
			// the Operating System is Windows
			if !AllowNonEmpty && runtime.GOOS != "windows" {
				err := checkMountEmpty(args[1])
				if err != nil {
					log.Fatalf("Fatal error: %v", err)
				}
			}

			err := Mount(fdst, args[1])
			if err != nil {
				log.Fatalf("Fatal error: %v", err)
			}
		},
	}

	// Register the command
	cmd.Root.AddCommand(commandDefintion)

	// Add flags
	flagSet := commandDefintion.Flags()
	flags.BoolVarP(flagSet, &DebugFUSE, "debug-fuse", "", DebugFUSE, "Debug the FUSE internals - needs -v.")
	// mount options
	flags.BoolVarP(flagSet, &AllowNonEmpty, "allow-non-empty", "", AllowNonEmpty, "Allow mounting over a non-empty directory.")
	flags.BoolVarP(flagSet, &AllowRoot, "allow-root", "", AllowRoot, "Allow access to root user.")
	flags.BoolVarP(flagSet, &AllowOther, "allow-other", "", AllowOther, "Allow access to other users.")
	flags.BoolVarP(flagSet, &DefaultPermissions, "default-permissions", "", DefaultPermissions, "Makes kernel enforce access control based on the file mode.")
	flags.BoolVarP(flagSet, &WritebackCache, "write-back-cache", "", WritebackCache, "Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.")
	flags.FVarP(flagSet, &MaxReadAhead, "max-read-ahead", "", "The number of bytes that can be prefetched for sequential reads.")
	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
	//flags.BoolVarP(flagSet, &foreground, "foreground", "", foreground, "Do not detach.")

	// Add in the generic flags
	vfsflags.AddFlags(flagSet)

	return commandDefintion
}
<|endoftext|>