<commit_before>package scm

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

func GitRootPath(path ...string) (string, error) {
	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
	if len(path) > 0 {
		cmd.Dir = path[0]
	}

	b, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("Unable to parse repository path, %s %s", string(b), err)
	}

	s := strings.TrimSpace(string(b))
	if s == "" {
		return "", fmt.Errorf("Unable to parse repository path, %s", err)
	}

	return s, nil
}

func GitBranch(wd ...string) (string, error) {
	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return "", fmt.Errorf("Unable to parse branch name, %s %s", string(b), err)
	}
	return strings.TrimSpace(string(b)), nil

}

func GitEmail(wd ...string) (string, error) {
	cmd := exec.Command("git", "config", "--get", "user.email")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return "", fmt.Errorf("Unable to get user email, %s %s", string(b), err)
	}
	return strings.TrimSpace(string(b)), nil
}

func GitLastLog(wd ...string) (string, error) {
	cmd := exec.Command("git", "log", "-1", "--oneline", "--raw")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		// if there are no git commits yet it will fail
		// ignoring this error
		return "", nil
	}
	return string(b), err
}

func GitParseMessage(m string) (uuid, msg string, files []string) {
	l := strings.Split(m, "\n")
	files = make([]string, 0)
	for i, v := range l {
		if i == 0 {
			s := strings.SplitN(v, " ", 2)
			uuid = s[0]
			msg = s[1]
		} else {
			if strings.TrimSpace(v) != "" {
				s := strings.Split(v, "\t")
				files = append(files, s[1])
			}
		}
	}
	return
}

func GitAddNote(n string, nameSpace string, wd ...string) error {
	cmd := exec.Command("git", "notes", fmt.Sprintf("--ref=%s", nameSpace), "add", "-f", "-m", n)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	if b, err := cmd.Output(); err != nil {
		return fmt.Errorf("Unable to add git note, %s %s", string(b), err)
	}
	return nil
}

func GitNote(commitID string, nameSpace string, wd ...string) (string, error) {
	cmd := exec.Command("git", "notes", fmt.Sprintf("--ref=%s", nameSpace), "show", commitID)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	b, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("Unable to get git note, %s %s", string(b), err)
	}
	return string(b), nil
}

func GitLog(commitID string, wd ...string) ([]string, error) {
	cmd := exec.Command("git", "log", "-1", "--pretty", "--format=%an,%ad,%h,%s", commitID)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return []string{}, fmt.Errorf("Unable to execute git log, %s %s", string(b), err)
	}
	parts := strings.SplitN(string(b), ",", 4)
	if len(parts) < 4 {
		return []string{}, fmt.Errorf("Unable to parse log output, %s %s", string(b), err)
	}
	return parts, err
}

func GitConfig(settings map[string]string, wd ...string) error {
	for k, v := range settings {
		cmd := exec.Command("git", "config", "-l")
		if len(wd) > 0 {
			cmd.Dir = wd[0]
		}
		var (
			b   []byte
			err error
		)
		if b, err = cmd.Output(); err != nil {
			return fmt.Errorf("Unable to run git config -l, %s %s", string(b), err)
		}
		if !strings.Contains(string(b), fmt.Sprintf("%s=%s", k, v)) {
			cmd := exec.Command("git", "config", "--add", k, v)
			if len(wd) > 0 {
				cmd.Dir = wd[0]
			}
			if b, err := cmd.Output(); err != nil {
				return fmt.Errorf("Unable to run git config --add %s %s, %s %s", k, v, string(b), err)
			}
		}
	}
	return nil
}

func GitTracked(f string, wd ...string) (bool, error) {
	cmd := exec.Command("git", "ls-files", f)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return false, fmt.Errorf("Unable to determine git tracked status for %s, %s %s", f, string(b), err)
	}
	return strings.TrimSpace(string(b)) != "", nil
}

func GitModified(f string, staging bool, wd ...string) (bool, error) {
	var cmd *exec.Cmd
	if staging {
		cmd = exec.Command("git", "diff", "--name-only", "--cached", f)
	} else {
		cmd = exec.Command("git", "diff", "--name-only", f)
	}
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return false, fmt.Errorf("Unable to determine git modified status for %s, %s %s", f, string(b), err)
	}
	return strings.TrimSpace(string(b)) != "", nil
}

func GitHasStaged(wd ...string) (bool, error) {
	var cmd *exec.Cmd
	cmd = exec.Command("git", "diff", "--name-only", "--cached")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return false, fmt.Errorf("Unable to determine git staged status, %s %s", string(b), err)
	}
	return strings.TrimSpace(string(b)) != "", nil
}

func GitSetHooks(hooks map[string]string, wd ...string) error {
	for hook, command := range hooks {
		var (
			p   string
			err error
		)

		if len(wd) > 0 {
			p = wd[0]
		} else {
			p, err = os.Getwd()
			if err != nil {
				return err
			}
		}
		fp := path.Join(p, ".git", "hooks", hook)

		var output string
		if _, err := os.Stat(fp); !os.IsNotExist(err) {
			b, err := ioutil.ReadFile(fp)
			if err != nil {
				return err
			}
			output = string(b)

			if strings.Contains(output, command+"\n") {
				// if file already exists this will make sure it's executable
				if err := os.Chmod(fp, 0755); err != nil {
					return err
				}
				return nil
			}
		}

		if err = ioutil.WriteFile(
			fp, []byte(fmt.Sprintf("%s\n%s\n", output, command)), 0755); err != nil {
			return err
		}
		// if file already exists this will make sure it's executable
		if err := os.Chmod(fp, 0755); err != nil {
			return err
		}
	}

	return nil
}

func GitIgnore(ignore string, wd ...string) error {
	var (
		p   string
		err error
	)

	if len(wd) > 0 {
		p = wd[0]
	} else {
		p, err = os.Getwd()
		if err != nil {
			return err
		}
	}
	fp := path.Join(p, ".gitignore")

	var output string
	if _, err := os.Stat(fp); !os.IsNotExist(err) {
		b, err := ioutil.ReadFile(fp)
		if err != nil {
			return err
		}
		output = string(b)

		if strings.Contains(output, ignore+"\n") {
			return nil
		}
	}

	if err = ioutil.WriteFile(
		fp, []byte(fmt.Sprintf("%s\n%s\n", output, ignore)), 0644); err != nil {
		return err
	}
	return nil
}
<commit_msg>For git diff, used '--' to separate paths from revisions<commit_after>package scm

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

func GitRootPath(path ...string) (string, error) {
	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
	if len(path) > 0 {
		cmd.Dir = path[0]
	}

	b, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("Unable to parse repository path, %s %s", string(b), err)
	}

	s := strings.TrimSpace(string(b))
	if s == "" {
		return "", fmt.Errorf("Unable to parse repository path, %s", err)
	}

	return s, nil
}

func GitBranch(wd ...string) (string, error) {
	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return "", fmt.Errorf("Unable to parse branch name, %s %s", string(b), err)
	}
	return strings.TrimSpace(string(b)), nil

}

func GitEmail(wd ...string) (string, error) {
	cmd := exec.Command("git", "config", "--get", "user.email")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return "", fmt.Errorf("Unable to get user email, %s %s", string(b), err)
	}
	return strings.TrimSpace(string(b)), nil
}

func GitLastLog(wd ...string) (string, error) {
	cmd := exec.Command("git", "log", "-1", "--oneline", "--raw")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		// if there are no git commits yet it will fail
		// ignoring this error
		return "", nil
	}
	return string(b), err
}

func GitParseMessage(m string) (uuid, msg string, files []string) {
	l := strings.Split(m, "\n")
	files = make([]string, 0)
	for i, v := range l {
		if i == 0 {
			s := strings.SplitN(v, " ", 2)
			uuid = s[0]
			msg = s[1]
		} else {
			if strings.TrimSpace(v) != "" {
				s := strings.Split(v, "\t")
				files = append(files, s[1])
			}
		}
	}
	return
}

func GitAddNote(n string, nameSpace string, wd ...string) error {
	cmd := exec.Command("git", "notes", fmt.Sprintf("--ref=%s", nameSpace), "add", "-f", "-m", n)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	if b, err := cmd.Output(); err != nil {
		return fmt.Errorf("Unable to add git note, %s %s", string(b), err)
	}
	return nil
}

func GitNote(commitID string, nameSpace string, wd ...string) (string, error) {
	cmd := exec.Command("git", "notes", fmt.Sprintf("--ref=%s", nameSpace), "show", commitID)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	b, err := cmd.Output()
	if err != nil {
		return "", fmt.Errorf("Unable to get git note, %s %s", string(b), err)
	}
	return string(b), nil
}

func GitLog(commitID string, wd ...string) ([]string, error) {
	cmd := exec.Command("git", "log", "-1", "--pretty", "--format=%an,%ad,%h,%s", commitID)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return []string{}, fmt.Errorf("Unable to execute git log, %s %s", string(b), err)
	}
	parts := strings.SplitN(string(b), ",", 4)
	if len(parts) < 4 {
		return []string{}, fmt.Errorf("Unable to parse log output, %s %s", string(b), err)
	}
	return parts, err
}

func GitConfig(settings map[string]string, wd ...string) error {
	for k, v := range settings {
		cmd := exec.Command("git", "config", "-l")
		if len(wd) > 0 {
			cmd.Dir = wd[0]
		}
		var (
			b   []byte
			err error
		)
		if b, err = cmd.Output(); err != nil {
			return fmt.Errorf("Unable to run git config -l, %s %s", string(b), err)
		}
		if !strings.Contains(string(b), fmt.Sprintf("%s=%s", k, v)) {
			cmd := exec.Command("git", "config", "--add", k, v)
			if len(wd) > 0 {
				cmd.Dir = wd[0]
			}
			if b, err := cmd.Output(); err != nil {
				return fmt.Errorf("Unable to run git config --add %s %s, %s %s", k, v, string(b), err)
			}
		}
	}
	return nil
}

func GitTracked(f string, wd ...string) (bool, error) {
	cmd := exec.Command("git", "ls-files", f)
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return false, fmt.Errorf("Unable to determine git tracked status for %s, %s %s", f, string(b), err)
	}
	return strings.TrimSpace(string(b)) != "", nil
}

func GitModified(f string, staging bool, wd ...string) (bool, error) {
	var cmd *exec.Cmd
	if staging {
		cmd = exec.Command("git", "diff", "--name-only", "--cached", "--", f)
	} else {
		cmd = exec.Command("git", "diff", "--name-only", "--", f)
	}
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return false, fmt.Errorf("Unable to determine git modified status for %s, %s %s", f, string(b), err)
	}
	return strings.TrimSpace(string(b)) != "", nil
}

func GitHasStaged(wd ...string) (bool, error) {
	var cmd *exec.Cmd
	cmd = exec.Command("git", "diff", "--name-only", "--cached")
	if len(wd) > 0 {
		cmd.Dir = wd[0]
	}
	var (
		b   []byte
		err error
	)
	if b, err = cmd.Output(); err != nil {
		return false, fmt.Errorf("Unable to determine git staged status, %s %s", string(b), err)
	}
	return strings.TrimSpace(string(b)) != "", nil
}

func GitSetHooks(hooks map[string]string, wd ...string) error {
	for hook, command := range hooks {
		var (
			p   string
			err error
		)

		if len(wd) > 0 {
			p = wd[0]
		} else {
			p, err = os.Getwd()
			if err != nil {
				return err
			}
		}
		fp := path.Join(p, ".git", "hooks", hook)

		var output string
		if _, err := os.Stat(fp); !os.IsNotExist(err) {
			b, err := ioutil.ReadFile(fp)
			if err != nil {
				return err
			}
			output = string(b)

			if strings.Contains(output, command+"\n") {
				// if file already exists this will make sure it's executable
				if err := os.Chmod(fp, 0755); err != nil {
					return err
				}
				return nil
			}
		}

		if err = ioutil.WriteFile(
			fp, []byte(fmt.Sprintf("%s\n%s\n", output, command)), 0755); err != nil {
			return err
		}
		// if file already exists this will make sure it's executable
		if err := os.Chmod(fp, 0755); err != nil {
			return err
		}
	}

	return nil
}

func GitIgnore(ignore string, wd ...string) error {
	var (
		p   string
		err error
	)

	if len(wd) > 0 {
		p = wd[0]
	} else {
		p, err = os.Getwd()
		if err != nil {
			return err
		}
	}
	fp := path.Join(p, ".gitignore")

	var output string
	if _, err := os.Stat(fp); !os.IsNotExist(err) {
		b, err := ioutil.ReadFile(fp)
		if err != nil {
			return err
		}
		output = string(b)

		if strings.Contains(output, ignore+"\n") {
			return nil
		}
	}

	if err = ioutil.WriteFile(
		fp, []byte(fmt.Sprintf("%s\n%s\n", output, ignore)), 0644); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/gorilla/websocket"
	"github.com/mohae/autofact/conf"
	"github.com/uber-go/zap"
)

const (
	addressVar = "address"
	aVar       = "a"
	portVar    = "port"
	pVar       = "p"
)

var (
	connFile = "autofact.json"
	// This is the default directory for autofact-client app data.
	autofactPath    = "$HOME/.autofact"
	autofactEnvName = "AUTOFACT_PATH"
	// default
	connConf   conf.Conn
	serverless bool // if the client is being run without a server
)

// TODO determine loglevel mapping to actual usage:
// Proposed:
//  DebugLevel == not used
//	InfoLevel == Gathered data
//  WarnLevel == Connection info an non-error messages: status type
//  ErrorLevel == Errors
//  PanicLevel == Panic: shouldn't be used
//  FatalLevel == Unrecoverable error that results in app shutdown
// TODO: implement data logging
var (
	log      zap.Logger
	loglevel = zap.LevelFlag("loglevel", zap.WarnLevel, "log level")
	logfile  string

	// These are globals so that they can be closed
	f        *os.File
	isStdErr bool
)

// TODO: reconcile these flags with config file usage.  Probably add contour
// to handle this after the next refactor of contour.
// TODO: make connectInterval/period handling consistent, e.g. should they be
// flags, what is precedence in relation to Conn?
func init() {
	flag.StringVar(&connConf.ServerAddress, addressVar, "127.0.0.1", "the server address")
	flag.StringVar(&connConf.ServerAddress, aVar, "127.0.0.1", "the server address (short)")
	flag.StringVar(&connConf.ServerPort, portVar, "8675", "the connection port")
	flag.StringVar(&connConf.ServerPort, pVar, "8675", "the connection port (short)")
	flag.StringVar(&logfile, "logfile", "autofact.log", "application log file; if empty stderr will be used")
	flag.StringVar(&logfile, "l", "autofact.log", "application log file; if empty stderr will be used")
	flag.BoolVar(&serverless, "serverless", false, "serverless: the client will run standalone and write the collected data to the log")
	connConf.ConnectInterval.Duration = 5 * time.Second
	connConf.ConnectPeriod.Duration = 15 * time.Minute
}

func main() {
	// Load the AUTOPATH value
	tmp := os.Getenv(autofactEnvName)
	if tmp != "" {
		autofactPath = tmp
	}
	autofactPath = os.ExpandEnv(autofactPath)

	// make sure the autofact path exists (create if it doesn't)
	err := os.MkdirAll(autofactPath, 0760)
	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to create AUTOFACT_PATH: %s\n", err)
		fmt.Fprintln(os.Stderr, "startup error: exiting")
		os.Exit(1)
	}

	// finalize the paths
	connFile = filepath.Join(autofactPath, connFile)

	// process the settings
	var connMsg string
	err = connConf.Load(connFile)
	if err != nil {
		// capture the error for logging once it is setup and continue.  An error
		// is not a show stopper as the file may not exist if this is the first
		// time autofact has run on this node.
		connMsg = fmt.Sprintf("using default settings")
	}

	// Parse the flags.
	flag.Parse()

	// now that everything is parsed; set up logging
	SetLogging()
	defer CloseLog()
	// if there was an error reading the connection configuration and this isn't
	// being run serverless, log it
	if connMsg != "" && !serverless {
		log.Warn(
			err.Error(),
			zap.String("conf", connMsg),
		)
	}

	// TODO add env var support

	// get a client
	c := NewClient(connConf)
	c.AutoPath = autofactPath

	// handle signals
	go handleSignals(c)

	// doneCh is used to signal that the connection has been closed
	doneCh := make(chan struct{})

	if !serverless {
		// connect to the Server
		c.ServerURL = url.URL{Scheme: "ws", Host: fmt.Sprintf("%s:%s", c.ServerAddress, c.ServerPort), Path: "/client"}

		// must have a connection before doing anything
		for i := 0; i < 3; i++ {
			connected := c.Connect()
			if connected {
				break
			}
			// retry on fail until retry attempts have been exceeded
		}
		if !c.IsConnected() {
			CloseLog() // defer doesn't run on fatal
			log.Fatal(
				"unable to connect",
				zap.String("server", c.ServerURL.String()),
			)
		}
	}

	// start the go routines for socket communications
	go c.Listen(doneCh)
	go c.MemInfo(doneCh)
	go c.CPUUtilization(doneCh)
	go c.NetUsage(doneCh)
	// start the connection handler
	go c.MessageWriter(doneCh)

	if !serverless {
		// if connected, save the conf: this will also save the ClientID
		err = c.Conn.Save()
		if err != nil {
			log.Error(
				err.Error(),
				zap.String("op", "save conn"),
				zap.String("file", c.Filename),
			)
		}
	}
	<-doneCh
}

func SetLogging() {
	// if logfile is empty, use Stderr
	var err error
	if logfile == "" {
		f = os.Stderr
		isStdErr = true
	} else {
		f, err = os.OpenFile(logfile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
		if err != nil {
			panic(err)
		}
	}
	log = zap.New(
		zap.NewJSONEncoder(
			zap.RFC3339Formatter("ts"),
		),
		zap.Output(f),
	)
	log.SetLevel(*loglevel)
}

// CloseLog closes the log file before exiting.
func CloseLog() {
	if !isStdErr && f != nil {
		f.Close()
	}
}

func handleSignals(c *Client) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	v := <-ch
	log.Info(
		"os signal received: shutting down autofact",
		zap.Object("signal", v.String()),
	)
	// If there's a connection send a close signal
	if c.IsConnected() {
		log.Debug(
			"closing connection",
			zap.String("op", "shutdown"),
		)
		c.WS.WriteMessage(websocket.CloseMessage, []byte(string(c.Conn.ID)+" shutting down"))
	}
	CloseLog()
	os.Exit(1)
}
<commit_msg>add data log stuff<commit_after>package main

import (
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/gorilla/websocket"
	"github.com/mohae/autofact/conf"
	czap "github.com/mohae/zap"
	"github.com/uber-go/zap"
)

const (
	addressVar = "address"
	aVar       = "a"
	portVar    = "port"
	pVar       = "p"
)

var (
	connFile = "autofact.json"
	// This is the default directory for autofact-client app data.
	autofactPath    = "$HOME/.autofact"
	autofactEnvName = "AUTOFACT_PATH"
	// default
	connConf   conf.Conn
	serverless bool // if the client is being run without a server
)

// TODO determine loglevel mapping to actual usage:
// Proposed:
//  DebugLevel == not used
//	InfoLevel == Gathered data
//  WarnLevel == Connection info an non-error messages: status type
//  ErrorLevel == Errors
//  PanicLevel == Panic: shouldn't be used
//  FatalLevel == Unrecoverable error that results in app shutdown
// TODO: implement data logging
var (
	log      zap.Logger
	loglevel = zap.LevelFlag("loglevel", zap.WarnLevel, "log level")
	logfile  string
	f        *os.File

	dataLog     czap.Logger // use mohae's fork to support level description override
	dataLogFile string
	dataFile    *os.File
	// These are globals so that they can be closed

	isStdErr bool
)

// TODO: reconcile these flags with config file usage.  Probably add contour
// to handle this after the next refactor of contour.
// TODO: make connectInterval/period handling consistent, e.g. should they be
// flags, what is precedence in relation to Conn?
func init() {
	flag.StringVar(&connConf.ServerAddress, addressVar, "127.0.0.1", "the server address")
	flag.StringVar(&connConf.ServerAddress, aVar, "127.0.0.1", "the server address (short)")
	flag.StringVar(&connConf.ServerPort, portVar, "8675", "the connection port")
	flag.StringVar(&connConf.ServerPort, pVar, "8675", "the connection port (short)")
	flag.StringVar(&logfile, "logfile", "autofact.log", "application log file; if empty stderr will be used")
	flag.StringVar(&logfile, "l", "autofact.log", "application log file; if empty stderr will be used")
	flag.StringVar(&dataLogFile, "datafile", "autodata.json", "the file that the collected data gets written when running serverless")
	flag.StringVar(&dataLogFile, "d", "autodata.json", "the file that the collected data gets written when running serverless")
	flag.BoolVar(&serverless, "serverless", false, "serverless: the client will run standalone and write the collected data to the log")
	connConf.ConnectInterval.Duration = 5 * time.Second
	connConf.ConnectPeriod.Duration = 15 * time.Minute

	// override czap description for InfoLevel
	czap.InfoString = "data"
}

func main() {
	// Load the AUTOPATH value
	tmp := os.Getenv(autofactEnvName)
	if tmp != "" {
		autofactPath = tmp
	}
	autofactPath = os.ExpandEnv(autofactPath)

	// make sure the autofact path exists (create if it doesn't)
	err := os.MkdirAll(autofactPath, 0760)
	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to create AUTOFACT_PATH: %s\n", err)
		fmt.Fprintln(os.Stderr, "startup error: exiting")
		os.Exit(1)
	}

	// finalize the paths
	connFile = filepath.Join(autofactPath, connFile)

	// process the settings
	var connMsg string
	err = connConf.Load(connFile)
	if err != nil {
		// capture the error for logging once it is setup and continue.  An error
		// is not a show stopper as the file may not exist if this is the first
		// time autofact has run on this node.
		connMsg = fmt.Sprintf("using default settings")
	}

	// Parse the flags.
	flag.Parse()

	// now that everything is parsed; set up logging
	SetLogging()
	defer CloseLog()
	// if there was an error reading the connection configuration and this isn't
	// being run serverless, log it
	if connMsg != "" && !serverless {
		log.Warn(
			err.Error(),
			zap.String("conf", connMsg),
		)
	}

	// TODO add env var support

	// get a client
	c := NewClient(connConf)
	c.AutoPath = autofactPath

	// handle signals
	go handleSignals(c)

	// doneCh is used to signal that the connection has been closed
	doneCh := make(chan struct{})

	if serverless { // open the datafile to use
		if dataLogFile == "" {
			log.Fatal(
				"filename is not specified",
				zap.String("op", "open data file"),
			)
		}
		SetDataLog()
		defer dataFile.Close()
	} else { // connect to the server
		// connect to the Server
		c.ServerURL = url.URL{Scheme: "ws", Host: fmt.Sprintf("%s:%s", c.ServerAddress, c.ServerPort), Path: "/client"}

		// must have a connection before doing anything
		for i := 0; i < 3; i++ {
			connected := c.Connect()
			if connected {
				break
			}
			// retry on fail until retry attempts have been exceeded
		}
		if !c.IsConnected() {
			CloseLog() // defer doesn't run on fatal
			log.Fatal(
				"unable to connect",
				zap.String("server", c.ServerURL.String()),
			)
		}
	}

	// start the go routines for socket communications
	go c.Listen(doneCh)
	go c.MemInfo(doneCh)
	go c.CPUUtilization(doneCh)
	go c.NetUsage(doneCh)
	// start the connection handler
	go c.MessageWriter(doneCh)

	if !serverless {
		// if connected, save the conf: this will also save the ClientID
		err = c.Conn.Save()
		if err != nil {
			log.Error(
				err.Error(),
				zap.String("op", "save conn"),
				zap.String("file", c.Filename),
			)
		}
	}
	<-doneCh
}

func SetLogging() {
	// if logfile is empty, use Stderr
	var err error
	if logfile == "" {
		f = os.Stderr
		isStdErr = true
	} else {
		f, err = os.OpenFile(logfile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
		if err != nil {
			panic(err)
		}
	}
	log = zap.New(
		zap.NewJSONEncoder(
			zap.RFC3339Formatter("ts"),
		),
		zap.Output(f),
	)
	log.SetLevel(*loglevel)
}

func SetDataLog() {
	var err error
	dataFile, err = os.OpenFile(dataLogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if err != nil {
		log.Fatal(
			err.Error(),
			zap.String("op", "open datafile"),
			zap.String("filename", dataLogFile),
		)
	}
	dataLog = czap.New(
		czap.NewJSONEncoder(
			czap.RFC3339Formatter("ts"),
		),
		czap.Output(dataFile),
	)
	dataLog.SetLevel(czap.InfoLevel)
}

// CloseLog closes the log file before exiting.
func CloseLog() {
	if !isStdErr && f != nil {
		f.Close()
	}
	// If running serverless, close the data file.
	if serverless {
		dataFile.Close()
	}
}

func handleSignals(c *Client) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	v := <-ch
	log.Info(
		"os signal received: shutting down autofact",
		zap.Object("signal", v.String()),
	)
	// If there's a connection send a close signal
	if c.IsConnected() {
		log.Debug(
			"closing connection",
			zap.String("op", "shutdown"),
		)
		c.WS.WriteMessage(websocket.CloseMessage, []byte(string(c.Conn.ID)+" shutting down"))
	}
	CloseLog()

	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/gorilla/websocket"
	"github.com/mohae/autofact/conf"
	czap "github.com/mohae/zap"
	"github.com/uber-go/zap"
)

const (
	addressVar = "address"
	aVar       = "a"
	portVar    = "port"
	pVar       = "p"
)

var (
	connFile    = "autofact.json"
	collectFile = "autocollect.json"
	// This is the default directory for autofact-client app data.
	autofactPath    = "$HOME/.autofact"
	autofactEnvName = "AUTOFACT_PATH"
	// configuration info
	connConf    conf.Conn
	collectConf conf.Collect

	// client configuration: used for serverless

	serverless bool
)

// Vars for logging and local data output, if applicable.
var (
	log      zap.Logger // application log
	loglevel = zap.LevelFlag("loglevel", zap.WarnLevel, "log level")
	logDest  string
	logOut   *os.File

	data     czap.Logger // use mohae's fork to support level description override
	dataDest string
	dataOut  *os.File
)

// TODO: reconcile these flags with config file usage.  Probably add contour
// to handle this after the next refactor of contour.
// TODO: make connectInterval/period handling consistent, e.g. should they be
// flags, what is precedence in relation to Conn?
func init() {
	flag.StringVar(&connConf.ServerAddress, addressVar, "127.0.0.1", "the server address")
	flag.StringVar(&connConf.ServerAddress, aVar, "127.0.0.1", "the server address (short)")
	flag.StringVar(&connConf.ServerPort, portVar, "8675", "the connection port")
	flag.StringVar(&connConf.ServerPort, pVar, "8675", "the connection port (short)")
	flag.StringVar(&logDest, "logdestination", "stderr", "log output destination; if empty stderr will be used")
	flag.StringVar(&logDest, "l", "stderr", "log output; if empty stderr will be used")
	flag.StringVar(&dataDest, "datadestination", "stdout", "serverless mode data output destination, if empty stderr will be used")
	flag.StringVar(&dataDest, "d", "stdout", "serverless mode data output destination, if empty stderr will be used")
	flag.BoolVar(&serverless, "serverless", false, "serverless: the client will run standalone and write the collected data to the log")
	connConf.ConnectInterval.Duration = 5 * time.Second
	connConf.ConnectPeriod.Duration = 15 * time.Minute

	// override czap description for InfoLevel
	czap.WarnString = "data"
}

func main() {
	// Load the AUTOPATH value
	tmp := os.Getenv(autofactEnvName)
	if tmp != "" {
		autofactPath = tmp
	}
	autofactPath = os.ExpandEnv(autofactPath)

	// make sure the autofact path exists (create if it doesn't)
	err := os.MkdirAll(autofactPath, 0760)
	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to create AUTOFACT_PATH: %s\n", err)
		fmt.Fprintln(os.Stderr, "startup error: exiting")
		os.Exit(1)
	}

	// finalize the paths
	connFile = filepath.Join(autofactPath, connFile)

	// process the settings: this gets read first just in case flags override
	var connMsg string
	err = connConf.Load(connFile)
	if err != nil {
		// capture the error for logging once it is setup and continue.  An error
		// is not a show stopper as the file may not exist if this is the first
		// time autofact has run on this node.
		connMsg = fmt.Sprintf("using default connection settings")
	}

	// Parse the flags.
	flag.Parse()

	// now that everything is parsed; set up logging
	SetLogging()
	defer CloseOut()
	// if there was an error reading the connection configuration and this isn't
	// being run serverless, log it
	if connMsg != "" && !serverless {
		log.Warn(
			err.Error(),
			zap.String("op", fmt.Sprintf("load %s", connFile)),
			zap.String("conf", connMsg),
		)
	}

	// if serverless: load the collection configuration
	if serverless {
		collectFile = filepath.Join(autofactPath, collectFile)
		err = collectConf.Load(collectFile)
		if err != nil {
			log.Warn(
				err.Error(),
				zap.String("op", fmt.Sprintf("load %s", collectFile)),
				zap.String("conf", "using default collect settings"),
			)
			collectConf.UseDefaults()
			err = collectConf.SaveAsJSON(collectFile)
			if err != nil {
				log.Warn(
					err.Error(),
					zap.String("op", fmt.Sprintf("save %s", collectFile)),
				)
			}
		}
	}

	// TODO add env var support

	// get a client
	c := NewClient(connConf, collectFile)
	c.AutoPath = autofactPath

	// handle signals
	go handleSignals(c)

	// doneCh is used to signal that the connection has been closed
	doneCh := make(chan struct{})

	// Set up the output destination.
	if serverless { // open the datafile to use
		SetDataOut()
	} else { // connect to the server
		// connect to the Server
		c.ServerURL = url.URL{Scheme: "ws", Host: fmt.Sprintf("%s:%s", c.ServerAddress, c.ServerPort), Path: "/client"}

		// must have a connection before doing anything
		for i := 0; i < 3; i++ {
			connected := c.Connect()
			if connected {
				break
			}
			// retry on fail until retry attempts have been exceeded
		}
		if !c.IsConnected() {
			log.Error(
				"unable to connect",
				zap.String("server", c.ServerURL.String()),
			)
			CloseOut() // defer doesn't run on fatal
			os.Exit(1)
		}
		// if connected, save the conf: this will also save the ClientID
		err = c.Conn.Save()
		if err != nil {
			log.Error(
				err.Error(),
				zap.String("op", "save conn"),
				zap.String("file", c.Conn.Filename),
			)
		}
	}

	// set up the data processing
	if serverless {
		// since there isn't a server pull for healthbeat, a local ticker is started
		go c.HealthbeatLocal(doneCh)
	} else {
		// assign the
		c.LoadAvg = LoadAvgFB
		go c.Listen(doneCh)
		go c.MemInfo(doneCh)
		go c.CPUUtilization(doneCh)
		go c.NetUsage(doneCh)
		// start the connection handler
		go c.MessageWriter(doneCh)
	}
	<-doneCh
}

func SetLogging() {
	// if logfile is empty, use Stderr
	var err error
	if logDest == "" || logDest == "stderr" {
		logOut = os.Stderr
		goto newLog
	}
	if logDest == "stdout" {
		logOut = os.Stdout
		goto newLog
	}
	logOut, err = os.OpenFile(logDest, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if err != nil {
		panic(err)
	}
newLog:
	log = zap.New(
		zap.NewJSONEncoder(
			zap.RFC3339Formatter("ts"),
		),
		zap.Output(logOut),
	)
	log.SetLevel(*loglevel)
}

func SetDataOut() {
	var err error
	if dataDest == "" || dataDest == "stdout" {
		dataOut = os.Stdout
		goto newData
	}
	if dataDest == "stderr" {
		dataOut = os.Stderr
		goto newData
	}
	dataOut, err = os.OpenFile(dataDest, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if err != nil {
		log.Fatal(
			err.Error(),
			zap.String("op", "open datafile"),
			zap.String("filename", dataDest),
		)
	}
newData:
	data = czap.New(
		czap.NewJSONEncoder(
			czap.RFC3339Formatter("ts"),
		),
		czap.Output(dataOut),
	)
	data.SetLevel(czap.WarnLevel)
}

// CloseOut closes the local output destinations before shutdown.
func CloseOut() {
	if logOut != nil {
		logOut.Close()
	}
	// If running serverless, close the data file.
	if serverless {
		dataOut.Close()
	}
}

func handleSignals(c *Client) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	v := <-ch
	log.Info(
		"os signal received: shutting down autofact",
		zap.Object("signal", v.String()),
	)
	// If there's a connection send a close signal
	if c.IsConnected() {
		log.Debug(
			"closing connection",
			zap.String("op", "shutdown"),
		)
		c.WS.WriteMessage(websocket.CloseMessage, []byte(string(c.Conn.ID)+" shutting down"))
	}
	CloseOut()

	os.Exit(1)
}
<commit_msg>use the client's Collect instead of a local one<commit_after>package main

import (
	"flag"
	"fmt"
	"net/url"
	"os"
	"os/signal"
	"path/filepath"
	"time"

	"github.com/gorilla/websocket"
	"github.com/mohae/autofact/conf"
	czap "github.com/mohae/zap"
	"github.com/uber-go/zap"
)

const (
	addressVar = "address"
	aVar       = "a"
	portVar    = "port"
	pVar       = "p"
)

var (
	connFile    = "autofact.json"
	collectFile = "autocollect.json"
	// This is the default directory for autofact-client app data.
	autofactPath    = "$HOME/.autofact"
	autofactEnvName = "AUTOFACT_PATH"
	// configuration info
	connConf conf.Conn

	// client configuration: used for serverless

	serverless bool
)

// Vars for logging and local data output, if applicable.
var (
	log      zap.Logger // application log
	loglevel = zap.LevelFlag("loglevel", zap.WarnLevel, "log level")
	logDest  string
	logOut   *os.File

	data     czap.Logger // use mohae's fork to support level description override
	dataDest string
	dataOut  *os.File
)

// TODO: reconcile these flags with config file usage.  Probably add contour
// to handle this after the next refactor of contour.
// TODO: make connectInterval/period handling consistent, e.g. should they be
// flags, what is precedence in relation to Conn?
func init() {
	flag.StringVar(&connConf.ServerAddress, addressVar, "127.0.0.1", "the server address")
	flag.StringVar(&connConf.ServerAddress, aVar, "127.0.0.1", "the server address (short)")
	flag.StringVar(&connConf.ServerPort, portVar, "8675", "the connection port")
	flag.StringVar(&connConf.ServerPort, pVar, "8675", "the connection port (short)")
	flag.StringVar(&logDest, "logdestination", "stderr", "log output destination; if empty stderr will be used")
	flag.StringVar(&logDest, "l", "stderr", "log output; if empty stderr will be used")
	flag.StringVar(&dataDest, "datadestination", "stdout", "serverless mode data output destination, if empty stderr will be used")
	flag.StringVar(&dataDest, "d", "stdout", "serverless mode data output destination, if empty stderr will be used")
	flag.BoolVar(&serverless, "serverless", false, "serverless: the client will run standalone and write the collected data to the log")
	connConf.ConnectInterval.Duration = 5 * time.Second
	connConf.ConnectPeriod.Duration = 15 * time.Minute

	// override czap description for InfoLevel
	czap.WarnString = "data"
}

func main() {
	// Load the AUTOPATH value
	tmp := os.Getenv(autofactEnvName)
	if tmp != "" {
		autofactPath = tmp
	}
	autofactPath = os.ExpandEnv(autofactPath)

	// make sure the autofact path exists (create if it doesn't)
	err := os.MkdirAll(autofactPath, 0760)
	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to create AUTOFACT_PATH: %s\n", err)
		fmt.Fprintln(os.Stderr, "startup error: exiting")
		os.Exit(1)
	}

	// finalize the paths
	connFile = filepath.Join(autofactPath, connFile)

	// process the settings: this gets read first just in case flags override
	var connMsg string
	err = connConf.Load(connFile)
	if err != nil {
		// capture the error for logging once it is setup and continue.  An error
		// is not a show stopper as the file may not exist if this is the first
		// time autofact has run on this node.
		connMsg = fmt.Sprintf("using default connection settings")
	}

	// Parse the flags.
	flag.Parse()

	// now that everything is parsed; set up logging
	SetLogging()
	defer CloseOut()
	// if there was an error reading the connection configuration and this isn't
	// being run serverless, log it
	if connMsg != "" && !serverless {
		log.Warn(
			err.Error(),
			zap.String("op", fmt.Sprintf("load %s", connFile)),
			zap.String("conf", connMsg),
		)
	}

	// TODO add env var support

	// get a client
	c := NewClient(connConf, collectFile)
	c.AutoPath = autofactPath

	// if serverless: load the collection configuration
	if serverless {
		collectFile = filepath.Join(autofactPath, collectFile)
		err = c.Collect.Load(collectFile)
		if err != nil {
			log.Warn(
				err.Error(),
				zap.String("op", fmt.Sprintf("load %s", collectFile)),
				zap.String("conf", "using default collect settings"),
			)
			c.Collect.UseDefaults()
			err = c.Collect.SaveAsJSON(collectFile)
			if err != nil {
				log.Warn(
					err.Error(),
					zap.String("op", fmt.Sprintf("save %s", collectFile)),
				)
			}
		}
	}

	// handle signals
	go handleSignals(c)

	// doneCh is used to signal that the connection has been closed
	doneCh := make(chan struct{})

	// Set up the output destination.
	if serverless { // open the datafile to use
		SetDataOut()
	} else { // connect to the server
		// connect to the Server
		c.ServerURL = url.URL{Scheme: "ws", Host: fmt.Sprintf("%s:%s", c.ServerAddress, c.ServerPort), Path: "/client"}

		// must have a connection before doing anything
		for i := 0; i < 3; i++ {
			connected := c.Connect()
			if connected {
				break
			}
			// retry on fail until retry attempts have been exceeded
		}
		if !c.IsConnected() {
			log.Error(
				"unable to connect",
				zap.String("server", c.ServerURL.String()),
			)
			CloseOut() // defer doesn't run on fatal
			os.Exit(1)
		}
		// if connected, save the conf: this will also save the ClientID
		err = c.Conn.Save()
		if err != nil {
			log.Error(
				err.Error(),
				zap.String("op", "save conn"),
				zap.String("file", c.Conn.Filename),
			)
		}
	}

	// set up the data processing
	if serverless {
		// since there isn't a server pull for healthbeat, a local ticker is started
		go c.HealthbeatLocal(doneCh)
	} else {
		// assign the
		c.LoadAvg = LoadAvgFB
		go c.Listen(doneCh)
		go c.MemInfo(doneCh)
		go c.CPUUtilization(doneCh)
		go c.NetUsage(doneCh)
		// start the connection handler
		go c.MessageWriter(doneCh)
	}
	<-doneCh
}

func SetLogging() {
	// if logfile is empty, use Stderr
	var err error
	if logDest == "" || logDest == "stderr" {
		logOut = os.Stderr
		goto newLog
	}
	if logDest == "stdout" {
		logOut = os.Stdout
		goto newLog
	}
	logOut, err = os.OpenFile(logDest, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if err != nil {
		panic(err)
	}
newLog:
	log = zap.New(
		zap.NewJSONEncoder(
			zap.RFC3339Formatter("ts"),
		),
		zap.Output(logOut),
	)
	log.SetLevel(*loglevel)
}

func SetDataOut() {
	var err error
	if dataDest == "" || dataDest == "stdout" {
		dataOut = os.Stdout
		goto newData
	}
	if dataDest == "stderr" {
		dataOut = os.Stderr
		goto newData
	}
	dataOut, err = os.OpenFile(dataDest, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0664)
	if err != nil {
		log.Fatal(
			err.Error(),
			zap.String("op", "open datafile"),
			zap.String("filename", dataDest),
		)
	}
newData:
	data = czap.New(
		czap.NewJSONEncoder(
			czap.RFC3339Formatter("ts"),
		),
		czap.Output(dataOut),
	)
	data.SetLevel(czap.WarnLevel)
}

// CloseOut closes the local output destinations before shutdown.
func CloseOut() {
	if logOut != nil {
		logOut.Close()
	}
	// If running serverless, close the data file.
	if serverless {
		dataOut.Close()
	}
}

func handleSignals(c *Client) {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, os.Interrupt)
	v := <-ch
	log.Info(
		"os signal received: shutting down autofact",
		zap.Object("signal", v.String()),
	)
	// If there's a connection send a close signal
	if c.IsConnected() {
		log.Debug(
			"closing connection",
			zap.String("op", "shutdown"),
		)
		c.WS.WriteMessage(websocket.CloseMessage, []byte(string(c.Conn.ID)+" shutting down"))
	}
	CloseOut()

	os.Exit(1)
}
<|endoftext|>


<commit_before>// The camget tool fetches blobs, files, and directories.
//
// Examples
//
// Writes to stdout by default:
//
//   camget <blobref>                 // dump raw blob
//   camget -contents <file-blobref>  // dump file contents
//
// Like curl, lets you set output file/directory with -o:
//
//   camget -o <dir> <blobref>
//     (if <dir> exists and is directory, <blobref> must be a directory;
//      use -f to overwrite any files)
//
//   camget -o <filename> <file-blobref>
//
// TODO(bradfitz): camget isn't very fleshed out. In general, using 'cammount' to just
// mount a tree is an easier way to get files back.
package main

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/client"
	"camlistore.org/pkg/index"
	"camlistore.org/pkg/schema"
)

var (
	flagVerbose  = flag.Bool("verbose", false, "be verbose")
	flagCheck    = flag.Bool("check", false, "just check for the existence of listed blobs; returning 0 if all are present")
	flagOutput   = flag.String("o", "-", "Output file/directory to create.  Use -f to overwrite.")
	flagGraph    = flag.Bool("graph", false, "Output a graphviz directed graph .dot file of the provided root schema blob, to be rendered with 'dot -Tsvg -o graph.svg graph.dot'")
	flagContents = flag.Bool("contents", false, "If true and the target blobref is a 'bytes' or 'file' schema blob, the contents of that file are output instead.")
	flagShared   = flag.String("shared", "", "If non-empty, the URL of a \"share\" blob. The URL will be used as the root of future fetches. Only \"haveref\" shares are currently supported.")
)

func main() {
	client.AddFlags()
	flag.Parse()

	if *flagGraph && flag.NArg() != 1 {
		log.Fatalf("The --graph option requires exactly one parameter.")
	}

	var cl *client.Client
	var items []*blobref.BlobRef

	if *flagShared != "" {
		if client.ExplicitServer() != "" {
			log.Fatal("Can't use --shared with an explicit blobserver; blobserver is implicit from the --shared URL.")
		}
		if flag.NArg() != 0 {
			log.Fatal("No arguments permitted when using --shared")
		}
		cl1, target, err := client.NewFromShareRoot(*flagShared)
		if err != nil {
			log.Fatal(err)
		}
		cl = cl1
		items = append(items, target)
	} else {
		cl = client.NewOrFail()
		for n := 0; n < flag.NArg(); n++ {
			arg := flag.Arg(n)
			br := blobref.Parse(arg)
			if br == nil {
				log.Fatalf("Failed to parse argument %q as a blobref.", arg)
			}
			items = append(items, br)
		}
	}

	for _, br := range items {
		if *flagGraph {
			printGraph(cl, br)
			return
		}
		if *flagCheck {
			// TODO: do HEAD requests checking if the blobs exists.
			log.Fatal("not implemented")
			return
		}
		if *flagOutput == "-" {
			var rc io.ReadCloser
			var err error
			if *flagContents {
				seekFetcher := blobref.SeekerFromStreamingFetcher(cl)
				rc, err = schema.NewFileReader(seekFetcher, br)
			} else {
				rc, err = fetch(cl, br)
			}
			if err != nil {
				log.Fatal(err)
			}
			defer rc.Close()
			if _, err := io.Copy(os.Stdout, rc); err != nil {
				log.Fatalf("Failed reading %q: %v", br, err)
			}
			return
		}
		if err := smartFetch(cl, *flagOutput, br); err != nil {
			log.Fatal(err)
		}
	}
}

func fetch(cl *client.Client, br *blobref.BlobRef) (r io.ReadCloser, err error) {
	if *flagVerbose {
		log.Printf("Fetching %s", br.String())
	}
	r, _, err = cl.FetchStreaming(br)
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch %s: %s", br, err)
	}
	return r, err
}

// A little less than the sniffer will take, so we don't truncate.
const sniffSize = 900 * 1024

// smartFetch the things that blobs point to, not just blobs.
func smartFetch(cl *client.Client, targ string, br *blobref.BlobRef) error {
	rc, err := fetch(cl, br)
	if err != nil {
		return err
	}
	defer rc.Close()

	sniffer := new(index.BlobSniffer)
	_, err = io.CopyN(sniffer, rc, sniffSize)
	if err != nil && err != io.EOF {
		return err
	}

	sniffer.Parse()
	sc, ok := sniffer.Superset()

	if !ok {
		if *flagVerbose {
			log.Printf("Fetching opaque data %v into %q", br, targ)
		}

		// opaque data - put it in a file
		f, err := os.Create(targ)
		if err != nil {
			return fmt.Errorf("opaque: %v", err)
		}
		defer f.Close()
		body, _ := sniffer.Body()
		r := io.MultiReader(bytes.NewReader(body), rc)
		_, err = io.Copy(f, r)
		return err
	}

	sc.BlobRef = br

	switch sc.Type {
	case "directory":
		dir := filepath.Join(targ, sc.FileName)
		if *flagVerbose {
			log.Printf("Fetching directory %v into %s", br, dir)
		}
		if err := os.MkdirAll(dir, sc.FileMode()); err != nil {
			return err
		}
		if err := setFileMeta(dir, sc); err != nil {
			log.Print(err)
		}
		entries := blobref.Parse(sc.Entries)
		if entries == nil {
			return fmt.Errorf("bad entries blobref: %v", sc.Entries)
		}
		return smartFetch(cl, dir, entries)
	case "static-set":
		if *flagVerbose {
			log.Printf("Fetching directory entries %v into %s", br, targ)
		}

		// directory entries
		for _, m := range sc.Members {
			// TODO: do n at a time
			dref := blobref.Parse(m)
			if dref == nil {
				return fmt.Errorf("bad member blobref: %v", m)
			}
			if err := smartFetch(cl, targ, dref); err != nil {
				return err
			}
		}
		return nil
	case "file":
		seekFetcher := blobref.SeekerFromStreamingFetcher(cl)
		fr, err := schema.NewFileReader(seekFetcher, br)
		if err != nil {
			return fmt.Errorf("NewFileReader: %v", err)
		}
		defer fr.Close()

		name := filepath.Join(targ, sc.FileName)

		if fi, err := os.Stat(name); err == nil && fi.Size() == fi.Size() {
			if *flagVerbose {
				log.Printf("Skipping %s; already exists.", name)
				return nil
			}
		}

		if *flagVerbose {
			log.Printf("Writing %s to %s ...", br, name)
		}

		f, err := os.Create(name)
		if err != nil {
			return fmt.Errorf("file type: %v", err)
		}
		defer f.Close()
		if _, err := io.Copy(f, fr); err != nil {
			return fmt.Errorf("Copying %s to %s: %v", br, name, err)
		}
		if err := setFileMeta(name, sc); err != nil {
			log.Print(err)
		}
		return nil
	default:
		return errors.New("unknown blob type: " + sc.Type)
	}
	panic("unreachable")
}

func setFileMeta(name string, sc *schema.Superset) error {
	err1 := os.Chmod(name, sc.FileMode())
	var err2 error
	if mt := sc.ModTime(); !mt.IsZero() {
		err2 = os.Chtimes(name, mt, mt)
	}
	err3 := os.Chown(name, sc.UnixOwnerId, sc.UnixGroupId)
	// Return first non-nil error for logging.
	for _, err := range []error{err1, err2, err3} {
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>camget: do 10 at once. better, much pref work remains.<commit_after>// The camget tool fetches blobs, files, and directories.
//
// Examples
//
// Writes to stdout by default:
//
//   camget <blobref>                 // dump raw blob
//   camget -contents <file-blobref>  // dump file contents
//
// Like curl, lets you set output file/directory with -o:
//
//   camget -o <dir> <blobref>
//     (if <dir> exists and is directory, <blobref> must be a directory;
//      use -f to overwrite any files)
//
//   camget -o <filename> <file-blobref>
//
// TODO(bradfitz): camget isn't very fleshed out. In general, using 'cammount' to just
// mount a tree is an easier way to get files back.
package main

/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"

	"camlistore.org/pkg/blobref"
	"camlistore.org/pkg/client"
	"camlistore.org/pkg/index"
	"camlistore.org/pkg/schema"
)

var (
	flagVerbose  = flag.Bool("verbose", false, "be verbose")
	flagCheck    = flag.Bool("check", false, "just check for the existence of listed blobs; returning 0 if all are present")
	flagOutput   = flag.String("o", "-", "Output file/directory to create.  Use -f to overwrite.")
	flagGraph    = flag.Bool("graph", false, "Output a graphviz directed graph .dot file of the provided root schema blob, to be rendered with 'dot -Tsvg -o graph.svg graph.dot'")
	flagContents = flag.Bool("contents", false, "If true and the target blobref is a 'bytes' or 'file' schema blob, the contents of that file are output instead.")
	flagShared   = flag.String("shared", "", "If non-empty, the URL of a \"share\" blob. The URL will be used as the root of future fetches. Only \"haveref\" shares are currently supported.")
)

func main() {
	client.AddFlags()
	flag.Parse()

	if *flagGraph && flag.NArg() != 1 {
		log.Fatalf("The --graph option requires exactly one parameter.")
	}

	var cl *client.Client
	var items []*blobref.BlobRef

	if *flagShared != "" {
		if client.ExplicitServer() != "" {
			log.Fatal("Can't use --shared with an explicit blobserver; blobserver is implicit from the --shared URL.")
		}
		if flag.NArg() != 0 {
			log.Fatal("No arguments permitted when using --shared")
		}
		cl1, target, err := client.NewFromShareRoot(*flagShared)
		if err != nil {
			log.Fatal(err)
		}
		cl = cl1
		items = append(items, target)
	} else {
		cl = client.NewOrFail()
		for n := 0; n < flag.NArg(); n++ {
			arg := flag.Arg(n)
			br := blobref.Parse(arg)
			if br == nil {
				log.Fatalf("Failed to parse argument %q as a blobref.", arg)
			}
			items = append(items, br)
		}
	}

	for _, br := range items {
		if *flagGraph {
			printGraph(cl, br)
			return
		}
		if *flagCheck {
			// TODO: do HEAD requests checking if the blobs exists.
			log.Fatal("not implemented")
			return
		}
		if *flagOutput == "-" {
			var rc io.ReadCloser
			var err error
			if *flagContents {
				seekFetcher := blobref.SeekerFromStreamingFetcher(cl)
				rc, err = schema.NewFileReader(seekFetcher, br)
			} else {
				rc, err = fetch(cl, br)
			}
			if err != nil {
				log.Fatal(err)
			}
			defer rc.Close()
			if _, err := io.Copy(os.Stdout, rc); err != nil {
				log.Fatalf("Failed reading %q: %v", br, err)
			}
			return
		}
		if err := smartFetch(cl, *flagOutput, br); err != nil {
			log.Fatal(err)
		}
	}
}

func fetch(cl *client.Client, br *blobref.BlobRef) (r io.ReadCloser, err error) {
	if *flagVerbose {
		log.Printf("Fetching %s", br.String())
	}
	r, _, err = cl.FetchStreaming(br)
	if err != nil {
		return nil, fmt.Errorf("Failed to fetch %s: %s", br, err)
	}
	return r, err
}

// A little less than the sniffer will take, so we don't truncate.
const sniffSize = 900 * 1024

// smartFetch the things that blobs point to, not just blobs.
func smartFetch(cl *client.Client, targ string, br *blobref.BlobRef) error {
	rc, err := fetch(cl, br)
	if err != nil {
		return err
	}
	defer rc.Close()

	sniffer := new(index.BlobSniffer)
	_, err = io.CopyN(sniffer, rc, sniffSize)
	if err != nil && err != io.EOF {
		return err
	}

	sniffer.Parse()
	sc, ok := sniffer.Superset()

	if !ok {
		if *flagVerbose {
			log.Printf("Fetching opaque data %v into %q", br, targ)
		}

		// opaque data - put it in a file
		f, err := os.Create(targ)
		if err != nil {
			return fmt.Errorf("opaque: %v", err)
		}
		defer f.Close()
		body, _ := sniffer.Body()
		r := io.MultiReader(bytes.NewReader(body), rc)
		_, err = io.Copy(f, r)
		return err
	}

	sc.BlobRef = br

	switch sc.Type {
	case "directory":
		dir := filepath.Join(targ, sc.FileName)
		if *flagVerbose {
			log.Printf("Fetching directory %v into %s", br, dir)
		}
		if err := os.MkdirAll(dir, sc.FileMode()); err != nil {
			return err
		}
		if err := setFileMeta(dir, sc); err != nil {
			log.Print(err)
		}
		entries := blobref.Parse(sc.Entries)
		if entries == nil {
			return fmt.Errorf("bad entries blobref: %v", sc.Entries)
		}
		return smartFetch(cl, dir, entries)
	case "static-set":
		if *flagVerbose {
			log.Printf("Fetching directory entries %v into %s", br, targ)
		}

		// directory entries
		const numWorkers = 10
		type work struct {
			br *blobref.BlobRef
			errc chan<- error
		}
		workc := make(chan work)
		defer close(workc)
		for i := 0; i < numWorkers; i++ {
			go func() {
				for wi := range workc {
					wi.errc <- smartFetch(cl, targ, wi.br)
				}
			}()
		}
		var errcs []<-chan error
		for _, m := range sc.Members {
			dref := blobref.Parse(m)
			if dref == nil {
				return fmt.Errorf("bad member blobref: %v", m)
			}
			errc := make(chan error, 1)
			errcs = append(errcs, errc)
			workc <- work{dref, errc}
		}
		for _, errc := range errcs {
			if err := <-errc; err != nil {
				return err
			}
		}
		return nil
	case "file":
		seekFetcher := blobref.SeekerFromStreamingFetcher(cl)
		fr, err := schema.NewFileReader(seekFetcher, br)
		if err != nil {
			return fmt.Errorf("NewFileReader: %v", err)
		}
		defer fr.Close()

		name := filepath.Join(targ, sc.FileName)

		if fi, err := os.Stat(name); err == nil && fi.Size() == fi.Size() {
			if *flagVerbose {
				log.Printf("Skipping %s; already exists.", name)
				return nil
			}
		}

		if *flagVerbose {
			log.Printf("Writing %s to %s ...", br, name)
		}

		f, err := os.Create(name)
		if err != nil {
			return fmt.Errorf("file type: %v", err)
		}
		defer f.Close()
		if _, err := io.Copy(f, fr); err != nil {
			return fmt.Errorf("Copying %s to %s: %v", br, name, err)
		}
		if err := setFileMeta(name, sc); err != nil {
			log.Print(err)
		}
		return nil
	default:
		return errors.New("unknown blob type: " + sc.Type)
	}
	panic("unreachable")
}

func setFileMeta(name string, sc *schema.Superset) error {
	err1 := os.Chmod(name, sc.FileMode())
	var err2 error
	if mt := sc.ModTime(); !mt.IsZero() {
		err2 = os.Chtimes(name, mt, mt)
	}
	err3 := os.Chown(name, sc.UnixOwnerId, sc.UnixGroupId)
	// Return first non-nil error for logging.
	for _, err := range []error{err1, err2, err3} {
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"sort"
	"strings"

	"github.com/junkblocker/codesearch/index"
)

const (
	DEFAULT_FOLLOW_SYMLINKS             = true
	DEFAULT_MAX_FILE_LENGTH             = 1 << 30
	DEFAULT_MAX_LINE_LENGTH             = 2000
	DEFAULT_MAX_TEXT_TRIGRAMS           = 30000
	DEFAULT_MAX_INVALID_UTF8_PERCENTAGE = 0.1
)

var usageMessage = `usage: cindex [options] [path...]

Options:

  -verbose     print extra information
  -list        list indexed paths and exit
  -reset       discard existing index
  -indexpath FILE
               use specified FILE as the index path. Overrides $CSEARCHINDEX.
  -cpuprofile FILE
               write CPU profile to FILE
  -logskip     print why a file was skipped from indexing
  -follow-symlinks
               follow symlinked files and directories also (Default: %v)
  -maxFileLen BYTES
               skip indexing a file if longer than this size in bytes (Default: %v)
  -maxlinelen BYTES
               skip indexing a file if it has a line longer than this size in bytes (Default: %v)
  -maxtrigrams COUNT
               skip indexing a file if it has more than this number of trigrams (Default: %v)
  -maxinvalidutf8ratio RATIO
               skip indexing a file if it has more than this ratio of invalid UTF-8 sequences (Default: %v)
  -exclude FILE
               path to file containing a list of file patterns to exclude from indexing

cindex prepares the trigram index for use by csearch.  The index is the
file named by $CSEARCHINDEX, or else $HOME/.csearchindex.

The simplest invocation is

	cindex path...

which adds the file or directory tree named by each path to the index.
For example:

	cindex $HOME/src /usr/include

or, equivalently:

	cindex $HOME/src
	cindex /usr/include

If cindex is invoked with no paths, it reindexes the paths that have
already been added, in case the files have changed.  Thus, 'cindex' by
itself is a useful command to run in a nightly cron job.

By default cindex adds the named paths to the index but preserves
information about other paths that might already be indexed
(the ones printed by cindex -list).  The -reset flag causes cindex to
delete the existing index before indexing the new paths.
With no path arguments, cindex -reset removes the index.
`

func usage() {
	fmt.Fprintf(os.Stderr, usageMessage, DEFAULT_FOLLOW_SYMLINKS, DEFAULT_MAX_FILE_LENGTH, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_TEXT_TRIGRAMS, DEFAULT_MAX_INVALID_UTF8_PERCENTAGE)
	os.Exit(2)
}

var (
	listFlag           = flag.Bool("list", false, "list indexed paths and exit")
	resetFlag          = flag.Bool("reset", false, "discard existing index")
	verboseFlag        = flag.Bool("verbose", false, "print extra information")
	cpuProfile         = flag.String("cpuprofile", "", "write cpu profile to this file")
	indexPath          = flag.String("indexpath", "", "specifies index path")
	logSkipFlag        = flag.Bool("logskip", false, "print why a file was skipped from indexing")
	followSymlinksFlag = flag.Bool("follow-symlinks", DEFAULT_FOLLOW_SYMLINKS, "follow symlinked files and directories also")
	exclude            = flag.String("exclude", "", "path to file containing a list of file patterns to exclude from indexing")
	// Tuning variables for detecting text files.
	// A file is assumed not to be text files (and thus not indexed) if
	// 1) if it contains an invalid UTF-8 sequences
	// 2) if it is longer than maxFileLength bytes
	// 3) if it contains a line longer than maxLineLen bytes,
	// or
	// 4) if it contains more than maxTextTrigrams distinct trigrams.
	maxFileLen          = flag.Int64("maxfilelen", DEFAULT_MAX_FILE_LENGTH, "skip indexing a file if longer than this size in bytes")
	maxLineLen          = flag.Int("maxlinelen", DEFAULT_MAX_LINE_LENGTH, "skip indexing a file if it has a line longer than this size in bytes")
	maxTextTrigrams     = flag.Int("maxtrigrams", DEFAULT_MAX_TEXT_TRIGRAMS, "skip indexing a file if it has more than this number of trigrams")
	maxInvalidUTF8Ratio = flag.Float64("maxinvalidutf8ratio", DEFAULT_MAX_INVALID_UTF8_PERCENTAGE, "skip indexing a file if it has more than this ratio of invalid UTF-8 sequences")

	excludePatterns = []string{
		".csearchindex",
	}
)

func walk(arg string, symlinkFrom string, out chan string, logskip bool) {
	filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
		if basedir, elem := filepath.Split(path); elem != "" {
			exclude := false
			for _, pattern := range excludePatterns {
				exclude, err = filepath.Match(pattern, elem)
				if err != nil {
					log.Fatal(err)
				}
				if exclude {
					break
				}
			}

			// Skip various temporary or "hidden" files or directories.
			if info != nil && info.IsDir() {
				if exclude {
					if logskip {
						if symlinkFrom != "" {
							log.Printf("%s: skipped. VCS or backup directory", symlinkFrom+path[len(arg):])
						} else {
							log.Printf("%s: skipped. VCS or backup directory", path)
						}
					}
					return filepath.SkipDir
				}
			} else {
				if exclude {
					if logskip {
						if symlinkFrom != "" {
							log.Printf("%s: skipped. Backup or undesirable file", symlinkFrom+path[len(arg):])
						} else {
							log.Printf("%s: skipped. Backup or undesirable file", path)
						}
					}
					return nil
				}
				if info != nil && info.Mode()&os.ModeSymlink != 0 {
					if !*followSymlinksFlag {
						if logskip {
							log.Printf("%s: skipped. Symlink", path)
						}
						return nil
					}
					var symlinkAs string
					if basedir[len(basedir)-1] == os.PathSeparator {
						symlinkAs = basedir + elem
					} else {
						symlinkAs = basedir + string(os.PathSeparator) + elem
					}
					if symlinkFrom != "" {
						symlinkAs = symlinkFrom + symlinkAs[len(arg):]
					}
					if p, err := filepath.EvalSymlinks(symlinkAs); err != nil {
						if symlinkFrom != "" {
							log.Printf("%s: skipped. Symlink could not be resolved", symlinkFrom+path[len(arg):])
						} else {
							log.Printf("%s: skipped. Symlink could not be resolved", path)
						}
					} else {
						walk(p, symlinkAs, out, logskip)
					}
					return nil
				}
			}
		}
		if err != nil {
			if symlinkFrom != "" {
				log.Printf("%s: skipped. Error: %s", symlinkFrom+path[len(arg):], err)
			} else {
				log.Printf("%s: skipped. Error: %s", path, err)
			}
			return nil
		}
		if info != nil {
			if info.Mode()&os.ModeType == 0 {
				if symlinkFrom == "" {
					out <- path
				} else {
					out <- symlinkFrom + path[len(arg):]
				}
			} else if !info.IsDir() {
				if logskip {
					if symlinkFrom != "" {
						log.Printf("%s: skipped. Unsupported path type", symlinkFrom+path[len(arg):])
					} else {
						log.Printf("%s: skipped. Unsupported path type", path)
					}
				}
			}
		} else {
			if logskip {
				if symlinkFrom != "" {
					log.Printf("%s: skipped. Could not stat.", symlinkFrom+path[len(arg):])
				} else {
					log.Printf("%s: skipped. Could not stat.", path)
				}
			}
		}
		return nil
	})
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if *indexPath != "" {
		if err := os.Setenv("CSEARCHINDEX", *indexPath); err != nil {
			log.Fatal(err)
		}
	}

	if *listFlag {
		master := index.File()
		if stat, err := os.Stat(master); err != nil || stat == nil {
			log.Fatal("Index " + master + " is not accessible")
		} else if stat.IsDir() || !stat.Mode().IsRegular() {
			log.Fatal("Index " + master + " must point to an index file")
		}
		ix := index.Open(master)
		for _, arg := range ix.Paths() {
			fmt.Printf("%s\n", arg)
		}
		return
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *resetFlag && len(args) == 0 {
		master := index.File()
		stat, err := os.Stat(master)
		if err != nil {
			// does not exist so nothing to do
			return
		}
		if stat != nil && !stat.IsDir() && stat.Mode().IsRegular() {
			os.Remove(master)
			return
		} else {
			log.Fatal("Invalid index path " + master)
		}
	}

	if *exclude != "" {
		data, err := ioutil.ReadFile(*exclude)
		if err != nil {
			log.Fatal(err)
		}
		excludePatterns = append(excludePatterns, strings.Split(string(data), "\n")...)
		for i, pattern := range excludePatterns {
			excludePatterns[i] = strings.TrimSpace(pattern)
		}
	}

	if len(args) == 0 {
		ix := index.Open(index.File())
		for _, arg := range ix.Paths() {
			args = append(args, arg)
		}
		ix.Close()
	}

	// Translate paths to absolute paths so that we can
	// generate the file list in sorted order.
	for i, arg := range args {
		a, err := filepath.Abs(arg)
		if err != nil {
			log.Printf("%s: %s", arg, err)
			args[i] = ""
			continue
		}
		args[i] = a
	}
	sort.Strings(args)

	for len(args) > 0 && args[0] == "" {
		args = args[1:]
	}

	master := index.File()
	if stat, err := os.Stat(master); err != nil {
		// Does not exist.
		*resetFlag = true
	} else {
		if stat != nil && (stat.IsDir() || !stat.Mode().IsRegular()) {
			log.Fatal("Invalid index path " + master)
		}

	}
	file := master
	if !*resetFlag {
		file += "~"
	}

	ix := index.Create(file)
	ix.Verbose = *verboseFlag
	ix.LogSkip = *logSkipFlag
	ix.MaxFileLen = *maxFileLen
	ix.MaxLineLen = *maxLineLen
	ix.MaxTextTrigrams = *maxTextTrigrams
	ix.MaxInvalidUTF8Ratio = *maxInvalidUTF8Ratio
	ix.AddPaths(args)

	walkChan := make(chan string)
	doneChan := make(chan bool)

	go func() {
		seen := make(map[string]bool)
		for {
			select {
			case path := <-walkChan:
				if !seen[path] {
					seen[path] = true
					ix.AddFile(path)
				}
			case <-doneChan:
				return
			}
		}
	}()
	for _, arg := range args {
		log.Printf("index %s", arg)
		walk(arg, "", walkChan, *logSkipFlag)
	}
	doneChan <- true
	log.Printf("flush index")
	ix.Flush()

	if !*resetFlag {
		log.Printf("merge %s %s", master, file)
		index.Merge(file+"~", master, file)
		os.Remove(file)
		os.Remove(master)
		if err := os.Rename(file+"~", master); err != nil {
			log.Fatalf("failed to merge indexes: %s", err)
		}
	}
	log.Printf("done")
	return
}
<commit_msg>Add convenient tilde expansion to ignore file path<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime/pprof"
	"sort"
	"strings"

	"github.com/junkblocker/codesearch/index"
)

const (
	DEFAULT_FOLLOW_SYMLINKS             = true
	DEFAULT_MAX_FILE_LENGTH             = 1 << 30
	DEFAULT_MAX_LINE_LENGTH             = 2000
	DEFAULT_MAX_TEXT_TRIGRAMS           = 30000
	DEFAULT_MAX_INVALID_UTF8_PERCENTAGE = 0.1
)

var usageMessage = `usage: cindex [options] [path...]

Options:

  -verbose     print extra information
  -list        list indexed paths and exit
  -reset       discard existing index
  -indexpath FILE
               use specified FILE as the index path. Overrides $CSEARCHINDEX.
  -cpuprofile FILE
               write CPU profile to FILE
  -logskip     print why a file was skipped from indexing
  -follow-symlinks
               follow symlinked files and directories also (Default: %v)
  -maxFileLen BYTES
               skip indexing a file if longer than this size in bytes (Default: %v)
  -maxlinelen BYTES
               skip indexing a file if it has a line longer than this size in bytes (Default: %v)
  -maxtrigrams COUNT
               skip indexing a file if it has more than this number of trigrams (Default: %v)
  -maxinvalidutf8ratio RATIO
               skip indexing a file if it has more than this ratio of invalid UTF-8 sequences (Default: %v)
  -exclude FILE
               path to file containing a list of file patterns to exclude from indexing

cindex prepares the trigram index for use by csearch.  The index is the
file named by $CSEARCHINDEX, or else $HOME/.csearchindex.

The simplest invocation is

	cindex path...

which adds the file or directory tree named by each path to the index.
For example:

	cindex $HOME/src /usr/include

or, equivalently:

	cindex $HOME/src
	cindex /usr/include

If cindex is invoked with no paths, it reindexes the paths that have
already been added, in case the files have changed.  Thus, 'cindex' by
itself is a useful command to run in a nightly cron job.

By default cindex adds the named paths to the index but preserves
information about other paths that might already be indexed
(the ones printed by cindex -list).  The -reset flag causes cindex to
delete the existing index before indexing the new paths.
With no path arguments, cindex -reset removes the index.
`

func usage() {
	fmt.Fprintf(os.Stderr, usageMessage, DEFAULT_FOLLOW_SYMLINKS, DEFAULT_MAX_FILE_LENGTH, DEFAULT_MAX_LINE_LENGTH, DEFAULT_MAX_TEXT_TRIGRAMS, DEFAULT_MAX_INVALID_UTF8_PERCENTAGE)
	os.Exit(2)
}

var (
	listFlag           = flag.Bool("list", false, "list indexed paths and exit")
	resetFlag          = flag.Bool("reset", false, "discard existing index")
	verboseFlag        = flag.Bool("verbose", false, "print extra information")
	cpuProfile         = flag.String("cpuprofile", "", "write cpu profile to this file")
	indexPath          = flag.String("indexpath", "", "specifies index path")
	logSkipFlag        = flag.Bool("logskip", false, "print why a file was skipped from indexing")
	followSymlinksFlag = flag.Bool("follow-symlinks", DEFAULT_FOLLOW_SYMLINKS, "follow symlinked files and directories also")
	exclude            = flag.String("exclude", "", "path to file containing a list of file patterns to exclude from indexing")
	// Tuning variables for detecting text files.
	// A file is assumed not to be text files (and thus not indexed) if
	// 1) if it contains an invalid UTF-8 sequences
	// 2) if it is longer than maxFileLength bytes
	// 3) if it contains a line longer than maxLineLen bytes,
	// or
	// 4) if it contains more than maxTextTrigrams distinct trigrams.
	maxFileLen          = flag.Int64("maxfilelen", DEFAULT_MAX_FILE_LENGTH, "skip indexing a file if longer than this size in bytes")
	maxLineLen          = flag.Int("maxlinelen", DEFAULT_MAX_LINE_LENGTH, "skip indexing a file if it has a line longer than this size in bytes")
	maxTextTrigrams     = flag.Int("maxtrigrams", DEFAULT_MAX_TEXT_TRIGRAMS, "skip indexing a file if it has more than this number of trigrams")
	maxInvalidUTF8Ratio = flag.Float64("maxinvalidutf8ratio", DEFAULT_MAX_INVALID_UTF8_PERCENTAGE, "skip indexing a file if it has more than this ratio of invalid UTF-8 sequences")

	excludePatterns = []string{
		".csearchindex",
	}
)

func walk(arg string, symlinkFrom string, out chan string, logskip bool) {
	filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
		if basedir, elem := filepath.Split(path); elem != "" {
			exclude := false
			for _, pattern := range excludePatterns {
				exclude, err = filepath.Match(pattern, elem)
				if err != nil {
					log.Fatal(err)
				}
				if exclude {
					break
				}
			}

			// Skip various temporary or "hidden" files or directories.
			if info != nil && info.IsDir() {
				if exclude {
					if logskip {
						if symlinkFrom != "" {
							log.Printf("%s: skipped. VCS or backup directory", symlinkFrom+path[len(arg):])
						} else {
							log.Printf("%s: skipped. VCS or backup directory", path)
						}
					}
					return filepath.SkipDir
				}
			} else {
				if exclude {
					if logskip {
						if symlinkFrom != "" {
							log.Printf("%s: skipped. Backup or undesirable file", symlinkFrom+path[len(arg):])
						} else {
							log.Printf("%s: skipped. Backup or undesirable file", path)
						}
					}
					return nil
				}
				if info != nil && info.Mode()&os.ModeSymlink != 0 {
					if !*followSymlinksFlag {
						if logskip {
							log.Printf("%s: skipped. Symlink", path)
						}
						return nil
					}
					var symlinkAs string
					if basedir[len(basedir)-1] == os.PathSeparator {
						symlinkAs = basedir + elem
					} else {
						symlinkAs = basedir + string(os.PathSeparator) + elem
					}
					if symlinkFrom != "" {
						symlinkAs = symlinkFrom + symlinkAs[len(arg):]
					}
					if p, err := filepath.EvalSymlinks(symlinkAs); err != nil {
						if symlinkFrom != "" {
							log.Printf("%s: skipped. Symlink could not be resolved", symlinkFrom+path[len(arg):])
						} else {
							log.Printf("%s: skipped. Symlink could not be resolved", path)
						}
					} else {
						walk(p, symlinkAs, out, logskip)
					}
					return nil
				}
			}
		}
		if err != nil {
			if symlinkFrom != "" {
				log.Printf("%s: skipped. Error: %s", symlinkFrom+path[len(arg):], err)
			} else {
				log.Printf("%s: skipped. Error: %s", path, err)
			}
			return nil
		}
		if info != nil {
			if info.Mode()&os.ModeType == 0 {
				if symlinkFrom == "" {
					out <- path
				} else {
					out <- symlinkFrom + path[len(arg):]
				}
			} else if !info.IsDir() {
				if logskip {
					if symlinkFrom != "" {
						log.Printf("%s: skipped. Unsupported path type", symlinkFrom+path[len(arg):])
					} else {
						log.Printf("%s: skipped. Unsupported path type", path)
					}
				}
			}
		} else {
			if logskip {
				if symlinkFrom != "" {
					log.Printf("%s: skipped. Could not stat.", symlinkFrom+path[len(arg):])
				} else {
					log.Printf("%s: skipped. Could not stat.", path)
				}
			}
		}
		return nil
	})
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if *indexPath != "" {
		if err := os.Setenv("CSEARCHINDEX", *indexPath); err != nil {
			log.Fatal(err)
		}
	}

	if *listFlag {
		master := index.File()
		if stat, err := os.Stat(master); err != nil || stat == nil {
			log.Fatal("Index " + master + " is not accessible")
		} else if stat.IsDir() || !stat.Mode().IsRegular() {
			log.Fatal("Index " + master + " must point to an index file")
		}
		ix := index.Open(master)
		for _, arg := range ix.Paths() {
			fmt.Printf("%s\n", arg)
		}
		return
	}

	if *cpuProfile != "" {
		f, err := os.Create(*cpuProfile)
		if err != nil {
			log.Fatal(err)
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *resetFlag && len(args) == 0 {
		master := index.File()
		stat, err := os.Stat(master)
		if err != nil {
			// does not exist so nothing to do
			return
		}
		if stat != nil && !stat.IsDir() && stat.Mode().IsRegular() {
			os.Remove(master)
			return
		} else {
			log.Fatal("Invalid index path " + master)
		}
	}

	if *exclude != "" {
		var excludePath string
		if (*exclude)[:2] == "~/" {
			usr, err := user.Current()
			if err != nil {
				log.Fatal(err)
			}
			excludePath = filepath.Join(usr.HomeDir, (*exclude)[2:])
		} else {
			excludePath = *exclude
		}
		if *logSkipFlag {
			log.Printf("Loading exclude patterns from %s", excludePath)
		}
		data, err := ioutil.ReadFile(excludePath)
		if err != nil {
			log.Fatal(err)
		}
		excludePatterns = append(excludePatterns, strings.Split(string(data), "\n")...)
		for i, pattern := range excludePatterns {
			excludePatterns[i] = strings.TrimSpace(pattern)
		}
	}

	if len(args) == 0 {
		ix := index.Open(index.File())
		for _, arg := range ix.Paths() {
			args = append(args, arg)
		}
		ix.Close()
	}

	// Translate paths to absolute paths so that we can
	// generate the file list in sorted order.
	for i, arg := range args {
		a, err := filepath.Abs(arg)
		if err != nil {
			log.Printf("%s: %s", arg, err)
			args[i] = ""
			continue
		}
		args[i] = a
	}
	sort.Strings(args)

	for len(args) > 0 && args[0] == "" {
		args = args[1:]
	}

	master := index.File()
	if stat, err := os.Stat(master); err != nil {
		// Does not exist.
		*resetFlag = true
	} else {
		if stat != nil && (stat.IsDir() || !stat.Mode().IsRegular()) {
			log.Fatal("Invalid index path " + master)
		}

	}
	file := master
	if !*resetFlag {
		file += "~"
	}

	ix := index.Create(file)
	ix.Verbose = *verboseFlag
	ix.LogSkip = *logSkipFlag
	ix.MaxFileLen = *maxFileLen
	ix.MaxLineLen = *maxLineLen
	ix.MaxTextTrigrams = *maxTextTrigrams
	ix.MaxInvalidUTF8Ratio = *maxInvalidUTF8Ratio
	ix.AddPaths(args)

	walkChan := make(chan string)
	doneChan := make(chan bool)

	go func() {
		seen := make(map[string]bool)
		for {
			select {
			case path := <-walkChan:
				if !seen[path] {
					seen[path] = true
					ix.AddFile(path)
				}
			case <-doneChan:
				return
			}
		}
	}()
	for _, arg := range args {
		log.Printf("index %s", arg)
		walk(arg, "", walkChan, *logSkipFlag)
	}
	doneChan <- true
	log.Printf("flush index")
	ix.Flush()

	if !*resetFlag {
		log.Printf("merge %s %s", master, file)
		index.Merge(file+"~", master, file)
		os.Remove(file)
		os.Remove(master)
		if err := os.Rename(file+"~", master); err != nil {
			log.Fatalf("failed to merge indexes: %s", err)
		}
	}
	log.Printf("done")
	return
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @authors
 * 	Jeffrey Wilcke <i@jev.io>
 */
package main

import (
	"bufio"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/ethereum/go-ethereum/cmd/utils"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/eth"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
	"github.com/peterh/liner"
)

const (
	ClientIdentifier = "Ethereum(G)"
	Version          = "0.9.0"
)

var (
	clilogger = logger.NewLogger("CLI")
	app       = utils.NewApp(Version, "the go-ethereum command line interface")
)

func init() {
	app.Action = run
	app.HideVersion = true // we have a command to print the version
	app.Commands = []cli.Command{
		{
			Action: version,
			Name:   "version",
			Usage:  "print ethereum version numbers",
			Description: `
The output of this command is supposed to be machine-readable.
`,
		},
		{
			Action: accountList,
			Name:   "account",
			Usage:  "manage accounts",
			Subcommands: []cli.Command{
				{
					Action: accountList,
					Name:   "list",
					Usage:  "print account addresses",
				},
				{
					Action: accountCreate,
					Name:   "new",
					Usage:  "create a new account",
				},
			},
		},
		{
			Action: dump,
			Name:   "dump",
			Usage:  `dump a specific block from storage`,
			Description: `
The arguments are interpreted as block numbers or hashes.
Use "ethereum dump 0" to dump the genesis block.
`,
		},
		{
			Action: runjs,
			Name:   "js",
			Usage:  `interactive JavaScript console`,
			Description: `
In the console, you can use the eth object to interact
with the running ethereum stack. The API does not match
ethereum.js.

A JavaScript file can be provided as the argument. The
runtime will execute the file and exit.
`,
		},
		{
			Action: importchain,
			Name:   "import",
			Usage:  `import a blockchain file`,
		},
		{
			Action: exportchain,
			Name:   "export",
			Usage:  `export blockchain into file`,
		},
	}
	app.Flags = []cli.Flag{
		utils.UnlockedAccountFlag,
		utils.BootnodesFlag,
		utils.DataDirFlag,
		utils.ListenPortFlag,
		utils.LogFileFlag,
		utils.LogFormatFlag,
		utils.LogLevelFlag,
		utils.MaxPeersFlag,
		utils.MinerThreadsFlag,
		utils.MiningEnabledFlag,
		utils.NATFlag,
		utils.NodeKeyFileFlag,
		utils.NodeKeyHexFlag,
		utils.RPCEnabledFlag,
		utils.RPCListenAddrFlag,
		utils.RPCPortFlag,
		utils.UnencryptedKeysFlag,
		utils.VMDebugFlag,
		//utils.VMTypeFlag,
	}

	// missing:
	// flag.StringVar(&ConfigFile, "conf", defaultConfigFile, "config file")
	// flag.BoolVar(&DiffTool, "difftool", false, "creates output for diff'ing. Sets LogLevel=0")
	// flag.StringVar(&DiffType, "diff", "all", "sets the level of diff output [vm, all]. Has no effect if difftool=false")

	// potential subcommands:
	// flag.StringVar(&SecretFile, "import", "", "imports the file given (hex or mnemonic formats)")
	// flag.StringVar(&ExportDir, "export", "", "exports the session keyring to files in the directory given")
	// flag.BoolVar(&GenAddr, "genaddr", false, "create a new priv/pub key")
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	defer logger.Flush()
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(ctx *cli.Context) {
	fmt.Printf("Welcome to the FRONTIER\n")
	utils.HandleInterrupt()
	eth, err := utils.GetEthereum(ClientIdentifier, Version, ctx)
	if err != nil {
		utils.Fatalf("%v", err)
	}

	startEth(ctx, eth)
	// this blocks the thread
	eth.WaitForShutdown()
}

func runjs(ctx *cli.Context) {
	eth, err := utils.GetEthereum(ClientIdentifier, Version, ctx)
	if err != nil {
		utils.Fatalf("%v", err)
	}

	startEth(ctx, eth)
	repl := newJSRE(eth)
	if len(ctx.Args()) == 0 {
		repl.interactive()
	} else {
		for _, file := range ctx.Args() {
			repl.exec(file)
		}
	}
	eth.Stop()
	eth.WaitForShutdown()
}

func startEth(ctx *cli.Context, eth *eth.Ethereum) {
	utils.StartEthereum(eth)

	// Load startup keys. XXX we are going to need a different format
	account := ctx.GlobalString(utils.UnlockedAccountFlag.Name)
	if len(account) > 0 {
		split := strings.Split(account, ":")
		if len(split) != 2 {
			utils.Fatalf("Illegal 'unlock' format (address:password)")
		}
		am := eth.AccountManager()
		// Attempt to unlock the account
		err := am.Unlock(ethutil.Hex2Bytes(split[0]), split[1])
		if err != nil {
			utils.Fatalf("Unlock account failed '%v'", err)
		}
	}
	// Start auxiliary services if enabled.
	if ctx.GlobalBool(utils.RPCEnabledFlag.Name) {
		utils.StartRPC(eth, ctx)
	}
	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) {
		eth.StartMining()
	}
}

func accountList(ctx *cli.Context) {
	am := utils.GetAccountManager(ctx)
	accts, err := am.Accounts()
	if err != nil {
		utils.Fatalf("Could not list accounts: %v", err)
	}
	for _, acct := range accts {
		fmt.Printf("Address: %#x\n", acct)
	}
}

func accountCreate(ctx *cli.Context) {
	am := utils.GetAccountManager(ctx)
	passphrase := ""
	if !ctx.GlobalBool(utils.UnencryptedKeysFlag.Name) {
		fmt.Println("The new account will be encrypted with a passphrase.")
		fmt.Println("Please enter a passphrase now.")
		auth, err := readPassword("Passphrase: ", true)
		if err != nil {
			utils.Fatalf("%v", err)
		}
		confirm, err := readPassword("Repeat Passphrase: ", false)
		if err != nil {
			utils.Fatalf("%v", err)
		}
		if auth != confirm {
			utils.Fatalf("Passphrases did not match.")
		}
		passphrase = auth
	}
	acct, err := am.NewAccount(passphrase)
	if err != nil {
		utils.Fatalf("Could not create the account: %v", err)
	}
	fmt.Printf("Address: %#x\n", acct.Address)
}

func importchain(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		utils.Fatalf("This command requires an argument.")
	}
	chainmgr, _, _ := utils.GetChain(ctx)
	start := time.Now()
	err := utils.ImportChain(chainmgr, ctx.Args().First())
	if err != nil {
		utils.Fatalf("Import error: %v\n", err)
	}
	fmt.Printf("Import done in %v", time.Since(start))
	return
}

func exportchain(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		utils.Fatalf("This command requires an argument.")
	}
	chainmgr, _, _ := utils.GetChain(ctx)
	start := time.Now()
	err := utils.ExportChain(chainmgr, ctx.Args().First())
	if err != nil {
		utils.Fatalf("Export error: %v\n", err)
	}
	fmt.Printf("Export done in %v", time.Since(start))
	return
}

func dump(ctx *cli.Context) {
	chainmgr, _, stateDb := utils.GetChain(ctx)
	for _, arg := range ctx.Args() {
		var block *types.Block
		if hashish(arg) {
			block = chainmgr.GetBlock(ethutil.Hex2Bytes(arg))
		} else {
			num, _ := strconv.Atoi(arg)
			block = chainmgr.GetBlockByNumber(uint64(num))
		}
		if block == nil {
			fmt.Println("{}")
			utils.Fatalf("block not found")
		} else {
			statedb := state.New(block.Root(), stateDb)
			fmt.Printf("%s\n", statedb.Dump())
			// fmt.Println(block)
		}
	}
}

func version(c *cli.Context) {
	fmt.Printf(`%v
Version: %v
Protocol Version: %d
Network Id: %d
GO: %s
OS: %s
GOPATH=%s
GOROOT=%s
`, ClientIdentifier, Version, eth.ProtocolVersion, eth.NetworkId, runtime.Version(), runtime.GOOS, os.Getenv("GOPATH"), runtime.GOROOT())
}

// hashish returns true for strings that look like hashes.
func hashish(x string) bool {
	_, err := strconv.Atoi(x)
	return err != nil
}

func readPassword(prompt string, warnTerm bool) (string, error) {
	if liner.TerminalSupported() {
		lr := liner.NewLiner()
		defer lr.Close()
		return lr.PasswordPrompt(prompt)
	}
	if warnTerm {
		fmt.Println("!! Unsupported terminal, password will be echoed.")
	}
	fmt.Print(prompt)
	input, err := bufio.NewReader(os.Stdin).ReadString('\n')
	fmt.Println()
	return input, err
}
<commit_msg>Support for addresses with or without 0x. Fixes #472<commit_after>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
/**
 * @authors
 * 	Jeffrey Wilcke <i@jev.io>
 */
package main

import (
	"bufio"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"github.com/ethereum/go-ethereum/cmd/utils"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/eth"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/state"
	"github.com/peterh/liner"
)

const (
	ClientIdentifier = "Ethereum(G)"
	Version          = "0.9.0"
)

var (
	clilogger = logger.NewLogger("CLI")
	app       = utils.NewApp(Version, "the go-ethereum command line interface")
)

func init() {
	app.Action = run
	app.HideVersion = true // we have a command to print the version
	app.Commands = []cli.Command{
		{
			Action: version,
			Name:   "version",
			Usage:  "print ethereum version numbers",
			Description: `
The output of this command is supposed to be machine-readable.
`,
		},
		{
			Action: accountList,
			Name:   "account",
			Usage:  "manage accounts",
			Subcommands: []cli.Command{
				{
					Action: accountList,
					Name:   "list",
					Usage:  "print account addresses",
				},
				{
					Action: accountCreate,
					Name:   "new",
					Usage:  "create a new account",
				},
			},
		},
		{
			Action: dump,
			Name:   "dump",
			Usage:  `dump a specific block from storage`,
			Description: `
The arguments are interpreted as block numbers or hashes.
Use "ethereum dump 0" to dump the genesis block.
`,
		},
		{
			Action: runjs,
			Name:   "js",
			Usage:  `interactive JavaScript console`,
			Description: `
In the console, you can use the eth object to interact
with the running ethereum stack. The API does not match
ethereum.js.

A JavaScript file can be provided as the argument. The
runtime will execute the file and exit.
`,
		},
		{
			Action: importchain,
			Name:   "import",
			Usage:  `import a blockchain file`,
		},
		{
			Action: exportchain,
			Name:   "export",
			Usage:  `export blockchain into file`,
		},
	}
	app.Flags = []cli.Flag{
		utils.UnlockedAccountFlag,
		utils.BootnodesFlag,
		utils.DataDirFlag,
		utils.ListenPortFlag,
		utils.LogFileFlag,
		utils.LogFormatFlag,
		utils.LogLevelFlag,
		utils.MaxPeersFlag,
		utils.MinerThreadsFlag,
		utils.MiningEnabledFlag,
		utils.NATFlag,
		utils.NodeKeyFileFlag,
		utils.NodeKeyHexFlag,
		utils.RPCEnabledFlag,
		utils.RPCListenAddrFlag,
		utils.RPCPortFlag,
		utils.UnencryptedKeysFlag,
		utils.VMDebugFlag,
		//utils.VMTypeFlag,
	}

	// missing:
	// flag.StringVar(&ConfigFile, "conf", defaultConfigFile, "config file")
	// flag.BoolVar(&DiffTool, "difftool", false, "creates output for diff'ing. Sets LogLevel=0")
	// flag.StringVar(&DiffType, "diff", "all", "sets the level of diff output [vm, all]. Has no effect if difftool=false")

	// potential subcommands:
	// flag.StringVar(&SecretFile, "import", "", "imports the file given (hex or mnemonic formats)")
	// flag.StringVar(&ExportDir, "export", "", "exports the session keyring to files in the directory given")
	// flag.BoolVar(&GenAddr, "genaddr", false, "create a new priv/pub key")
}

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	defer logger.Flush()
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

func run(ctx *cli.Context) {
	fmt.Printf("Welcome to the FRONTIER\n")
	utils.HandleInterrupt()
	eth, err := utils.GetEthereum(ClientIdentifier, Version, ctx)
	if err != nil {
		utils.Fatalf("%v", err)
	}

	startEth(ctx, eth)
	// this blocks the thread
	eth.WaitForShutdown()
}

func runjs(ctx *cli.Context) {
	eth, err := utils.GetEthereum(ClientIdentifier, Version, ctx)
	if err != nil {
		utils.Fatalf("%v", err)
	}

	startEth(ctx, eth)
	repl := newJSRE(eth)
	if len(ctx.Args()) == 0 {
		repl.interactive()
	} else {
		for _, file := range ctx.Args() {
			repl.exec(file)
		}
	}
	eth.Stop()
	eth.WaitForShutdown()
}

func startEth(ctx *cli.Context, eth *eth.Ethereum) {
	utils.StartEthereum(eth)

	// Load startup keys. XXX we are going to need a different format
	account := ctx.GlobalString(utils.UnlockedAccountFlag.Name)
	if len(account) > 0 {
		split := strings.Split(account, ":")
		if len(split) != 2 {
			utils.Fatalf("Illegal 'unlock' format (address:password)")
		}
		am := eth.AccountManager()
		// Attempt to unlock the account
		err := am.Unlock(ethutil.FromHex(split[0]), split[1])
		if err != nil {
			utils.Fatalf("Unlock account failed '%v'", err)
		}
	}
	// Start auxiliary services if enabled.
	if ctx.GlobalBool(utils.RPCEnabledFlag.Name) {
		utils.StartRPC(eth, ctx)
	}
	if ctx.GlobalBool(utils.MiningEnabledFlag.Name) {
		eth.StartMining()
	}
}

func accountList(ctx *cli.Context) {
	am := utils.GetAccountManager(ctx)
	accts, err := am.Accounts()
	if err != nil {
		utils.Fatalf("Could not list accounts: %v", err)
	}
	for _, acct := range accts {
		fmt.Printf("Address: %#x\n", acct)
	}
}

func accountCreate(ctx *cli.Context) {
	am := utils.GetAccountManager(ctx)
	passphrase := ""
	if !ctx.GlobalBool(utils.UnencryptedKeysFlag.Name) {
		fmt.Println("The new account will be encrypted with a passphrase.")
		fmt.Println("Please enter a passphrase now.")
		auth, err := readPassword("Passphrase: ", true)
		if err != nil {
			utils.Fatalf("%v", err)
		}
		confirm, err := readPassword("Repeat Passphrase: ", false)
		if err != nil {
			utils.Fatalf("%v", err)
		}
		if auth != confirm {
			utils.Fatalf("Passphrases did not match.")
		}
		passphrase = auth
	}
	acct, err := am.NewAccount(passphrase)
	if err != nil {
		utils.Fatalf("Could not create the account: %v", err)
	}
	fmt.Printf("Address: %#x\n", acct.Address)
}

func importchain(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		utils.Fatalf("This command requires an argument.")
	}
	chainmgr, _, _ := utils.GetChain(ctx)
	start := time.Now()
	err := utils.ImportChain(chainmgr, ctx.Args().First())
	if err != nil {
		utils.Fatalf("Import error: %v\n", err)
	}
	fmt.Printf("Import done in %v", time.Since(start))
	return
}

func exportchain(ctx *cli.Context) {
	if len(ctx.Args()) != 1 {
		utils.Fatalf("This command requires an argument.")
	}
	chainmgr, _, _ := utils.GetChain(ctx)
	start := time.Now()
	err := utils.ExportChain(chainmgr, ctx.Args().First())
	if err != nil {
		utils.Fatalf("Export error: %v\n", err)
	}
	fmt.Printf("Export done in %v", time.Since(start))
	return
}

func dump(ctx *cli.Context) {
	chainmgr, _, stateDb := utils.GetChain(ctx)
	for _, arg := range ctx.Args() {
		var block *types.Block
		if hashish(arg) {
			block = chainmgr.GetBlock(ethutil.Hex2Bytes(arg))
		} else {
			num, _ := strconv.Atoi(arg)
			block = chainmgr.GetBlockByNumber(uint64(num))
		}
		if block == nil {
			fmt.Println("{}")
			utils.Fatalf("block not found")
		} else {
			statedb := state.New(block.Root(), stateDb)
			fmt.Printf("%s\n", statedb.Dump())
			// fmt.Println(block)
		}
	}
}

func version(c *cli.Context) {
	fmt.Printf(`%v
Version: %v
Protocol Version: %d
Network Id: %d
GO: %s
OS: %s
GOPATH=%s
GOROOT=%s
`, ClientIdentifier, Version, eth.ProtocolVersion, eth.NetworkId, runtime.Version(), runtime.GOOS, os.Getenv("GOPATH"), runtime.GOROOT())
}

// hashish returns true for strings that look like hashes.
func hashish(x string) bool {
	_, err := strconv.Atoi(x)
	return err != nil
}

func readPassword(prompt string, warnTerm bool) (string, error) {
	if liner.TerminalSupported() {
		lr := liner.NewLiner()
		defer lr.Close()
		return lr.PasswordPrompt(prompt)
	}
	if warnTerm {
		fmt.Println("!! Unsupported terminal, password will be echoed.")
	}
	fmt.Print(prompt)
	input, err := bufio.NewReader(os.Stdin).ReadString('\n')
	fmt.Println()
	return input, err
}
<|endoftext|>


<commit_before>// groot-ls recursively dumps the hierarchy tree of a ROOT file
package main

import (
	"flag"
	"fmt"
	//"log"
	"os"
	//"runtime/pprof"
	"strings"

	"github.com/sbinet/go-root/pkg/groot"
)

var fname = flag.String("f", "", "ROOT file to inspect")
var detailed = flag.Bool("detailed", false, "enable detailed dump (of trees)")
//var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

var s_tee = "|--"
var s_bot = "`--"

func normpath(path []string) string {
	name := strings.Join(path, "/")
	if len(name) > 2 && name[:2] == "//" {
		name = name[1:]
	}
	return name
}

func inspect(dir *groot.Directory, path []string, indent string) {
	name := normpath(path)
	if dir == nil {
		fmt.Printf("err: invalid directory [%s]\n", name)
		return
	}
	keys := dir.Keys()
	nkeys := len(keys)
	str := s_tee
	//fmt.Printf("%s%s -> #%d key(s)\n", indent, name, len(keys))
	for i, k := range keys {
		if i+1 >= nkeys {
			str = s_bot
		}
		switch v := k.Value().(type) {
		default:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str, k.Name(), k.Title(), k.Class())

		case *groot.Directory:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str,
				k.Name(), k.Title(), k.Class())
			path := append(path, k.Name())
			inspect(v, path, indent+"    ")

		case *groot.Tree:
			nbranches := len(v.Branches())
			fmt.Printf("%s%s %s title='%s' entries=%v nbranches=%v type=%s\n",
				indent, str,
				k.Name(), k.Title(), v.Entries(), nbranches, k.Class())
			if *detailed {
				strbr := s_tee
				for i,branch := range v.Branches() {
					if i+1 >= nbranches {
						strbr = s_bot
					}
					fmt.Printf(" %s%s%s %s type=%s\n",
						indent, "   ", strbr, branch.Name(), branch.Class())
				}
			}
		}
	}
}

func main() {
	fmt.Printf(":: groot-ls ::\n")
	flag.Parse()

	// if *cpuprofile != "" {
    //     f, err := os.Create(*cpuprofile)
    //     if err != nil {
    //         log.Fatal(err)
    //     }
    //     pprof.StartCPUProfile(f)
    //     defer pprof.StopCPUProfile()
    // }

	if *fname == "" {
		fmt.Printf("**error** you have to give a (valid) path to a ROOT file\n")
		os.Exit(1)
	}

	f, err := groot.NewFileReader(*fname)
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	if f == nil {
		fmt.Printf("**error** invalid file pointer\n")
		os.Exit(1)
	}

	fmt.Printf("file: '%s' (version=%v)\n", f.Name(), f.Version())

	dir := f.Dir()
	inspect(dir, []string{"/"}, "")

	fmt.Printf("::bye.\n")
}

// EOF
<commit_msg>gofmt<commit_after>// groot-ls recursively dumps the hierarchy tree of a ROOT file
package main

import (
	"flag"
	"fmt"
	//"log"
	"os"
	//"runtime/pprof"
	"strings"

	"github.com/sbinet/go-root/pkg/groot"
)

var fname = flag.String("f", "", "ROOT file to inspect")
var detailed = flag.Bool("detailed", false, "enable detailed dump (of trees)")

//var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")

var s_tee = "|--"
var s_bot = "`--"

func normpath(path []string) string {
	name := strings.Join(path, "/")
	if len(name) > 2 && name[:2] == "//" {
		name = name[1:]
	}
	return name
}

func inspect(dir *groot.Directory, path []string, indent string) {
	name := normpath(path)
	if dir == nil {
		fmt.Printf("err: invalid directory [%s]\n", name)
		return
	}
	keys := dir.Keys()
	nkeys := len(keys)
	str := s_tee
	//fmt.Printf("%s%s -> #%d key(s)\n", indent, name, len(keys))
	for i, k := range keys {
		if i+1 >= nkeys {
			str = s_bot
		}
		switch v := k.Value().(type) {
		default:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str, k.Name(), k.Title(), k.Class())

		case *groot.Directory:
			fmt.Printf("%s%s %s title='%s' type=%s\n",
				indent, str,
				k.Name(), k.Title(), k.Class())
			path := append(path, k.Name())
			inspect(v, path, indent+"    ")

		case *groot.Tree:
			nbranches := len(v.Branches())
			fmt.Printf("%s%s %s title='%s' entries=%v nbranches=%v type=%s\n",
				indent, str,
				k.Name(), k.Title(), v.Entries(), nbranches, k.Class())
			if *detailed {
				strbr := s_tee
				for i, branch := range v.Branches() {
					if i+1 >= nbranches {
						strbr = s_bot
					}
					fmt.Printf(" %s%s%s %s type=%s\n",
						indent, "   ", strbr, branch.Name(), branch.Class())
				}
			}
		}
	}
}

func main() {
	fmt.Printf(":: groot-ls ::\n")
	flag.Parse()

	// if *cpuprofile != "" {
	//     f, err := os.Create(*cpuprofile)
	//     if err != nil {
	//         log.Fatal(err)
	//     }
	//     pprof.StartCPUProfile(f)
	//     defer pprof.StopCPUProfile()
	// }

	if *fname == "" {
		fmt.Printf("**error** you have to give a (valid) path to a ROOT file\n")
		os.Exit(1)
	}

	f, err := groot.NewFileReader(*fname)
	if err != nil {
		fmt.Printf("**error** %v\n", err)
		os.Exit(1)
	}

	if f == nil {
		fmt.Printf("**error** invalid file pointer\n")
		os.Exit(1)
	}

	fmt.Printf("file: '%s' (version=%v)\n", f.Name(), f.Version())

	dir := f.Dir()
	inspect(dir, []string{"/"}, "")

	fmt.Printf("::bye.\n")
}

// EOF
<|endoftext|>